diff --git a/.github/workflows/aurora_performance.yml b/.github/workflows/aurora_performance.yml index 72ebb9317..5d0bd0825 100644 --- a/.github/workflows/aurora_performance.yml +++ b/.github/workflows/aurora_performance.yml @@ -18,25 +18,25 @@ jobs: steps: - name: Clone repository - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 50 - name: "Set up JDK 8" - uses: actions/setup-java@v3 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Install dependencies run: npm install --no-save - name: Configure AWS Credentials id: creds - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} role-session-name: nodejs_aurora_perf_tests @@ -59,7 +59,7 @@ jobs: - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@v1.3 + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - name: "Remove Github Action IP" if: always() @@ -73,7 +73,7 @@ jobs: - name: Archive Performance results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.db }}-performance-results path: ./tests/integration/container/reports diff --git a/.github/workflows/dependabot_auto_merge.yml b/.github/workflows/dependabot_auto_merge.yml index c7574baba..536f793be 100644 --- a/.github/workflows/dependabot_auto_merge.yml +++ b/.github/workflows/dependabot_auto_merge.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1 + uses: dependabot/fetch-metadata@25dd0e34f4fe68f24cc83900b1fe3fe149efef98 # v2 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Approve PR diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index db65f2f6b..2218d9eac 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - chore/remove-aliases paths-ignore: - '**/*.md' - '**/*.jpg' @@ -48,8 +49,8 @@ jobs: environment: integration-tests steps: - run: echo "Fork PR approved by maintainer" - run-integration-tests: - name: Run Integration Tests + run-integration-tests-default: + name: Run Integration Tests (Default) needs: [ approve, approve-fork ] if: | always() && @@ -59,29 +60,28 @@ jobs: strategy: fail-fast: false matrix: - versions: [ "default", "latest" ] - dbEngine: ["aurora-mysql", "aurora-postgres", "multi-az-mysql", "multi-az-postgres" ] + dbEngine: ["aurora-mysql", "aurora-postgres"] steps: - name: 'Clone repository' - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: "Set up JDK 8" - uses: actions/setup-java@v3 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Install dependencies run: npm install --no-save - name: Configure AWS Credentials id: creds - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} role-session-name: nodejs_int_default_tests @@ -97,13 +97,13 @@ jobs: AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} - AURORA_MYSQL_DB_ENGINE_VERSION: ${{ matrix.dbEngine }} - AURORA_PG_DB_ENGINE_VERSION: ${{ matrix.versions }} + AURORA_MYSQL_DB_ENGINE_VERSION: default + AURORA_PG_DB_ENGINE_VERSION: default - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@v1.3 + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - name: "Remove Github Action IP" if: always() @@ -117,8 +117,79 @@ jobs: - name: Archive results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: integration-report-default-${{ matrix.dbEngine }}-${{ matrix.versions}} + name: integration-report-default-${{ matrix.dbEngine }} + path: ./tests/integration/container/reports + retention-days: 5 + + run-integration-tests-latest: + name: Run Integration Tests (Latest) + runs-on: ubuntu-latest + needs: run-integration-tests-default + if: | + always() && + needs.run-integration-tests-default.result == 'success' + strategy: + fail-fast: false + matrix: + dbEngine: ["aurora-mysql", "aurora-postgres" ] + + steps: + - name: Clone repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: "Set up JDK 8" + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + distribution: "corretto" + java-version: 8 + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "22.x" + - name: Install dependencies + run: npm install --no-save + + - name: Configure AWS Credentials + id: creds + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: nodejs_int_latest_tests + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + output-credentials: true + + - name: Run Integration Tests + run: | + ./gradlew --no-parallel --no-daemon test-${{ matrix.dbEngine }} --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} + AURORA_MYSQL_DB_ENGINE_VERSION: latest + AURORA_PG_DB_ENGINE_VERSION: latest + + - name: "Get Github Action IP" + if: always() + id: ip + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 + + - name: "Remove Github Action IP" + if: always() + run: | + aws ec2 revoke-security-group-ingress \ + --group-name default \ + --protocol -1 \ + --port -1 \ + --cidr ${{ steps.ip.outputs.ipv4 }}/32 \ + 2>&1 > /dev/null; + + - name: Archive results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: integration-report-latest-${{ matrix.dbEngine }} path: ./tests/integration/container/reports retention-days: 5 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02c8389b9..7ba7b23ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,6 +7,7 @@ on: pull_request: branches: - "*" + - dev/v3 permissions: contents: read @@ -15,11 +16,11 @@ jobs: run-checks-and-unit-tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Install dependencies run: npm install --no-save - name: Run eslint - linting diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aedac0853..dcd54a9a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,11 +11,11 @@ jobs: packages: write steps: - name: "Clone Repository" - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: "Set up Node.js" - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" registry-url: "https://registry.npmjs.org" - name: "Install dependencies" run: npm install --no-save diff --git a/.github/workflows/release_draft.yml b/.github/workflows/release_draft.yml index fb2c2207a..85973b38a 100644 --- a/.github/workflows/release_draft.yml +++ b/.github/workflows/release_draft.yml @@ -19,11 +19,11 @@ jobs: runs-on: ubuntu-latest steps: - name: "Clone Repository" - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: "Set up Node.js" - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: "Install dependencies" run: npm install --no-save - name: "Run eslint - linting" @@ -46,7 +46,7 @@ jobs: touch RELEASE_DETAILS.md echo "$RELEASE_DETAILS" > RELEASE_DETAILS.md - name: "Upload to Draft Release" - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd # v1 with: draft: true name: "AWS Advanced NodeJS Wrapper - v${{ env.RELEASE_VERSION }}" diff --git a/.github/workflows/remove-old-artifacts.yml b/.github/workflows/remove-old-artifacts.yml index 5899e8671..71d497c8d 100644 --- a/.github/workflows/remove-old-artifacts.yml +++ b/.github/workflows/remove-old-artifacts.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Remove Old Artifacts - uses: c-hive/gha-remove-artifacts@v1 + uses: c-hive/gha-remove-artifacts@44fc7acaf1b3d0987da0e8d4707a989d80e9554b # v1 with: age: "1 week" skip-tags: true diff --git a/.github/workflows/run-autoscaling-tests.yml b/.github/workflows/run-autoscaling-tests.yml index 32d243db2..0c65d3aec 100644 --- a/.github/workflows/run-autoscaling-tests.yml +++ b/.github/workflows/run-autoscaling-tests.yml @@ -21,21 +21,21 @@ jobs: dbEngine: [ "mysql", "postgres" ] steps: - name: Clone repository - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 50 - name: "Set up JDK 8" - uses: actions/setup-java@v3 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Configure AWS credentials id: creds - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} role-session-name: nodejs_autoscaling_tests @@ -56,7 +56,7 @@ jobs: - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@v1.3 + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - name: "Remove Github Action IP" if: always() run: | @@ -68,7 +68,7 @@ jobs: 2>&1 > /dev/null; - name: Archive results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: autoscaling-report path: ./tests/integration/container/reports diff --git a/common/lib/authentication/aws_secrets_manager_plugin.ts b/common/lib/authentication/aws_secrets_manager_plugin.ts index f8e06e174..631b9f068 100644 --- a/common/lib/authentication/aws_secrets_manager_plugin.ts +++ b/common/lib/authentication/aws_secrets_manager_plugin.ts @@ -37,7 +37,7 @@ export class AwsSecretsManagerPlugin extends AbstractConnectionPlugin implements private static readonly TELEMETRY_UPDATE_SECRETS = "fetch credentials"; private static readonly TELEMETRY_FETCH_CREDENTIALS_COUNTER = "secretsManager.fetchCredentials.count"; private static SUBSCRIBED_METHODS: Set = new Set(["connect", "forceConnect"]); - private static SECRETS_ARN_PATTERN: RegExp = new RegExp("^arn:aws:secretsmanager:(?[^:\\n]*):[^:\\n]*:([^:/\\n]*[:/])?(.*)$"); + private static SECRETS_ARN_PATTERN: RegExp = new RegExp("^arn:aws[^:]*:secretsmanager:(?[^:\\n]*):[^:\\n]*:([^:/\\n]*[:/])?(.*)$"); private readonly pluginService: PluginService; private readonly fetchCredentialsCounter; private readonly expirationSec: number; @@ -129,20 +129,18 @@ export class AwsSecretsManagerPlugin extends AbstractConnectionPlugin implements this.pluginService.updateConfigWithProperties(props); return await connectFunc(); } catch (error) { - if (error instanceof Error) { - if ((error.message.includes("password authentication failed") || error.message.includes("Access denied")) && !secretWasFetched) { - // Login unsuccessful with cached credentials - // Try to re-fetch credentials and try again - - secretWasFetched = await this.updateSecret(true); - if (secretWasFetched) { - WrapperProperties.USER.set(props, this.secret?.username ?? ""); - WrapperProperties.PASSWORD.set(props, this.secret?.password ?? ""); - return await connectFunc(); - } + if ((error.message.includes("password authentication failed") || error.message.includes("Access denied")) && !secretWasFetched) { + // Login unsuccessful with cached credentials + // Try to re-fetch credentials and try again + + secretWasFetched = await this.updateSecret(true); + if (secretWasFetched) { + WrapperProperties.USER.set(props, this.secret?.username ?? ""); + WrapperProperties.PASSWORD.set(props, this.secret?.password ?? ""); + return await connectFunc(); } - logger.debug(Messages.get("AwsSecretsManagerConnectionPlugin.unhandledError", error.name, error.message)); } + logger.debug(Messages.get("AwsSecretsManagerConnectionPlugin.unhandledError", error.name, error.message)); throw error; } } diff --git a/common/lib/authentication/aws_secrets_manager_plugin_factory.ts b/common/lib/authentication/aws_secrets_manager_plugin_factory.ts index eb0ad30d3..6b3242cd1 100644 --- a/common/lib/authentication/aws_secrets_manager_plugin_factory.ts +++ b/common/lib/authentication/aws_secrets_manager_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; import { ConnectionPlugin } from "../connection_plugin"; import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; +import { FullServicesContainer } from "../utils/full_services_container"; export class AwsSecretsManagerPluginFactory extends ConnectionPluginFactory { private static awsSecretsManagerPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!AwsSecretsManagerPluginFactory.awsSecretsManagerPlugin) { AwsSecretsManagerPluginFactory.awsSecretsManagerPlugin = await import("./aws_secrets_manager_plugin"); } - return new AwsSecretsManagerPluginFactory.awsSecretsManagerPlugin.AwsSecretsManagerPlugin(pluginService, new Map(properties)); + return new AwsSecretsManagerPluginFactory.awsSecretsManagerPlugin.AwsSecretsManagerPlugin(servicesContainer.pluginService, new Map(properties)); } catch (error: any) { if (error.code === "MODULE_NOT_FOUND") { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "AwsSecretsManagerPlugin")); diff --git a/common/lib/authentication/iam_authentication_plugin.ts b/common/lib/authentication/iam_authentication_plugin.ts index d3a128a0e..95da8a8bc 100644 --- a/common/lib/authentication/iam_authentication_plugin.ts +++ b/common/lib/authentication/iam_authentication_plugin.ts @@ -26,19 +26,26 @@ import { IamAuthUtils, TokenInfo } from "../utils/iam_auth_utils"; import { ClientWrapper } from "../client_wrapper"; import { RegionUtils } from "../utils/region_utils"; import { CanReleaseResources } from "../can_release_resources"; +import { RdsUrlType } from "../utils/rds_url_type"; +import { RdsUtils } from "../utils/rds_utils"; +import { GDBRegionUtils } from "../utils/gdb_region_utils"; export class IamAuthenticationPlugin extends AbstractConnectionPlugin implements CanReleaseResources { private static readonly SUBSCRIBED_METHODS = new Set(["connect", "forceConnect"]); protected static readonly tokenCache = new Map(); private readonly telemetryFactory; private readonly fetchTokenCounter; - private pluginService: PluginService; + private readonly pluginService: PluginService; + private readonly rdsUtils: RdsUtils = new RdsUtils(); + protected regionUtils: RegionUtils; + protected readonly iamAuthUtils: IamAuthUtils; - constructor(pluginService: PluginService) { + constructor(pluginService: PluginService, iamAuthUtils: IamAuthUtils = new IamAuthUtils()) { super(); this.pluginService = pluginService; this.telemetryFactory = this.pluginService.getTelemetryFactory(); this.fetchTokenCounter = this.telemetryFactory.createCounter("iam.fetchTokenCount"); + this.iamAuthUtils = iamAuthUtils; } getSubscribedMethods(): Set { @@ -74,14 +81,22 @@ export class IamAuthenticationPlugin extends AbstractConnectionPlugin implements throw new AwsWrapperError(`${WrapperProperties.USER.name} is null or empty`); } - const host = IamAuthUtils.getIamHost(props, hostInfo); - const region: string = RegionUtils.getRegion(props.get(WrapperProperties.IAM_REGION.name), host); - const port = IamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getCurrentClient().defaultPort); + const host = this.iamAuthUtils.getIamHost(props, hostInfo); + const port = this.iamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getCurrentClient().defaultPort); + + const type: RdsUrlType = this.rdsUtils.identifyRdsType(host.host); + this.regionUtils = type == RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER ? new GDBRegionUtils() : new RegionUtils(); + const region: string | null = await this.regionUtils.getRegion(WrapperProperties.IAM_REGION.name, host, props); + + if (!region) { + throw new AwsWrapperError(Messages.get("SamlAuthPlugin.unableToDetermineRegion", WrapperProperties.IAM_REGION.name)); + } + const tokenExpirationSec = WrapperProperties.IAM_TOKEN_EXPIRATION.get(props); if (tokenExpirationSec < 0) { throw new AwsWrapperError(Messages.get("AuthenticationToken.tokenExpirationLessThanZero")); } - const cacheKey: string = IamAuthUtils.getCacheKey(port, user, host, region); + const cacheKey: string = this.iamAuthUtils.getCacheKey(port, user, host.host, region); const tokenInfo = IamAuthenticationPlugin.tokenCache.get(cacheKey); const isCachedToken: boolean = tokenInfo !== undefined && !tokenInfo.isExpired(); @@ -91,8 +106,8 @@ export class IamAuthenticationPlugin extends AbstractConnectionPlugin implements WrapperProperties.PASSWORD.set(props, tokenInfo.token); } else { const tokenExpiry: number = Date.now() + tokenExpirationSec * 1000; - const token = await IamAuthUtils.generateAuthenticationToken( - host, + const token = await this.iamAuthUtils.generateAuthenticationToken( + host.host, port, region, user, @@ -118,8 +133,8 @@ export class IamAuthenticationPlugin extends AbstractConnectionPlugin implements // Try to generate a new token and try to connect again const tokenExpiry: number = Date.now() + tokenExpirationSec * 1000; - const token = await IamAuthUtils.generateAuthenticationToken( - host, + const token = await this.iamAuthUtils.generateAuthenticationToken( + host.host, port, region, user, diff --git a/common/lib/authentication/iam_authentication_plugin_factory.ts b/common/lib/authentication/iam_authentication_plugin_factory.ts index da5e0a32b..6ececd67b 100644 --- a/common/lib/authentication/iam_authentication_plugin_factory.ts +++ b/common/lib/authentication/iam_authentication_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; import { ConnectionPlugin } from "../connection_plugin"; import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; +import { FullServicesContainer } from "../utils/full_services_container"; export class IamAuthenticationPluginFactory extends ConnectionPluginFactory { private static iamAuthenticationPlugin: any; - async getInstance(pluginService: PluginService, properties: object): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: object): Promise { try { if (!IamAuthenticationPluginFactory.iamAuthenticationPlugin) { IamAuthenticationPluginFactory.iamAuthenticationPlugin = await import("./iam_authentication_plugin"); } - return new IamAuthenticationPluginFactory.iamAuthenticationPlugin.IamAuthenticationPlugin(pluginService); + return new IamAuthenticationPluginFactory.iamAuthenticationPlugin.IamAuthenticationPlugin(servicesContainer.pluginService); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "IamAuthenticationPlugin")); } diff --git a/common/lib/aws_client.ts b/common/lib/aws_client.ts index b1e3b1258..9f772be13 100644 --- a/common/lib/aws_client.ts +++ b/common/lib/aws_client.ts @@ -14,8 +14,7 @@ limitations under the License. */ -import { PluginServiceManagerContainer } from "./plugin_service_manager_container"; -import { PluginService, PluginServiceImpl } from "./plugin_service"; +import { PluginService } from "./plugin_service"; import { DatabaseDialect, DatabaseType } from "./database_dialect/database_dialect"; import { ConnectionUrlParser } from "./utils/connection_url_parser"; import { HostListProvider } from "./host_list_provider/host_list_provider"; @@ -23,23 +22,31 @@ import { PluginManager } from "./plugin_manager"; import pkgStream from "stream"; import { ClientWrapper } from "./client_wrapper"; -import { ConnectionProviderManager } from "./connection_provider_manager"; import { DefaultTelemetryFactory } from "./utils/telemetry/default_telemetry_factory"; import { TelemetryFactory } from "./utils/telemetry/telemetry_factory"; import { DriverDialect } from "./driver_dialect/driver_dialect"; import { WrapperProperties } from "./wrapper_property"; import { DriverConfigurationProfiles } from "./profile/driver_configuration_profiles"; import { ConfigurationProfile } from "./profile/configuration_profile"; -import { AwsWrapperError, TransactionIsolationLevel, ConnectionProvider } from "./"; +import { AwsWrapperError, ConnectionProvider, TransactionIsolationLevel } from "./"; import { Messages } from "./utils/messages"; import { HostListProviderService } from "./host_list_provider_service"; import { SessionStateClient } from "./session_state_client"; -import { DriverConnectionProvider } from "./driver_connection_provider"; +import { ServiceUtils } from "./utils/service_utils"; +import { StorageService } from "./utils/storage/storage_service"; +import { MonitorService } from "./utils/monitoring/monitor_service"; +import { CoreServicesContainer } from "./utils/core_services_container"; +import { FullServicesContainer } from "./utils/full_services_container"; +import { EventPublisher } from "./utils/events/event"; const { EventEmitter } = pkgStream; export abstract class AwsClient extends EventEmitter implements SessionStateClient { private _defaultPort: number = -1; + private readonly fullServiceContainer: FullServicesContainer; + private readonly storageService: StorageService; + private readonly monitorService: MonitorService; + private readonly eventPublisher: EventPublisher; protected telemetryFactory: TelemetryFactory; protected pluginManager: PluginManager; protected pluginService: PluginService; @@ -64,6 +71,8 @@ export abstract class AwsClient extends EventEmitter implements SessionStateClie this.properties = new Map(Object.entries(config)); + this.storageService = CoreServicesContainer.getInstance().storageService; + const profileName = WrapperProperties.PROFILE_NAME.get(this.properties); if (profileName && profileName.length > 0) { this._configurationProfile = DriverConfigurationProfiles.getProfileConfiguration(profileName); @@ -98,22 +107,27 @@ export abstract class AwsClient extends EventEmitter implements SessionStateClie } } + const coreServicesContainer: CoreServicesContainer = CoreServicesContainer.getInstance(); + this.storageService = coreServicesContainer.storageService; + this.monitorService = coreServicesContainer.monitorService; + this.eventPublisher = coreServicesContainer.eventPublisher; this.telemetryFactory = new DefaultTelemetryFactory(this.properties); - const container = new PluginServiceManagerContainer(); - this.pluginService = new PluginServiceImpl( - container, + + this.fullServiceContainer = ServiceUtils.instance.createStandardServiceContainer( + this.storageService, + this.monitorService, + this.eventPublisher, this, + this.properties, dbType, knownDialectsByCode, - this.properties, - this._configurationProfile?.getDriverDialect() ?? driverDialect - ); - this.pluginManager = new PluginManager( - container, - this.properties, - new ConnectionProviderManager(connectionProvider ?? new DriverConnectionProvider(), WrapperProperties.CONNECTION_PROVIDER.get(this.properties)), - this.telemetryFactory + this._configurationProfile?.getDriverDialect() ?? driverDialect, + this.telemetryFactory, + connectionProvider ); + + this.pluginService = this.fullServiceContainer.pluginService; + this.pluginManager = this.fullServiceContainer.pluginManager; } private async setup() { @@ -125,12 +139,12 @@ export abstract class AwsClient extends EventEmitter implements SessionStateClie await this.setup(); const hostListProvider: HostListProvider = this.pluginService .getDialect() - .getHostListProvider(this.properties, this.properties.get("host"), (this.pluginService)); + .getHostListProvider(this.properties, this.properties.get("host"), this.fullServiceContainer); this.pluginService.setHostListProvider(hostListProvider); await this.pluginService.refreshHostList(); const initialHostInfo = this.pluginService.getInitialConnectionHostInfo(); if (initialHostInfo != null) { - await this.pluginManager.initHostProvider(initialHostInfo, this.properties, (this.pluginService)); + await this.pluginManager.initHostProvider(initialHostInfo, this.properties, this.fullServiceContainer.hostListProviderService); await this.pluginService.refreshHostList(); } } @@ -154,23 +168,23 @@ export abstract class AwsClient extends EventEmitter implements SessionStateClie abstract setReadOnly(readOnly: boolean): Promise; - abstract isReadOnly(): boolean; + abstract isReadOnly(): boolean | undefined; abstract setAutoCommit(autoCommit: boolean): Promise; - abstract getAutoCommit(): boolean; + abstract getAutoCommit(): boolean | undefined; abstract setTransactionIsolation(level: TransactionIsolationLevel): Promise; - abstract getTransactionIsolation(): TransactionIsolationLevel; + abstract getTransactionIsolation(): TransactionIsolationLevel | undefined; abstract setSchema(schema: any): Promise; - abstract getSchema(): string; + abstract getSchema(): string | undefined; abstract setCatalog(catalog: string): Promise; - abstract getCatalog(): string; + abstract getCatalog(): string | undefined; abstract end(): Promise; diff --git a/common/lib/plugin_service_manager_container.ts b/common/lib/connection_info.ts similarity index 51% rename from common/lib/plugin_service_manager_container.ts rename to common/lib/connection_info.ts index e82728227..77b8e6507 100644 --- a/common/lib/plugin_service_manager_container.ts +++ b/common/lib/connection_info.ts @@ -14,26 +14,22 @@ limitations under the License. */ -import { PluginService } from "./plugin_service"; -import { PluginManager } from "./plugin_manager"; +import { ClientWrapper } from "./client_wrapper"; -export class PluginServiceManagerContainer { - private _pluginService?: PluginService | null; - private _pluginManager?: PluginManager | null; +export class ConnectionInfo { + private readonly _client: ClientWrapper; + private readonly _isPooled: boolean; - get pluginService(): PluginService | null { - return this._pluginService ?? null; + constructor(client: ClientWrapper, isPooled: boolean) { + this._client = client; + this._isPooled = isPooled; } - set pluginService(service: PluginService | null) { - this._pluginService = service; + get client(): ClientWrapper { + return this._client; } - get pluginManager(): PluginManager | null { - return this._pluginManager ?? null; - } - - set pluginManager(service: PluginManager | null) { - this._pluginManager = service; + get isPooled(): boolean { + return this._isPooled; } } diff --git a/common/lib/connection_plugin_chain_builder.ts b/common/lib/connection_plugin_chain_builder.ts index 9cc01baa1..1f4fd2777 100644 --- a/common/lib/connection_plugin_chain_builder.ts +++ b/common/lib/connection_plugin_chain_builder.ts @@ -29,9 +29,9 @@ import { FailoverPluginFactory } from "./plugins/failover/failover_plugin_factor import { Failover2PluginFactory } from "./plugins/failover2/failover2_plugin_factory"; import { StaleDnsPluginFactory } from "./plugins/stale_dns/stale_dns_plugin_factory"; import { FederatedAuthPluginFactory } from "./plugins/federated_auth/federated_auth_plugin_factory"; -import { ReadWriteSplittingPluginFactory } from "./plugins/read_write_splitting_plugin_factory"; +import { ReadWriteSplittingPluginFactory } from "./plugins/read_write_splitting/read_write_splitting_plugin_factory"; import { OktaAuthPluginFactory } from "./plugins/federated_auth/okta_auth_plugin_factory"; -import { HostMonitoringPluginFactory } from "./plugins/efm/host_monitoring_plugin_factory"; +import { HostMonitoringPluginFactory } from "./plugins/efm/v1/host_monitoring_plugin_factory"; import { AuroraInitialConnectionStrategyFactory } from "./plugins/aurora_initial_connection_strategy_plugin_factory"; import { AuroraConnectionTrackerPluginFactory } from "./plugins/connection_tracker/aurora_connection_tracker_plugin_factory"; import { ConnectionProviderManager } from "./connection_provider_manager"; @@ -41,8 +41,11 @@ import { LimitlessConnectionPluginFactory } from "./plugins/limitless/limitless_ import { FastestResponseStrategyPluginFactory } from "./plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory"; import { CustomEndpointPluginFactory } from "./plugins/custom_endpoint/custom_endpoint_plugin_factory"; import { ConfigurationProfile } from "./profile/configuration_profile"; -import { HostMonitoring2PluginFactory } from "./plugins/efm2/host_monitoring2_plugin_factory"; +import { HostMonitoring2PluginFactory } from "./plugins/efm/v2/host_monitoring2_plugin_factory"; import { BlueGreenPluginFactory } from "./plugins/bluegreen/blue_green_plugin_factory"; +import { GlobalDbFailoverPluginFactory } from "./plugins/gdb_failover/global_db_failover_plugin_factory"; +import { FullServicesContainer } from "./utils/full_services_container"; +import { GdbReadWriteSplittingPluginFactory } from "./plugins/read_write_splitting/gdb_read_write_splitting_plugin_factory"; /* Type alias used for plugin factory sorting. It holds a reference to a plugin @@ -63,8 +66,10 @@ export class ConnectionPluginChainBuilder { ["staleDns", { factory: StaleDnsPluginFactory, weight: 500 }], ["bg", { factory: BlueGreenPluginFactory, weight: 550 }], ["readWriteSplitting", { factory: ReadWriteSplittingPluginFactory, weight: 600 }], + ["gdbReadWriteSplitting", { factory: GdbReadWriteSplittingPluginFactory, weight: 610 }], ["failover", { factory: FailoverPluginFactory, weight: 700 }], ["failover2", { factory: Failover2PluginFactory, weight: 710 }], + ["gdbFailover", { factory: GlobalDbFailoverPluginFactory, weight: 720 }], ["efm", { factory: HostMonitoringPluginFactory, weight: 800 }], ["efm2", { factory: HostMonitoring2PluginFactory, weight: 810 }], ["fastestResponseStrategy", { factory: FastestResponseStrategyPluginFactory, weight: 900 }], @@ -84,8 +89,10 @@ export class ConnectionPluginChainBuilder { [StaleDnsPluginFactory, 500], [BlueGreenPluginFactory, 550], [ReadWriteSplittingPluginFactory, 600], + [GdbReadWriteSplittingPluginFactory, 610], [FailoverPluginFactory, 700], [Failover2PluginFactory, 710], + [GlobalDbFailoverPluginFactory, 720], [HostMonitoringPluginFactory, 800], [HostMonitoring2PluginFactory, 810], [LimitlessConnectionPluginFactory, 950], @@ -99,7 +106,7 @@ export class ConnectionPluginChainBuilder { ]); static async getPlugins( - pluginService: PluginService, + servicesContainer: FullServicesContainer, props: Map, connectionProviderManager: ConnectionProviderManager, configurationProfile: ConfigurationProfile | null @@ -162,10 +169,10 @@ export class ConnectionPluginChainBuilder { for (const pluginFactoryInfo of pluginFactoryInfoList) { const factoryObj = new pluginFactoryInfo.factory(); - plugins.push(await factoryObj.getInstance(pluginService, props)); + plugins.push(await factoryObj.getInstance(servicesContainer, props)); } - plugins.push(new DefaultPlugin(pluginService, connectionProviderManager)); + plugins.push(new DefaultPlugin(servicesContainer, connectionProviderManager)); return plugins; } diff --git a/common/lib/connection_provider.ts b/common/lib/connection_provider.ts index 8da16f5d9..9f60b31ba 100644 --- a/common/lib/connection_provider.ts +++ b/common/lib/connection_provider.ts @@ -17,10 +17,10 @@ import { HostRole } from "./host_role"; import { HostInfo } from "./host_info"; import { PluginService } from "./plugin_service"; -import { ClientWrapper } from "./client_wrapper"; +import { ConnectionInfo } from "./connection_info"; export interface ConnectionProvider { - connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise; + connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise; acceptsUrl(hostInfo: HostInfo, props: Map): boolean; acceptsStrategy(role: HostRole, strategy: string): boolean; getHostInfoByStrategy(hosts: HostInfo[], role: HostRole, strategy: string, props?: Map): HostInfo; diff --git a/common/lib/database_dialect/database_dialect.ts b/common/lib/database_dialect/database_dialect.ts index 22f2cf28a..2077934af 100644 --- a/common/lib/database_dialect/database_dialect.ts +++ b/common/lib/database_dialect/database_dialect.ts @@ -21,6 +21,7 @@ import { FailoverRestriction } from "../plugins/failover/failover_restriction"; import { ErrorHandler } from "../error_handler"; import { TransactionIsolationLevel } from "../utils/transaction_isolation_level"; import { HostRole } from "../host_role"; +import { FullServicesContainer } from "../utils/full_services_container"; export enum DatabaseType { MYSQL, @@ -41,7 +42,7 @@ export interface DatabaseDialect { getErrorHandler(): ErrorHandler; getHostRole(targetClient: ClientWrapper): Promise; isDialect(targetClient: ClientWrapper): Promise; - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider; + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider; isClientValid(targetClient: ClientWrapper): Promise; getDatabaseType(): DatabaseType; getDialectName(): string; diff --git a/common/lib/database_dialect/database_dialect_codes.ts b/common/lib/database_dialect/database_dialect_codes.ts index 4815f48ec..4351a6cb5 100644 --- a/common/lib/database_dialect/database_dialect_codes.ts +++ b/common/lib/database_dialect/database_dialect_codes.ts @@ -15,11 +15,13 @@ */ export class DatabaseDialectCodes { + static readonly GLOBAL_AURORA_MYSQL: string = "global-aurora-mysql"; static readonly AURORA_MYSQL: string = "aurora-mysql"; static readonly RDS_MYSQL: string = "rds-mysql"; static readonly MYSQL: string = "mysql"; // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html static readonly RDS_MULTI_AZ_MYSQL: string = "rds-multi-az-mysql"; + static readonly GLOBAL_AURORA_PG: string = "global-aurora-pg"; static readonly AURORA_PG: string = "aurora-pg"; static readonly RDS_PG: string = "rds-pg"; // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html diff --git a/common/lib/database_dialect/database_dialect_manager.ts b/common/lib/database_dialect/database_dialect_manager.ts index e8127f2a0..b462e60bd 100644 --- a/common/lib/database_dialect/database_dialect_manager.ts +++ b/common/lib/database_dialect/database_dialect_manager.ts @@ -95,6 +95,14 @@ export class DatabaseDialectManager implements DatabaseDialectProvider { if (this.dbType === DatabaseType.MYSQL) { const type = this.rdsHelper.identifyRdsType(host); + if (type == RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER) { + this.canUpdate = false; + this.dialectCode = DatabaseDialectCodes.GLOBAL_AURORA_MYSQL; + this.dialect = this.knownDialectsByCode.get(DatabaseDialectCodes.GLOBAL_AURORA_MYSQL); + this.logCurrentDialect(); + return this.dialect; + } + if (type.isRdsCluster) { this.canUpdate = true; this.dialectCode = DatabaseDialectCodes.AURORA_MYSQL; @@ -128,6 +136,14 @@ export class DatabaseDialectManager implements DatabaseDialectProvider { return this.dialect; } + if (type == RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER) { + this.canUpdate = false; + this.dialectCode = DatabaseDialectCodes.GLOBAL_AURORA_PG; + this.dialect = this.knownDialectsByCode.get(DatabaseDialectCodes.GLOBAL_AURORA_PG); + this.logCurrentDialect(); + return this.dialect; + } + if (type.isRdsCluster) { this.canUpdate = true; this.dialectCode = DatabaseDialectCodes.AURORA_PG; diff --git a/common/lib/topology_aware_database_dialect.ts b/common/lib/database_dialect/topology_aware_database_dialect.ts similarity index 60% rename from common/lib/topology_aware_database_dialect.ts rename to common/lib/database_dialect/topology_aware_database_dialect.ts index e5d6bc1c9..df0553c8c 100644 --- a/common/lib/topology_aware_database_dialect.ts +++ b/common/lib/database_dialect/topology_aware_database_dialect.ts @@ -14,13 +14,12 @@ limitations under the License. */ -import { HostInfo } from "./host_info"; -import { HostListProvider } from "./host_list_provider/host_list_provider"; -import { HostRole } from "./host_role"; -import { ClientWrapper } from "./client_wrapper"; +import { HostRole } from "../host_role"; +import { ClientWrapper } from "../client_wrapper"; +import { TopologyQueryResult } from "../host_list_provider/topology_utils"; export interface TopologyAwareDatabaseDialect { - queryForTopology(client: ClientWrapper, hostListProvider: HostListProvider): Promise; + queryForTopology(client: ClientWrapper): Promise; identifyConnection(targetClient: ClientWrapper): Promise; @@ -28,4 +27,14 @@ export interface TopologyAwareDatabaseDialect { // Returns the host id of the targetClient if it is connected to a writer, null otherwise. getWriterId(targetClient: ClientWrapper): Promise; + + getInstanceId(targetClient: ClientWrapper): Promise<[string, string]>; +} + +export interface GlobalAuroraTopologyDialect extends TopologyAwareDatabaseDialect { + getRegionByInstanceId(targetClient: ClientWrapper, instanceId: string): Promise; +} + +export function isDialectTopologyAware(dialect: any): dialect is TopologyAwareDatabaseDialect { + return dialect; } diff --git a/common/lib/driver_connection_provider.ts b/common/lib/driver_connection_provider.ts index 7affca522..6a4cd0340 100644 --- a/common/lib/driver_connection_provider.ts +++ b/common/lib/driver_connection_provider.ts @@ -30,6 +30,7 @@ import { logger } from "../logutils"; import { ClientWrapper } from "./client_wrapper"; import { RoundRobinHostSelector } from "./round_robin_host_selector"; import { DriverDialect } from "./driver_dialect/driver_dialect"; +import { ConnectionInfo } from "./connection_info"; export class DriverConnectionProvider implements ConnectionProvider { private static readonly acceptedStrategies: Map = new Map([ @@ -46,7 +47,7 @@ export class DriverConnectionProvider implements ConnectionProvider { return true; } - async connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise { + async connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise { let resultTargetClient; const resultProps = new Map(props); resultProps.set(WrapperProperties.HOST.name, hostInfo.host); @@ -92,7 +93,7 @@ export class DriverConnectionProvider implements ConnectionProvider { resultTargetClient = driverDialect.connect(hostInfo, resultProps); } pluginService.attachErrorListener(resultTargetClient); - return resultTargetClient; + return new ConnectionInfo(resultTargetClient, false); } getHostInfoByStrategy(hosts: HostInfo[], role: HostRole, strategy: string, props?: Map): HostInfo { diff --git a/common/lib/driver_dialect/driver_dialect.ts b/common/lib/driver_dialect/driver_dialect.ts index 347805448..05bf05911 100644 --- a/common/lib/driver_dialect/driver_dialect.ts +++ b/common/lib/driver_dialect/driver_dialect.ts @@ -30,7 +30,7 @@ export interface DriverDialect { setConnectTimeout(props: Map, wrapperConnectTimeout?: any): void; - setQueryTimeout(props: Map, sql?: any, wrapperConnectTimeout?: any): void; + setQueryTimeout(props: Map, sql?: any, wrapperQueryTimeout?: any): void; setKeepAliveProperties(props: Map, keepAliveProps: any): void; diff --git a/common/lib/highest_weight_host_selector.ts b/common/lib/highest_weight_host_selector.ts index 370d68773..75d42d48c 100644 --- a/common/lib/highest_weight_host_selector.ts +++ b/common/lib/highest_weight_host_selector.ts @@ -26,7 +26,7 @@ export class HighestWeightHostSelector implements HostSelector { getHost(hosts: HostInfo[], role: HostRole, props?: Map): HostInfo { const eligibleHosts: HostInfo[] = hosts - .filter((host: HostInfo) => host.role === role && host.availability === HostAvailability.AVAILABLE) + .filter((host: HostInfo) => (role === null || host.role === role) && host.availability === HostAvailability.AVAILABLE) .sort((hostA: HostInfo, hostB: HostInfo) => (hostA.weight > hostB.weight ? -1 : hostA.weight < hostB.weight ? 1 : 0)); if (eligibleHosts.length === 0) { diff --git a/common/lib/host_availability/host_availability_cache_item.ts b/common/lib/host_availability/host_availability_cache_item.ts new file mode 100644 index 000000000..c068f2de3 --- /dev/null +++ b/common/lib/host_availability/host_availability_cache_item.ts @@ -0,0 +1,25 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { HostAvailability } from "./host_availability"; + +export class HostAvailabilityCacheItem { + readonly availability: HostAvailability; + + constructor(availability: HostAvailability) { + this.availability = availability; + } +} diff --git a/common/lib/host_info.ts b/common/lib/host_info.ts index 566f5cfaa..4d332a384 100644 --- a/common/lib/host_info.ts +++ b/common/lib/host_info.ts @@ -25,15 +25,13 @@ export class HostInfo { public static readonly NO_PORT: number = -1; public static readonly DEFAULT_WEIGHT: number = 100; - readonly host: string; + readonly host: string; // full domain name readonly port: number; role: HostRole; readonly weight: number; // Greater or equal 0. Lesser the weight, the healthier host. readonly lastUpdateTime: number; availability: HostAvailability; - aliases: Set = new Set(); - allAliases: Set = new Set(); - hostId: string; + hostId: string; // id; could be a host name, host domain name, or a unique string hostAvailabilityStrategy: HostAvailabilityStrategy; constructor( @@ -57,7 +55,6 @@ export class HostInfo { this.weight = weight; this.lastUpdateTime = lastUpdateTime; this.hostAvailabilityStrategy = hostAvailabilityStrategy; - this.allAliases.add(this.asAlias).add(this.host); this.hostId = hostId; } @@ -65,42 +62,6 @@ export class HostInfo { return this.port != HostInfo.NO_PORT; } - getHostAndPort(): string { - return this.isPortSpecified() ? this.host + ":" + this.port : this.host; - } - - addAlias(...alias: string[]) { - if (!alias || alias.length < 1) { - return; - } - - alias.forEach((x) => { - this.aliases.add(x); - this.allAliases.add(x); - }); - } - - removeAlias(aliases: string[]) { - if (!aliases || aliases.length < 1) { - return; - } - - aliases.forEach((x) => { - this.aliases.delete(x); - this.allAliases.delete(x); - }); - } - - resetAliases() { - this.aliases.clear(); - this.allAliases.clear(); - this.allAliases.add(this.asAlias); - } - - get asAlias() { - return this.isPortSpecified() ? `${this.host}:${this.port}` : this.host; - } - get url() { let url = this.hostAndPort; if (!url.endsWith("/")) { diff --git a/common/lib/host_list_provider/aurora_topology_utils.ts b/common/lib/host_list_provider/aurora_topology_utils.ts new file mode 100644 index 000000000..5a16ac359 --- /dev/null +++ b/common/lib/host_list_provider/aurora_topology_utils.ts @@ -0,0 +1,68 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TopologyQueryResult, TopologyUtils } from "./topology_utils"; +import { ClientWrapper } from "../client_wrapper"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { HostInfo } from "../host_info"; +import { isDialectTopologyAware } from "../database_dialect/topology_aware_database_dialect"; +import { Messages } from "../utils/messages"; + +/** + * TopologyUtils implementation for Aurora clusters using a single HostInfo template. + */ +export class AuroraTopologyUtils extends TopologyUtils { + async queryForTopology( + targetClient: ClientWrapper, + dialect: DatabaseDialect, + initialHost: HostInfo, + clusterInstanceTemplate: HostInfo + ): Promise { + if (!isDialectTopologyAware(dialect)) { + throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); + } + + return await dialect + .queryForTopology(targetClient) + .then((res: TopologyQueryResult[]) => this.verifyWriter(this.createHosts(res, initialHost, clusterInstanceTemplate))); + } + + public createHosts(topologyQueryResults: TopologyQueryResult[], initialHost: HostInfo, clusterInstanceTemplate: HostInfo): HostInfo[] { + const hostsMap = new Map(); + topologyQueryResults.forEach((row) => { + const lastUpdateTime = row.lastUpdateTime ?? Date.now(); + + const host = this.createHost( + row.id, + row.host, + row.isWriter, + row.weight, + lastUpdateTime, + initialHost, + clusterInstanceTemplate, + row.endpoint, + row.port + ); + + const existing = hostsMap.get(host.host); + if (!existing || existing.lastUpdateTime < host.lastUpdateTime) { + hostsMap.set(host.host, host); + } + }); + + return Array.from(hostsMap.values()); + } +} diff --git a/common/lib/host_list_provider/connection_string_host_list_provider.ts b/common/lib/host_list_provider/connection_string_host_list_provider.ts index 0885d9b3c..a7973111a 100644 --- a/common/lib/host_list_provider/connection_string_host_list_provider.ts +++ b/common/lib/host_list_provider/connection_string_host_list_provider.ts @@ -65,17 +65,12 @@ export class ConnectionStringHostListProvider implements StaticHostListProvider this.isInitialized = true; } - refresh(): Promise; - refresh(client: ClientWrapper): Promise; - refresh(client?: ClientWrapper): Promise; - refresh(client?: ClientWrapper | undefined): Promise { + refresh(): Promise { this.init(); return Promise.resolve(this.hostList); } - forceRefresh(): Promise; - forceRefresh(client: ClientWrapper): Promise; - forceRefresh(client?: ClientWrapper): Promise { + forceRefresh(): Promise { this.init(); return Promise.resolve(this.hostList); } @@ -89,7 +84,7 @@ export class ConnectionStringHostListProvider implements StaticHostListProvider return null; } const instance = await this.hostListProviderService.getDialect().getHostAliasAndParseResults(client.client); - const topology = await this.refresh(client.client); + const topology = await this.refresh(); if (!topology || topology.length == 0) { return null; } @@ -97,19 +92,6 @@ export class ConnectionStringHostListProvider implements StaticHostListProvider return topology.filter((hostInfo) => instance === hostInfo.hostId)[0]; } - createHost(host: string, isWriter: boolean, weight: number, lastUpdateTime: number, port?: number): HostInfo { - return this.hostListProviderService - .getHostInfoBuilder() - .withHost(host ?? "") - .withPort(port ?? this.initialPort) - .withRole(isWriter ? HostRole.WRITER : HostRole.READER) - .withAvailability(HostAvailability.AVAILABLE) - .withWeight(weight) - .withLastUpdateTime(lastUpdateTime) - .withHostId(host) - .build(); - } - getHostProviderType(): string { return this.constructor.name; } diff --git a/common/lib/host_list_provider/global_aurora_host_list_provider.ts b/common/lib/host_list_provider/global_aurora_host_list_provider.ts new file mode 100644 index 000000000..7b17e3a5f --- /dev/null +++ b/common/lib/host_list_provider/global_aurora_host_list_provider.ts @@ -0,0 +1,71 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { RdsHostListProvider } from "./rds_host_list_provider"; +import { FullServicesContainer } from "../utils/full_services_container"; +import { HostInfo } from "../host_info"; +import { WrapperProperties } from "../wrapper_property"; +import { ClusterTopologyMonitor, ClusterTopologyMonitorImpl } from "./monitoring/cluster_topology_monitor"; +import { GlobalAuroraTopologyMonitor } from "./monitoring/global_aurora_topology_monitor"; +import { MonitorInitializer } from "../utils/monitoring/monitor"; +import { ClientWrapper } from "../client_wrapper"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { parseInstanceTemplates } from "../utils/utils"; + +export class GlobalAuroraHostListProvider extends RdsHostListProvider { + protected instanceTemplatesByRegion: Map; + protected override initSettings(): void { + super.initSettings(); + + const instanceTemplates = WrapperProperties.GLOBAL_CLUSTER_INSTANCE_HOST_PATTERNS.get(this.properties); + this.instanceTemplatesByRegion = parseInstanceTemplates( + instanceTemplates, + (hostPattern: string) => this.validateHostPatternSetting(hostPattern), + () => this.hostListProviderService.getHostInfoBuilder() + ); + } + + protected override async getOrCreateMonitor(): Promise { + const initializer: MonitorInitializer = { + createMonitor: (servicesContainer: FullServicesContainer): ClusterTopologyMonitor => { + return new GlobalAuroraTopologyMonitor( + servicesContainer, + this.topologyUtils, + this.clusterId, + this.initialHost, + this.properties, + this.clusterInstanceTemplate, + this.refreshRateNano, + this.highRefreshRateNano, + this.instanceTemplatesByRegion + ); + } + }; + + return await this.servicesContainers.monitorService.runIfAbsent( + ClusterTopologyMonitorImpl, + this.clusterId, + this.servicesContainers, + this.properties, + initializer + ); + } + + override async getCurrentTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { + this.init(); + return await this.topologyUtils.queryForTopology(targetClient, dialect, this.initialHost, this.instanceTemplatesByRegion); + } +} diff --git a/common/lib/host_list_provider/global_topology_utils.ts b/common/lib/host_list_provider/global_topology_utils.ts new file mode 100644 index 000000000..7b197dc15 --- /dev/null +++ b/common/lib/host_list_provider/global_topology_utils.ts @@ -0,0 +1,93 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TopologyQueryResult, TopologyUtils } from "./topology_utils"; +import { ClientWrapper } from "../client_wrapper"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { HostInfo } from "../host_info"; +import { isDialectTopologyAware } from "../database_dialect/topology_aware_database_dialect"; +import { Messages } from "../utils/messages"; +import { AwsWrapperError } from "../utils/errors"; + +export interface GdbTopologyUtils { + getRegion(instanceId: string, targetClient: ClientWrapper, dialect: DatabaseDialect): Promise; +} + +export class GlobalTopologyUtils extends TopologyUtils implements GdbTopologyUtils { + async queryForTopology( + targetClient: ClientWrapper, + dialect: DatabaseDialect, + initialHost: HostInfo, + instanceTemplateByRegion: Map + ): Promise { + if (!isDialectTopologyAware(dialect)) { + throw new AwsWrapperError(Messages.get("RdsHostListProvider.incorrectDialect")); + } + + return await dialect + .queryForTopology(targetClient) + .then((res: TopologyQueryResult[]) => this.verifyWriter(this.createHostsWithTemplateMap(res, initialHost, instanceTemplateByRegion))); + } + + async getRegion(instanceId: string, targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { + if (!isDialectTopologyAware(dialect)) { + throw new AwsWrapperError(Messages.get("RdsHostListProvider.incorrectDialect")); + } + + const results = await dialect.queryForTopology(targetClient); + const match = results.find((row) => row.id === instanceId); + return match?.awsRegion ?? null; + } + + private createHostsWithTemplateMap( + topologyQueryResults: TopologyQueryResult[], + initialHost: HostInfo, + instanceTemplateByRegion: Map + ): HostInfo[] { + const hostsMap = new Map(); + topologyQueryResults.forEach((row) => { + if (!row.awsRegion) { + throw new AwsWrapperError(Messages.get("GlobalTopologyUtils.missingRegion", row.host)); + } + const clusterInstanceTemplate = instanceTemplateByRegion.get(row.awsRegion); + + if (!clusterInstanceTemplate) { + throw new AwsWrapperError(Messages.get("GlobalTopologyUtils.missingTemplateForRegion", row.awsRegion, row.host)); + } + + const lastUpdateTime = row.lastUpdateTime ?? Date.now(); + + const host = this.createHost( + row.id, + row.host, + row.isWriter, + row.weight, + lastUpdateTime, + initialHost, + clusterInstanceTemplate, + row.endpoint, + row.port + ); + + const existing = hostsMap.get(host.host); + if (!existing || existing.lastUpdateTime < host.lastUpdateTime) { + hostsMap.set(host.host, host); + } + }); + + return Array.from(hostsMap.values()); + } +} diff --git a/common/lib/host_list_provider/host_list_provider.ts b/common/lib/host_list_provider/host_list_provider.ts index 2c2c24eca..b7e1005a2 100644 --- a/common/lib/host_list_provider/host_list_provider.ts +++ b/common/lib/host_list_provider/host_list_provider.ts @@ -19,32 +19,22 @@ import { HostRole } from "../host_role"; import { DatabaseDialect } from "../database_dialect/database_dialect"; import { ClientWrapper } from "../client_wrapper"; -export type DynamicHostListProvider = HostListProvider; - export type StaticHostListProvider = HostListProvider; -export interface BlockingHostListProvider extends HostListProvider { - forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise; - - clearAll(): Promise; -} - export interface HostListProvider { refresh(): Promise; - refresh(client: ClientWrapper): Promise; - forceRefresh(): Promise; - forceRefresh(client: ClientWrapper): Promise; - getHostRole(client: ClientWrapper, dialect: DatabaseDialect): Promise; - identifyConnection(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise; - - createHost(host: string, isWriter: boolean, weight: number, lastUpdateTime: number, port?: number): HostInfo; + identifyConnection(targetClient: ClientWrapper): Promise; getHostProviderType(): string; getClusterId(): string; } + +export interface DynamicHostListProvider extends HostListProvider { + forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise; +} diff --git a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts index 7e7263d69..462416781 100644 --- a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts +++ b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts @@ -15,45 +15,78 @@ */ import { HostInfo } from "../../host_info"; -import { CacheMap } from "../../utils/cache_map"; import { PluginService } from "../../plugin_service"; import { HostAvailability } from "../../host_availability/host_availability"; -import { logTopology, sleep } from "../../utils/utils"; +import { convertMsToNanos, convertNanosToMs, getTimeInNanos, logTopology, sleep } from "../../utils/utils"; import { logger } from "../../../logutils"; import { HostRole } from "../../host_role"; import { ClientWrapper } from "../../client_wrapper"; -import { AwsWrapperError } from "../../utils/errors"; -import { MonitoringRdsHostListProvider } from "./monitoring_host_list_provider"; +import { AwsTimeoutError, AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; - -export interface ClusterTopologyMonitor { +import { Topology } from "../topology"; +import { StorageService } from "../../utils/storage/storage_service"; +import { TopologyUtils } from "../topology_utils"; +import { RdsUtils } from "../../utils/rds_utils"; +import { AbstractMonitor, Monitor } from "../../utils/monitoring/monitor"; +import { FullServicesContainer } from "../../utils/full_services_container"; +import { HostListProviderService } from "../../host_list_provider_service"; +import { Event, EventSubscriber } from "../../utils/events/event"; +import { MonitorResetEvent } from "../../utils/events/monitor_reset_event"; +import { ServiceUtils } from "../../utils/service_utils"; +import { WrapperProperties } from "../../wrapper_property"; + +export interface ClusterTopologyMonitor extends Monitor, EventSubscriber { forceRefresh(client: ClientWrapper, timeoutMs: number): Promise; close(): Promise; - forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise; + /** + * Initiates a topology update. + * + * @param verifyTopology defines whether extra measures should be taken to verify the topology. If false, the + * method will return as soon as topology is successfully retrieved from any instance. If + * true, extra steps are taken to verify the topology is accurate. + * @param timeoutMs timeout in msec to wait until the topology gets refreshed (if verifyWriter has a value of + * false) or verified (if verifyTopology has a value of true). + * @return true if successful, false if unsuccessful or the timeout is reached + * @throws AwsWrapperError if wrapper timed out while fetching the topology. + */ + forceMonitoringRefresh(verifyTopology: boolean, timeoutMs: number): Promise; + + canDispose(): boolean; } -export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { - private static readonly TOPOLOGY_CACHE_EXPIRATION_NANOS: number = 5 * 60 * 1_000_000_000; // 5 minutes. - static readonly MONITORING_PROPERTY_PREFIX: string = "topology_monitoring_"; +export class ClusterTopologyMonitorImpl extends AbstractMonitor implements ClusterTopologyMonitor { + private static readonly MONITOR_TERMINATION_TIMEOUT_SEC: number = 30; + private static readonly STABLE_TOPOLOGIES_DURATION_NS: bigint = convertMsToNanos(15000); // 15 seconds. + protected static readonly DEFAULT_CONNECTION_TIMEOUT_MS: number = 5000; + protected static readonly DEFAULT_QUERY_TIMEOUT_MS: number = 5000; private readonly clusterId: string; - private readonly initialHostInfo: HostInfo; + protected readonly initialHostInfo: HostInfo; + private readonly servicesContainer: FullServicesContainer; private readonly _monitoringProperties: Map; private readonly _pluginService: PluginService; - private readonly _hostListProvider: MonitoringRdsHostListProvider; - private readonly refreshRateMs: number; - private readonly highRefreshRateMs: number; - - private topologyMap: CacheMap; - private writerHostInfo: HostInfo = null; + protected readonly hostListProviderService: HostListProviderService; + private readonly refreshRateNs: number; + private readonly highRefreshRateNs: number; + private readonly storageService: StorageService; + private readonly rdsUtils: RdsUtils = new RdsUtils(); + protected readonly instanceTemplate: HostInfo; + + private writerHostInfo: HostInfo | null = null; private isVerifiedWriterConnection: boolean = false; - private monitoringClient: ClientWrapper = null; - private highRefreshRateEndTimeMs: number = -1; - private highRefreshPeriodAfterPanicMs: number = 30000; // 30 seconds. - private ignoreNewTopologyRequestsEndTimeMs: number = -1; - private ignoreTopologyRequestMs: number = 10000; // 10 seconds. + private monitoringClient: ClientWrapper | null = null; + private highRefreshRateEndTimeNs: bigint = BigInt(0); + + public readonly topologyUtils: TopologyUtils; + public readonly readerTopologiesById: Map = new Map(); + public readonly completedOneCycle: Map = new Map(); + // When comparing topologies, we don't want to check HostInfo.weight, which is used in HostInfo#equals. + // We use this function to compare the other fields. + protected readonly hostInfoExtractor = (host: HostInfo): string => { + return `${host.host}:${host.port}:${host.availability}:${host.role}`; + }; // Tracking of the host monitors. private hostMonitors: Map = new Map(); @@ -63,43 +96,50 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { public hostMonitorsLatestTopology: HostInfo[] = []; // Controls for stopping asynchronous monitoring tasks. - private stopMonitoring: boolean = false; public hostMonitorsStop: boolean = false; - private untrackedPromises: Promise[] = []; // Signals to other methods that asynchronous tasks have completed/should be completed. private requestToUpdateTopology: boolean = false; + private submittedHosts: Map> = new Map(); + private stableTopologiesStartNs: bigint; constructor( + servicesContainer: FullServicesContainer, + topologyUtils: TopologyUtils, clusterId: string, - topologyMap: CacheMap, initialHostInfo: HostInfo, props: Map, - pluginService: PluginService, - hostListProvider: MonitoringRdsHostListProvider, - refreshRateMs: number, - highRefreshRateMs: number + instanceTemplate: HostInfo, + refreshRateNs: number, + highRefreshRateNs: number ) { + super(ClusterTopologyMonitorImpl.MONITOR_TERMINATION_TIMEOUT_SEC); + this.topologyUtils = topologyUtils; this.clusterId = clusterId; - this.topologyMap = topologyMap; this.initialHostInfo = initialHostInfo; - this._pluginService = pluginService; - this._hostListProvider = hostListProvider; - this.refreshRateMs = refreshRateMs; - this.highRefreshRateMs = highRefreshRateMs; + this.instanceTemplate = instanceTemplate; + this.servicesContainer = servicesContainer; + this.storageService = this.servicesContainer.storageService; + this._pluginService = this.servicesContainer.pluginService; + this.hostListProviderService = this.servicesContainer.hostListProviderService; + this.refreshRateNs = refreshRateNs; + this.highRefreshRateNs = highRefreshRateNs; this._monitoringProperties = new Map(props); for (const [key, val] of props) { - if (key.startsWith(ClusterTopologyMonitorImpl.MONITORING_PROPERTY_PREFIX)) { - this._monitoringProperties.set(key.substring(ClusterTopologyMonitorImpl.MONITORING_PROPERTY_PREFIX.length), val); + if (key.startsWith(WrapperProperties.TOPOLOGY_MONITORING_PROPERTY_PREFIX)) { + this._monitoringProperties.set(key.substring(WrapperProperties.TOPOLOGY_MONITORING_PROPERTY_PREFIX.length), val); this._monitoringProperties.delete(key); } } - this.untrackedPromises.push(this.run()); - } - get hostListProvider(): MonitoringRdsHostListProvider { - return this._hostListProvider; + const connectTimeout = + this._monitoringProperties.get(WrapperProperties.WRAPPER_CONNECT_TIMEOUT.name) ?? ClusterTopologyMonitorImpl.DEFAULT_CONNECTION_TIMEOUT_MS; + const queryTimeout = + this._monitoringProperties.get(WrapperProperties.WRAPPER_QUERY_TIMEOUT.name) ?? ClusterTopologyMonitorImpl.DEFAULT_QUERY_TIMEOUT_MS; + const driverDialect = this._pluginService.getDriverDialect(); + driverDialect.setConnectTimeout(this._monitoringProperties, connectTimeout); + driverDialect.setQueryTimeout(this._monitoringProperties, undefined, queryTimeout); } get pluginService(): PluginService { @@ -111,27 +151,35 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { } async close(): Promise { - this.stopMonitoring = true; this.hostMonitorsStop = true; this.requestToUpdateTopology = true; - await Promise.all(this.untrackedPromises); - await this.closeConnection(this.monitoringClient); - await this.closeConnection(this.hostMonitorsWriterClient); - await this.closeConnection(this.hostMonitorsReaderClient); - this.untrackedPromises = []; + await Promise.all(this.submittedHosts.values()); + + const monitoringClientToClose = this.monitoringClient; + const hostMonitorsWriterClientToClose = this.hostMonitorsWriterClient; + const hostMonitorsReaderClientToClose = this.hostMonitorsReaderClient; + + this.monitoringClient = null; + this.hostMonitorsWriterClient = null; + this.hostMonitorsReaderClient = null; + + await this.closeConnection(monitoringClientToClose); + if (hostMonitorsWriterClientToClose && hostMonitorsWriterClientToClose !== monitoringClientToClose) { + await this.closeConnection(hostMonitorsWriterClientToClose); + } + if ( + hostMonitorsReaderClientToClose && + hostMonitorsReaderClientToClose !== monitoringClientToClose && + hostMonitorsReaderClientToClose !== hostMonitorsWriterClientToClose + ) { + await this.closeConnection(hostMonitorsReaderClientToClose); + } + + this.submittedHosts.clear(); this.hostMonitors.clear(); } async forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise { - if (Date.now() < this.ignoreNewTopologyRequestsEndTimeMs) { - // Previous failover has just completed, use results without triggering new update. - const currentHosts = this.topologyMap.get(this.clusterId); - if (currentHosts !== null) { - logger.info(Messages.get("ClusterTopologyMonitoring.ignoringNewTopologyRequest")); - return currentHosts; - } - } - if (shouldVerifyWriter) { this.isVerifiedWriterConnection = false; if (this.monitoringClient) { @@ -147,18 +195,19 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { async forceRefresh(client: ClientWrapper, timeoutMs: number): Promise { if (this.isVerifiedWriterConnection) { + // Get the monitoring task to refresh the topology using a verified connection. return await this.waitTillTopologyGetsUpdated(timeoutMs); } - // Otherwise use provided unverified connection to update topology. + // Otherwise, use the provided unverified connection to update the topology. return await this.fetchTopologyAndUpdateCache(client); } async waitTillTopologyGetsUpdated(timeoutMs: number): Promise { - // Signal to any monitor that might be in delay, that topology should be updated. + // Notify the monitoring task, which may be sleeping, that topology should be refreshed immediately. this.requestToUpdateTopology = true; - const currentHosts: HostInfo[] = this.topologyMap.get(this.clusterId); + const currentHosts: HostInfo[] = this.getStoredHosts(); if (timeoutMs === 0) { logger.info(logTopology(currentHosts, Messages.get("ClusterTopologyMonitoring.timeoutSetToZero"))); @@ -168,12 +217,12 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { const endTime = Date.now() + timeoutMs; let latestHosts: HostInfo[]; - while ((latestHosts = this.topologyMap.get(this.clusterId)) === currentHosts && Date.now() < endTime) { + while ((latestHosts = this.getStoredHosts()) === currentHosts && Date.now() < endTime) { await sleep(1000); } if (Date.now() >= endTime) { - throw new AwsWrapperError(Messages.get("ClusterTopologyMonitor.timeoutError", timeoutMs.toString())); + throw new AwsTimeoutError(Messages.get("ClusterTopologyMonitor.timeoutError", timeoutMs.toString())); } return latestHosts; } @@ -184,7 +233,7 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { } try { - const hosts: HostInfo[] = await this._hostListProvider.sqlQueryForTopology(client); + const hosts: HostInfo[] = await this.queryForTopology(client); if (hosts) { this.updateTopologyCache(hosts); } @@ -196,12 +245,11 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { } private async openAnyClientAndUpdateTopology(): Promise { - let writerVerifiedByThisTask = false; if (!this.monitoringClient) { let client: ClientWrapper; try { - client = await this._pluginService.forceConnect(this.initialHostInfo, this._monitoringProperties); - } catch { + client = await this.servicesContainer.pluginService.forceConnect(this.initialHostInfo, this._monitoringProperties); + } catch (connectError) { // Unable to connect to host; return null; } @@ -210,31 +258,30 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { this.monitoringClient = client; logger.debug(Messages.get("ClusterTopologyMonitor.openedMonitoringConnection", this.initialHostInfo.host)); try { - const writerId = await this.getWriterHostIdIfConnected(this.monitoringClient, this.initialHostInfo.hostId); - if (writerId) { + if (await this.topologyUtils.isWriterInstance(this.monitoringClient)) { this.isVerifiedWriterConnection = true; - this.writerHostInfo = this.initialHostInfo; - logger.info(Messages.get("ClusterTopologyMonitor.writerMonitoringConnection", this.initialHostInfo.host)); - writerVerifiedByThisTask = true; + + if (this.rdsUtils.isRdsInstance(this.initialHostInfo.host)) { + this.writerHostInfo = this.initialHostInfo; + logger.info(Messages.get("ClusterTopologyMonitor.writerMonitoringConnection", this.writerHostInfo.host)); + } else { + const pair: [string, string] = await this.topologyUtils.getInstanceId(this.monitoringClient); + const instanceTemplate: HostInfo = await this.getInstanceTemplate(pair[1], this.monitoringClient); + this.writerHostInfo = this.topologyUtils.createHost(pair[0], pair[1], true, 0, Date.now(), this.initialHostInfo, instanceTemplate); + logger.debug(Messages.get("ClusterTopologyMonitor.writerMonitoringConnection", this.writerHostInfo.host)); + } } } catch (error) { // Do nothing. logger.error(Messages.get("ClusterTopologyMonitor.invalidWriterQuery", error?.message)); } - } else { + } else if (client) { // Monitoring connection already set by another task, close the new connection. await this.closeConnection(client); } } const hosts: HostInfo[] = await this.fetchTopologyAndUpdateCache(this.monitoringClient); - if (writerVerifiedByThisTask) { - if (this.ignoreNewTopologyRequestsEndTimeMs === -1) { - this.ignoreNewTopologyRequestsEndTimeMs = 0; - } else { - this.ignoreNewTopologyRequestsEndTimeMs = Date.now() + this.ignoreTopologyRequestMs; - } - } if (hosts === null) { this.isVerifiedWriterConnection = false; @@ -243,289 +290,540 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { return hosts; } + protected getInstanceTemplate(hostId: string, targetClient: ClientWrapper): Promise { + return Promise.resolve(this.instanceTemplate); + } + + queryForTopology(client: ClientWrapper): Promise { + return this.topologyUtils.queryForTopology(client, this.pluginService.getDialect(), this.initialHostInfo, this.instanceTemplate); + } + + updateHostsAvailability(hosts: HostInfo[]): void { + if (!hosts) { + return; + } + + hosts.forEach((host) => { + host.setAvailability(this.readerTopologiesById.has(host.hostId) ? HostAvailability.AVAILABLE : HostAvailability.NOT_AVAILABLE); + }); + } + updateTopologyCache(hosts: HostInfo[]): void { - this.topologyMap.put(this.clusterId, hosts, ClusterTopologyMonitorImpl.TOPOLOGY_CACHE_EXPIRATION_NANOS); + this.storageService.set(this.clusterId, new Topology(hosts)); this.requestToUpdateTopology = false; } - async getWriterHostIdIfConnected(client: ClientWrapper, hostId: string): Promise { - const writerHost: string = await this.hostListProvider.getWriterId(client); - // Returns the hostId of the writer if client is connected to that writer, otherwise returns null. - return writerHost === hostId ? writerHost : null; + protected clearTopologyCache(): void { + this.servicesContainer.storageService.remove(Topology, this.clusterId); } - async closeConnection(client: ClientWrapper): Promise { - if (client !== null) { - await client.abort(); - client = null; - } + async closeConnection(client: ClientWrapper | null): Promise { + await client?.abort(); } async updateMonitoringClient(newClient: ClientWrapper | null): Promise { const clientToClose = this.monitoringClient; this.monitoringClient = newClient; - if (clientToClose) { - await clientToClose.abort(); - } + await clientToClose?.abort(); } - private isInPanicMode(): boolean { - return !this.monitoringClient || !this.isVerifiedWriterConnection; + async stop(): Promise { + this._stop = true; + this.hostMonitorsStop = true; + + await Promise.all(this.submittedHosts.values()); + + await this.closeHostMonitors(); + + const hostMonitorsWriterClientToClose = this.hostMonitorsWriterClient; + const hostMonitorsReaderClientToClose = this.hostMonitorsReaderClient; + const monitoringClientToClose = this.monitoringClient; + + this.hostMonitorsWriterClient = null; + this.hostMonitorsReaderClient = null; + this.monitoringClient = null; + + await this.closeConnection(hostMonitorsWriterClientToClose); + if (hostMonitorsReaderClientToClose && hostMonitorsReaderClientToClose !== hostMonitorsWriterClientToClose) { + await this.closeConnection(hostMonitorsReaderClientToClose); + } + if ( + monitoringClientToClose && + monitoringClientToClose !== hostMonitorsWriterClientToClose && + monitoringClientToClose !== hostMonitorsReaderClientToClose + ) { + await this.closeConnection(monitoringClientToClose); + } + + this.submittedHosts.clear(); + + return super.stop(); } - async run(): Promise { - logger.debug(Messages.get("ClusterTopologyMonitor.startMonitoring")); + async monitor(): Promise { try { - while (!this.stopMonitoring) { + logger.debug(Messages.get("ClusterTopologyMonitor.startMonitoring", this.clusterId, this.initialHostInfo.host)); + this.servicesContainer.eventPublisher.subscribe(this, new Set([MonitorResetEvent])); + + while (!this._stop) { + this.lastActivityTimestampNanos = getTimeInNanos(); + if (this.isInPanicMode()) { - // Panic Mode: high refresh rate in effect. + if (this.submittedHosts.size === 0) { + logger.debug(Messages.get("ClusterTopologyMonitor.startingHostMonitoringTasks")); - if (this.hostMonitors.size === 0) { - // Initialize host tasks. - logger.debug(Messages.get("ClusterTopologyMonitor.startingHostMonitors")); + // Start host monitoring tasks. this.hostMonitorsStop = false; - if (this.hostMonitorsReaderClient !== null) { - await this.closeConnection(this.hostMonitorsReaderClient); - } - if (this.hostMonitorsWriterClient !== null) { - await this.closeConnection(this.hostMonitorsWriterClient); - } - this.hostMonitorsWriterClient = null; - this.hostMonitorsReaderClient = null; + await this.hostMonitorClientCleanUp(); this.hostMonitorsWriterInfo = null; this.hostMonitorsLatestTopology = []; - // Use any client to gather topology information. - let hosts: HostInfo[] = this.topologyMap.get(this.clusterId); - if (!hosts) { + let hosts: HostInfo[] = this.getStoredHosts(); + if (hosts === null) { + // Use any available connection to get the topology. hosts = await this.openAnyClientAndUpdateTopology(); } - // Set up host monitors. - if (hosts && !this.isVerifiedWriterConnection) { - for (const hostInfo of hosts) { - if (!this.hostMonitors.has(hostInfo.host)) { - const hostMonitor = new HostMonitor(this, hostInfo, this.writerHostInfo); - const hostRun = hostMonitor.run(); - this.hostMonitors.set(hostInfo.host, hostMonitor); - this.untrackedPromises.push(hostRun); - } + await this.closeHostMonitors(); + + if (!(hosts !== null && !this.isVerifiedWriterConnection)) { + await this.delay(true); + continue; + } + + for (const hostInfo of hosts) { + if (!this.submittedHosts.get(hostInfo.host)) { + const minimalServiceContainer = ServiceUtils.instance.createMinimalServiceContainerFrom( + this.servicesContainer, + this._monitoringProperties + ); + await minimalServiceContainer.pluginManager.init(); + const hostMonitor = new HostMonitor(minimalServiceContainer, this, hostInfo, this.writerHostInfo); + const promise = hostMonitor.run(); + this.submittedHosts.set(hostInfo.host, promise); } } - // If topology is not correctly updated, will try on the next round. + + // We will try again in the next iteration. } else { - // Host monitors already running, check if a writer has been detected. - const writerClient = this.hostMonitorsWriterClient; - const writerHostInfo = this.hostMonitorsWriterInfo; - if (writerClient && writerHostInfo && writerHostInfo !== this.writerHostInfo) { - // Writer detected, update monitoringClient. - logger.info(Messages.get("ClusterTopologyMonitor.writerPickedUpFromHostMonitors", writerHostInfo.hostId)); - await this.updateMonitoringClient(writerClient); - this.writerHostInfo = writerHostInfo; + // The host monitors are running, so we check if the writer has been detected. + const writerClient: ClientWrapper | null = this.hostMonitorsWriterClient; + const writerClientHostInfo: HostInfo | null = this.hostMonitorsWriterInfo; + + if (writerClient && writerClientHostInfo) { + logger.debug(Messages.get("ClusterTopologyMonitor.writerPickedUpFromHostMonitors", writerClientHostInfo.toString())); + + const oldMonitoringClient = this.monitoringClient; + this.monitoringClient = writerClient; + this.writerHostInfo = writerClientHostInfo; this.isVerifiedWriterConnection = true; - if (this.ignoreNewTopologyRequestsEndTimeMs === -1) { - this.ignoreNewTopologyRequestsEndTimeMs = 0; - } else { - this.ignoreNewTopologyRequestsEndTimeMs = Date.now() + this.ignoreTopologyRequestMs; - } - if (this.highRefreshRateEndTimeMs === -1) { - this.highRefreshRateEndTimeMs = 0; - } else { - this.highRefreshRateEndTimeMs = Date.now() + this.highRefreshPeriodAfterPanicMs; - } + this.highRefreshRateEndTimeNs = getTimeInNanos() + BigInt(this.highRefreshRateNs); - // Stop monitoring of each host, writer detected. this.hostMonitorsStop = true; - this.hostMonitors.clear(); + await this.closeHostMonitors(); + this.submittedHosts.clear(); + this.stableTopologiesStartNs = BigInt(0); + this.readerTopologiesById.clear(); + this.completedOneCycle.clear(); + + // Close the old monitoring client that was replaced by the new writer client. + if (oldMonitoringClient && oldMonitoringClient !== this.monitoringClient) { + await this.closeConnection(oldMonitoringClient); + } + + await this.delay(true); continue; } else { - // No writer detected, update host monitors with any new hosts in the topology. - const hosts: HostInfo[] = this.hostMonitorsLatestTopology; - if (hosts !== null && !this.hostMonitorsStop) { + // Update host monitors with the new instances in the topology. + const hosts: HostInfo[] | null = this.hostMonitorsLatestTopology; + if (hosts && !this.hostMonitorsStop) { for (const hostInfo of hosts) { - if (!this.hostMonitors.has(hostInfo.host)) { - const hostMonitor = new HostMonitor(this, hostInfo, this.writerHostInfo); - const hostRun = hostMonitor.run(); - this.hostMonitors.set(hostInfo.host, hostMonitor); - this.untrackedPromises.push(hostRun); + if (!this.submittedHosts.get(hostInfo.host)) { + const minimalServiceContainer = ServiceUtils.instance.createMinimalServiceContainerFrom( + this.servicesContainer, + this._monitoringProperties + ); + await minimalServiceContainer.pluginManager.init(); + // Intentionally not calling await on hostMonitor.run(). + const hostMonitor = new HostMonitor(minimalServiceContainer, this, hostInfo, this.writerHostInfo); + const promise = hostMonitor.run(); + this.submittedHosts.set(hostInfo.host, promise); } } } } } - // Trigger a delay before retrying. + + this.checkForStableReaderTopologies(); await this.delay(true); } else { - // Regular mode: lower refresh rate than panic mode. - if (this.hostMonitors.size !== 0) { - // Stop host monitors. - this.hostMonitorsStop = true; - this.hostMonitors.clear(); + // We are in regular mode. + if (this.submittedHosts.size !== 0) { + await this.closeHostMonitors(); + this.submittedHosts.clear(); + this.stableTopologiesStartNs = BigInt(0); + this.readerTopologiesById.clear(); + this.completedOneCycle.clear(); } - const hosts = await this.fetchTopologyAndUpdateCache(this.monitoringClient); + + const hosts: HostInfo[] = await this.fetchTopologyAndUpdateCache(this.monitoringClient); if (hosts === null) { - // Unable to gather topology, switch to panic mode. + // Attempt to fetch topology failed, so we switch to panic mode. + const clientToClose = this.monitoringClient; + this.monitoringClient = null; + await this.closeConnection(clientToClose); this.isVerifiedWriterConnection = false; - await this.updateMonitoringClient(null); + this.writerHostInfo = null; + await this.delay(false); continue; } - if (this.highRefreshRateEndTimeMs > 0 && Date.now() > this.highRefreshRateEndTimeMs) { - this.highRefreshRateEndTimeMs = 0; + + if (this.highRefreshRateEndTimeNs > 0 && getTimeInNanos() > this.highRefreshRateEndTimeNs) { + this.highRefreshRateEndTimeNs = BigInt(0); } - if (this.highRefreshRateEndTimeMs < 0) { - // Log topology when not in high refresh rate. - this.logTopology(`[clusterTopologyMonitor] `); + + // We avoid logging the topology while using the high refresh rate because it is too noisy. + if (this.highRefreshRateEndTimeNs === BigInt(0)) { + logger.debug(logTopology(this.getStoredHosts(), "")); } - // Set an easily interruptible delay between topology refreshes. + await this.delay(false); } - if (this.ignoreNewTopologyRequestsEndTimeMs > 0 && Date.now() > this.ignoreNewTopologyRequestsEndTimeMs) { - this.ignoreNewTopologyRequestsEndTimeMs = 0; - } } - } catch (error) { - logger.error(Messages.get("ClusterTopologyMonitor.errorDuringMonitoring", error?.message)); } finally { - this.stopMonitoring = true; - await this.updateMonitoringClient(null); - logger.debug(Messages.get("ClusterTopologyMonitor.endMonitoring")); + this._stop = true; + await this.closeHostMonitors(); + await this.hostMonitorClientCleanUp(); + + this.servicesContainer.eventPublisher.unsubscribe(this, new Set([MonitorResetEvent])); + + logger.debug(Messages.get("ClusterTopologyMonitor.stopHostMonitoringTask", this.initialHostInfo.host)); } + + return Promise.resolve(); } - private async delay(useHighRefreshRate: boolean) { - if (Date.now() < this.highRefreshRateEndTimeMs) { - useHighRefreshRate = true; + protected checkForStableReaderTopologies(): void { + const latestHosts: HostInfo[] = this.getStoredHosts(); + if (!latestHosts || latestHosts.length === 0) { + this.stableTopologiesStartNs = BigInt(0); + return; } - const endTime = Date.now() + (useHighRefreshRate ? this.highRefreshRateMs : this.refreshRateMs); - await sleep(50); - while (Date.now() < endTime && !this.requestToUpdateTopology) { - await sleep(50); + + const readerIds: string[] = latestHosts.map((host) => host.hostId); + for (const id of readerIds) { + const completedCycle = this.completedOneCycle.get(id) ?? false; + if (!completedCycle) { + // Not all reader monitors have completed a cycle. We shouldn't conclude that reader topologies are stable until + // each reader monitor has made at least one attempt to fetch topology information, even if unsuccessful. + this.stableTopologiesStartNs = BigInt(0); + return; + } + } + + const readerTopologyValues = Array.from(this.readerTopologiesById.values()); + const readerTopology: HostInfo[] | undefined = readerTopologyValues.length > 0 ? readerTopologyValues[0] : undefined; + if (!readerTopology) { + // readerTopologiesById has been cleared since checking its size. + this.stableTopologiesStartNs = BigInt(0); + return; + } + + // Check whether the topologies match. HostInfos are compared using their host, port, role, and availability fields. + // Using the first HostInfo in the topology as the reference. + // Note that monitors that encounter errors will remove their entry from the map, so only entries from + // successful monitors are checked. + const reference = JSON.stringify(readerTopology.map(this.hostInfoExtractor).sort()); + const allTopologiesMatch = readerTopologyValues.every((hosts) => JSON.stringify(hosts.map(this.hostInfoExtractor).sort()) === reference); + + if (!allTopologiesMatch) { + // The topologies detected by each reader do not match. + this.stableTopologiesStartNs = BigInt(0); + return; + } + + // All reader topologies match. + if (this.stableTopologiesStartNs === BigInt(0)) { + this.stableTopologiesStartNs = getTimeInNanos(); + } + + if (getTimeInNanos() > this.stableTopologiesStartNs + ClusterTopologyMonitorImpl.STABLE_TOPOLOGIES_DURATION_NS) { + // Reader topologies have been consistent for STABLE_TOPOLOGIES_DURATION_NS, so the topology should be accurate. + this.stableTopologiesStartNs = BigInt(0); + this.updateHostsAvailability(readerTopology); + logger.debug( + logTopology( + readerTopology, + Messages.get( + "ClusterTopologyMonitor.matchingReaderTopologies", + String(convertNanosToMs(ClusterTopologyMonitorImpl.STABLE_TOPOLOGIES_DURATION_NS)) + ) + ) + ); + this.updateTopologyCache(readerTopology); + } + } + + protected async reset(): Promise { + logger.debug(Messages.get("ClusterTopologyMonitor.reset", this.clusterId, this.initialHostInfo.host)); + + this.hostMonitorsStop = true; + await this.closeHostMonitors(); + await this.hostMonitorClientCleanUp(); + this.hostMonitorsStop = false; + this.submittedHosts.clear(); + this.stableTopologiesStartNs = BigInt(0); + this.readerTopologiesById.clear(); + this.completedOneCycle.clear(); + + this.hostMonitorsWriterInfo = null; + this.hostMonitorsLatestTopology = []; + + await this.updateMonitoringClient(null); + this.isVerifiedWriterConnection = false; + this.writerHostInfo = null; + this.highRefreshRateEndTimeNs = BigInt(0); + this.requestToUpdateTopology = false; + this.clearTopologyCache(); + + // This breaks any waiting/sleeping cycles in the monitoring task. + this.requestToUpdateTopology = true; + } + + async processEvent(event: Event): Promise { + if (event instanceof MonitorResetEvent) { + logger.debug(Messages.get("ClusterTopologyMonitor.resetEventReceived")); + const resetEvent = event as MonitorResetEvent; + if (resetEvent.clusterId === this.clusterId) { + await this.reset(); + } + } + } + + protected async hostMonitorClientCleanUp(): Promise { + const writerClientToClose = this.hostMonitorsWriterClient; + const readerClientToClose = this.hostMonitorsReaderClient; + + this.hostMonitorsWriterClient = null; + this.hostMonitorsReaderClient = null; + + if (writerClientToClose && this.monitoringClient !== writerClientToClose) { + try { + await this.closeConnection(writerClientToClose); + } catch (e: any) { + // Ignore + } } + + if (readerClientToClose && this.monitoringClient !== readerClientToClose && writerClientToClose !== readerClientToClose) { + try { + await this.closeConnection(readerClientToClose); + } catch (e: any) { + // Ignore + } + } + } + + protected async closeHostMonitors(): Promise { + await Promise.all(this.submittedHosts.values()); + this.submittedHosts.clear(); + await this.hostMonitorClientCleanUp(); } - logTopology(msgPrefix: string) { - const hosts: HostInfo[] = this.topologyMap.get(this.clusterId); - if (hosts && hosts.length !== 0) { - logger.debug(logTopology(hosts, msgPrefix)); + private isInPanicMode(): boolean { + return !this.monitoringClient || !this.isVerifiedWriterConnection; + } + + private getStoredHosts(): HostInfo[] | null { + return this.storageService.get(Topology, this.clusterId)?.hosts ?? null; + } + + private async delay(useHighRefreshRate: boolean): Promise { + if (getTimeInNanos() < this.highRefreshRateEndTimeNs) { + useHighRefreshRate = true; + } + const delayNs = useHighRefreshRate ? this.highRefreshRateNs : this.refreshRateNs; + const endTime: bigint = getTimeInNanos() + BigInt(delayNs); + await sleep(50); + while (getTimeInNanos() < endTime && !this.requestToUpdateTopology && !this._stop) { + await sleep(50); } } } export class HostMonitor { + private static readonly INITIAL_BACKOFF_MS = 100; + private static readonly MAX_BACKOFF_MS = 10000; + + protected readonly servicesContainer: FullServicesContainer; protected readonly monitor: ClusterTopologyMonitorImpl; protected readonly hostInfo: HostInfo; - protected readonly writerHostInfo: HostInfo; + protected readonly writerHostInfo: HostInfo | null; protected writerChanged: boolean = false; + protected connectionAttempts: number = 0; + protected client: ClientWrapper | null = null; - constructor(monitor: ClusterTopologyMonitorImpl, hostInfo: HostInfo, writerHostInfo: HostInfo) { + constructor(servicesContainer: FullServicesContainer, monitor: ClusterTopologyMonitorImpl, hostInfo: HostInfo, writerHostInfo: HostInfo | null) { + this.servicesContainer = servicesContainer; this.monitor = monitor; this.hostInfo = hostInfo; this.writerHostInfo = writerHostInfo; } async run() { - let client: ClientWrapper | null = null; let updateTopology: boolean = false; const startTime: number = Date.now(); logger.debug(Messages.get("HostMonitor.startMonitoring", this.hostInfo.hostId)); + const pluginService = this.servicesContainer.pluginService; try { while (!this.monitor.hostMonitorsStop) { - if (!client) { + if (!this.client) { try { - client = await this.monitor.pluginService.forceConnect(this.hostInfo, this.monitor.monitoringProperties); - this.monitor.pluginService.setAvailability(this.hostInfo.allAliases, HostAvailability.AVAILABLE); + this.client = await pluginService.forceConnect(this.hostInfo, this.monitor.monitoringProperties); + this.connectionAttempts = 0; } catch (error) { - this.monitor.pluginService.setAvailability(this.hostInfo.allAliases, HostAvailability.NOT_AVAILABLE); + // A problem occurred while connecting. + if (pluginService.isNetworkError(error)) { + // It's a network issue that's expected during a cluster failover. + // We will try again on the next iteration. + await sleep(100); + this.monitor.completedOneCycle.set(this.hostInfo.hostId, true); + this.monitor.readerTopologiesById.delete(this.hostInfo.hostId); + continue; + } else if (pluginService.isLoginError(error)) { + throw new AwsWrapperError(Messages.get("HostMonitor.loginErrorDuringMonitoring"), error); + } else { + // It might be some transient error. Let's try again. + // If the error repeats, we will try again after a longer delay. + const backoff = this.calculateBackoffWithJitter(this.connectionAttempts++); + await sleep(backoff); + this.monitor.completedOneCycle.set(this.hostInfo.hostId, true); + this.monitor.readerTopologiesById.delete(this.hostInfo.hostId); + continue; + } } } - if (client) { - let writerId = null; + if (this.client) { + let isWriter: boolean = false; try { - writerId = await this.monitor.getWriterHostIdIfConnected(client, this.hostInfo.hostId); + isWriter = await this.monitor.topologyUtils.isWriterInstance(this.client); } catch (error) { logger.error(Messages.get("ClusterTopologyMonitor.invalidWriterQuery", error?.message)); - await this.monitor.closeConnection(client); - client = null; + await this.monitor.closeConnection(this.client); + this.client = null; } - if (writerId) { - // First connection after failover may be stale. - if ((await this.monitor.pluginService.getHostRole(client)) !== HostRole.WRITER) { - logger.debug(Messages.get("HostMonitor.writerIsStale", writerId)); - writerId = null; + if (isWriter) { + try { + // First connection after failover may be stale. + const hostRole = await this.monitor.pluginService.getHostRole(this.client); + if (hostRole !== HostRole.WRITER) { + isWriter = false; + } + } catch (error: any) { + // Invalid connection, retry. + this.monitor.completedOneCycle.set(this.hostInfo.hostId, true); + this.monitor.readerTopologiesById.delete(this.hostInfo.hostId); + continue; } } - if (writerId) { + if (isWriter) { + // This prevents us from closing the connection in the finally block. if (this.monitor.hostMonitorsWriterClient) { - await this.monitor.closeConnection(client); + // The writer connection is already set up, probably by another host monitor. + await this.monitor.closeConnection(this.client); } else { - logger.debug(Messages.get("HostMonitor.detectedWriter", writerId, this.hostInfo.host)); - const updatedHosts: HostInfo[] = await this.monitor.fetchTopologyAndUpdateCache(client); - if (updatedHosts && this.monitor.hostMonitorsWriterClient === null) { - this.monitor.hostMonitorsWriterClient = client; - this.monitor.hostMonitorsWriterInfo = this.hostInfo; - this.monitor.hostMonitorsStop = true; - this.monitor.logTopology(`[hostMonitor ${this.hostInfo.hostId}] `); - } else { - await this.monitor.closeConnection(client); - } + // Successfully updated the host monitor writer connection. + logger.debug(Messages.get("HostMonitor.detectedWriter", this.hostInfo.hostId, this.hostInfo.url)); + + this.servicesContainer.importantEventService.registerEvent(() => + Messages.get("HostMonitor.detectedWriter", this.hostInfo.hostId, this.hostInfo.url) + ); + + await this.monitor.fetchTopologyAndUpdateCache(this.client); + this.hostInfo.setAvailability(HostAvailability.AVAILABLE); + this.monitor.hostMonitorsWriterClient = this.client; + this.monitor.hostMonitorsWriterInfo = this.hostInfo; + // Connection is already assigned to this.monitor.hostMonitorsWriterClient + // so we need to reset client without closing it. + this.client = null; + this.monitor.hostMonitorsStop = true; + logger.debug(logTopology(this.monitor.hostMonitorsLatestTopology, `[hostMonitor ${this.hostInfo.hostId}] `)); } - client = null; return; - } else if (client) { + } else if (this.client) { // Client is a reader. if (!this.monitor.hostMonitorsWriterClient) { - // While the writer hasn't been identified, reader client can update topology. + // We can use this reader connection to update the topology while we wait for the writer connection to + // be established. if (updateTopology) { - await this.readerTaskFetchTopology(client, this.writerHostInfo); - } else if (this.monitor.hostMonitorsReaderClient === null) { - this.monitor.hostMonitorsReaderClient = client; + await this.readerTaskFetchTopology(this.client, this.writerHostInfo); + } else if (!this.monitor.hostMonitorsReaderClient) { + this.monitor.hostMonitorsReaderClient = this.client; updateTopology = true; - await this.readerTaskFetchTopology(client, this.writerHostInfo); + await this.readerTaskFetchTopology(this.client, this.writerHostInfo); + } else { + await this.readerTaskFetchTopology(this.client, this.writerHostInfo); } } } } + + this.monitor.completedOneCycle.set(this.hostInfo.hostId, true); await sleep(100); } } catch (error) { // Close the monitor. } finally { - await this.monitor.closeConnection(client); + this.monitor.completedOneCycle.set(this.hostInfo.hostId, true); + this.monitor.readerTopologiesById.delete(this.hostInfo.hostId); + + await this.monitor.closeConnection(this.client); logger.debug(Messages.get("HostMonitor.endMonitoring", this.hostInfo.hostId, (Date.now() - startTime).toString())); } } - private async readerTaskFetchTopology(client: any, writerHostInfo: HostInfo) { + private async readerTaskFetchTopology(client: ClientWrapper, writerHostInfo: HostInfo | null) { if (!client) { return; } - let hosts: HostInfo[]; + let hosts: HostInfo[] | null; try { - hosts = await this.monitor.hostListProvider.sqlQueryForTopology(client); - if (hosts === null || hosts.length === 0) { + hosts = await this.monitor.queryForTopology(client); + if (!hosts) { return; } - this.monitor.hostMonitorsLatestTopology = hosts; } catch (error) { return; } + // Share this topology so that the main monitoring task can adjust the node monitoring tasks. + this.monitor.hostMonitorsLatestTopology = hosts; + this.monitor.readerTopologiesById.set(this.hostInfo.hostId, hosts); + if (this.writerChanged) { + this.monitor.updateHostsAvailability(hosts); this.monitor.updateTopologyCache(hosts); logger.debug(logTopology(hosts, `[hostMonitor ${this.hostInfo.hostId}] `)); return; } - const latestWriterHostInfo: HostInfo = hosts.find((x) => x.role === HostRole.WRITER); - if (latestWriterHostInfo && writerHostInfo && latestWriterHostInfo.getHostAndPort() !== writerHostInfo.getHostAndPort()) { + const latestWriterHostInfo = hosts.find((x) => x.role === HostRole.WRITER); + if (latestWriterHostInfo && writerHostInfo && latestWriterHostInfo.hostAndPort !== writerHostInfo.hostAndPort) { this.writerChanged = true; - logger.debug(Messages.get("HostMonitor.writerHostChanged", writerHostInfo.getHostAndPort(), latestWriterHostInfo.getHostAndPort())); + logger.debug(Messages.get("HostMonitor.writerHostChanged", writerHostInfo.hostAndPort, latestWriterHostInfo.hostAndPort)); + this.monitor.updateHostsAvailability(hosts); this.monitor.updateTopologyCache(hosts); logger.debug(logTopology(hosts, `[hostMonitor ${this.hostInfo.hostId}] `)); } } + + private calculateBackoffWithJitter(attempt: number): number { + let backoff = HostMonitor.INITIAL_BACKOFF_MS * Math.round(Math.pow(2, Math.min(attempt, 6))); + backoff = Math.min(backoff, HostMonitor.MAX_BACKOFF_MS); + return Math.round(backoff * (0.5 + Math.random() * 0.5)); + } } diff --git a/common/lib/host_list_provider/monitoring/global_aurora_topology_monitor.ts b/common/lib/host_list_provider/monitoring/global_aurora_topology_monitor.ts new file mode 100644 index 000000000..9582c5222 --- /dev/null +++ b/common/lib/host_list_provider/monitoring/global_aurora_topology_monitor.ts @@ -0,0 +1,69 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ClusterTopologyMonitorImpl } from "./cluster_topology_monitor"; +import { GdbTopologyUtils, GlobalTopologyUtils } from "../global_topology_utils"; +import { FullServicesContainer } from "../../utils/full_services_container"; +import { HostInfo } from "../../host_info"; +import { ClientWrapper } from "../../client_wrapper"; +import { AwsWrapperError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { TopologyUtils } from "../topology_utils"; + +function isGdbTopologyUtils(utils: TopologyUtils): utils is TopologyUtils & GdbTopologyUtils { + return "getRegion" in utils && typeof (utils as unknown as GdbTopologyUtils).getRegion === "function"; +} + +export class GlobalAuroraTopologyMonitor extends ClusterTopologyMonitorImpl { + protected readonly instanceTemplatesByRegion: Map; + declare public readonly topologyUtils: TopologyUtils; + + constructor( + servicesContainer: FullServicesContainer, + topologyUtils: TopologyUtils, + clusterId: string, + initialHostInfo: HostInfo, + properties: Map, + instanceTemplate: HostInfo, + refreshRateNano: number, + highRefreshRateNano: number, + instanceTemplatesByRegion: Map + ) { + super(servicesContainer, topologyUtils, clusterId, initialHostInfo, properties, instanceTemplate, refreshRateNano, highRefreshRateNano); + + this.instanceTemplatesByRegion = instanceTemplatesByRegion; + this.topologyUtils = topologyUtils; + } + + protected override async getInstanceTemplate(hostId: string, targetClient: ClientWrapper): Promise { + if (!isGdbTopologyUtils(this.topologyUtils)) { + throw new AwsWrapperError(Messages.get("GlobalAuroraTopologyMonitor.invalidTopologyUtils")); + } + + const dialect = this.hostListProviderService.getDialect(); + const region = await this.topologyUtils.getRegion(hostId, targetClient, dialect); + + if (region) { + const instanceTemplate = this.instanceTemplatesByRegion.get(region); + if (!instanceTemplate) { + throw new AwsWrapperError(Messages.get("GlobalAuroraTopologyMonitor.cannotFindRegionTemplate", region)); + } + return instanceTemplate; + } + + return this.instanceTemplate; + } +} diff --git a/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts b/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts deleted file mode 100644 index 49c5ae8fb..000000000 --- a/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts +++ /dev/null @@ -1,108 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { RdsHostListProvider } from "../rds_host_list_provider"; -import { HostInfo, AwsWrapperError } from "../../"; -import { PluginService } from "../../plugin_service"; -import { ClusterTopologyMonitor, ClusterTopologyMonitorImpl } from "./cluster_topology_monitor"; -import { HostListProviderService } from "../../host_list_provider_service"; -import { ClientWrapper } from "../../client_wrapper"; -import { DatabaseDialect } from "../../database_dialect/database_dialect"; -import { Messages } from "../../utils/messages"; -import { WrapperProperties } from "../../wrapper_property"; -import { BlockingHostListProvider } from "../host_list_provider"; -import { logger } from "../../../logutils"; -import { SlidingExpirationCacheWithCleanupTask } from "../../utils/sliding_expiration_cache_with_cleanup_task"; -import { isDialectTopologyAware } from "../../utils/utils"; - -export class MonitoringRdsHostListProvider extends RdsHostListProvider implements BlockingHostListProvider { - static readonly CACHE_CLEANUP_NANOS: bigint = BigInt(60_000_000_000); // 1 minute. - static readonly MONITOR_EXPIRATION_NANOS: bigint = BigInt(15 * 60_000_000_000); // 15 minutes. - static readonly DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS = 5000; // 5 seconds. - - private static monitors: SlidingExpirationCacheWithCleanupTask = new SlidingExpirationCacheWithCleanupTask( - MonitoringRdsHostListProvider.CACHE_CLEANUP_NANOS, - () => true, - async (item: ClusterTopologyMonitor) => { - try { - await item.close(); - } catch { - // Ignore. - } - }, - "MonitoringRdsHostListProvider.monitors" - ); - - private readonly pluginService: PluginService; - - constructor(properties: Map, originalUrl: string, hostListProviderService: HostListProviderService, pluginService: PluginService) { - super(properties, originalUrl, hostListProviderService); - this.pluginService = pluginService; - } - - async clearAll(): Promise { - RdsHostListProvider.clearAll(); - await MonitoringRdsHostListProvider.monitors.clear(); - } - - async queryForTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { - const monitor: ClusterTopologyMonitor = this.initMonitor(); - - try { - return await monitor.forceRefresh(targetClient, MonitoringRdsHostListProvider.DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS); - } catch (error) { - logger.info(Messages.get("MonitoringHostListProvider.errorForceRefresh", error.message)); - return null; - } - } - - async sqlQueryForTopology(targetClient: ClientWrapper): Promise { - const dialect: DatabaseDialect = this.hostListProviderService.getDialect(); - if (!isDialectTopologyAware(dialect)) { - throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); - } - return await dialect.queryForTopology(targetClient, this).then((res: any) => this.processQueryResults(res)); - } - - async forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise { - const monitor: ClusterTopologyMonitor = this.initMonitor(); - - return await monitor.forceMonitoringRefresh(shouldVerifyWriter, timeoutMs); - } - - protected initMonitor(): ClusterTopologyMonitor { - const monitor: ClusterTopologyMonitor = MonitoringRdsHostListProvider.monitors.computeIfAbsent( - this.clusterId, - () => - new ClusterTopologyMonitorImpl( - this.clusterId, - MonitoringRdsHostListProvider.topologyCache, - this.initialHost, - this.properties, - this.pluginService, - this, - WrapperProperties.CLUSTER_TOPOLOGY_REFRESH_RATE_MS.get(this.properties), - WrapperProperties.CLUSTER_TOPOLOGY_HIGH_REFRESH_RATE_MS.get(this.properties) - ), - MonitoringRdsHostListProvider.MONITOR_EXPIRATION_NANOS - ); - - if (monitor === null) { - throw new AwsWrapperError(Messages.get("MonitoringHostListProvider.requiresMonitor")); - } - return monitor; - } -} diff --git a/common/lib/host_list_provider/rds_host_list_provider.ts b/common/lib/host_list_provider/rds_host_list_provider.ts index fd7353bbe..4185d7db7 100644 --- a/common/lib/host_list_provider/rds_host_list_provider.ts +++ b/common/lib/host_list_provider/rds_host_list_provider.ts @@ -21,48 +21,71 @@ import { RdsUrlType } from "../utils/rds_url_type"; import { RdsUtils } from "../utils/rds_utils"; import { HostListProviderService } from "../host_list_provider_service"; import { ConnectionUrlParser } from "../utils/connection_url_parser"; -import { AwsWrapperError } from "../utils/errors"; +import { AwsTimeoutError, AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; import { WrapperProperties } from "../wrapper_property"; import { logger } from "../../logutils"; -import { HostAvailability } from "../host_availability/host_availability"; -import { CacheMap } from "../utils/cache_map"; -import { isDialectTopologyAware, logTopology } from "../utils/utils"; +import { isDialectTopologyAware } from "../database_dialect/topology_aware_database_dialect"; import { DatabaseDialect } from "../database_dialect/database_dialect"; import { ClientWrapper } from "../client_wrapper"; +import { CoreServicesContainer } from "../utils/core_services_container"; +import { StorageService } from "../utils/storage/storage_service"; +import { Topology } from "./topology"; +import { TopologyUtils } from "./topology_utils"; +import { FullServicesContainer } from "../utils/full_services_container"; +import { PluginService } from "../plugin_service"; +import { ClusterTopologyMonitor, ClusterTopologyMonitorImpl } from "./monitoring/cluster_topology_monitor"; +import { MonitorInitializer } from "../utils/monitoring/monitor"; export class RdsHostListProvider implements DynamicHostListProvider { + private static readonly DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS: number = 5000; private readonly originalUrl: string; - private readonly rdsHelper: RdsUtils; + protected readonly rdsHelper: RdsUtils; + protected readonly servicesContainers: FullServicesContainer; + private readonly pluginService: PluginService; + private readonly storageService: StorageService; + protected readonly topologyUtils: TopologyUtils; protected readonly properties: Map; private rdsUrlType: RdsUrlType; private initialHostList: HostInfo[]; protected initialHost: HostInfo; - private refreshRateNano: number; - private suggestedClusterIdRefreshRateNano: number = 10 * 60 * 1_000_000_000; // 10 minutes - private hostList?: HostInfo[]; + protected refreshRateNano: number; + protected highRefreshRateNano: number; protected readonly connectionUrlParser: ConnectionUrlParser; protected readonly hostListProviderService: HostListProviderService; - public static readonly suggestedPrimaryClusterIdCache: CacheMap = new CacheMap(); - - public static readonly primaryClusterIdCache: CacheMap = new CacheMap(); - public static readonly topologyCache: CacheMap = new CacheMap(); public clusterId: string = Date.now().toString(); public isInitialized: boolean = false; - public isPrimaryClusterId?: boolean; public clusterInstanceTemplate?: HostInfo; - constructor(properties: Map, originalUrl: string, hostListProviderService: HostListProviderService) { + constructor(properties: Map, originalUrl: string, topologyUtils: TopologyUtils, servicesContainers: FullServicesContainer) { this.rdsHelper = new RdsUtils(); - this.hostListProviderService = hostListProviderService; - this.connectionUrlParser = hostListProviderService.getConnectionUrlParser(); + this.topologyUtils = topologyUtils; + this.servicesContainers = servicesContainers; + this.pluginService = this.servicesContainers.pluginService; + this.storageService = this.servicesContainers.storageService; + this.hostListProviderService = this.servicesContainers.hostListProviderService; + this.connectionUrlParser = this.hostListProviderService.getConnectionUrlParser(); this.originalUrl = originalUrl; this.properties = properties; + this.refreshRateNano = WrapperProperties.CLUSTER_TOPOLOGY_REFRESH_RATE_MS.get(this.properties) * 1000000; + this.highRefreshRateNano = WrapperProperties.CLUSTER_TOPOLOGY_HIGH_REFRESH_RATE_MS.get(this.properties) * 1000000; + } + + init(): void { + if (this.isInitialized) { + return; + } + + this.initSettings(); + + this.isInitialized = true; + } - let port = WrapperProperties.PORT.get(properties); + protected initSettings(): void { + let port = WrapperProperties.PORT.get(this.properties); if (port == null) { - port = hostListProviderService.getDialect().getDefaultPort(); + port = this.hostListProviderService.getDialect().getDefaultPort(); } this.initialHostList = this.connectionUrlParser.getHostsFromConnectionUrl(this.originalUrl, false, port, () => @@ -74,17 +97,8 @@ export class RdsHostListProvider implements DynamicHostListProvider { this.initialHost = this.initialHostList[0]; this.hostListProviderService.setInitialConnectionHostInfo(this.initialHost); - this.refreshRateNano = WrapperProperties.CLUSTER_TOPOLOGY_REFRESH_RATE_MS.get(this.properties) * 1000000; - this.rdsUrlType = this.rdsHelper.identifyRdsType(this.initialHost.host); - } - - init(): void { - if (this.isInitialized) { - return; - } - - this.isPrimaryClusterId = false; + this.clusterId = WrapperProperties.CLUSTER_ID.get(this.properties); const hostInfoBuilder = this.hostListProviderService.getHostInfoBuilder(); this.clusterInstanceTemplate = hostInfoBuilder @@ -93,58 +107,62 @@ export class RdsHostListProvider implements DynamicHostListProvider { .build(); this.validateHostPatternSetting(this.clusterInstanceTemplate.host); + this.rdsUrlType = this.rdsHelper.identifyRdsType(this.initialHost.host); + } - const clusterIdSetting: string = WrapperProperties.CLUSTER_ID.get(this.properties); - if (clusterIdSetting) { - this.clusterId = clusterIdSetting; - } else if (this.rdsUrlType === RdsUrlType.RDS_PROXY) { - // Each proxy is associated with a single cluster, so it's safe to use RDS Proxy Url as cluster - // identification - this.clusterId = this.initialHost.url; - } else if (this.rdsUrlType.isRds) { - const clusterSuggestedResult: ClusterSuggestedResult | null = this.getSuggestedClusterId(this.initialHost.hostAndPort); - if (clusterSuggestedResult && clusterSuggestedResult.clusterId) { - this.clusterId = clusterSuggestedResult.clusterId; - this.isPrimaryClusterId = clusterSuggestedResult.isPrimaryClusterId; - } else { - const clusterRdsHostUrl: string | null = this.rdsHelper.getRdsClusterHostUrl(this.initialHost.host); - if (clusterRdsHostUrl) { - this.clusterId = this.clusterInstanceTemplate.isPortSpecified() - ? `${clusterRdsHostUrl}:${this.clusterInstanceTemplate.port}` - : clusterRdsHostUrl; - this.isPrimaryClusterId = true; - RdsHostListProvider.primaryClusterIdCache.put(this.clusterId, true, this.suggestedClusterIdRefreshRateNano); - } + protected async getOrCreateMonitor(): Promise { + const initializer: MonitorInitializer = { + createMonitor: (servicesContainer: FullServicesContainer): ClusterTopologyMonitor => { + return new ClusterTopologyMonitorImpl( + servicesContainer, + this.topologyUtils, + this.clusterId, + this.initialHost, + this.properties, + this.clusterInstanceTemplate, + this.refreshRateNano, + this.highRefreshRateNano + ); } - } + }; + + return await this.servicesContainers.monitorService.runIfAbsent( + ClusterTopologyMonitorImpl, + this.clusterId, + this.servicesContainers, + this.properties, + initializer + ); + } - this.isInitialized = true; + async forceRefresh(): Promise { + return this.forceMonitoringRefresh(false, RdsHostListProvider.DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS); } - async forceRefresh(): Promise; - async forceRefresh(targetClient: ClientWrapper): Promise; - async forceRefresh(targetClient?: ClientWrapper): Promise { + async forceMonitoringRefresh(verifyTopology: boolean, timeoutMs: number): Promise { this.init(); - const currentClient = targetClient ?? this.hostListProviderService.getCurrentClient().targetClient; - if (currentClient) { - const results: FetchTopologyResult = await this.getTopology(currentClient, true); - this.hostList = results.hosts; - return Array.from(this.hostList); + if (!this.pluginService.isDialectConfirmed()) { + // We need to confirm the dialect before creating a topology monitor so that it uses the correct SQL queries. + // Return the original hosts parsed from the connection string. + return this.initialHostList; } - throw new AwsWrapperError("Could not retrieve targetClient."); - } - async getHostRole(client: ClientWrapper, dialect: DatabaseDialect): Promise { - if (!isDialectTopologyAware(dialect)) { - throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); + const hosts = await this.forceRefreshMonitor(verifyTopology, timeoutMs); + if (hosts && hosts.length > 0) { + return hosts; } - if (client) { - return await dialect.getHostRole(client); - } else { - throw new AwsWrapperError(Messages.get("AwsClient.targetClientNotDefined")); + // Check for cached topology as a fallback. + const storedTopology = this.getStoredTopology(); + if (storedTopology && storedTopology.length > 0) { + return storedTopology; } + return this.initialHostList; + } + + async getHostRole(client: ClientWrapper, _dialect: DatabaseDialect): Promise { + return this.topologyUtils.getHostRole(client); } async getWriterId(client: ClientWrapper): Promise { @@ -155,214 +173,112 @@ export class RdsHostListProvider implements DynamicHostListProvider { if (client) { return await dialect.getWriterId(client); - } else { - throw new AwsWrapperError(Messages.get("AwsClient.targetClientNotDefined")); - } - } - - async identifyConnection(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { - if (!isDialectTopologyAware(dialect)) { - throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); } - const instanceName = await dialect.identifyConnection(targetClient); - - return this.refresh(targetClient).then((topology) => { - const matches = topology.filter((host) => host.hostId === instanceName); - return matches.length === 0 ? null : matches[0]; - }); - } - async refresh(): Promise; - async refresh(targetClient: ClientWrapper): Promise; - async refresh(targetClient?: ClientWrapper): Promise { - this.init(); - - const currentClient = targetClient ?? this.hostListProviderService.getCurrentClient().targetClient; - const results: FetchTopologyResult = await this.getTopology(currentClient, false); - logger.debug(logTopology(results.hosts, results.isCachedData ? "[From cache] " : "")); - this.hostList = results.hosts; - return this.hostList; + throw new AwsWrapperError(Messages.get("AwsClient.targetClientNotDefined")); } - async getTopology(targetClient: ClientWrapper | undefined, forceUpdate: boolean): Promise { - this.init(); - - if (!this.clusterId) { - throw new AwsWrapperError("no cluster id"); - } - - const suggestedPrimaryClusterId: string | null = RdsHostListProvider.suggestedPrimaryClusterIdCache.get(this.clusterId); - if (suggestedPrimaryClusterId && this.clusterId !== suggestedPrimaryClusterId) { - this.clusterId = suggestedPrimaryClusterId; - this.isPrimaryClusterId = true; + async identifyConnection(targetClient: ClientWrapper): Promise { + const instanceIds: [string, string] = await this.topologyUtils.getInstanceId(targetClient); + if (instanceIds.some((id) => !id)) { + return null; } - const cachedHosts: HostInfo[] | null = RdsHostListProvider.topologyCache.get(this.clusterId); - - // This clusterId is a primary one and is about to create a new entry in the cache. - // When a primary entry is created it needs to be suggested for other (non-primary) entries. - // Remember a flag to do suggestion after cache is updated. - const needToSuggest: boolean = !cachedHosts && this.isPrimaryClusterId === true; - if (!cachedHosts || forceUpdate) { - // need to re-fetch the topology. - if (!targetClient || !(await this.hostListProviderService.isClientValid(targetClient))) { - return new FetchTopologyResult(false, this.initialHostList); - } + let topology = await this.refresh(); + let isForcedRefresh = false; - const hosts = await this.queryForTopology(targetClient, this.hostListProviderService.getDialect()); - if (hosts && hosts.length > 0) { - RdsHostListProvider.topologyCache.put(this.clusterId, hosts, this.refreshRateNano); - if (needToSuggest) { - this.suggestPrimaryCluster(hosts); - } - return new FetchTopologyResult(false, hosts); - } + if (!topology) { + topology = await this.forceRefresh(); + isForcedRefresh = true; } - if (!cachedHosts) { - return new FetchTopologyResult(false, this.initialHostList); - } else { - return new FetchTopologyResult(true, cachedHosts); + if (!topology) { + return null; } - } - private getSuggestedClusterId(hostAndPort: string): ClusterSuggestedResult | null { - for (const [key, hosts] of RdsHostListProvider.topologyCache.getEntries()) { - const isPrimaryCluster: boolean = RdsHostListProvider.primaryClusterIdCache.get(key, false, this.suggestedClusterIdRefreshRateNano) ?? false; - if (key === hostAndPort) { - return new ClusterSuggestedResult(hostAndPort, isPrimaryCluster); - } + const instanceName = instanceIds[1]; + let matches = topology.filter((host) => host.hostId === instanceName); + const foundHost = matches.length === 0 ? null : matches[0]; - if (hosts) { - for (const hostInfo of hosts) { - if (hostInfo.hostAndPort === hostAndPort) { - logger.debug(Messages.get("RdsHostListProvider.suggestedClusterId", key, hostAndPort)); - return new ClusterSuggestedResult(key, isPrimaryCluster); - } - } + if (!foundHost && !isForcedRefresh) { + topology = await this.forceRefresh(); + if (!topology) { + return null; } } - return null; - } - suggestPrimaryCluster(primaryClusterHosts: HostInfo[]): void { - if (!primaryClusterHosts) { - return; - } + matches = topology.filter((host) => host.hostId === instanceName); + return matches.length === 0 ? null : matches[0]; + } - const primaryClusterHostUrls: Set = new Set(); - primaryClusterHosts.forEach((hostInfo) => { - primaryClusterHostUrls.add(hostInfo.url); - }); - - for (const [clusterId, clusterHosts] of RdsHostListProvider.topologyCache.getEntries()) { - const isPrimaryCluster: boolean | null = RdsHostListProvider.primaryClusterIdCache.get( - clusterId, - false, - this.suggestedClusterIdRefreshRateNano - ); - const suggestedPrimaryClusterId: string | null = RdsHostListProvider.suggestedPrimaryClusterIdCache.get(clusterId); - if (isPrimaryCluster || suggestedPrimaryClusterId || !clusterHosts) { - continue; - } + async refresh(): Promise { + this.init(); - for (const clusterHost of clusterHosts) { - if (primaryClusterHostUrls.has(clusterHost.url)) { - RdsHostListProvider.suggestedPrimaryClusterIdCache.put(clusterId, this.clusterId, this.suggestedClusterIdRefreshRateNano); - break; - } - } - } + const results: FetchTopologyResult = await this.getTopology(); + return results.hosts; } - async queryForTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { - if (!isDialectTopologyAware(dialect)) { - throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); - } + async getTopology(): Promise { + this.init(); - return await dialect.queryForTopology(targetClient, this).then((res: any) => this.processQueryResults(res)); - } + const storedTopology: HostInfo[] | null = this.getStoredTopology(); - protected async processQueryResults(result: HostInfo[]): Promise { - const hostMap: Map = new Map(); + if (!storedTopology) { + // need to re-fetch the topology. - let hosts: HostInfo[] = []; - const writers: HostInfo[] = []; - result.forEach((host) => { - hostMap.set(host.host, host); - }); + if (!this.pluginService.isDialectConfirmed()) { + // We need to confirm the dialect before creating a topology monitor so that it uses the correct SQL queries. + // We will return the original hosts parsed from the connections string until the dialect has been confirmed. + return new FetchTopologyResult(false, this.initialHostList); + } - hostMap.forEach((host) => { - if (host.role !== HostRole.WRITER) { - hosts.push(host); - } else { - writers.push(host); + const hosts = await this.forceRefreshMonitor(false, RdsHostListProvider.DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS); + if (hosts && hosts.length > 0) { + return new FetchTopologyResult(false, hosts); } - }); + } - const writerCount: number = writers.length; - if (writerCount === 0) { - hosts = []; - } else if (writerCount === 1) { - hosts.push(writers[0]); + if (!storedTopology) { + return new FetchTopologyResult(false, this.initialHostList); } else { - const sortedWriters: HostInfo[] = writers.sort((a, b) => { - return b.lastUpdateTime - a.lastUpdateTime; // reverse order - }); - - hosts.push(sortedWriters[0]); + return new FetchTopologyResult(true, storedTopology); } - - return hosts; } - createHost(host: string, isWriter: boolean, weight: number, lastUpdateTime: number, port?: number): HostInfo { - host = !host ? "?" : host; - const endpoint: string | null = this.getHostEndpoint(host); - if (!port) { - port = this.clusterInstanceTemplate?.isPortSpecified() ? this.clusterInstanceTemplate?.port : this.initialHost?.port; - } - - return this.hostListProviderService - .getHostInfoBuilder() - .withHost(endpoint ?? "") - .withPort(port ?? -1) - .withRole(isWriter ? HostRole.WRITER : HostRole.READER) - .withAvailability(HostAvailability.AVAILABLE) - .withWeight(weight) - .withLastUpdateTime(lastUpdateTime) - .withHostId(host) - .build(); + async getCurrentTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { + this.init(); + return await this.topologyUtils.queryForTopology(targetClient, dialect, this.initialHost, this.clusterInstanceTemplate); } - private getHostEndpoint(hostName: string): string | null { - if (!this.clusterInstanceTemplate || !this.clusterInstanceTemplate.host) { - return null; + protected async forceRefreshMonitor(verifyTopology: boolean, timeoutMs: number): Promise { + const monitor = await this.getOrCreateMonitor(); + try { + return await monitor.forceMonitoringRefresh(verifyTopology, timeoutMs); + } catch (error) { + if (error instanceof AwsTimeoutError) { + return null; + } + throw error; } - const host = this.clusterInstanceTemplate.host; - return host.replace("?", hostName); } - getCachedTopology(): HostInfo[] | null { + getStoredTopology(): HostInfo[] | null { if (!this.clusterId) { return null; } - return RdsHostListProvider.topologyCache.get(this.clusterId) ?? null; - } - static clearAll(): void { - RdsHostListProvider.topologyCache.clear(); - RdsHostListProvider.primaryClusterIdCache.clear(); - RdsHostListProvider.suggestedPrimaryClusterIdCache.clear(); + const topology: Topology = this.storageService.get(Topology, this.clusterId); + + return topology == null ? null : topology.hosts; } clear(): void { if (this.clusterId) { - RdsHostListProvider.topologyCache.delete(this.clusterId); + this.servicesContainers.storageService.remove(Topology, this.clusterId); } } - private validateHostPatternSetting(hostPattern: string) { + protected validateHostPatternSetting(hostPattern: string) { if (!this.rdsHelper.isDnsPatternValid(hostPattern)) { const message: string = Messages.get("RdsHostListProvider.invalidPattern.suggestedClusterId"); logger.error(message); @@ -370,7 +286,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { } const rdsUrlType: RdsUrlType = this.rdsHelper.identifyRdsType(hostPattern); - if (rdsUrlType == RdsUrlType.RDS_PROXY) { + if (rdsUrlType == RdsUrlType.RDS_PROXY || rdsUrlType == RdsUrlType.RDS_PROXY_ENDPOINT) { const message: string = Messages.get("RdsHostListProvider.clusterInstanceHostPatternNotSupportedForRDSProxy"); logger.error(message); throw new AwsWrapperError(message); @@ -406,13 +322,3 @@ export class FetchTopologyResult { this.isCachedData = isCachedData; } } - -class ClusterSuggestedResult { - clusterId: string; - isPrimaryClusterId: boolean; - - constructor(clusterId: string, isPrimaryClusterId: boolean) { - this.clusterId = clusterId; - this.isPrimaryClusterId = isPrimaryClusterId; - } -} diff --git a/common/lib/host_list_provider/topology.ts b/common/lib/host_list_provider/topology.ts new file mode 100644 index 000000000..b7d4e344b --- /dev/null +++ b/common/lib/host_list_provider/topology.ts @@ -0,0 +1,29 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { HostInfo } from "../host_info"; + +export class Topology { + private readonly _hosts: HostInfo[]; + + constructor(hosts: HostInfo[]) { + this._hosts = hosts; + } + + get hosts(): HostInfo[] { + return this._hosts; + } +} diff --git a/common/lib/host_list_provider/topology_utils.ts b/common/lib/host_list_provider/topology_utils.ts new file mode 100644 index 000000000..3cddf4860 --- /dev/null +++ b/common/lib/host_list_provider/topology_utils.ts @@ -0,0 +1,226 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ClientWrapper } from "../client_wrapper"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { HostInfo } from "../host_info"; +import { Messages } from "../utils/messages"; +import { HostRole } from "../host_role"; +import { HostAvailability } from "../host_availability/host_availability"; +import { HostInfoBuilder } from "../host_info_builder"; +import { AwsWrapperError } from "../utils/errors"; +import { TopologyAwareDatabaseDialect } from "../database_dialect/topology_aware_database_dialect"; + +/** + * Type representing an instance template - either a single HostInfo or a Map of region to HostInfo. + */ +export type InstanceTemplate = HostInfo | Map; + +/** + * Options for creating a TopologyQueryResult instance. + */ +export interface TopologyQueryResultOptions { + host: string; + isWriter: boolean; + weight: number; + lastUpdateTime?: number; + port?: number; + id?: string; + endpoint?: string; + awsRegion?: string; +} + +/** + * Represents the result of a topology query for a single database instance. + * Contains information about the instance's role, weight, and connection details. + */ +export class TopologyQueryResult { + host: string; + isWriter: boolean; + weight: number; + lastUpdateTime?: number; + port?: number; + id?: string; + endpoint?: string; + awsRegion?: string; + + constructor(options: TopologyQueryResultOptions) { + this.host = options.host; + this.isWriter = options.isWriter; + this.weight = options.weight; + this.lastUpdateTime = options.lastUpdateTime; + this.port = options.port; + this.id = options.id; + this.endpoint = options.endpoint; + this.awsRegion = options.awsRegion; + } +} + +/** + * An abstract class defining utility methods that can be used to retrieve and process a variety of database topology + * information. This class can be overridden to define logic specific to various database engine deployments + * (e.g. Aurora, Multi-AZ, Global Aurora etc.). + */ +export abstract class TopologyUtils { + protected readonly dialect: TopologyAwareDatabaseDialect; + protected readonly hostInfoBuilder: HostInfoBuilder; + + constructor(dialect: TopologyAwareDatabaseDialect, hostInfoBuilder: HostInfoBuilder) { + this.dialect = dialect; + this.hostInfoBuilder = hostInfoBuilder; + } + + /** + * Query the database for information for each instance in the database topology. + * + * @param targetClient the client wrapper to use to query the database. + * @param dialect the database dialect to use for the topology query. + * @param initialHost the initial host info. + * @param instanceTemplate the template for constructing host info objects. + * @returns a list of {@link HostInfo} objects representing the results of the topology query. + * @throws TypeError if the dialect is not topology-aware. + */ + abstract queryForTopology( + targetClient: ClientWrapper, + dialect: DatabaseDialect, + initialHost: HostInfo, + instanceTemplate: InstanceTemplate + ): Promise; + + public createHost( + instanceId: string | undefined, + instanceName: string | undefined, + isWriter: boolean, + weight: number, + lastUpdateTime: number, + initialHost: HostInfo, + instanceTemplate: HostInfo, + endpoint?: string, + port?: number + ): HostInfo { + const hostname = !instanceName ? "?" : instanceName; + const finalInstanceId = instanceId ?? hostname; + + if (!finalInstanceId) { + throw new AwsWrapperError(Messages.get("TopologyUtils.instanceIdRequired")); + } + + const finalEndpoint = endpoint ?? this.getHostEndpoint(hostname, instanceTemplate) ?? ""; + const finalPort = port ?? (instanceTemplate?.isPortSpecified() ? instanceTemplate?.port : initialHost?.port); + + const host: HostInfo = this.hostInfoBuilder + .withHost(finalEndpoint) + .withPort(finalPort ?? HostInfo.NO_PORT) + .withRole(isWriter ? HostRole.WRITER : HostRole.READER) + .withAvailability(HostAvailability.AVAILABLE) + .withWeight(weight) + .withLastUpdateTime(lastUpdateTime) + .withHostId(finalInstanceId) + .build(); + return host; + } + + /** + * Gets the host endpoint by replacing the placeholder in the cluster instance template. + */ + protected getHostEndpoint(hostName: string, clusterInstanceTemplate: HostInfo): string | null { + if (!clusterInstanceTemplate || !clusterInstanceTemplate.host) { + return null; + } + const host = clusterInstanceTemplate.host; + return host.replace("?", hostName); + } + + /** + * Verifies that the topology contains exactly one writer instance. + * If multiple writers are found, selects the most recently updated one. + */ + protected async verifyWriter(allHosts: HostInfo[]): Promise { + if (allHosts === null || allHosts.length === 0) { + return null; + } + + const hosts: HostInfo[] = []; + const writers: HostInfo[] = []; + + for (const host of allHosts) { + if (host.role === HostRole.WRITER) { + writers.push(host); + } else { + hosts.push(host); + } + } + + const writerCount = writers.length; + if (writerCount === 0) { + return null; + } else if (writerCount === 1) { + hosts.push(writers[0]); + } else { + // Assume the latest updated writer instance is the current writer. Other potential writers will be ignored. + const sortedWriters: HostInfo[] = writers.sort((a, b) => { + return b.lastUpdateTime - a.lastUpdateTime; // reverse order + }); + hosts.push(sortedWriters[0]); + } + + return hosts; + } + + /** + * Identifies instances across different database types using instanceId and instanceName values. + * + * Database types handle these identifiers differently: + * - Aurora: Uses the instance name as both instanceId and instanceName + * Example: "test-instance-1" for both values + * - RDS Cluster: Uses distinct values for instanceId and instanceName + * Example: + * instanceId: "db-WQFQKBTL2LQUPIEFIFBGENS4ZQ" + * instanceName: "test-multiaz-instance-1" + * + * @param client the client wrapper to query. + * @returns a tuple of [instanceId, instanceName], or null if the query fails. + */ + getInstanceId(client: ClientWrapper): Promise<[string, string]> { + return this.dialect.getInstanceId(client); + } + + /** + * Evaluate whether the given connection is to a writer instance. + * + * @param client the client wrapper to evaluate. + * @returns true if the connection is to a writer instance, false otherwise. + * @throws Error if an exception occurs when querying the database or processing the database response. + */ + async isWriterInstance(client: ClientWrapper): Promise { + return (await this.dialect.getWriterId(client)) != null; + } + + /** + * Evaluate the database role of the given connection, either {@link HostRole.WRITER} or {@link HostRole.READER}. + * + * @param client the client wrapper to evaluate. + * @returns the database role of the given connection. + * @throws Error if an exception occurs when querying the database or processing the database response. + */ + async getHostRole(client: ClientWrapper): Promise { + try { + return await this.dialect.getHostRole(client); + } catch (error: any) { + throw new AwsWrapperError(Messages.get("TopologyUtils.errorGettingHostRole")); + } + } +} diff --git a/common/lib/host_list_provider_service.ts b/common/lib/host_list_provider_service.ts index 094750d92..155377723 100644 --- a/common/lib/host_list_provider_service.ts +++ b/common/lib/host_list_provider_service.ts @@ -14,7 +14,7 @@ limitations under the License. */ -import { BlockingHostListProvider, HostListProvider } from "./host_list_provider/host_list_provider"; +import { HostListProvider } from "./host_list_provider/host_list_provider"; import { HostInfo } from "./host_info"; import { AwsClient } from "./aws_client"; import { DatabaseDialect } from "./database_dialect/database_dialect"; @@ -28,7 +28,7 @@ export interface HostListProviderService { setHostListProvider(hostListProvider: HostListProvider): void; - isStaticHostListProvider(): boolean; + isDynamicHostListProvider(): boolean; setInitialConnectionHostInfo(initialConnectionHostInfo: HostInfo): void; @@ -53,6 +53,4 @@ export interface HostListProviderService { getTelemetryFactory(): TelemetryFactory; setAllowedAndBlockedHosts(allowedAndBlockedHosts: AllowedAndBlockedHosts): void; - - isBlockingHostListProvider(arg: any): arg is BlockingHostListProvider; } diff --git a/common/lib/host_selector.ts b/common/lib/host_selector.ts index ac995cf37..e32fbb934 100644 --- a/common/lib/host_selector.ts +++ b/common/lib/host_selector.ts @@ -18,5 +18,5 @@ import { HostInfo } from "./host_info"; import { HostRole } from "./host_role"; export interface HostSelector { - getHost(hosts: HostInfo[], role: HostRole, props?: Map): HostInfo; + getHost(hosts: HostInfo[], role: HostRole | null, props?: Map): HostInfo; } diff --git a/common/lib/internal_pooled_connection_provider.ts b/common/lib/internal_pooled_connection_provider.ts index 300442ad8..a104bfe28 100644 --- a/common/lib/internal_pooled_connection_provider.ts +++ b/common/lib/internal_pooled_connection_provider.ts @@ -27,7 +27,7 @@ import { lookup, LookupAddress } from "dns"; import { promisify } from "util"; import { HostInfoBuilder } from "./host_info_builder"; import { RdsUrlType } from "./utils/rds_url_type"; -import { AwsWrapperError } from "./index"; +import { AwsWrapperError } from "./utils/errors"; import { Messages } from "./utils/messages"; import { HostSelector } from "./host_selector"; import { RandomHostSelector } from "./random_host_selector"; @@ -39,6 +39,7 @@ import { LeastConnectionsHostSelector } from "./least_connections_host_selector" import { PoolClientWrapper } from "./pool_client_wrapper"; import { logger } from "../logutils"; import { SlidingExpirationCacheWithCleanupTask } from "./utils/sliding_expiration_cache_with_cleanup_task"; +import { ConnectionInfo } from "./connection_info"; export class InternalPooledConnectionProvider implements PooledConnectionProvider, CanReleaseResources { static readonly CACHE_CLEANUP_NANOS: bigint = BigInt(10 * 60_000_000_000); // 10 minutes @@ -79,7 +80,7 @@ export class InternalPooledConnectionProvider implements PooledConnectionProvide return RdsUrlType.RDS_INSTANCE === urlType; } - async connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise { + async connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise { const resultProps = new Map(props); resultProps.set(WrapperProperties.HOST.name, hostInfo.host); if (hostInfo.isPortSpecified()) { @@ -122,7 +123,7 @@ export class InternalPooledConnectionProvider implements PooledConnectionProvide const poolClient = await this.getPoolConnection(connectionHostInfo, props); pluginService.attachErrorListener(poolClient); - return poolClient; + return new ConnectionInfo(poolClient, true); } async getPoolConnection(hostInfo: HostInfo, props: Map) { @@ -179,7 +180,7 @@ export class InternalPooledConnectionProvider implements PooledConnectionProvide return; } - const connections = Array.from(InternalPooledConnectionProvider.databasePools.entries).map(([v, k]) => [v, k.item.constructor.name]); + const connections = Array.from(InternalPooledConnectionProvider.databasePools.entries).map(([v, k]) => [v, k.get().constructor.name]); logger.debug(`Internal Pooled Connections: [\r\n${connections.join("\r\n")}\r\n]`); } diff --git a/common/lib/least_connections_host_selector.ts b/common/lib/least_connections_host_selector.ts index 58a6ed867..f02f355e9 100644 --- a/common/lib/least_connections_host_selector.ts +++ b/common/lib/least_connections_host_selector.ts @@ -32,7 +32,7 @@ export class LeastConnectionsHostSelector implements HostSelector { getHost(hosts: HostInfo[], role: HostRole, props?: Map): HostInfo { const eligibleHosts: HostInfo[] = hosts - .filter((host: HostInfo) => host.role === role && host.availability === HostAvailability.AVAILABLE) + .filter((host: HostInfo) => (role === null || host.role === role) && host.availability === HostAvailability.AVAILABLE) .sort((hostA: HostInfo, hostB: HostInfo) => { const hostACount = this.getNumConnections(hostA, LeastConnectionsHostSelector.databasePools); const hostBCount = this.getNumConnections(hostB, LeastConnectionsHostSelector.databasePools); diff --git a/common/lib/mysql_client_wrapper.ts b/common/lib/mysql_client_wrapper.ts index eb438d533..52367053e 100644 --- a/common/lib/mysql_client_wrapper.ts +++ b/common/lib/mysql_client_wrapper.ts @@ -66,7 +66,7 @@ export class MySQLClientWrapper implements ClientWrapper { async abort(): Promise { try { - return await ClientUtils.queryWithTimeout(this.client?.destroy(), this.properties); + this.client?.destroy(); } catch (error: any) { // ignore } diff --git a/common/lib/partial_plugin_service.ts b/common/lib/partial_plugin_service.ts new file mode 100644 index 000000000..4203b2309 --- /dev/null +++ b/common/lib/partial_plugin_service.ts @@ -0,0 +1,532 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { PluginService } from "./plugin_service"; +import { HostInfo } from "./host_info"; +import { AwsClient } from "./aws_client"; +import { DynamicHostListProvider, HostListProvider } from "./host_list_provider/host_list_provider"; +import { ConnectionUrlParser } from "./utils/connection_url_parser"; +import { DatabaseDialect } from "./database_dialect/database_dialect"; +import { HostInfoBuilder } from "./host_info_builder"; +import { AwsTimeoutError, AwsWrapperError, UnsupportedMethodError } from "./"; +import { HostAvailability } from "./host_availability/host_availability"; +import { HostAvailabilityCacheItem } from "./host_availability/host_availability_cache_item"; +import { HostChangeOptions } from "./host_change_options"; +import { HostRole } from "./host_role"; +import { SessionStateService } from "./session_state_service"; +import { HostAvailabilityStrategyFactory } from "./host_availability/host_availability_strategy_factory"; +import { ClientWrapper } from "./client_wrapper"; +import { logger } from "../logutils"; +import { Messages } from "./utils/messages"; +import { getWriter, logTopology } from "./utils/utils"; +import { TelemetryFactory } from "./utils/telemetry/telemetry_factory"; +import { DriverDialect } from "./driver_dialect/driver_dialect"; +import { AllowedAndBlockedHosts } from "./allowed_and_blocked_hosts"; +import { ConnectionPlugin } from "./connection_plugin"; +import { FullServicesContainer } from "./utils/full_services_container"; +import { HostListProviderService } from "./host_list_provider_service"; +import { StorageService } from "./utils/storage/storage_service"; +import { CoreServicesContainer } from "./utils/core_services_container"; +import type { TrackedConnectionListHost } from "./plugins/connection_tracker/tracked_connection_list"; + +/** + * A PluginService containing some methods that are not intended to be called. This class is intended to be used + * by monitors, which require a PluginService, but are not expected to need or use some of the methods defined + * by the PluginService interface. The methods that are not expected to be called will throw an + * UnsupportedOperationException when called. + */ +export class PartialPluginService implements PluginService, HostListProviderService { + private static readonly DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS = 5000; // 5 seconds + + protected readonly servicesContainer: FullServicesContainer; + protected readonly storageService: StorageService; + protected readonly props: Map; + protected hostListProvider: HostListProvider | null = null; + protected hosts: HostInfo[] = []; + protected currentHostInfo: HostInfo | null = null; + protected initialConnectionHostInfo: HostInfo | null = null; + protected isInTransactionFlag: boolean = false; + protected readonly dialect: DatabaseDialect; + protected readonly driverDialect: DriverDialect; + protected allowedAndBlockedHosts: AllowedAndBlockedHosts | null = null; + private _isPooledClient: boolean = false; + private _trackedConnectionHost: TrackedConnectionListHost | null = null; + private connectionUrlParser: ConnectionUrlParser; + + constructor( + servicesContainer: FullServicesContainer, + props: Map, + dialect: DatabaseDialect, + driverDialect: DriverDialect, + connectionUrlParser: ConnectionUrlParser + ) { + this.servicesContainer = servicesContainer; + this.storageService = servicesContainer.storageService; + this.servicesContainer.hostListProviderService = this; + this.servicesContainer.pluginService = this; + + this.props = props; + this.dialect = dialect; + this.driverDialect = driverDialect; + this.connectionUrlParser = connectionUrlParser; + + this.hostListProvider = this.dialect.getHostListProvider(this.props, this.props.get("host"), this.servicesContainer); + } + + getCurrentClient(): AwsClient { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getCurrentClient")); + } + + getCurrentHostInfo(): HostInfo | null { + if (!this.currentHostInfo) { + this.currentHostInfo = this.initialConnectionHostInfo; + + if (!this.currentHostInfo) { + if (this.getAllHosts().length === 0) { + throw new AwsWrapperError(Messages.get("PluginService.hostListEmpty")); + } + + const writerHost = getWriter(this.getAllHosts()); + if (writerHost) { + this.currentHostInfo = writerHost; + const allowedHosts = this.getHosts(); + if (!allowedHosts.some((hostInfo: HostInfo) => hostInfo.host === writerHost.host && hostInfo.port === writerHost.port)) { + throw new AwsWrapperError( + Messages.get( + "PluginService.currentHostNotAllowed", + this.currentHostInfo ? this.currentHostInfo.host : "", + logTopology(allowedHosts, "") + ) + ); + } + } + + if (!this.currentHostInfo) { + const hosts = this.getHosts(); + if (hosts.length > 0) { + this.currentHostInfo = hosts[0]; + } + } + } + + if (!this.currentHostInfo) { + throw new AwsWrapperError(Messages.get("PluginService.currentHostNotDefined")); + } + + logger.debug(`Set current host to: ${this.currentHostInfo.host}`); + } + + return this.currentHostInfo; + } + + setCurrentHostInfo(value: HostInfo): void { + this.currentHostInfo = value; + } + + setInitialConnectionHostInfo(initialConnectionHostInfo: HostInfo): void { + this.initialConnectionHostInfo = initialConnectionHostInfo; + } + + getInitialConnectionHostInfo(): HostInfo | null { + return this.initialConnectionHostInfo; + } + + acceptsStrategy(role: HostRole, strategy: string): boolean { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "acceptsStrategy")); + } + + getHostInfoByStrategy(role: HostRole, strategy: string, hosts?: HostInfo[]): HostInfo | undefined { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getHostInfoByStrategy")); + } + + getHostRole(client: any): Promise | undefined { + return this.dialect.getHostRole(client); + } + + getDriverDialect(): DriverDialect { + return this.driverDialect; + } + + getConnectionUrlParser(): ConnectionUrlParser { + return this.connectionUrlParser; + } + + setCurrentClient(newClient: ClientWrapper, hostInfo: HostInfo): Promise> { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "setCurrentClient")); + } + + protected compare(hostInfoA: HostInfo, hostInfoB: HostInfo): Set { + const changes: Set = new Set(); + + if (hostInfoA.host !== hostInfoB.host || hostInfoA.port !== hostInfoB.port) { + changes.add(HostChangeOptions.HOSTNAME); + } + + if (hostInfoA.role !== hostInfoB.role) { + if (hostInfoB.role === HostRole.WRITER) { + changes.add(HostChangeOptions.PROMOTED_TO_WRITER); + } else if (hostInfoB.role === HostRole.READER) { + changes.add(HostChangeOptions.PROMOTED_TO_READER); + } + } + + if (hostInfoA.availability !== hostInfoB.availability) { + if (hostInfoB.availability === HostAvailability.AVAILABLE) { + changes.add(HostChangeOptions.WENT_UP); + } else if (hostInfoB.availability === HostAvailability.NOT_AVAILABLE) { + changes.add(HostChangeOptions.WENT_DOWN); + } + } + + if (changes.size > 0) { + changes.add(HostChangeOptions.HOST_CHANGED); + } + + return changes; + } + + getAllHosts(): HostInfo[] { + return this.hosts; + } + + getHosts(): HostInfo[] { + const hostPermissions = this.allowedAndBlockedHosts; + if (!hostPermissions) { + return this.hosts; + } + + let hosts = this.hosts; + const allowedHostIds = hostPermissions.getAllowedHostIds(); + const blockedHostIds = hostPermissions.getBlockedHostIds(); + + if (allowedHostIds && allowedHostIds.size > 0) { + hosts = hosts.filter((host: HostInfo) => allowedHostIds.has(host.hostId)); + } + + if (blockedHostIds && blockedHostIds.size > 0) { + hosts = hosts.filter((host: HostInfo) => !blockedHostIds.has(host.hostId)); + } + + return hosts; + } + + setAvailability(hostInfo: HostInfo, availability: HostAvailability): void { + const hostsToChange = [ + ...new Set( + this.getAllHosts().filter( + (host: HostInfo) => (hostInfo.hostId != null && hostInfo.hostId === host.hostId) || (hostInfo.host != null && hostInfo.host === host.host) + ) + ) + ]; + + if (hostsToChange.length === 0) { + return; + } + + const changes = new Map>(); + for (const host of hostsToChange) { + const currentAvailability = host.getAvailability(); + host.availability = availability; + this.storageService.set(host.url, new HostAvailabilityCacheItem(availability)); + if (currentAvailability !== availability) { + let hostChanges: Set; + if (availability === HostAvailability.AVAILABLE) { + hostChanges = new Set([HostChangeOptions.WENT_UP, HostChangeOptions.HOST_CHANGED]); + } else { + hostChanges = new Set([HostChangeOptions.WENT_DOWN, HostChangeOptions.HOST_CHANGED]); + } + changes.set(host.url, hostChanges); + } + } + + if (changes.size > 0) { + this.servicesContainer.pluginManager?.notifyHostListChanged(changes); + } + } + + isInTransaction(): boolean { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "isInTransaction")); + } + + isDialectConfirmed(): boolean { + return true; + } + + setInTransaction(inTransaction: boolean): void { + this.isInTransactionFlag = inTransaction; + } + + getHostListProvider(): HostListProvider | null { + return this.hostListProvider; + } + + async refreshHostList(): Promise { + const updatedHostList = await this.getHostListProvider()?.refresh(); + if (updatedHostList && updatedHostList !== this.hosts) { + this.updateHostAvailability(updatedHostList); + this.setHostList(this.hosts, updatedHostList); + } + } + + async forceRefreshHostList(): Promise { + await this.forceMonitoringRefresh(false, PartialPluginService.DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS); + } + + async forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise { + const hostListProvider = this.getHostListProvider(); + + if (!this.isDynamicHostListProvider()) { + const providerName = hostListProvider?.constructor.name ?? "null"; + throw new UnsupportedMethodError(Messages.get("PluginService.requiredDynamicHostListProvider", providerName)); + } + + try { + const updatedHostList = await (hostListProvider as DynamicHostListProvider).forceMonitoringRefresh(shouldVerifyWriter, timeoutMs); + if (updatedHostList) { + this.updateHostAvailability(updatedHostList); + this.setHostList(this.hosts, updatedHostList); + return true; + } + } catch (err) { + if (err instanceof AwsTimeoutError) { + logger.debug(Messages.get("PluginService.forceMonitoringRefreshTimeout", timeoutMs.toString())); + } + } + + return false; + } + + protected setHostList(oldHosts: HostInfo[] | null, newHosts: HostInfo[] | null): void { + const oldHostMap: Map = oldHosts ? new Map(oldHosts.map((e) => [e.url, e])) : new Map(); + + const newHostMap: Map = newHosts ? new Map(newHosts.map((e) => [e.url, e])) : new Map(); + + const changes: Map> = new Map(); + + oldHostMap.forEach((value, key) => { + const correspondingNewHost = newHostMap.get(key); + if (!correspondingNewHost) { + changes.set(key, new Set([HostChangeOptions.HOST_DELETED])); + } else { + const hostChanges = this.compare(value, correspondingNewHost); + if (hostChanges.size > 0) { + changes.set(key, hostChanges); + } + } + }); + + newHostMap.forEach((value, key) => { + if (!oldHostMap.has(key)) { + changes.set(key, new Set([HostChangeOptions.HOST_ADDED])); + } + }); + + if (changes.size > 0) { + this.hosts = newHosts ? newHosts : []; + this.servicesContainer.pluginManager?.notifyHostListChanged(changes); + } + } + + isDynamicHostListProvider(): boolean { + const provider = this.getHostListProvider(); + return provider !== null && typeof (provider as any).forceMonitoringRefresh === "function"; + } + + setHostListProvider(hostListProvider: HostListProvider): void { + this.hostListProvider = hostListProvider; + } + + connect(hostInfo: HostInfo, props: Map): Promise; + connect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin | null): Promise; + connect(hostInfo: HostInfo, props: Map, pluginToSkip?: ConnectionPlugin | null): Promise { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "connect")); + } + + forceConnect(hostInfo: HostInfo, props: Map): Promise; + forceConnect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin | null): Promise; + forceConnect(hostInfo: HostInfo, props: Map, pluginToSkip?: ConnectionPlugin | null): Promise { + const pluginManager = this.servicesContainer.pluginManager; + if (!pluginManager) { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "forceConnect")); + } + return pluginManager.forceConnect(hostInfo, props, true, pluginToSkip ?? null); + } + + protected updateHostAvailability(hosts: HostInfo[]): void { + hosts.forEach((host) => { + const cacheItem = this.storageService.get(HostAvailabilityCacheItem, host.url); + if (cacheItem != null) { + host.availability = cacheItem.availability; + } + }); + } + + // Error handler methods + isLoginError(e: Error): boolean { + return this.dialect.getErrorHandler().isLoginError(e); + } + + isNetworkError(e: Error): boolean { + return this.dialect.getErrorHandler().isNetworkError(e); + } + + isSyntaxError(e: Error): boolean { + return this.dialect.getErrorHandler().isSyntaxError(e); + } + + hasLoginError(): boolean { + return this.dialect.getErrorHandler().hasLoginError(); + } + + hasNetworkError(): boolean { + return this.dialect.getErrorHandler().hasNetworkError(); + } + + getUnexpectedError(): Error | null { + return this.dialect.getErrorHandler().getUnexpectedError(); + } + + attachErrorListener(clientWrapper: ClientWrapper | undefined): void { + this.dialect.getErrorHandler().attachErrorListener(clientWrapper); + } + + attachNoOpErrorListener(clientWrapper: ClientWrapper | undefined): void { + this.dialect.getErrorHandler().attachNoOpErrorListener(clientWrapper); + } + + removeErrorListener(clientWrapper: ClientWrapper | undefined): void { + this.dialect.getErrorHandler().removeErrorListener(clientWrapper); + } + + getDialect(): DatabaseDialect { + return this.dialect; + } + + async updateDialect(targetClient: ClientWrapper): Promise { + // Do nothing. This method is called after connecting in DefaultConnectionPlugin but the dialect passed to the + // constructor should already be updated and verified. + } + + async identifyConnection(targetClient: ClientWrapper): Promise { + const provider = this.getHostListProvider(); + if (!provider) { + return Promise.reject(new AwsWrapperError(Messages.get("PluginService.errorIdentifyConnection"))); + } + return provider.identifyConnection(targetClient); + } + + getRoutedHostInfo(): HostInfo | null { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getRoutedHostInfo")); + } + + setRoutedHostInfo(routedHostInfo: HostInfo | null) { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "setRoutedHostInfo")); + } + + getHostInfoBuilder(): HostInfoBuilder { + return new HostInfoBuilder({ hostAvailabilityStrategy: new HostAvailabilityStrategyFactory().create(this.props) }); + } + + getProperties(): Map { + return this.props; + } + + getTelemetryFactory(): TelemetryFactory { + const pluginManager = this.servicesContainer.pluginManager; + if (!pluginManager) { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getTelemetryFactory")); + } + return pluginManager.getTelemetryFactory(); + } + + getSessionStateService(): SessionStateService { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getSessionStateService")); + } + + async updateState(sql: string): Promise { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "updateState")); + } + + updateInTransaction(sql: string): void { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "updateInTransaction")); + } + + async isClientValid(targetClient: ClientWrapper): Promise { + return await this.getDialect().isClientValid(targetClient); + } + + async abortCurrentClient(): Promise { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "abortCurrentClient")); + } + + async abortTargetClient(targetClient: ClientWrapper | undefined | null): Promise { + if (targetClient) { + await targetClient.abort(); + } + } + + updateConfigWithProperties(props: Map): void { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "updateConfigWithProperties")); + } + + setAllowedAndBlockedHosts(allowedAndBlockedHosts: AllowedAndBlockedHosts): void { + this.allowedAndBlockedHosts = allowedAndBlockedHosts; + } + + setStatus(clazz: any, status: T | null, clusterBound: boolean): void; + setStatus(clazz: any, status: T | null, key: string): void; + setStatus(clazz: any, status: T | null, clusterBoundOrKey: boolean | string): void { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "setStatus")); + } + + getStatus(clazz: any, clusterBound: boolean): T; + getStatus(clazz: any, key: string): T; + getStatus(clazz: any, clusterBoundOrKey: boolean | string): T { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getStatus")); + } + + isPluginInUse(plugin: any): boolean { + try { + return this.servicesContainer.pluginManager?.isPluginInUse(plugin) ?? false; + } catch (e) { + return false; + } + } + + getPlugin(pluginClazz: new (...args: any[]) => T): T | null { + return this.servicesContainer.pluginManager?.unwrapPlugin(pluginClazz) ?? null; + } + + static clearCache(): void { + CoreServicesContainer.getInstance().storageService.clear(HostAvailabilityCacheItem); + } + + isPooledClient(): boolean { + return this._isPooledClient; + } + + setIsPooledClient(isPooledClient: boolean): void { + this._isPooledClient = isPooledClient; + } + + getTrackedConnectionHost(): TrackedConnectionListHost | null { + return this._trackedConnectionHost; + } + + setTrackedConnectionHost(host: TrackedConnectionListHost | null): void { + this._trackedConnectionHost = host; + } +} diff --git a/common/lib/pg_client_wrapper.ts b/common/lib/pg_client_wrapper.ts index 99e3ec76f..a9c2e3204 100644 --- a/common/lib/pg_client_wrapper.ts +++ b/common/lib/pg_client_wrapper.ts @@ -17,6 +17,7 @@ import { ClientWrapper } from "./client_wrapper"; import { HostInfo } from "./host_info"; import { uniqueId } from "../logutils"; +import { ClientUtils } from "./utils/client_utils"; /* This an internal wrapper class for a target community driver client created by the NodePostgresPgDriverDialect. @@ -45,8 +46,8 @@ export class PgClientWrapper implements ClientWrapper { return this.client?.query(config, values, callback); } - queryWithTimeout(sql: string): Promise { - return this.client?.queryWithTimeout(this.client.query(sql), this.properties); + async queryWithTimeout(sql: string): Promise { + return await ClientUtils.queryWithTimeout(this.client.query(sql), this.properties); } end(): Promise { @@ -59,7 +60,7 @@ export class PgClientWrapper implements ClientWrapper { async abort(): Promise { try { - return await this.end(); + this.client?.connection?.stream?.destroy(); } catch (error: any) { // Ignore } diff --git a/common/lib/plugin_factory.ts b/common/lib/plugin_factory.ts index 7360ba1dc..1e69580b6 100644 --- a/common/lib/plugin_factory.ts +++ b/common/lib/plugin_factory.ts @@ -16,9 +16,10 @@ import { PluginService } from "./plugin_service"; import { ConnectionPlugin } from "./connection_plugin"; +import { FullServicesContainer } from "./utils/full_services_container"; export class ConnectionPluginFactory { - getInstance(pluginService: PluginService, properties: Map): Promise { + getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { return; } } diff --git a/common/lib/plugin_manager.ts b/common/lib/plugin_manager.ts index 492566962..1edffdf54 100644 --- a/common/lib/plugin_manager.ts +++ b/common/lib/plugin_manager.ts @@ -19,7 +19,6 @@ import { HostInfo } from "./host_info"; import { ConnectionPluginChainBuilder } from "./connection_plugin_chain_builder"; import { AwsWrapperError } from "./utils/errors"; import { Messages } from "./utils/messages"; -import { PluginServiceManagerContainer } from "./plugin_service_manager_container"; import { HostListProviderService } from "./host_list_provider_service"; import { HostChangeOptions } from "./host_change_options"; import { OldConnectionSuggestionAction } from "./old_connection_suggestion_action"; @@ -32,6 +31,8 @@ import { TelemetryTraceLevel } from "./utils/telemetry/telemetry_trace_level"; import { ConnectionProvider } from "./connection_provider"; import { ConnectionPluginFactory } from "./plugin_factory"; import { ConfigurationProfile } from "./profile/configuration_profile"; +import { FullServicesContainer } from "./utils/full_services_container"; +import { CoreServicesContainer } from "./utils/core_services_container"; type PluginFunc = (plugin: ConnectionPlugin, targetFunc: () => Promise) => Promise; @@ -79,17 +80,16 @@ export class PluginManager { private readonly props: Map; private _plugins: ConnectionPlugin[] = []; private readonly connectionProviderManager: ConnectionProviderManager; - private pluginServiceManagerContainer: PluginServiceManagerContainer; + private fullServicesContainer: FullServicesContainer; protected telemetryFactory: TelemetryFactory; constructor( - pluginServiceManagerContainer: PluginServiceManagerContainer, + fullServicesContainer: FullServicesContainer, props: Map, connectionProviderManager: ConnectionProviderManager, telemetryFactory: TelemetryFactory ) { - this.pluginServiceManagerContainer = pluginServiceManagerContainer; - this.pluginServiceManagerContainer.pluginManager = this; + this.fullServicesContainer = fullServicesContainer; this.connectionProviderManager = connectionProviderManager; this.props = props; this.telemetryFactory = telemetryFactory; @@ -98,17 +98,15 @@ export class PluginManager { async init(configurationProfile?: ConfigurationProfile | null): Promise; async init(configurationProfile: ConfigurationProfile | null, plugins: ConnectionPlugin[]): Promise; async init(configurationProfile: ConfigurationProfile | null, plugins?: ConnectionPlugin[]) { - if (this.pluginServiceManagerContainer.pluginService != null) { - if (plugins) { - this._plugins = plugins; - } else { - this._plugins = await ConnectionPluginChainBuilder.getPlugins( - this.pluginServiceManagerContainer.pluginService, - this.props, - this.connectionProviderManager, - configurationProfile - ); - } + if (plugins) { + this._plugins = plugins; + } else { + this._plugins = await ConnectionPluginChainBuilder.getPlugins( + this.fullServicesContainer, + this.props, + this.connectionProviderManager, + configurationProfile + ); } for (const plugin of this._plugins) { PluginManager.PLUGINS.add(plugin); @@ -128,8 +126,8 @@ export class PluginManager { } const telemetryContext = this.telemetryFactory.openTelemetryContext(methodName, TelemetryTraceLevel.NESTED); - const currentClient: ClientWrapper = this.pluginServiceManagerContainer.pluginService.getCurrentClient().targetClient; - this.pluginServiceManagerContainer.pluginService.attachNoOpErrorListener(currentClient); + const currentClient: ClientWrapper = this.fullServicesContainer.pluginService.getCurrentClient().targetClient; + this.fullServicesContainer.pluginService.attachNoOpErrorListener(currentClient); try { return await telemetryContext.start(() => { return this.executeWithSubscribedPlugins( @@ -142,7 +140,7 @@ export class PluginManager { ); }); } finally { - this.pluginServiceManagerContainer.pluginService.attachErrorListener(currentClient); + this.fullServicesContainer.pluginService.attachErrorListener(currentClient); } } @@ -395,6 +393,7 @@ export class PluginManager { } PluginManager.STRATEGY_PLUGIN_CHAIN_CACHE.clear(); + await CoreServicesContainer.releaseResources(); PluginManager.PLUGINS = new Set(); } @@ -443,7 +442,7 @@ export class PluginManager { for (const p of this._plugins) { if (p instanceof iface) { - return p as any; + return p as T; } } return null; diff --git a/common/lib/plugin_service.ts b/common/lib/plugin_service.ts index e2aae5062..1b80affbb 100644 --- a/common/lib/plugin_service.ts +++ b/common/lib/plugin_service.ts @@ -14,18 +14,18 @@ limitations under the License. */ -import { PluginServiceManagerContainer } from "./plugin_service_manager_container"; import { ErrorHandler } from "./error_handler"; import { HostInfo } from "./host_info"; import { AwsClient } from "./aws_client"; import { HostListProviderService } from "./host_list_provider_service"; -import { BlockingHostListProvider, HostListProvider } from "./host_list_provider/host_list_provider"; +import { DynamicHostListProvider, HostListProvider } from "./host_list_provider/host_list_provider"; import { ConnectionUrlParser } from "./utils/connection_url_parser"; import { DatabaseDialect, DatabaseType } from "./database_dialect/database_dialect"; import { HostInfoBuilder } from "./host_info_builder"; -import { AwsWrapperError } from "./"; +import { AwsTimeoutError, AwsWrapperError, UnsupportedMethodError } from "./utils/errors"; import { HostAvailability } from "./host_availability/host_availability"; -import { CacheMap } from "./utils/cache_map"; +import { HostAvailabilityCacheItem } from "./host_availability/host_availability_cache_item"; +import { StatusCacheItem } from "./utils/status_cache_item"; import { HostChangeOptions } from "./host_change_options"; import { HostRole } from "./host_role"; import { WrapperProperties } from "./wrapper_property"; @@ -45,6 +45,9 @@ import { TelemetryFactory } from "./utils/telemetry/telemetry_factory"; import { DriverDialect } from "./driver_dialect/driver_dialect"; import { AllowedAndBlockedHosts } from "./allowed_and_blocked_hosts"; import { ConnectionPlugin } from "./connection_plugin"; +import { FullServicesContainer } from "./utils/full_services_container"; +import { StorageService } from "./utils/storage/storage_service"; +import type { TrackedConnectionListHost } from "./plugins/connection_tracker/tracked_connection_list"; export interface PluginService extends ErrorHandler { isInTransaction(): boolean; @@ -73,40 +76,40 @@ export interface PluginService extends ErrorHandler { getDialect(): DatabaseDialect; + isDialectConfirmed(): boolean; + getDriverDialect(): DriverDialect; getHostInfoBuilder(): HostInfoBuilder; - isStaticHostListProvider(): boolean; + isDynamicHostListProvider(): boolean; acceptsStrategy(role: HostRole, strategy: string): boolean; forceRefreshHostList(): Promise; - forceRefreshHostList(targetClient: ClientWrapper): Promise; - - forceRefreshHostList(targetClient?: ClientWrapper): Promise; - forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise; refreshHostList(): Promise; - refreshHostList(targetClient: ClientWrapper): Promise; - - refreshHostList(targetClient?: ClientWrapper): Promise; - getAllHosts(): HostInfo[]; getHosts(): HostInfo[]; - setAvailability(hostAliases: Set, availability: HostAvailability): void; + setAvailability(hostInfo: HostInfo, availability: HostAvailability): void; updateConfigWithProperties(props: Map): void; - fillAliases(targetClient: ClientWrapper, hostInfo: HostInfo): Promise; + getRoutedHostInfo(): HostInfo | null; + + setRoutedHostInfo(routedHostInfo: HostInfo | null); identifyConnection(targetClient: ClientWrapper): Promise; + identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo | null): Promise; + + identifyConnection(targetClient: ClientWrapper, connectionHostInfo?: HostInfo | null): Promise; + connect(hostInfo: HostInfo, props: Map): Promise; connect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin | null): Promise; @@ -150,30 +153,41 @@ export interface PluginService extends ErrorHandler { getStatus(clazz: any, key: string): T; isPluginInUse(plugin: any): boolean; + + isPooledClient(): boolean; + + setIsPooledClient(isPooledClient: boolean): void; + + getTrackedConnectionHost(): TrackedConnectionListHost | null; + + setTrackedConnectionHost(host: TrackedConnectionListHost | null): void; } export class PluginServiceImpl implements PluginService, HostListProviderService { - private static readonly DEFAULT_HOST_AVAILABILITY_CACHE_EXPIRE_NANO = 5 * 60_000_000_000; // 5 minutes + private static readonly DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS = 5000; // 5 seconds. private readonly _currentClient: AwsClient; private _currentHostInfo?: HostInfo; private _hostListProvider?: HostListProvider; private _initialConnectionHostInfo?: HostInfo; private _isInTransaction: boolean = false; - private pluginServiceManagerContainer: PluginServiceManagerContainer; + private servicesContainer: FullServicesContainer; protected hosts: HostInfo[] = []; + protected routedHostInfo: HostInfo | null; private dbDialectProvider: DatabaseDialectProvider; private readonly initialHost: string; private dialect: DatabaseDialect; + private _isDialectConfirmed: boolean = false; private readonly driverDialect: DriverDialect; protected readonly sessionStateService: SessionStateService; - protected static readonly hostAvailabilityExpiringCache: CacheMap = new CacheMap(); + protected storageService: StorageService; readonly props: Map; private allowedAndBlockedHosts: AllowedAndBlockedHosts | null = null; - protected static readonly statusesExpiringCache: CacheMap = new CacheMap(); - protected static readonly DEFAULT_STATUS_CACHE_EXPIRE_NANO: number = 3_600_000_000_000; // 60 minutes + + protected _isPooledClient: boolean = false; + protected _trackedConnectionHost: TrackedConnectionListHost | null = null; constructor( - container: PluginServiceManagerContainer, + container: FullServicesContainer, client: AwsClient, dbType: DatabaseType, knownDialectsByCode: Map, @@ -181,12 +195,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService driverDialect: DriverDialect ) { this._currentClient = client; - this.pluginServiceManagerContainer = container; + this.servicesContainer = container; + this.storageService = container.storageService; this.props = props; this.dbDialectProvider = new DatabaseDialectManager(knownDialectsByCode, dbType, this.props); this.driverDialect = driverDialect; this.initialHost = props.get(WrapperProperties.HOST.name); - container.pluginService = this; this.dialect = WrapperProperties.CUSTOM_DATABASE_DIALECT.get(this.props) ?? this.dbDialectProvider.getDialect(this.props); this.sessionStateService = new SessionStateServiceImpl(this, this.props); @@ -217,7 +231,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService } getHostInfoByStrategy(role: HostRole, strategy: string, hosts?: HostInfo[]): HostInfo | undefined { - const pluginManager = this.pluginServiceManagerContainer.pluginManager; + const pluginManager = this.servicesContainer.pluginManager; return pluginManager?.getHostInfoByStrategy(role, strategy, hosts); } @@ -279,6 +293,31 @@ export class PluginServiceImpl implements PluginService, HostListProviderService return this.dialect; } + private static readonly DIALECT_CONFIRMED_STATUS_KEY = "DialectConfirmed"; + + private getDialectConfirmedCacheKey(): string { + let clusterId = WrapperProperties.CLUSTER_ID.defaultValue; + try { + clusterId = this._hostListProvider?.getClusterId() ?? WrapperProperties.CLUSTER_ID.defaultValue; + } catch (e) { + // May fail if the host list provider does not support getClusterId. In this case use the default value. + } + return `${clusterId}::${PluginServiceImpl.DIALECT_CONFIRMED_STATUS_KEY}`; + } + + isDialectConfirmed(): boolean { + if (this._isDialectConfirmed) { + return true; + } + + const cacheItem = this.storageService.get(StatusCacheItem, this.getDialectConfirmedCacheKey()); + if (cacheItem && cacheItem.status === true) { + this._isDialectConfirmed = true; + } + + return this._isDialectConfirmed; + } + getDriverDialect(): DriverDialect { return this.driverDialect; } @@ -287,58 +326,46 @@ export class PluginServiceImpl implements PluginService, HostListProviderService return new HostInfoBuilder({ hostAvailabilityStrategy: new HostAvailabilityStrategyFactory().create(this.props) }); } - isStaticHostListProvider(): boolean { - return false; + isDynamicHostListProvider(): boolean { + const provider = this.getHostListProvider(); + return provider !== null && typeof (provider as any).forceMonitoringRefresh === "function"; } acceptsStrategy(role: HostRole, strategy: string): boolean { - return this.pluginServiceManagerContainer.pluginManager?.acceptsStrategy(role, strategy) ?? false; + return this.servicesContainer.pluginManager?.acceptsStrategy(role, strategy) ?? false; } - async forceRefreshHostList(): Promise; - async forceRefreshHostList(targetClient: ClientWrapper): Promise; - async forceRefreshHostList(targetClient?: ClientWrapper): Promise { - const updatedHostList = targetClient - ? await this.getHostListProvider()?.forceRefresh(targetClient) - : await this.getHostListProvider()?.forceRefresh(); - if (updatedHostList && updatedHostList !== this.hosts) { - this.updateHostAvailability(updatedHostList); - await this.setHostList(this.hosts, updatedHostList); - } + async forceRefreshHostList(): Promise { + await this.forceMonitoringRefresh(false, PluginServiceImpl.DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS); } async forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise { const hostListProvider: HostListProvider = this.getHostListProvider(); - if (!this.isBlockingHostListProvider(hostListProvider)) { - logger.info(Messages.get("PluginService.requiredBlockingHostListProvider", typeof hostListProvider)); - throw new AwsWrapperError(Messages.get("PluginService.requiredBlockingHostListProvider", typeof hostListProvider)); + + if (!this.isDynamicHostListProvider()) { + const providerName = hostListProvider?.constructor.name ?? "null"; + throw new UnsupportedMethodError(Messages.get("PluginService.requiredDynamicHostListProvider", providerName)); } try { - const updatedHostList: HostInfo[] = await hostListProvider.forceMonitoringRefresh(shouldVerifyWriter, timeoutMs); + const updatedHostList: HostInfo[] = await (hostListProvider as DynamicHostListProvider).forceMonitoringRefresh(shouldVerifyWriter, timeoutMs); if (updatedHostList) { - if (updatedHostList !== this.hosts) { - this.updateHostAvailability(updatedHostList); - await this.setHostList(this.hosts, updatedHostList); - } + this.updateHostAvailability(updatedHostList); + await this.setHostList(this.hosts, updatedHostList); return true; } } catch (err) { - // Do nothing. - logger.info(Messages.get("PluginService.forceMonitoringRefreshTimeout", timeoutMs.toString())); + if (err instanceof AwsTimeoutError) { + // Do nothing. + logger.info(Messages.get("PluginService.forceMonitoringRefreshTimeout", timeoutMs.toString())); + } } return false; } - isBlockingHostListProvider(arg: any): arg is BlockingHostListProvider { - return arg != null && typeof arg.clearAll === "function" && typeof arg.forceMonitoringRefresh === "function"; - } - - async refreshHostList(): Promise; - async refreshHostList(targetClient: ClientWrapper): Promise; - async refreshHostList(targetClient?: ClientWrapper): Promise { - const updatedHostList = targetClient ? await this.getHostListProvider()?.refresh(targetClient) : await this.getHostListProvider()?.refresh(); + async refreshHostList(): Promise { + const updatedHostList = await this.getHostListProvider()?.refresh(); if (updatedHostList && updatedHostList !== this.hosts) { this.updateHostAvailability(updatedHostList); await this.setHostList(this.hosts, updatedHostList); @@ -347,9 +374,9 @@ export class PluginServiceImpl implements PluginService, HostListProviderService private updateHostAvailability(hosts: HostInfo[]) { hosts.forEach((host) => { - const availability = PluginServiceImpl.hostAvailabilityExpiringCache.get(host.url); - if (availability != null) { - host.availability = availability; + const cacheItem = this.storageService.get(HostAvailabilityCacheItem, host.url); + if (cacheItem != null) { + host.availability = cacheItem.availability; } }); } @@ -415,7 +442,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService if (changes.size > 0) { this.hosts = newHosts ? newHosts : []; - await this.pluginServiceManagerContainer.pluginManager!.notifyHostListChanged(changes); + await this.servicesContainer.pluginManager!.notifyHostListChanged(changes); } } @@ -444,28 +471,23 @@ export class PluginServiceImpl implements PluginService, HostListProviderService return hosts; } - setAvailability(hostAliases: Set, availability: HostAvailability) { - if (hostAliases.size === 0) { - return; - } - + setAvailability(hostInfo: HostInfo, availability: HostAvailability) { const hostsToChange = [ ...new Set( this.getAllHosts().filter( - (host: HostInfo) => hostAliases.has(host.asAlias) || [...host.aliases].some((hostAlias: string) => hostAliases.has(hostAlias)) + (host: HostInfo) => (hostInfo.hostId != null && hostInfo.hostId === host.hostId) || (hostInfo.host != null && hostInfo.host === host.host) ) ) ]; if (hostsToChange.length === 0) { - logger.debug(Messages.get("PluginService.hostsChangeListEmpty")); return; } const changes = new Map>(); for (const host of hostsToChange) { const currentAvailability = host.getAvailability(); - PluginServiceImpl.hostAvailabilityExpiringCache.put(host.url, availability, PluginServiceImpl.DEFAULT_HOST_AVAILABILITY_CACHE_EXPIRE_NANO); + this.storageService.set(host.url, new HostAvailabilityCacheItem(availability)); if (currentAvailability !== availability) { let hostChanges = new Set(); if (availability === HostAvailability.AVAILABLE) { @@ -482,30 +504,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService this._currentClient.config = Object.fromEntries(props.entries()); } - async fillAliases(targetClient: ClientWrapper, hostInfo: HostInfo) { - if (hostInfo == null) { - return; - } - - if (hostInfo.aliases.size > 0) { - logger.debug(Messages.get("PluginService.nonEmptyAliases", [...hostInfo.aliases].join(", "))); - return; - } - - hostInfo.addAlias(hostInfo.asAlias); - - // Add the host name and port, this host name is usually the internal IP address. - try { - const res: string = await this.dialect.getHostAliasAndParseResults(targetClient); - hostInfo.addAlias(res); - } catch (error) { - logger.debug(Messages.get("PluginService.failedToRetrieveHostPort")); - } + getRoutedHostInfo(): HostInfo | null { + return this.routedHostInfo; + } - const host: HostInfo | void | null = await this.identifyConnection(targetClient); - if (host) { - hostInfo.addAlias(...host.allAliases); - } + setRoutedHostInfo(routedHostInfo: HostInfo | null) { + this.routedHostInfo = routedHostInfo; } identifyConnection(targetClient: ClientWrapper): Promise { @@ -513,19 +517,19 @@ export class PluginServiceImpl implements PluginService, HostListProviderService if (!provider) { return Promise.reject(); } - return provider.identifyConnection(targetClient, this.dialect); + return provider.identifyConnection(targetClient); } connect(hostInfo: HostInfo, props: Map): Promise; connect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin): Promise; connect(hostInfo: HostInfo, props: Map, pluginToSkip?: ConnectionPlugin): Promise { - return this.pluginServiceManagerContainer.pluginManager!.connect(hostInfo, props, false, pluginToSkip); + return this.servicesContainer.pluginManager!.connect(hostInfo, props, false, pluginToSkip); } forceConnect(hostInfo: HostInfo, props: Map): Promise; forceConnect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin): Promise; forceConnect(hostInfo: HostInfo, props: Map, pluginToSkip?: ConnectionPlugin): Promise { - return this.pluginServiceManagerContainer.pluginManager!.forceConnect(hostInfo, props, false, pluginToSkip); + return this.servicesContainer.pluginManager!.forceConnect(hostInfo, props, false, pluginToSkip); } async setCurrentClient(newClient: ClientWrapper, hostInfo: HostInfo): Promise> { @@ -535,8 +539,8 @@ export class PluginServiceImpl implements PluginService, HostListProviderService this.sessionStateService.reset(); const changes = new Set([HostChangeOptions.INITIAL_CONNECTION]); - if (this.pluginServiceManagerContainer.pluginManager) { - await this.pluginServiceManagerContainer.pluginManager.notifyConnectionChanged(changes, null); + if (this.servicesContainer.pluginManager) { + await this.servicesContainer.pluginManager.notifyConnectionChanged(changes, null); } return changes; @@ -563,8 +567,10 @@ export class PluginServiceImpl implements PluginService, HostListProviderService } } - const pluginOpinions: Set = - await this.pluginServiceManagerContainer.pluginManager!.notifyConnectionChanged(changes, null); + const pluginOpinions: Set = await this.servicesContainer.pluginManager!.notifyConnectionChanged( + changes, + null + ); const shouldCloseConnection = changes.has(HostChangeOptions.CONNECTION_OBJECT_CHANGED) && @@ -638,11 +644,14 @@ export class PluginServiceImpl implements PluginService, HostListProviderService const originalDialect = this.dialect; this.dialect = await this.dbDialectProvider.getDialectForUpdate(targetClient, this.initialHost, this.props.get(WrapperProperties.HOST.name)); + this._isDialectConfirmed = true; + this.storageService.set(this.getDialectConfirmedCacheKey(), new StatusCacheItem(true)); + if (originalDialect === this.dialect) { return; } - this._hostListProvider = this.dialect.getHostListProvider(this.props, this.props.get(WrapperProperties.HOST.name), this); + this._hostListProvider = this.dialect.getHostListProvider(this.props, this.props.get(WrapperProperties.HOST.name), this.servicesContainer); } private async updateReadOnly(statements: string[]) { @@ -685,7 +694,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService } getTelemetryFactory(): TelemetryFactory { - return this.pluginServiceManagerContainer.pluginManager!.getTelemetryFactory(); + return this.servicesContainer.pluginManager!.getTelemetryFactory(); } /* Error Handler interface implementation */ @@ -730,15 +739,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService this.allowedAndBlockedHosts = allowedAndBlockedHosts; } - static clearHostAvailabilityCache(): void { - PluginServiceImpl.hostAvailabilityExpiringCache.clear(); - } - getStatus(clazz: any, clusterBound: boolean): T; getStatus(clazz: any, key: string): T; getStatus(clazz: any, clusterBound: boolean | string): T { if (typeof clusterBound === "string") { - return PluginServiceImpl.statusesExpiringCache.get(this.getStatusCacheKey(clazz, clusterBound)); + const cacheItem = this.storageService.get(StatusCacheItem, this.getStatusCacheKey(clazz, clusterBound)); + return cacheItem ? cacheItem.status : null; } let clusterId: string = null; if (clusterBound) { @@ -761,9 +767,9 @@ export class PluginServiceImpl implements PluginService, HostListProviderService if (typeof clusterBound === "string") { const cacheKey: string = this.getStatusCacheKey(clazz, clusterBound); if (!status) { - PluginServiceImpl.statusesExpiringCache.delete(cacheKey); + this.storageService.remove(StatusCacheItem, cacheKey); } else { - PluginServiceImpl.statusesExpiringCache.put(cacheKey, status, PluginServiceImpl.DEFAULT_STATUS_CACHE_EXPIRE_NANO); + this.storageService.set(cacheKey, new StatusCacheItem(status)); } return; } @@ -780,6 +786,22 @@ export class PluginServiceImpl implements PluginService, HostListProviderService } isPluginInUse(plugin: any) { - return this.pluginServiceManagerContainer.pluginManager!.isPluginInUse(plugin); + return this.servicesContainer.pluginManager!.isPluginInUse(plugin); + } + + isPooledClient(): boolean { + return this._isPooledClient; + } + + setIsPooledClient(isPooledClient: boolean): void { + this._isPooledClient = isPooledClient; + } + + getTrackedConnectionHost(): TrackedConnectionListHost | null { + return this._trackedConnectionHost; + } + + setTrackedConnectionHost(host: TrackedConnectionListHost | null): void { + this._trackedConnectionHost = host; } } diff --git a/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts b/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts index 2a56e606c..cdabc7d1a 100644 --- a/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts +++ b/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts @@ -110,7 +110,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (writerCandidate === null || this.rdsUtils.isRdsClusterDns(writerCandidate.host)) { // Writer is not found. It seems that topology is outdated. writerCandidateClient = await connectFunc(); - await this.pluginService.forceRefreshHostList(writerCandidateClient); + await this.pluginService.forceRefreshHostList(); writerCandidate = await this.pluginService.identifyConnection(writerCandidateClient); if (writerCandidate) { @@ -121,9 +121,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu continue; } - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(writerCandidate); - } + this.pluginService.setRoutedHostInfo(writerCandidate); } return writerCandidateClient; } @@ -132,23 +130,21 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if ((await this.pluginService.getHostRole(writerCandidateClient)) !== HostRole.WRITER) { // If the new connection resolves to a reader instance, this means the topology is outdated. // Force refresh to update the topology. - await this.pluginService.forceRefreshHostList(writerCandidateClient); + await this.pluginService.forceRefreshHostList(); await this.pluginService.abortTargetClient(writerCandidateClient); await sleep(retryDelayMs); continue; } // Writer connection is valid and verified. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(writerCandidate); - } + this.pluginService.setRoutedHostInfo(writerCandidate); return writerCandidateClient; } catch (error: any) { await this.pluginService.abortTargetClient(writerCandidateClient); if (this.pluginService.isLoginError(error) || !writerCandidate) { throw error; } else if (writerCandidate) { - this.pluginService.setAvailability(writerCandidate.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(writerCandidate, HostAvailability.NOT_AVAILABLE); } } } @@ -177,7 +173,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (readerCandidate === null || this.rdsUtils.isRdsClusterDns(readerCandidate.host)) { // Reader is not found. It seems that topology is outdated. readerCandidateClient = await connectFunc(); - await this.pluginService.forceRefreshHostList(readerCandidateClient); + await this.pluginService.forceRefreshHostList(); readerCandidate = await this.pluginService.identifyConnection(readerCandidateClient); if (readerCandidate) { @@ -185,9 +181,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (this.hasNoReaders()) { // It seems that cluster has no readers. Simulate Aurora reader cluster endpoint logic // and return the current (writer) client. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(readerCandidate); - } + this.pluginService.setRoutedHostInfo(readerCandidate); return readerCandidateClient; } await this.pluginService.abortTargetClient(readerCandidateClient); @@ -196,9 +190,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu } // Reader connection is valid and verified. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(readerCandidate); - } + this.pluginService.setRoutedHostInfo(readerCandidate); } else { logger.debug("Reader candidate not found"); } @@ -209,7 +201,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if ((await this.pluginService.getHostRole(readerCandidateClient)) !== HostRole.READER) { // If the new connection resolves to a writer instance, this means the topology is outdated. // Force refresh to update the topology. - await this.pluginService.forceRefreshHostList(readerCandidateClient); + await this.pluginService.forceRefreshHostList(); if (this.hasNoReaders()) { // It seems that cluster has no readers. Simulate Aurora reader cluster endpoint logic @@ -233,7 +225,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (this.pluginService.isLoginError(error) || !readerCandidate) { throw error; } else if (readerCandidate) { - this.pluginService.setAvailability(readerCandidate.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(readerCandidate, HostAvailability.NOT_AVAILABLE); } } } diff --git a/common/lib/plugins/aurora_initial_connection_strategy_plugin_factory.ts b/common/lib/plugins/aurora_initial_connection_strategy_plugin_factory.ts index d13d287f7..b12e4d7b7 100644 --- a/common/lib/plugins/aurora_initial_connection_strategy_plugin_factory.ts +++ b/common/lib/plugins/aurora_initial_connection_strategy_plugin_factory.ts @@ -15,19 +15,21 @@ */ import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; +import { FullServicesContainer } from "../utils/full_services_container"; export class AuroraInitialConnectionStrategyFactory extends ConnectionPluginFactory { private static auroraInitialConnectionStrategyPlugin: any; - async getInstance(pluginService: PluginService, props: Map) { + async getInstance(servicesContainer: FullServicesContainer, props: Map) { try { if (!AuroraInitialConnectionStrategyFactory.auroraInitialConnectionStrategyPlugin) { AuroraInitialConnectionStrategyFactory.auroraInitialConnectionStrategyPlugin = await import("./aurora_initial_connection_strategy_plugin"); } - return new AuroraInitialConnectionStrategyFactory.auroraInitialConnectionStrategyPlugin.AuroraInitialConnectionStrategyPlugin(pluginService); + return new AuroraInitialConnectionStrategyFactory.auroraInitialConnectionStrategyPlugin.AuroraInitialConnectionStrategyPlugin( + servicesContainer.pluginService + ); } catch (error: any) { throw new AwsWrapperError( Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "AuroraInitialConnectionStrategyPlugin") diff --git a/common/lib/plugins/bluegreen/blue_green_interim_status.ts b/common/lib/plugins/bluegreen/blue_green_interim_status.ts index c2c979317..4cf7bb3fa 100644 --- a/common/lib/plugins/bluegreen/blue_green_interim_status.ts +++ b/common/lib/plugins/bluegreen/blue_green_interim_status.ts @@ -104,7 +104,7 @@ export class BlueGreenInterimStatus { this.startTopology == null ? "" : this.startTopology - .map((x) => x.getHostAndPort() + x.role) + .map((x) => x.hostAndPort + x.role) .sort() .join(",") ); @@ -114,7 +114,7 @@ export class BlueGreenInterimStatus { this.currentTopology == null ? "" : this.currentTopology - .map((x) => x.getHostAndPort() + x.role) + .map((x) => x.hostAndPort + x.role) .sort() .join(",") ); diff --git a/common/lib/plugins/bluegreen/blue_green_plugin.ts b/common/lib/plugins/bluegreen/blue_green_plugin.ts index 4f7b90ece..ddcd34e89 100644 --- a/common/lib/plugins/bluegreen/blue_green_plugin.ts +++ b/common/lib/plugins/bluegreen/blue_green_plugin.ts @@ -27,9 +27,10 @@ import { IamAuthenticationPlugin } from "../../authentication/iam_authentication import { BlueGreenRole } from "./blue_green_role"; import { ExecuteRouting, RoutingResultHolder } from "./routing/execute_routing"; import { CanReleaseResources } from "../../can_release_resources"; +import { FullServicesContainer } from "../../utils/full_services_container"; export interface BlueGreenProviderSupplier { - create(pluginService: PluginService, props: Map, bgdId: string): BlueGreenStatusProvider; + create(servicesContainer: FullServicesContainer, props: Map, bgdId: string): BlueGreenStatusProvider; } export class BlueGreenPlugin extends AbstractConnectionPlugin implements CanReleaseResources { @@ -42,6 +43,7 @@ export class BlueGreenPlugin extends AbstractConnectionPlugin implements CanRele private static readonly CLOSED_METHOD_NAMES: Set = new Set(["end", "abort"]); protected readonly pluginService: PluginService; + protected readonly servicesContainer: FullServicesContainer; protected readonly properties: Map; protected bgProviderSupplier: BlueGreenProviderSupplier; protected bgStatus: BlueGreenStatus = null; @@ -53,18 +55,19 @@ export class BlueGreenPlugin extends AbstractConnectionPlugin implements CanRele protected endTimeNano: bigint = BigInt(0); private static provider: Map = new Map(); - constructor(pluginService: PluginService, properties: Map, bgProviderSupplier: BlueGreenProviderSupplier = null) { + constructor(servicesContainer: FullServicesContainer, properties: Map, bgProviderSupplier: BlueGreenProviderSupplier = null) { super(); if (!bgProviderSupplier) { bgProviderSupplier = { - create: (pluginService: PluginService, props: Map, bgdId: string): BlueGreenStatusProvider => { - return new BlueGreenStatusProvider(pluginService, props, bgdId); + create: (servicesContainer: FullServicesContainer, props: Map, bgdId: string): BlueGreenStatusProvider => { + return new BlueGreenStatusProvider(servicesContainer, props, bgdId); } }; } this.properties = properties; - this.pluginService = pluginService; + this.servicesContainer = servicesContainer; + this.pluginService = servicesContainer.pluginService; this.bgProviderSupplier = bgProviderSupplier; this.bgdId = WrapperProperties.BGD_ID.get(this.properties).trim().toLowerCase(); } @@ -176,7 +179,7 @@ export class BlueGreenPlugin extends AbstractConnectionPlugin implements CanRele this.startTimeNano = getTimeInNanos(); - while (routing && result && !result.isPresent()) { + while (routing && (!result || !result.isPresent())) { result = await routing.apply(this, methodName, methodFunc, methodArgs, this.properties, this.pluginService); if (!result?.isPresent()) { this.bgStatus = this.pluginService.getStatus(BlueGreenStatus, this.bgdId); @@ -215,7 +218,7 @@ export class BlueGreenPlugin extends AbstractConnectionPlugin implements CanRele private initProvider() { const provider = BlueGreenPlugin.provider.get(this.bgdId); if (!provider) { - const provider = this.bgProviderSupplier.create(this.pluginService, this.properties, this.bgdId); + const provider = this.bgProviderSupplier.create(this.servicesContainer, this.properties, this.bgdId); BlueGreenPlugin.provider.set(this.bgdId, provider); } } diff --git a/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts b/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts index f5ba48a3c..c7f581067 100644 --- a/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts +++ b/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class BlueGreenPluginFactory extends ConnectionPluginFactory { private static blueGreenPlugin: any; - async getInstance(pluginService: PluginService, props: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, props: Map): Promise { try { if (!BlueGreenPluginFactory.blueGreenPlugin) { BlueGreenPluginFactory.blueGreenPlugin = await import("./blue_green_plugin"); } - return new BlueGreenPluginFactory.blueGreenPlugin.BlueGreenPlugin(pluginService, props); + return new BlueGreenPluginFactory.blueGreenPlugin.BlueGreenPlugin(servicesContainer, props); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "BlueGreenPluginFactory")); } diff --git a/common/lib/plugins/bluegreen/blue_green_status.ts b/common/lib/plugins/bluegreen/blue_green_status.ts index 1c17463f3..783e28407 100644 --- a/common/lib/plugins/bluegreen/blue_green_status.ts +++ b/common/lib/plugins/bluegreen/blue_green_status.ts @@ -19,7 +19,6 @@ import { ConnectRouting } from "./routing/connect_routing"; import { ExecuteRouting } from "./routing/execute_routing"; import { BlueGreenRole } from "./blue_green_role"; import { HostInfo } from "../../host_info"; -import { Pair } from "../../utils/utils"; export class BlueGreenStatus { private readonly bgdId: string; @@ -28,7 +27,7 @@ export class BlueGreenStatus { private readonly _unmodifiableExecuteRouting: readonly ExecuteRouting[]; private readonly _roleByHost: Map; - private readonly _correspondingHosts: Map>; + private readonly _correspondingHosts: Map; constructor( bgdId: string, @@ -36,7 +35,7 @@ export class BlueGreenStatus { unmodifiableConnectRouting?: ConnectRouting[], unmodifiableExecuteRouting?: ExecuteRouting[], roleByHost?: Map, - correspondingHosts?: Map> + correspondingHosts?: Map ) { this.bgdId = bgdId; this._currentPhase = phase; @@ -62,7 +61,7 @@ export class BlueGreenStatus { return this._roleByHost; } - get correspondingHosts(): Map> { + get correspondingHosts(): Map { return this._correspondingHosts; } diff --git a/common/lib/plugins/bluegreen/blue_green_status_monitor.ts b/common/lib/plugins/bluegreen/blue_green_status_monitor.ts index f320c84f7..0bd698580 100644 --- a/common/lib/plugins/bluegreen/blue_green_status_monitor.ts +++ b/common/lib/plugins/bluegreen/blue_green_status_monitor.ts @@ -36,6 +36,7 @@ import { HostListProviderService } from "../../host_list_provider_service"; import { StatusInfo } from "./status_info"; import { DatabaseDialect } from "../../database_dialect/database_dialect"; import { AwsWrapperError } from "../../utils/errors"; +import { FullServicesContainer } from "../../utils/full_services_container"; export interface OnBlueGreenStatusChange { onBlueGreenStatusChanged(role: BlueGreenRole, interimStatus: BlueGreenInterimStatus): void; @@ -50,6 +51,7 @@ export class BlueGreenStatusMonitor { protected static readonly knownVersions: Set = new Set([BlueGreenStatusMonitor.latestKnownVersion]); protected readonly blueGreenDialect: BlueGreenDialect; + protected readonly servicesContainer: FullServicesContainer; protected readonly pluginService: PluginService; protected readonly bgdId: string; protected readonly props: Map; @@ -99,7 +101,7 @@ export class BlueGreenStatusMonitor { role: BlueGreenRole, bgdId: string, initialHostInfo: HostInfo, - pluginService: PluginService, + servicesContainer: FullServicesContainer, props: Map, statusCheckIntervalMap: Map, onBlueGreenStatusChangeFunc: OnBlueGreenStatusChange @@ -107,7 +109,8 @@ export class BlueGreenStatusMonitor { this.role = role; this.bgdId = bgdId; this.initialHostInfo = initialHostInfo; - this.pluginService = pluginService; + this.servicesContainer = servicesContainer; + this.pluginService = this.servicesContainer.pluginService; this.props = props; this.statusCheckIntervalMap = statusCheckIntervalMap; this.onBlueGreenStatusChangeFunc = onBlueGreenStatusChangeFunc; @@ -296,12 +299,7 @@ export class BlueGreenStatusMonitor { return; } - const client: ClientWrapper = this.clientWrapper; - if (await this.isConnectionClosed(client)) { - return; - } - - this.currentTopology = await this.hostListProvider.forceRefresh(client); + this.currentTopology = await this.hostListProvider.forceRefresh(); if (this.collectedTopology) { this.startTopology = this.currentTopology; } @@ -518,7 +516,7 @@ export class BlueGreenStatusMonitor { if (connectionHostInfoCopy) { this.hostListProvider = this.pluginService .getDialect() - .getHostListProvider(hostListProperties, connectionHostInfoCopy.host, this.pluginService as unknown as HostListProviderService); + .getHostListProvider(hostListProperties, connectionHostInfoCopy.host, this.servicesContainer); } else { logger.warn(Messages.get("Bgd.hostInfoNull")); } diff --git a/common/lib/plugins/bluegreen/blue_green_status_provider.ts b/common/lib/plugins/bluegreen/blue_green_status_provider.ts index bf899e110..5a80116d9 100644 --- a/common/lib/plugins/bluegreen/blue_green_status_provider.ts +++ b/common/lib/plugins/bluegreen/blue_green_status_provider.ts @@ -20,7 +20,7 @@ import { SimpleHostAvailabilityStrategy } from "../../host_availability/simple_h import { BlueGreenStatusMonitor } from "./blue_green_status_monitor"; import { BlueGreenInterimStatus } from "./blue_green_interim_status"; import { HostInfo } from "../../host_info"; -import { convertMsToNanos, getTimeInNanos, Pair } from "../../utils/utils"; +import { convertMsToNanos, getTimeInNanos } from "../../utils/utils"; import { BlueGreenRole } from "./blue_green_role"; import { BlueGreenStatus } from "./blue_green_status"; import { BlueGreenPhase } from "./blue_green_phase"; @@ -41,9 +41,10 @@ import { SuspendExecuteRouting } from "./routing/suspend_execute_routing"; import { SuspendUntilCorrespondingHostFoundConnectRouting } from "./routing/suspend_until_corresponding_host_found_connect_routing"; import { RejectConnectRouting } from "./routing/reject_connect_routing"; import { getValueHash } from "./blue_green_utils"; +import { FullServicesContainer } from "../../utils/full_services_container"; +import { StorageService } from "../../utils/storage/storage_service"; export class BlueGreenStatusProvider { - static readonly MONITORING_PROPERTY_PREFIX = "blue_green_monitoring_"; private static readonly DEFAULT_CONNECT_TIMEOUT_MS = 10_000; // 10 seconds private static readonly DEFAULT_QUERY_TIMEOUT_MS = 10_000; // 10 seconds @@ -54,8 +55,8 @@ export class BlueGreenStatusProvider { protected interimStatuses: BlueGreenInterimStatus[] = [null, null]; protected hostIpAddresses: Map = new Map(); - // The second parameter of Pair is null when no corresponding host is found. - protected readonly correspondingHosts: Map> = new Map(); + // The second element is null when no corresponding host is found. + protected readonly correspondingHosts: Map = new Map(); // all known host names; host with no port protected readonly roleByHost: Map = new Map(); @@ -76,14 +77,18 @@ export class BlueGreenStatusProvider { protected readonly switchoverTimeoutNanos: bigint; protected readonly suspendNewBlueConnectionsWhenInProgress: boolean; + protected readonly servicesContainer: FullServicesContainer; + protected readonly storageService: StorageService; protected readonly pluginService: PluginService; protected readonly properties: Map; protected readonly bgdId: string; protected phaseTimeNanos: Map = new Map(); protected readonly rdsUtils: RdsUtils = new RdsUtils(); - constructor(pluginService: PluginService, properties: Map, bgdId: string) { - this.pluginService = pluginService; + constructor(servicesContainer: FullServicesContainer, properties: Map, bgdId: string) { + this.servicesContainer = servicesContainer; + this.pluginService = this.servicesContainer.pluginService; + this.storageService = this.servicesContainer.storageService; this.properties = properties; this.bgdId = bgdId; @@ -107,7 +112,7 @@ export class BlueGreenStatusProvider { BlueGreenRole.SOURCE, this.bgdId, this.pluginService.getCurrentHostInfo(), - this.pluginService, + this.servicesContainer, this.getMonitoringProperties(), this.statusCheckIntervalMap, { onBlueGreenStatusChanged: (role, status) => this.prepareStatus(role, status) } @@ -117,7 +122,7 @@ export class BlueGreenStatusProvider { BlueGreenRole.TARGET, this.bgdId, this.pluginService.getCurrentHostInfo(), - this.pluginService, + this.servicesContainer, this.getMonitoringProperties(), this.statusCheckIntervalMap, { onBlueGreenStatusChanged: (role, status) => this.prepareStatus(role, status) } @@ -128,7 +133,7 @@ export class BlueGreenStatusProvider { const monitoringConnProperties: Map = new Map(this.properties); for (const key of monitoringConnProperties.keys()) { - if (!key.startsWith(BlueGreenStatusProvider.MONITORING_PROPERTY_PREFIX)) { + if (!key.startsWith(WrapperProperties.BG_MONITORING_PROPERTY_PREFIX)) { continue; } @@ -234,19 +239,19 @@ export class BlueGreenStatusProvider { if (blueWriterHostInfo) { // greenWriterHostInfo can be null but that will be handled properly by corresponding routing. - this.correspondingHosts.set(blueWriterHostInfo.host, new Pair(blueWriterHostInfo, greenWriterHostInfo)); + this.correspondingHosts.set(blueWriterHostInfo.host, [blueWriterHostInfo, greenWriterHostInfo]); } if (sortedBlueReaderHostInfos?.length > 0) { if (sortedGreenReaderHostInfos?.length > 0) { let greenIndex: number = 0; sortedBlueReaderHostInfos.forEach((blueHostInfo) => { - this.correspondingHosts.set(blueHostInfo.host, new Pair(blueHostInfo, sortedGreenReaderHostInfos.at(greenIndex++))); + this.correspondingHosts.set(blueHostInfo.host, [blueHostInfo, sortedGreenReaderHostInfos.at(greenIndex++)]); greenIndex %= sortedGreenReaderHostInfos.length; }); } else { sortedBlueReaderHostInfos.forEach((blueHostInfo) => { - this.correspondingHosts.set(blueHostInfo.host, new Pair(blueHostInfo, greenWriterHostInfo)); + this.correspondingHosts.set(blueHostInfo.host, [blueHostInfo, greenWriterHostInfo]); }); } } @@ -268,10 +273,10 @@ export class BlueGreenStatusProvider { if (blueClusterHost !== null && greenClusterHost !== null) { if (!this.correspondingHosts.has(blueClusterHost)) { - this.correspondingHosts.set( - blueClusterHost, - new Pair(this.hostInfoBuilder.withHost(blueClusterHost).build(), this.hostInfoBuilder.withHost(greenClusterHost).build()) - ); + this.correspondingHosts.set(blueClusterHost, [ + this.hostInfoBuilder.withHost(blueClusterHost).build(), + this.hostInfoBuilder.withHost(greenClusterHost).build() + ]); } } @@ -288,10 +293,10 @@ export class BlueGreenStatusProvider { if (blueClusterReaderHost !== null && greenClusterReaderHost !== null) { if (!this.correspondingHosts.has(blueClusterReaderHost)) { - this.correspondingHosts.set( - blueClusterReaderHost, - new Pair(this.hostInfoBuilder.withHost(blueClusterReaderHost).build(), this.hostInfoBuilder.withHost(greenClusterReaderHost).build()) - ); + this.correspondingHosts.set(blueClusterReaderHost, [ + this.hostInfoBuilder.withHost(blueClusterReaderHost).build(), + this.hostInfoBuilder.withHost(greenClusterReaderHost).build() + ]); } } @@ -308,10 +313,10 @@ export class BlueGreenStatusProvider { }); if (greenHost) { if (!this.correspondingHosts.has(blueHost)) { - this.correspondingHosts.set( - blueHost, - new Pair(this.hostInfoBuilder.withHost(blueHost).build(), this.hostInfoBuilder.withHost(greenHost).build()) - ); + this.correspondingHosts.set(blueHost, [ + this.hostInfoBuilder.withHost(blueHost).build(), + this.hostInfoBuilder.withHost(greenHost).build() + ]); } } } @@ -483,7 +488,7 @@ export class BlueGreenStatusProvider { Array.from(this.roleByHost.entries()) .filter(([host, role]) => role === BlueGreenRole.SOURCE && this.correspondingHosts.has(host)) .forEach(([host, role]) => { - const hostSpec = this.correspondingHosts.get(host).left; + const hostSpec = this.correspondingHosts.get(host)[0]; const blueIp = this.hostIpAddresses.get(hostSpec.host); const substituteHostSpecWithIp = !blueIp ? hostSpec : this.hostInfoBuilder.copyFrom(hostSpec).withHost(blueIp).build(); @@ -627,9 +632,9 @@ export class BlueGreenStatusProvider { .forEach(([host, role]) => { const blueHost: string = host; const isBlueHostInstance: boolean = this.rdsUtils.isRdsInstance(blueHost); - const pair: Pair | undefined = this.correspondingHosts?.get(host); - const blueHostInfo: HostInfo | undefined = pair?.left; - const greenHostInfo: HostInfo | undefined = pair?.right; + const pair: [HostInfo, HostInfo | null] | undefined = this.correspondingHosts?.get(host); + const blueHostInfo: HostInfo | undefined = pair?.[0]; + const greenHostInfo: HostInfo | undefined = pair?.[1]; if (!greenHostInfo) { // A corresponding host is not found. We need to suspend this call. @@ -872,7 +877,7 @@ export class BlueGreenStatusProvider { logger.debug( "Corresponding hosts:\n" + Array.from(this.correspondingHosts.entries()) - .map(([key, value]) => ` ${key} -> ${value.right == null ? "" : value.right.getHostAndPort()}`) + .map(([key, value]) => ` ${key} -> ${value[1] == null ? "" : value[1].hostAndPort}`) .join("\n") ); diff --git a/common/lib/plugins/bluegreen/routing/base_connect_routing.ts b/common/lib/plugins/bluegreen/routing/base_connect_routing.ts index 5dfdd25cc..9a0deeccc 100644 --- a/common/lib/plugins/bluegreen/routing/base_connect_routing.ts +++ b/common/lib/plugins/bluegreen/routing/base_connect_routing.ts @@ -25,7 +25,7 @@ import { PluginService } from "../../../plugin_service"; export abstract class BaseConnectRouting extends BaseRouting implements ConnectRouting { isMatch(hostInfo: HostInfo, hostRole: BlueGreenRole): boolean { return ( - (this.hostAndPort === null || this.hostAndPort === (hostInfo ?? hostInfo.getHostAndPort().toLowerCase())) && + (this.hostAndPort === null || this.hostAndPort === (hostInfo ?? hostInfo.hostAndPort.toLowerCase())) && (this.role === null || this.role === hostRole) ); } diff --git a/common/lib/plugins/bluegreen/routing/base_execute_routing.ts b/common/lib/plugins/bluegreen/routing/base_execute_routing.ts index 80d7d50e5..e631442ba 100644 --- a/common/lib/plugins/bluegreen/routing/base_execute_routing.ts +++ b/common/lib/plugins/bluegreen/routing/base_execute_routing.ts @@ -24,7 +24,7 @@ import { ConnectionPlugin } from "../../../connection_plugin"; export abstract class BaseExecuteRouting extends BaseRouting implements ExecuteRouting { isMatch(hostInfo: HostInfo, hostRole: BlueGreenRole): boolean { return ( - (this.hostAndPort === null || this.hostAndPort === (hostInfo ?? hostInfo.getHostAndPort().toLowerCase())) && + (this.hostAndPort === null || this.hostAndPort === (hostInfo ?? hostInfo.hostAndPort.toLowerCase())) && (this.role === null || this.role === hostRole) ); } diff --git a/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts b/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts index 1d453e461..812873d7a 100644 --- a/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts +++ b/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts @@ -73,7 +73,6 @@ export class SubstituteConnectRouting extends BaseConnectRouting { .withHostId(iamHost.hostId) .withAvailability(HostAvailability.AVAILABLE) .build(); - reroutedHostInfo.addAlias(iamHost.host); const reroutedProperties: Map = new Map(properties); reroutedProperties.set(WrapperProperties.HOST.name, iamHost.host); @@ -99,12 +98,12 @@ export class SubstituteConnectRouting extends BaseConnectRouting { // try with another IAM host } } - throw new AwsWrapperError(Messages.get("Bgd.inProgressCantOpenConnection", this.substituteHost.getHostAndPort())); + throw new AwsWrapperError(Messages.get("Bgd.inProgressCantOpenConnection", this.substituteHost.hostAndPort)); } toString(): string { - return `${this.constructor.name} [${this.hostAndPort ?? ""}, ${this.role?.name ?? ""}, substitute: ${this.substituteHost?.getHostAndPort() ?? ""}, iamHosts: ${ - this.iamHosts?.map((host) => host.getHostAndPort()).join(", ") ?? "" + return `${this.constructor.name} [${this.hostAndPort ?? ""}, ${this.role?.name ?? ""}, substitute: ${this.substituteHost?.hostAndPort ?? ""}, iamHosts: ${ + this.iamHosts?.map((host) => host.hostAndPort).join(", ") ?? "" }]`; } } diff --git a/common/lib/plugins/bluegreen/routing/suspend_until_corresponding_host_found_connect_routing.ts b/common/lib/plugins/bluegreen/routing/suspend_until_corresponding_host_found_connect_routing.ts index 7ce13cb1e..ecf7cab26 100644 --- a/common/lib/plugins/bluegreen/routing/suspend_until_corresponding_host_found_connect_routing.ts +++ b/common/lib/plugins/bluegreen/routing/suspend_until_corresponding_host_found_connect_routing.ts @@ -26,7 +26,7 @@ import { TelemetryFactory } from "../../../utils/telemetry/telemetry_factory"; import { TelemetryContext } from "../../../utils/telemetry/telemetry_context"; import { TelemetryTraceLevel } from "../../../utils/telemetry/telemetry_trace_level"; import { BlueGreenStatus } from "../blue_green_status"; -import { convertMsToNanos, convertNanosToMs, getTimeInNanos, Pair } from "../../../utils/utils"; +import { convertMsToNanos, convertNanosToMs, getTimeInNanos } from "../../../utils/utils"; import { WrapperProperties } from "../../../wrapper_property"; import { BlueGreenPhase } from "../blue_green_phase"; import { AwsWrapperError } from "../../../utils/errors"; @@ -60,7 +60,7 @@ export class SuspendUntilCorrespondingHostFoundConnectRouting extends BaseConnec return await telemetryContext.start(async () => { let bgStatus: BlueGreenStatus = pluginService.getStatus(BlueGreenStatus, this.bgdId); - let correspondingPair: Pair = bgStatus?.correspondingHosts.get(hostInfo.host); + let correspondingPair: [HostInfo, HostInfo] | undefined = bgStatus?.correspondingHosts.get(hostInfo.host); const timeoutNanos: bigint = convertMsToNanos(WrapperProperties.BG_CONNECT_TIMEOUT_MS.get(properties)); const suspendStartTime: bigint = getTimeInNanos(); @@ -70,7 +70,7 @@ export class SuspendUntilCorrespondingHostFoundConnectRouting extends BaseConnec getTimeInNanos() <= endTime && bgStatus != null && bgStatus.currentPhase !== BlueGreenPhase.COMPLETED && - (!correspondingPair || !correspondingPair.right) + (!correspondingPair || !correspondingPair[1]) ) { await this.delay(SuspendUntilCorrespondingHostFoundConnectRouting.SLEEP_TIME_MS, bgStatus, pluginService, this.bgdId); diff --git a/common/lib/plugins/connect_time_plugin_factory.ts b/common/lib/plugins/connect_time_plugin_factory.ts index bc333ccbc..f2362458c 100644 --- a/common/lib/plugins/connect_time_plugin_factory.ts +++ b/common/lib/plugins/connect_time_plugin_factory.ts @@ -15,15 +15,15 @@ */ import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; import { ConnectionPlugin } from "../connection_plugin"; import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; +import { FullServicesContainer } from "../utils/full_services_container"; export class ConnectTimePluginFactory extends ConnectionPluginFactory { private static connectTimePlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!ConnectTimePluginFactory.connectTimePlugin) { ConnectTimePluginFactory.connectTimePlugin = await import("./connect_time_plugin"); diff --git a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts index 651bbea84..d256d8bce 100644 --- a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts +++ b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts @@ -28,7 +28,15 @@ import { HostRole } from "../../host_role"; import { OpenedConnectionTracker } from "./opened_connection_tracker"; export class AuroraConnectionTrackerPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - private static readonly subscribedMethods = new Set(["notifyHostListChanged", "connect", "query", "rollback"]); + private static readonly subscribedMethods = new Set([ + ...SubscribedMethodHelper.NETWORK_BOUND_METHODS, + "end", + "abort", + "notifyHostListChanged" + ]); + private static readonly CLOSING_METHODS = new Set(["end", "abort"]); + private static readonly TOPOLOGY_CHANGES_EXPECTED_TIME_NS = BigInt(3 * 60 * 1_000_000_000); + private static hostListRefreshEndTimeNs: bigint = 0n; private readonly pluginService: PluginService; private readonly rdsUtils: RdsUtils; @@ -56,46 +64,92 @@ export class AuroraConnectionTrackerPlugin extends AbstractConnectionPlugin impl connectFunc: () => Promise ): Promise { const targetClient = await connectFunc(); - - if (targetClient) { - const type: RdsUrlType = this.rdsUtils.identifyRdsType(hostInfo.host); - if (type.isRdsCluster) { - hostInfo.resetAliases(); - await this.pluginService.fillAliases(targetClient, hostInfo); + let connectionHostInfo: HostInfo = this.pluginService.getRoutedHostInfo() ?? hostInfo; + + if (targetClient && !this.pluginService.isPooledClient()) { + const type: RdsUrlType = this.rdsUtils.identifyRdsType(connectionHostInfo.host); + if (type.isRdsCluster || type === RdsUrlType.OTHER || type === RdsUrlType.IP_ADDRESS) { + const identifiedHostInfo: HostInfo | null = await this.pluginService.identifyConnection(targetClient, connectionHostInfo); + if (identifiedHostInfo) { + connectionHostInfo = identifiedHostInfo; + await this.pluginService.setRoutedHostInfo(connectionHostInfo); + } } - await this.tracker.populateOpenedConnectionQueue(hostInfo, targetClient); + const host = this.tracker.populateOpenedConnectionQueue(connectionHostInfo, targetClient); + this.pluginService.setTrackedConnectionHost(host); } return targetClient; } override async execute(methodName: string, methodFunc: () => Promise, methodArgs: any[]): Promise { + const currentHostInfo = this.pluginService.getCurrentHostInfo(); this.rememberWriter(); + const isClosing = AuroraConnectionTrackerPlugin.CLOSING_METHODS.has(methodName); + try { + if (!isClosing) { + let needRefreshHostList = false; + const localRefreshEndTimeNs = AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs; + if (localRefreshEndTimeNs > 0n) { + if (localRefreshEndTimeNs > process.hrtime.bigint()) { + needRefreshHostList = true; + } else { + AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs = 0n; + } + } + if (this.needUpdateCurrentWriter || needRefreshHostList) { + await this.checkWriterChanged(needRefreshHostList); + } + } + const result = await methodFunc(); - if (this.needUpdateCurrentWriter) { - await this.checkWriterChanged(); + + if (isClosing) { + const host = this.pluginService.getTrackedConnectionHost(); + if (host) { + this.tracker.removeConnectionTracking(host); + this.pluginService.setTrackedConnectionHost(null); + } else if (currentHostInfo) { + this.tracker.removeConnectionTrackingByHost(currentHostInfo, this.pluginService.getCurrentClient()?.targetClient); + } } return result; } catch (error) { if (error instanceof FailoverError) { - await this.checkWriterChanged(); + AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs = + process.hrtime.bigint() + AuroraConnectionTrackerPlugin.TOPOLOGY_CHANGES_EXPECTED_TIME_NS; + // This call may effectively close/abort the current connection. + await this.checkWriterChanged(true); } throw error; } } - private async checkWriterChanged(): Promise { + private async checkWriterChanged(needRefreshHostList: boolean): Promise { + if (needRefreshHostList) { + try { + await this.pluginService.refreshHostList(); + } catch (error) { + // Ignore: continue with whatever topology is currently available. + } + } + const hostInfoAfterFailover = this.getWriter(this.pluginService.getAllHosts()); + if (hostInfoAfterFailover === null) { + return; + } + if (this.currentWriter === null) { this.currentWriter = hostInfoAfterFailover; this.needUpdateCurrentWriter = false; - } else if (!this.currentWriter.equals(hostInfoAfterFailover!)) { - // writer changed + } else if (this.currentWriter.hostAndPort !== hostInfoAfterFailover.hostAndPort) { + // The writer changed. await this.tracker.invalidateAllConnections(this.currentWriter); this.tracker.logOpenedConnections(); this.currentWriter = hostInfoAfterFailover; this.needUpdateCurrentWriter = false; + AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs = 0n; } } diff --git a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin_factory.ts b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin_factory.ts index 4eb2b6e1b..b7ce5b357 100644 --- a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin_factory.ts +++ b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class AuroraConnectionTrackerPluginFactory extends ConnectionPluginFactory { private static auroraConnectionTrackerPlugin: any; - async getInstance(pluginService: PluginService, props: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, props: Map): Promise { try { if (!AuroraConnectionTrackerPluginFactory.auroraConnectionTrackerPlugin) { AuroraConnectionTrackerPluginFactory.auroraConnectionTrackerPlugin = await import("./aurora_connection_tracker_plugin"); } - return new AuroraConnectionTrackerPluginFactory.auroraConnectionTrackerPlugin.AuroraConnectionTrackerPlugin(pluginService); + return new AuroraConnectionTrackerPluginFactory.auroraConnectionTrackerPlugin.AuroraConnectionTrackerPlugin(servicesContainer.pluginService); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "AuroraConnectionTrackerPlugin")); } diff --git a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts index 364a123be..6efa08391 100644 --- a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts +++ b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts @@ -21,9 +21,10 @@ import { logger } from "../../../logutils"; import { MapUtils } from "../../utils/map_utils"; import { Messages } from "../../utils/messages"; import { PluginService } from "../../plugin_service"; +import { TrackedConnectionList, TrackedConnectionListHost } from "./tracked_connection_list"; export class OpenedConnectionTracker { - static readonly openedConnections: Map>> = new Map>>(); + static readonly openedConnections: Map = new Map(); readonly pluginService: PluginService; private static readonly rdsUtils = new RdsUtils(); @@ -31,101 +32,113 @@ export class OpenedConnectionTracker { this.pluginService = pluginService; } - async populateOpenedConnectionQueue(hostInfo: HostInfo, client: ClientWrapper): Promise { - const aliases = hostInfo.aliases; + populateOpenedConnectionQueue(hostInfo: HostInfo, client: ClientWrapper): TrackedConnectionListHost | null { + if (!hostInfo || !client) { + return null; + } - // Check if the connection was established using an instance endpoint + // Check if the connection was established using an instance endpoint. if (OpenedConnectionTracker.rdsUtils.isRdsInstance(hostInfo.host)) { - this.trackConnection(hostInfo.getHostAndPort(), client); - return; + const host = this.trackConnection(hostInfo.hostAndPort, client); + this.logOpenedConnections(); + return host; } - const instanceEndpoint = [...aliases] - .filter((x) => OpenedConnectionTracker.rdsUtils.isRdsInstance(OpenedConnectionTracker.rdsUtils.removePort(x))) - .reduce((max, s) => (s > max ? s : max), ""); + // It might be a custom domain name. Let's track by hostId and custom domain name. + let lastHost: TrackedConnectionListHost | null = null; + if (hostInfo.hostId) { + lastHost = this.trackConnection(hostInfo.hostId, client); + } + if (hostInfo.hostAndPort) { + lastHost = this.trackConnection(hostInfo.hostAndPort, client); + } + this.logOpenedConnections(); + return lastHost; + } - if (!instanceEndpoint) { - logger.debug(Messages.get("OpenedConnectionTracker.unableToPopulateOpenedConnectionQueue", hostInfo.host)); + async invalidateAllConnections(hostInfo: HostInfo): Promise { + if (!hostInfo) { return; } + await this.invalidateAllConnectionsMultipleHosts(hostInfo.hostAndPort, hostInfo.host, hostInfo.hostId); + } - this.trackConnection(instanceEndpoint, client); + async invalidateAllConnectionsMultipleHosts(...keys: string[]): Promise { + for (const key of keys) { + if (!key) { + continue; + } + try { + const connectionList = OpenedConnectionTracker.openedConnections.get(key); + this.logConnectionList(key, connectionList); + await this.invalidateConnections(connectionList); + } catch (error) { + // Ignore and continue with the remaining keys. + } + } } - async invalidateAllConnections(hostInfo: HostInfo): Promise { - await this.invalidateAllConnectionsMultipleHosts(hostInfo.asAlias); - await this.invalidateAllConnectionsMultipleHosts(...Array.from(hostInfo.aliases)); + removeConnectionTracking(host: TrackedConnectionListHost | null): void { + host?.remove(); } - async invalidateAllConnectionsMultipleHosts(...hosts: string[]): Promise { - try { - const instanceEndpoint = hosts - .filter((x) => OpenedConnectionTracker.rdsUtils.isRdsInstance(OpenedConnectionTracker.rdsUtils.removePort(x))) - .at(0); - if (!instanceEndpoint) { - return; - } - const connectionQueue = OpenedConnectionTracker.openedConnections.get(instanceEndpoint); - this.logConnectionQueue(instanceEndpoint, connectionQueue!); - await this.invalidateConnections(connectionQueue!); - } catch (error) { - // ignore + removeConnectionTrackingByHost(hostInfo: HostInfo, client: ClientWrapper | undefined | null): void { + const hostAndPort = OpenedConnectionTracker.rdsUtils.isRdsInstance(hostInfo.host) ? hostInfo.hostAndPort : null; + if (!hostAndPort) { + return; + } + + const connectionList = OpenedConnectionTracker.openedConnections.get(hostAndPort); + if (connectionList) { + connectionList.removeIf((ref) => { + const conn = ref.deref(); + return !conn || conn === client; + }); } } - private trackConnection(instanceEndpoint: string, client: ClientWrapper): void { - const connectionQueue = MapUtils.computeIfAbsent( - OpenedConnectionTracker.openedConnections, - instanceEndpoint, - (k) => new Array>() - ); - connectionQueue!.push(new WeakRef(client)); - this.logOpenedConnections(); + private trackConnection(instanceEndpoint: string, client: ClientWrapper): TrackedConnectionListHost { + const connectionList = MapUtils.computeIfAbsent(OpenedConnectionTracker.openedConnections, instanceEndpoint, (_) => new TrackedConnectionList()); + return connectionList!.add(client); } - private async invalidateConnections(connectionQueue: Array>): Promise { - let clientRef: WeakRef | undefined; - while ((clientRef = connectionQueue?.shift()) != null) { - const client = clientRef?.deref() ?? null; - if (!client) { - continue; - } + private async invalidateConnections(connectionList: TrackedConnectionList | undefined): Promise { + if (!connectionList || connectionList.isEmpty()) { + return; + } + + const connections = connectionList.drainAll(); + for (const client of connections) { await this.pluginService.abortTargetClient(client); } } logOpenedConnections(): void { - let str = ""; - const hostList = []; - - for (const queue of OpenedConnectionTracker.openedConnections.values()) { - if (queue.length !== 0) { - for (const connRef of queue) { - const conn = connRef?.deref() ?? null; - if (conn) { - hostList.push(`${conn.id} - ${conn.hostInfo.toString()}`); - } - } - str = hostList.join("\n\t"); + const hostList: string[] = []; + for (const connectionList of OpenedConnectionTracker.openedConnections.values()) { + for (const conn of connectionList.getConnections()) { + hostList.push(`${conn.id} - ${conn.hostInfo.toString()}`); } } - logger.debug(`Opened Connections Tracked: \n\t${str}`); + logger.debug(`Opened Connections Tracked: \n\t${hostList.join("\n\t")}`); } - private logConnectionQueue(host: string, queue: Array>): void { - if (!queue || queue.length === 0) { + private logConnectionList(host: string, connectionList: TrackedConnectionList | undefined): void { + if (!connectionList || connectionList.isEmpty()) { return; } - logger.debug(Messages.get("OpenedConnectionTracker.invalidatingConnections", `${host}\n[${queue.map((x) => x.deref()!.hostInfo).join()}\n]`)); + const connections = connectionList.getConnections().map((conn) => conn.hostInfo); + logger.debug(Messages.get("OpenedConnectionTracker.invalidatingConnections", `${host}\n[${connections.join()}\n]`)); } pruneNullConnections(): void { - for (const [key, queue] of OpenedConnectionTracker.openedConnections) { - OpenedConnectionTracker.openedConnections.set( - key, - queue.filter((connWeakRef: WeakRef) => connWeakRef?.deref() ?? null) - ); + for (const connectionList of OpenedConnectionTracker.openedConnections.values()) { + connectionList.removeIf((ref) => !ref.deref()); } } + + static clearCache(): void { + OpenedConnectionTracker.openedConnections.clear(); + } } diff --git a/common/lib/plugins/custom_endpoint/custom_endpoint_plugin.ts b/common/lib/plugins/custom_endpoint/custom_endpoint_plugin.ts index 99e5bfb42..70dee99cc 100644 --- a/common/lib/plugins/custom_endpoint/custom_endpoint_plugin.ts +++ b/common/lib/plugins/custom_endpoint/custom_endpoint_plugin.ts @@ -31,11 +31,14 @@ import { sleep } from "../../utils/utils"; import { CustomEndpointMonitor, CustomEndpointMonitorImpl } from "./custom_endpoint_monitor_impl"; import { SubscribedMethodHelper } from "../../utils/subscribed_method_helper"; import { CanReleaseResources } from "../../can_release_resources"; +import { RdsUrlType } from "../../utils/rds_url_type"; +import { GDBRegionUtils } from "../../utils/gdb_region_utils"; export class CustomEndpointPlugin extends AbstractConnectionPlugin implements CanReleaseResources { private static readonly TELEMETRY_WAIT_FOR_INFO_COUNTER = "customEndpoint.waitForInfo.counter"; private static SUBSCRIBED_METHODS: Set = new Set(SubscribedMethodHelper.NETWORK_BOUND_METHODS); private static readonly CACHE_CLEANUP_NANOS = BigInt(60_000_000_000); + private static readonly regionUtils: RegionUtils = new RegionUtils(); private static readonly rdsUtils = new RdsUtils(); protected static readonly monitors: SlidingExpirationCache = new SlidingExpirationCache( @@ -106,7 +109,7 @@ export class CustomEndpointPlugin extends AbstractConnectionPlugin implements Ca throw new AwsWrapperError(Messages.get("CustomEndpointPlugin.errorParsingEndpointIdentifier", this.customEndpointHostInfo.host)); } - this.region = RegionUtils.getRegion(props.get(WrapperProperties.CUSTOM_ENDPOINT_REGION.name), this.customEndpointHostInfo.host); + this.region = await CustomEndpointPlugin.regionUtils.getRegion(WrapperProperties.CUSTOM_ENDPOINT_REGION.name, this.customEndpointHostInfo, props); if (!this.region) { throw new AwsWrapperError(Messages.get("CustomEndpointPlugin.unableToDetermineRegion", WrapperProperties.CUSTOM_ENDPOINT_REGION.name)); } diff --git a/common/lib/plugins/custom_endpoint/custom_endpoint_plugin_factory.ts b/common/lib/plugins/custom_endpoint/custom_endpoint_plugin_factory.ts index 0e945c2ea..23a3ee502 100644 --- a/common/lib/plugins/custom_endpoint/custom_endpoint_plugin_factory.ts +++ b/common/lib/plugins/custom_endpoint/custom_endpoint_plugin_factory.ts @@ -15,19 +15,19 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class CustomEndpointPluginFactory extends ConnectionPluginFactory { private static customEndpointPlugin: any; - async getInstance(pluginService: PluginService, props: Map) { + async getInstance(servicesContainer: FullServicesContainer, props: Map) { try { if (!CustomEndpointPluginFactory.customEndpointPlugin) { CustomEndpointPluginFactory.customEndpointPlugin = await import("./custom_endpoint_plugin"); } - return new CustomEndpointPluginFactory.customEndpointPlugin.CustomEndpointPlugin(pluginService, props); + return new CustomEndpointPluginFactory.customEndpointPlugin.CustomEndpointPlugin(servicesContainer.pluginService, props); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "CustomEndpointPlugin")); } diff --git a/common/lib/plugins/custom_endpoint/custom_endpoint_role_type.ts b/common/lib/plugins/custom_endpoint/custom_endpoint_role_type.ts index b4e2abec5..5b89d8d55 100644 --- a/common/lib/plugins/custom_endpoint/custom_endpoint_role_type.ts +++ b/common/lib/plugins/custom_endpoint/custom_endpoint_role_type.ts @@ -15,17 +15,16 @@ */ export enum CustomEndpointRoleType { - ANY, - WRITER, - READER + ANY = "ANY", + WRITER = "WRITER", + READER = "READER", + UNKNOWN = "UNKNOWN" } -const nameToValue = new Map([ - ["ANY", CustomEndpointRoleType.ANY], - ["WRITER", CustomEndpointRoleType.WRITER], - ["READER", CustomEndpointRoleType.READER] -]); - -export function customEndpointRoleTypeFromValue(name: string): CustomEndpointRoleType { - return nameToValue.get(name.toUpperCase()) ?? CustomEndpointRoleType.ANY; +export function customEndpointRoleTypeFromValue(value: string | null | undefined): CustomEndpointRoleType { + if (!value) { + return CustomEndpointRoleType.UNKNOWN; + } + const normalized = value.toUpperCase(); + return Object.values(CustomEndpointRoleType).find((v) => v === normalized) ?? CustomEndpointRoleType.UNKNOWN; } diff --git a/common/lib/plugins/default_plugin.ts b/common/lib/plugins/default_plugin.ts index ddfcf5f97..ecd5ae0a7 100644 --- a/common/lib/plugins/default_plugin.ts +++ b/common/lib/plugins/default_plugin.ts @@ -29,15 +29,19 @@ import { AwsWrapperError } from "../utils/errors"; import { HostAvailability } from "../host_availability/host_availability"; import { ClientWrapper } from "../client_wrapper"; import { TelemetryTraceLevel } from "../utils/telemetry/telemetry_trace_level"; +import { ConnectionInfo } from "../connection_info"; +import { FullServicesContainer } from "../utils/full_services_container"; export class DefaultPlugin extends AbstractConnectionPlugin { id: string = uniqueId("_defaultPlugin"); + private readonly servicesContainer: FullServicesContainer; private readonly pluginService: PluginService; private readonly connectionProviderManager: ConnectionProviderManager; - constructor(pluginService: PluginService, connectionProviderManager: ConnectionProviderManager) { + constructor(servicesContainer: FullServicesContainer, connectionProviderManager: ConnectionProviderManager) { super(); - this.pluginService = pluginService; + this.servicesContainer = servicesContainer; + this.pluginService = servicesContainer.pluginService; this.connectionProviderManager = connectionProviderManager; } @@ -79,10 +83,11 @@ export class DefaultPlugin extends AbstractConnectionPlugin { TelemetryTraceLevel.NESTED ); - const result = await telemetryContext.start(async () => await connProvider.connect(hostInfo, this.pluginService, props)); - this.pluginService.setAvailability(hostInfo.allAliases, HostAvailability.AVAILABLE); - await this.pluginService.updateDialect(result); - return result; + const result: ConnectionInfo = await telemetryContext.start(async () => await connProvider.connect(hostInfo, this.pluginService, props)); + this.pluginService.setAvailability(hostInfo, HostAvailability.AVAILABLE); + this.pluginService.setIsPooledClient(result.isPooled); + await this.pluginService.updateDialect(result.client); + return result.client; } override async execute(methodName: string, methodFunc: () => Promise): Promise { diff --git a/common/lib/plugins/dev/developer_connection_plugin_factory.ts b/common/lib/plugins/dev/developer_connection_plugin_factory.ts index ef942d673..e03dfa328 100644 --- a/common/lib/plugins/dev/developer_connection_plugin_factory.ts +++ b/common/lib/plugins/dev/developer_connection_plugin_factory.ts @@ -15,21 +15,25 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { RdsUtils } from "../../utils/rds_utils"; import { Messages } from "../../utils/messages"; import { AwsWrapperError } from "../../utils/errors"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class DeveloperConnectionPluginFactory extends ConnectionPluginFactory { private static developerPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!DeveloperConnectionPluginFactory.developerPlugin) { DeveloperConnectionPluginFactory.developerPlugin = await import("./developer_connection_plugin"); } - return new DeveloperConnectionPluginFactory.developerPlugin.DeveloperConnectionPlugin(pluginService, properties, new RdsUtils()); + return new DeveloperConnectionPluginFactory.developerPlugin.DeveloperConnectionPlugin( + servicesContainer.pluginService, + properties, + new RdsUtils() + ); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "DeveloperConnectionPlugin")); } diff --git a/common/lib/plugins/efm/base/connection_context.ts b/common/lib/plugins/efm/base/connection_context.ts new file mode 100644 index 000000000..bfcaffd56 --- /dev/null +++ b/common/lib/plugins/efm/base/connection_context.ts @@ -0,0 +1,147 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ClientWrapper } from "../../../client_wrapper"; +import { TelemetryCounter } from "../../../utils/telemetry/telemetry_counter"; +import { logger } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { getCurrentTimeNano } from "../../../utils/utils"; + +/** + * Monitoring context for each connection. This contains each connection's criteria for whether a + * host should be considered unhealthy. The context is shared between the main task and the + * monitor task. + */ +export interface ConnectionContext { + readonly failureDetectionIntervalMillis: number; + readonly failureDetectionCount: number; + readonly expectedActiveMonitoringStartTimeNano: number; + + isActiveContext(): boolean; + isHostUnhealthy(): boolean; + setInactive(): void; + abortConnection(): Promise; + updateConnectionStatus(hostName: string, statusCheckStartTimeNano: number, statusCheckEndTimeNano: number, isValid: boolean): Promise; +} + +export class ConnectionContextImpl implements ConnectionContext { + readonly failureDetectionIntervalMillis: number; + readonly failureDetectionCount: number; + readonly expectedActiveMonitoringStartTimeNano: number; + + private readonly failureDetectionTimeMillis: number; + private readonly connectionToAbortRef: WeakRef; + private readonly abortedConnectionsCounter: TelemetryCounter; + private readonly startMonitorTimeNano: number; + + private _activeContext: boolean = true; + private _hostUnhealthy: boolean = false; + private invalidHostStartTimeNano: number = 0; + private failureCount: number = 0; + + constructor( + connectionToAbort: ClientWrapper, + failureDetectionTimeMillis: number, + failureDetectionIntervalMillis: number, + failureDetectionCount: number, + abortedConnectionsCounter: TelemetryCounter + ) { + this.connectionToAbortRef = new WeakRef(connectionToAbort); + this.failureDetectionTimeMillis = failureDetectionTimeMillis; + this.failureDetectionIntervalMillis = failureDetectionIntervalMillis; + this.failureDetectionCount = failureDetectionCount; + this.abortedConnectionsCounter = abortedConnectionsCounter; + this.startMonitorTimeNano = getCurrentTimeNano(); + this.expectedActiveMonitoringStartTimeNano = this.startMonitorTimeNano + this.failureDetectionTimeMillis * 1_000_000; + } + + isActiveContext(): boolean { + return this._activeContext; + } + + isHostUnhealthy(): boolean { + return this._hostUnhealthy; + } + + setInactive(): void { + this._activeContext = false; + } + + async abortConnection(): Promise { + const connectionToAbort = this.connectionToAbortRef.deref(); + if (connectionToAbort == null || !this._activeContext) { + return; + } + + try { + await connectionToAbort.abort(); + this.abortedConnectionsCounter.inc(); + } catch (error: any) { + // ignore + logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); + } + } + + /** + * Update whether the connection is still valid if the total elapsed time has passed the + * grace period. + */ + async updateConnectionStatus(hostName: string, statusCheckStartTimeNano: number, statusCheckEndTimeNano: number, isValid: boolean): Promise { + if (!this._activeContext) { + return; + } + + const totalElapsedTimeNano = statusCheckEndTimeNano - this.startMonitorTimeNano; + + if (totalElapsedTimeNano > this.failureDetectionTimeMillis * 1_000_000) { + await this.setConnectionValid(hostName, isValid, statusCheckStartTimeNano, statusCheckEndTimeNano); + } + } + + private async setConnectionValid( + hostName: string, + connectionValid: boolean, + statusCheckStartNano: number, + statusCheckEndNano: number + ): Promise { + if (!connectionValid) { + this.failureCount++; + + if (this.invalidHostStartTimeNano === 0) { + this.invalidHostStartTimeNano = statusCheckStartNano; + } + + const invalidHostDurationNano = statusCheckEndNano - this.invalidHostStartTimeNano; + const maxInvalidHostDurationNano = this.failureDetectionIntervalMillis * Math.max(0, this.failureDetectionCount) * 1_000_000; + + if (invalidHostDurationNano >= maxInvalidHostDurationNano) { + logger.debug(Messages.get("MonitorConnectionContext.hostDead", hostName)); + this._hostUnhealthy = true; + await this.abortConnection(); + return; + } + + logger.debug(Messages.get("MonitorConnectionContext.hostNotResponding", hostName)); + return; + } + + this.failureCount = 0; + this.invalidHostStartTimeNano = 0; + this._hostUnhealthy = false; + + logger.debug(Messages.get("MonitorConnectionContext.hostAlive", hostName)); + } +} diff --git a/common/lib/plugins/efm/base/host_monitor.ts b/common/lib/plugins/efm/base/host_monitor.ts new file mode 100644 index 000000000..9cebcd58e --- /dev/null +++ b/common/lib/plugins/efm/base/host_monitor.ts @@ -0,0 +1,230 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ConnectionContext } from "./connection_context"; +import { HostInfo } from "../../../host_info"; +import { PluginService } from "../../../plugin_service"; +import { ClientWrapper } from "../../../client_wrapper"; +import { WrapperProperties } from "../../../wrapper_property"; +import { logger } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { getCurrentTimeNano } from "../../../utils/utils"; +import { getTimeInNanos } from "../../../utils/utils"; +import { TelemetryCounter } from "../../../utils/telemetry/telemetry_counter"; +import { TelemetryFactory } from "../../../utils/telemetry/telemetry_factory"; +import { TelemetryTraceLevel } from "../../../utils/telemetry/telemetry_trace_level"; +import { AbstractMonitor, MonitorState } from "../../../utils/monitoring/monitor"; + +export interface HostMonitor { + startMonitoring(context: ConnectionContext): void; + + stopMonitoring(context: ConnectionContext): void; + + clearContexts(): void; + + isStopped(): boolean; + + run(): Promise; + + releaseResources(): Promise; +} + +type ConnectionStatus = [isValid: boolean, elapsedTimeNano: number]; + +export class HostMonitorImpl extends AbstractMonitor implements HostMonitor { + private static readonly SLEEP_WHEN_INACTIVE_MILLIS = 100; + private static readonly MIN_CONNECTION_CHECK_TIMEOUT_MILLIS = 3000; + private static readonly MONITOR_TERMINATION_TIMEOUT_SEC = 30; + + private readonly pluginService: PluginService; + private readonly telemetryFactory: TelemetryFactory; + private readonly hostInvalidCounter: TelemetryCounter; + private readonly properties: Map; + private readonly hostInfo: HostInfo; + private readonly monitorDisposalTimeMillis: number; + private contexts: ConnectionContext[] = []; + + private contextLastUsedTimestampNano: number; + private monitoringClient: ClientWrapper | null = null; + private delayTimeoutId: ReturnType | undefined; + private sleepTimeoutId: ReturnType | undefined; + + constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { + super(HostMonitorImpl.MONITOR_TERMINATION_TIMEOUT_SEC); + this.pluginService = pluginService; + this.telemetryFactory = this.pluginService.getTelemetryFactory(); + this.hostInfo = hostInfo; + this.properties = properties; + this.monitorDisposalTimeMillis = monitorDisposalTimeMillis; + this.hostInvalidCounter = this.telemetryFactory.createCounter(`efm.nodeUnhealthy.count.${hostInfo.hostId || hostInfo.host}`); + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + } + + startMonitoring(context: ConnectionContext): void { + if (this._stop) { + logger.warn(Messages.get("MonitorImpl.monitorIsStopped", this.hostInfo.host)); + } + + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + this.lastActivityTimestampNanos = getTimeInNanos(); + this.contexts.push(context); + } + + stopMonitoring(context: ConnectionContext): void { + if (!context) { + logger.warn(Messages.get("MonitorImpl.contextNullWarning")); + return; + } + + context.setInactive(); + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + this.lastActivityTimestampNanos = getTimeInNanos(); + } + + clearContexts(): void { + this.contexts.length = 0; + } + + isStopped(): boolean { + return this._stop || this.state === MonitorState.STOPPED; + } + + canDispose(): boolean { + return this.contexts.length === 0; + } + + async monitor(): Promise { + logger.debug(Messages.get("MonitorImpl.startMonitoring", this.hostInfo.host)); + + try { + while (!this._stop) { + try { + this.lastActivityTimestampNanos = getTimeInNanos(); + + const activeContexts = this.contexts.filter((ctx) => ctx.isActiveContext()); + + if (activeContexts.length > 0) { + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + + const statusCheckStartTimeNano = getCurrentTimeNano(); + const [isValid, elapsedTimeNano] = await this.checkConnectionStatus(); + + let delayMillis = -1; + + for (const context of activeContexts) { + if (!context.isActiveContext()) { + continue; + } + + await context.updateConnectionStatus(this.hostInfo.url, statusCheckStartTimeNano, statusCheckStartTimeNano + elapsedTimeNano, isValid); + + if ( + context.isActiveContext() && + !context.isHostUnhealthy() && + statusCheckStartTimeNano >= context.expectedActiveMonitoringStartTimeNano + ) { + if (delayMillis === -1 || delayMillis > context.failureDetectionIntervalMillis) { + delayMillis = context.failureDetectionIntervalMillis; + } + } + } + + this.contexts = this.contexts.filter((ctx) => ctx.isActiveContext() && !ctx.isHostUnhealthy()); + + if (delayMillis === -1) { + delayMillis = HostMonitorImpl.SLEEP_WHEN_INACTIVE_MILLIS; + } else { + delayMillis -= Math.round(elapsedTimeNano / 1_000_000); + if (delayMillis <= 0) { + delayMillis = HostMonitorImpl.MIN_CONNECTION_CHECK_TIMEOUT_MILLIS; + } + } + + await new Promise((resolve) => { + this.delayTimeoutId = setTimeout(resolve, delayMillis); + }); + } else { + if (getCurrentTimeNano() - this.contextLastUsedTimestampNano >= this.monitorDisposalTimeMillis * 1_000_000) { + break; + } + await new Promise((resolve) => { + this.sleepTimeoutId = setTimeout(resolve, HostMonitorImpl.SLEEP_WHEN_INACTIVE_MILLIS); + }); + } + } catch (error: any) { + logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringContinue", error.message)); + } + } + } catch (error: any) { + logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringStop", error.message)); + } + + logger.debug(Messages.get("MonitorImpl.stopMonitoring", this.hostInfo.host)); + } + + async close(): Promise { + this.contexts.length = 0; + await this.closeMonitoringClient(); + } + + async releaseResources(): Promise { + clearTimeout(this.delayTimeoutId); + clearTimeout(this.sleepTimeoutId); + await this.stop(); + } + + protected async checkConnectionStatus(): Promise { + const connectContext = this.telemetryFactory.openTelemetryContext("Connection status check", TelemetryTraceLevel.FORCE_TOP_LEVEL); + connectContext.setAttribute("url", this.hostInfo.host); + return await connectContext.start(async () => { + const startNanos = getCurrentTimeNano(); + try { + if (this.monitoringClient != null && (await this.pluginService.isClientValid(this.monitoringClient))) { + return [true, getCurrentTimeNano() - startNanos]; + } + + await this.closeMonitoringClient(); + + const monitoringConnProperties = new Map(this.properties); + for (const key of monitoringConnProperties.keys()) { + if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { + continue; + } + monitoringConnProperties.set(key.substring(WrapperProperties.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); + monitoringConnProperties.delete(key); + } + + this.monitoringClient = await this.pluginService.forceConnect(this.hostInfo, monitoringConnProperties); + return [true, getCurrentTimeNano() - startNanos]; + } catch (error: any) { + this.hostInvalidCounter.inc(); + await this.closeMonitoringClient(); + return [false, getCurrentTimeNano() - startNanos]; + } + }); + } + + private async closeMonitoringClient(): Promise { + if (this.monitoringClient) { + try { + await this.pluginService.abortTargetClient(this.monitoringClient); + } catch { + // ignore + } + this.monitoringClient = null; + } + } +} diff --git a/common/lib/plugins/efm/base/host_monitor_service.ts b/common/lib/plugins/efm/base/host_monitor_service.ts new file mode 100644 index 000000000..1c976a179 --- /dev/null +++ b/common/lib/plugins/efm/base/host_monitor_service.ts @@ -0,0 +1,117 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ConnectionContext, ConnectionContextImpl } from "./connection_context"; +import { HostMonitor, HostMonitorImpl } from "./host_monitor"; +import { HostInfo } from "../../../host_info"; +import { ClientWrapper } from "../../../client_wrapper"; +import { WrapperProperties } from "../../../wrapper_property"; +import { AwsWrapperError } from "../../../utils/errors"; +import { Messages } from "../../../utils/messages"; +import { MonitorService } from "../../../utils/monitoring/monitor_service"; +import { MonitorInitializer } from "../../../utils/monitoring/monitor"; +import { TelemetryCounter } from "../../../utils/telemetry/telemetry_counter"; +import { FullServicesContainer } from "../../../utils/full_services_container"; + +export interface HostMonitorService { + startMonitoring( + connectionToAbort: ClientWrapper, + hostInfo: HostInfo, + properties: Map, + failureDetectionTimeMillis: number, + failureDetectionIntervalMillis: number, + failureDetectionCount: number + ): Promise; + + stopMonitoring(context: ConnectionContext): void; + + releaseResources(): Promise; +} + +export class HostMonitorServiceImpl implements HostMonitorService { + private static readonly MONITOR_DISPOSAL_TIME_NS = BigInt(10 * 60 * 1_000_000_000); // 10 minutes + private static readonly INACTIVE_TIMEOUT_NS = BigInt(3 * 60 * 1_000_000_000); // 3 minutes + + private readonly servicesContainer: FullServicesContainer; + private readonly coreMonitorService: MonitorService; + private readonly abortedConnectionsCounter: TelemetryCounter; + + constructor(servicesContainer: FullServicesContainer) { + this.servicesContainer = servicesContainer; + this.coreMonitorService = servicesContainer.monitorService; + const telemetryFactory = servicesContainer.telemetryFactory; + this.abortedConnectionsCounter = telemetryFactory.createCounter("efm.connections.aborted"); + + this.coreMonitorService.registerMonitorTypeIfAbsent( + HostMonitorImpl, + HostMonitorServiceImpl.MONITOR_DISPOSAL_TIME_NS, + HostMonitorServiceImpl.INACTIVE_TIMEOUT_NS, + new Set(), + undefined + ); + } + + async startMonitoring( + connectionToAbort: ClientWrapper, + hostInfo: HostInfo, + properties: Map, + failureDetectionTimeMillis: number, + failureDetectionIntervalMillis: number, + failureDetectionCount: number + ): Promise { + const monitorKey = hostInfo.hostId || hostInfo.host; + + const monitor = await this.getMonitor(monitorKey, hostInfo, properties); + + if (!monitor) { + throw new AwsWrapperError(Messages.get("MonitorService.startMonitoringNullMonitor", hostInfo.host)); + } + + const context = new ConnectionContextImpl( + connectionToAbort, + failureDetectionTimeMillis, + failureDetectionIntervalMillis, + failureDetectionCount, + this.abortedConnectionsCounter + ); + + monitor.startMonitoring(context); + return context; + } + + stopMonitoring(context: ConnectionContext): void { + context.setInactive(); + } + + private async getMonitor(monitorKey: string, hostInfo: HostInfo, properties: Map): Promise { + const monitorDisposalTimeMillis = WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties); + + const initializer: MonitorInitializer = { + createMonitor: (_servicesContainer) => + new HostMonitorImpl(this.servicesContainer.pluginService, hostInfo, properties, monitorDisposalTimeMillis) + }; + + return await this.coreMonitorService.runIfAbsent(HostMonitorImpl, monitorKey, this.servicesContainer, properties, initializer); + } + + async releaseResources(): Promise { + await this.coreMonitorService.stopAndRemoveMonitors(HostMonitorImpl); + } + + static async clearMonitors(monitorService: MonitorService): Promise { + await monitorService.stopAndRemoveMonitors(HostMonitorImpl); + } +} diff --git a/common/lib/plugins/efm/monitor.ts b/common/lib/plugins/efm/monitor.ts deleted file mode 100644 index 3094427f6..000000000 --- a/common/lib/plugins/efm/monitor.ts +++ /dev/null @@ -1,290 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { PluginService } from "../../plugin_service"; -import { logger } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { ClientWrapper } from "../../client_wrapper"; -import { getCurrentTimeNano, sleep } from "../../utils/utils"; -import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; -import { WrapperProperties } from "../../wrapper_property"; - -export interface Monitor { - startMonitoring(context: MonitorConnectionContext): void; - - stopMonitoring(context: MonitorConnectionContext): void; - - clearContexts(): void; - - isStopped(): boolean; - - run(): Promise; - - releaseResources(): Promise; - - endMonitoringClient(): Promise; -} - -class ConnectionStatus { - isValid: boolean; - elapsedTimeNano: number; - - constructor(isValid: boolean, elapsedTimeNano: number) { - this.isValid = isValid; - this.elapsedTimeNano = elapsedTimeNano; - } -} - -export class MonitorImpl implements Monitor { - private readonly SLEEP_WHEN_INACTIVE_MILLIS: number = 100; - private readonly MIN_CONNECTION_CHECK_TIMEOUT_MILLIS: number = 3000; - - private readonly activeContexts: MonitorConnectionContext[] = []; - private readonly newContexts: MonitorConnectionContext[] = []; - - private readonly pluginService: PluginService; - private readonly properties: Map; - private readonly hostInfo: HostInfo; - private readonly monitorDisposalTimeMillis: number; - private readonly telemetryFactory: TelemetryFactory; - private readonly instanceInvalidCounter: TelemetryCounter; - private contextLastUsedTimestampNanos: number; - private started = false; - private stopped: boolean = false; - private cancelled: boolean = false; - private monitoringClient: ClientWrapper | null = null; - private delayMillisTimeoutId: any; - private sleepWhenInactiveTimeoutId: any; - - constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { - this.pluginService = pluginService; - this.telemetryFactory = this.pluginService.getTelemetryFactory(); - this.properties = properties; - this.hostInfo = hostInfo; - this.monitorDisposalTimeMillis = monitorDisposalTimeMillis; - this.contextLastUsedTimestampNanos = getCurrentTimeNano(); - const instanceId = this.hostInfo.hostId ?? this.hostInfo.host; - this.instanceInvalidCounter = this.telemetryFactory.createCounter(`efm.hostUnhealthy.count.${instanceId}`); - } - - startRun() { - this.run(); - this.started = true; - } - - startMonitoring(context: MonitorConnectionContext): void { - if (this.stopped) { - logger.warn(Messages.get("MonitorImpl.monitorIsStopped", this.hostInfo.host)); - } - - const currentTimeNanos: number = getCurrentTimeNano(); - context.startMonitorTimeNano = currentTimeNanos; - this.contextLastUsedTimestampNanos = currentTimeNanos; - this.newContexts.push(context); - if (!this.started) { - this.startRun(); - } - } - - stopMonitoring(context: MonitorConnectionContext): void { - if (context == null) { - logger.warn(Messages.get("MonitorImpl.contextNullWarning")); - return; - } - - context.isActiveContext = false; - this.contextLastUsedTimestampNanos = getCurrentTimeNano(); - } - - async run(): Promise { - logger.debug(Messages.get("MonitorImpl.startMonitoring", this.hostInfo.host)); - - try { - while (!this.cancelled) { - try { - let newMonitorContext: MonitorConnectionContext | undefined; - let firstAddedNewMonitorContext: MonitorConnectionContext | null = null; - const currentTimeNano: number = getCurrentTimeNano(); - while ((newMonitorContext = this.newContexts?.shift()) != null) { - if (firstAddedNewMonitorContext === newMonitorContext) { - this.newContexts.push(newMonitorContext); - - break; - } - - if (newMonitorContext.isActiveContext) { - if (newMonitorContext.expectedActiveMonitoringStartTimeNano > currentTimeNano) { - this.newContexts.push(newMonitorContext); - - firstAddedNewMonitorContext = firstAddedNewMonitorContext ?? newMonitorContext; - } else { - this.activeContexts.push(newMonitorContext); - } - } - } - - if (this.activeContexts.length > 0) { - this.contextLastUsedTimestampNanos = getCurrentTimeNano(); - - const statusCheckStartTimeNanos: number = getCurrentTimeNano(); - this.contextLastUsedTimestampNanos = statusCheckStartTimeNanos; - - const status: ConnectionStatus = await this.checkConnectionStatus(); - let delayMillis: number = -1; - - let monitorContext: MonitorConnectionContext | undefined; - let firstAddedMonitorContext: MonitorConnectionContext | null = null; - - while ((monitorContext = this.activeContexts?.shift()) != null) { - // If context is already invalid, just skip it. - if (!monitorContext.isActiveContext) { - continue; - } - - if (firstAddedMonitorContext == monitorContext) { - // This context is already processed by this loop. - // Add it to the array and exit this loop. - - this.activeContexts?.push(monitorContext); - break; - } - - // Otherwise, process this context. - await monitorContext.updateConnectionStatus( - this.hostInfo.url, - statusCheckStartTimeNanos, - statusCheckStartTimeNanos + status.elapsedTimeNano, - status.isValid - ); - - if (monitorContext.isActiveContext && !monitorContext.isHostUnhealthy) { - this.activeContexts?.push(monitorContext); - - if (firstAddedMonitorContext == null) { - firstAddedMonitorContext = monitorContext; - } - - if (delayMillis === -1 || delayMillis > monitorContext.failureDetectionIntervalMillis) { - delayMillis = monitorContext.failureDetectionIntervalMillis; - } - } - } - - if (delayMillis === -1) { - // No active contexts. - delayMillis = this.SLEEP_WHEN_INACTIVE_MILLIS; - } else { - delayMillis -= Math.round(status.elapsedTimeNano / 1_000_000); - // Check for minimum delay between host health check; - if (delayMillis <= 0) { - delayMillis = this.MIN_CONNECTION_CHECK_TIMEOUT_MILLIS; - } - } - - await new Promise((resolve) => { - this.delayMillisTimeoutId = setTimeout(resolve, delayMillis); - }); - } else { - if (getCurrentTimeNano() - this.contextLastUsedTimestampNanos >= this.monitorDisposalTimeMillis * 1_000_000) { - break; - } - await new Promise((resolve) => { - this.sleepWhenInactiveTimeoutId = setTimeout(resolve, this.SLEEP_WHEN_INACTIVE_MILLIS); - }); - } - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringContinue", error.message)); - } - } - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringStop", error.message)); - } finally { - this.stopped = true; - await this.endMonitoringClient(); - } - - logger.debug(Messages.get("MonitorImpl.stopMonitoring", this.hostInfo.host)); - } - - /** - * Check the status of the monitored server by sending a ping. - * - * @return whether the server is still alive and the elapsed time spent checking. - */ - async checkConnectionStatus(): Promise { - const connectContext = this.telemetryFactory.openTelemetryContext("Connection status check", TelemetryTraceLevel.FORCE_TOP_LEVEL); - connectContext.setAttribute("url", this.hostInfo.host); - return await connectContext.start(async () => { - const startNanos = getCurrentTimeNano(); - try { - const clientIsValid = this.monitoringClient && (await this.pluginService.isClientValid(this.monitoringClient)); - - if (this.monitoringClient !== null && clientIsValid) { - return Promise.resolve(new ConnectionStatus(clientIsValid, getCurrentTimeNano() - startNanos)); - } - - await this.endMonitoringClient(); - // Open a new connection. - const monitoringConnProperties: Map = new Map(this.properties); - for (const key of monitoringConnProperties.keys()) { - if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { - continue; - } - monitoringConnProperties.set(key.substring(WrapperProperties.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); - monitoringConnProperties.delete(key); - } - - logger.debug(`Opening a monitoring connection to ${this.hostInfo.url}`); - this.monitoringClient = await this.pluginService.forceConnect(this.hostInfo, monitoringConnProperties); - logger.debug(`Successfully opened monitoring connection to ${this.monitoringClient.id} - ${this.hostInfo.url}`); - return Promise.resolve(new ConnectionStatus(true, getCurrentTimeNano() - startNanos)); - } catch (error: any) { - this.instanceInvalidCounter.inc(); - await this.endMonitoringClient(); - return Promise.resolve(new ConnectionStatus(false, getCurrentTimeNano() - startNanos)); - } - }); - } - - clearContexts(): void { - this.activeContexts.length = 0; - this.newContexts.length = 0; - } - - isStopped(): boolean { - return this.stopped || this.cancelled; - } - - async releaseResources() { - this.cancelled = true; - clearTimeout(this.delayMillisTimeoutId); - clearTimeout(this.sleepWhenInactiveTimeoutId); - await this.endMonitoringClient(); - // Allow time for monitor loop to close. - await sleep(500); - } - - async endMonitoringClient() { - if (this.monitoringClient) { - await this.pluginService.abortTargetClient(this.monitoringClient); - this.monitoringClient = null; - } - } -} diff --git a/common/lib/plugins/efm/monitor_connection_context.ts b/common/lib/plugins/efm/monitor_connection_context.ts deleted file mode 100644 index a68893228..000000000 --- a/common/lib/plugins/efm/monitor_connection_context.ts +++ /dev/null @@ -1,155 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { Monitor } from "./monitor"; -import { logger, uniqueId } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { ClientWrapper } from "../../client_wrapper"; -import { sleep } from "../../utils/utils"; -import { AwsWrapperError } from "../../utils/errors"; -import { PluginService } from "../../plugin_service"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; - -export class MonitorConnectionContext { - readonly failureDetectionIntervalMillis: number; - private readonly failureDetectionTimeMillis: number; - private readonly failureDetectionCount: number; - private readonly telemetryAbortedConnectionCounter: TelemetryCounter; - readonly clientToAbort: ClientWrapper; - readonly monitor: Monitor; - readonly pluginService: PluginService; - - isActiveContext: boolean = true; - isHostUnhealthy: boolean = false; - startMonitorTimeNano: number = 0; - expectedActiveMonitoringStartTimeNano: number = 0; - private invalidHostStartTimeNano: number = 0; - private abortedConnectionCounter: number = 0; - failureCount: number = 0; - id: string = uniqueId("_monitorContext"); - - constructor( - monitor: Monitor, - clientToAbort: ClientWrapper, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number, - pluginService: PluginService, - telemetryAbortedConnectionCounter: TelemetryCounter - ) { - this.monitor = monitor; - this.clientToAbort = clientToAbort; - this.failureDetectionTimeMillis = failureDetectionTimeMillis; - this.failureDetectionIntervalMillis = failureDetectionIntervalMillis; - this.failureDetectionCount = failureDetectionCount; - this.pluginService = pluginService; - this.telemetryAbortedConnectionCounter = telemetryAbortedConnectionCounter; - } - - resetInvalidHostStartTimeNano(): void { - this.invalidHostStartTimeNano = 0; - } - - isInvalidHostStartTimeDefined(): boolean { - return this.invalidHostStartTimeNano > 0; - } - - async abortConnection(): Promise { - if (this.clientToAbort == null || !this.isActiveContext) { - return Promise.resolve(); - } - - try { - await this.pluginService.abortTargetClient(this.clientToAbort); - this.telemetryAbortedConnectionCounter.inc(); - } catch (error: any) { - // ignore - logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); - } - this.abortedConnectionCounter++; - } - - /** - * Update whether the connection is still valid if the total elapsed time has passed the grace period. - * @param hostName A host name for logging purposes. - * @param statusCheckStartNano The time when connection status check started in nanos. - * @param statusCheckEndNano The time when connection status check ended in nanos. - * @param isValid Whether the connection is valid. - */ - async updateConnectionStatus(hostName: string, statusCheckStartNano: number, statusCheckEndNano: number, isValid: boolean): Promise { - if (!this.isActiveContext) { - return; - } - - const totalElapsedTimeNano: number = statusCheckEndNano - this.startMonitorTimeNano; - if (totalElapsedTimeNano > this.failureDetectionTimeMillis * 1000000) { - await this.setConnectionValid(hostName, isValid, statusCheckStartNano, statusCheckEndNano); - } - } - - /** - * Set whether the connection to the server is still valid based on the monitoring settings set in the {@link AwsClient}. - * - *

These monitoring settings include: - * - *

    - *
  • {@code failureDetectionInterval} - *
  • {@code failureDetectionTime} - *
  • {@code failureDetectionCount} - *
- * - * @param hostName A host name for logging purposes. - * @param connectionValid Boolean indicating whether the server is still responsive. - * @param statusCheckStartNano The time when connection status check started in nanos. - * @param statusCheckEndNano The time when connection status check ended in nanos. - * @protected - */ - async setConnectionValid(hostName: string, connectionValid: boolean, statusCheckStartNano: number, statusCheckEndNano: number) { - if (!connectionValid) { - this.failureCount++; - - if (!this.isInvalidHostStartTimeDefined()) { - this.invalidHostStartTimeNano = statusCheckStartNano; - } - - const invalidHostDurationNano: number = statusCheckEndNano - this.invalidHostStartTimeNano; - const maxInvalidHostDurationMillis: number = this.failureDetectionIntervalMillis * Math.max(0, this.failureDetectionCount); - if (this.failureCount >= this.failureDetectionCount || invalidHostDurationNano >= maxInvalidHostDurationMillis * 1_000_000) { - logger.debug(Messages.get("MonitorConnectionContext.hostDead", hostName)); - this.isHostUnhealthy = true; - await this.abortConnection(); - return; - } - - logger.debug(Messages.get("MonitorConnectionContext.hostNotResponding", hostName)); - return; - } - - this.failureCount = 0; - this.resetInvalidHostStartTimeNano(); - this.isHostUnhealthy = false; - - logger.debug(Messages.get("MonitorConnectionContext.hostAlive", hostName)); - } - - async trackHealthStatus() { - while (!this.isHostUnhealthy && this.isActiveContext) { - await sleep(100); - } - - throw new AwsWrapperError("trackHealthStatus stopped"); - } -} diff --git a/common/lib/plugins/efm/monitor_service.ts b/common/lib/plugins/efm/monitor_service.ts deleted file mode 100644 index c81639f52..000000000 --- a/common/lib/plugins/efm/monitor_service.ts +++ /dev/null @@ -1,174 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { AwsWrapperError, IllegalArgumentError } from "../../utils/errors"; -import { Monitor, MonitorImpl } from "./monitor"; -import { WrapperProperties } from "../../wrapper_property"; -import { PluginService } from "../../plugin_service"; -import { Messages } from "../../utils/messages"; -import { SlidingExpirationCacheWithCleanupTask } from "../../utils/sliding_expiration_cache_with_cleanup_task"; -import { ClientWrapper } from "../../client_wrapper"; - -export interface MonitorService { - startMonitoring( - clientToAbort: ClientWrapper, - hostKeys: Set, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise; - - stopMonitoring(context: MonitorConnectionContext): Promise; - - stopMonitoringForAllConnections(hostKeys: Set): void; - - releaseResources(): Promise; -} - -export class MonitorServiceImpl implements MonitorService { - private static readonly CACHE_CLEANUP_NANOS = BigInt(60_000_000_000); - protected static readonly monitors: SlidingExpirationCacheWithCleanupTask = new SlidingExpirationCacheWithCleanupTask( - MonitorServiceImpl.CACHE_CLEANUP_NANOS, - undefined, - async (monitor: Monitor) => { - await monitor.releaseResources(); - }, - "efm/MonitorServiceImpl.monitors" - ); - private readonly pluginService: PluginService; - private cachedMonitorHostKeys: Set | undefined; - private cachedMonitorRef: WeakRef | undefined; - monitorSupplier = (pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) => - new MonitorImpl(pluginService, hostInfo, properties, monitorDisposalTimeMillis); - - constructor(pluginService: PluginService) { - this.pluginService = pluginService; - } - - async startMonitoring( - clientToAbort: ClientWrapper, - hostKeys: Set, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise { - if (hostKeys.size === 0) { - throw new IllegalArgumentError(Messages.get("MonitorService.emptyAliasSet", hostInfo.host)); - } - - let monitor: Monitor | null = this.cachedMonitorRef?.deref() ?? null; - - if (!monitor || (monitor && monitor.isStopped()) || this.cachedMonitorHostKeys?.size === 0 || this.cachedMonitorHostKeys !== hostKeys) { - monitor = await this.getMonitor(hostKeys, hostInfo, properties); - if (monitor) { - this.cachedMonitorRef = new WeakRef(monitor); - this.cachedMonitorHostKeys = hostKeys; - } - } - - const telemetryFactory = this.pluginService.getTelemetryFactory(); - const abortedConnectionsCounter = telemetryFactory.createCounter("efm.connections.aborted"); - - if (monitor) { - const context = new MonitorConnectionContext( - monitor, - clientToAbort, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount, - this.pluginService, - abortedConnectionsCounter - ); - monitor.startMonitoring(context); - return context; - } - - throw new AwsWrapperError(Messages.get("MonitorService.startMonitoringNullMonitor", hostInfo.host)); - } - - async stopMonitoring(context: MonitorConnectionContext) { - context.monitor.stopMonitoring(context); - await context.monitor.endMonitoringClient(); - } - - stopMonitoringForAllConnections(hostKeys: Set) { - let monitor: Monitor; - for (const hostKey of hostKeys) { - monitor = MonitorServiceImpl.monitors.get(hostKey); - if (monitor) { - monitor.clearContexts(); - return; - } - } - } - - async getMonitor(hostKeys: Set, hostInfo: HostInfo, properties: Map): Promise { - let monitor: Monitor; - let anyHostKey: string; - for (const hostKey of hostKeys) { - monitor = MonitorServiceImpl.monitors.get(hostKey); - anyHostKey = hostKey; - if (monitor) { - break; - } - } - - const cacheExpirationNanos = BigInt(WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties) * 1_000_000); - if (anyHostKey && (!monitor || (monitor && monitor.isStopped()))) { - monitor = MonitorServiceImpl.monitors.computeIfAbsent( - anyHostKey, - () => this.monitorSupplier(this.pluginService, hostInfo, properties, WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties)), - cacheExpirationNanos - ); - - if (monitor && monitor.isStopped()) { - await monitor.releaseResources(); - MonitorServiceImpl.monitors.remove(anyHostKey); - monitor = this.monitorSupplier(this.pluginService, hostInfo, properties, WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties)); - } - } - - if (monitor) { - this.populateMonitorMap(hostKeys, monitor, cacheExpirationNanos); - return monitor; - } - - return null; - } - - private populateMonitorMap(hostKeys: Set, monitor: Monitor, cacheExpirationNanos: bigint) { - for (const hostKey of hostKeys) { - MonitorServiceImpl.monitors.putIfAbsent(hostKey, monitor, cacheExpirationNanos); - } - } - - async releaseResources() { - await MonitorServiceImpl.monitors.clear(); - this.cachedMonitorHostKeys = undefined; - this.cachedMonitorRef = undefined; - } - - // Used for performance testing. - static async clearMonitors() { - await MonitorServiceImpl.monitors.clear(); - } -} diff --git a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts b/common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts similarity index 52% rename from common/lib/plugins/efm/host_monitoring_connection_plugin.ts rename to common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts index 753e2f695..3c7548d24 100644 --- a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts +++ b/common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts @@ -14,37 +14,42 @@ limitations under the License. */ -import { HostInfo, AwsWrapperError, UnavailableHostError, HostAvailability } from "../../"; -import { PluginService } from "../../plugin_service"; -import { HostChangeOptions } from "../../host_change_options"; -import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; -import { RdsUrlType } from "../../utils/rds_url_type"; -import { WrapperProperties } from "../../wrapper_property"; -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { logger, uniqueId } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { MonitorService, MonitorServiceImpl } from "./monitor_service"; -import { HostListProvider } from "../../host_list_provider/host_list_provider"; -import { CanReleaseResources } from "../../can_release_resources"; -import { SubscribedMethodHelper } from "../../utils/subscribed_method_helper"; -import { ClientWrapper } from "../../client_wrapper"; +import { HostInfo } from "../../../host_info"; +import { AwsWrapperError, UnavailableHostError } from "../../../utils/errors"; +import { PluginService } from "../../../plugin_service"; +import { HostChangeOptions } from "../../../host_change_options"; +import { OldConnectionSuggestionAction } from "../../../old_connection_suggestion_action"; +import { RdsUtils } from "../../../utils/rds_utils"; +import { AbstractConnectionPlugin } from "../../../abstract_connection_plugin"; +import { RdsUrlType } from "../../../utils/rds_url_type"; +import { WrapperProperties } from "../../../wrapper_property"; +import { ConnectionContext } from "../base/connection_context"; +import { HostMonitorService, HostMonitorServiceImpl } from "../base/host_monitor_service"; +import { logger, uniqueId } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { HostListProvider } from "../../../host_list_provider/host_list_provider"; +import { CanReleaseResources } from "../../../can_release_resources"; +import { SubscribedMethodHelper } from "../../../utils/subscribed_method_helper"; +import { ClientWrapper } from "../../../client_wrapper"; +import { FullServicesContainer } from "../../../utils/full_services_container"; +import { sleep } from "../../../utils/utils"; export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin implements CanReleaseResources { + private static readonly UNHEALTHY_STATE = Symbol("unhealthy"); + id: string = uniqueId("_efmPlugin"); - private readonly properties: Map; - private pluginService: PluginService; - private rdsUtils: RdsUtils; + protected readonly properties: Map; + protected readonly pluginService: PluginService; + protected readonly rdsUtils: RdsUtils; + protected readonly monitorService: HostMonitorService; private monitoringHostInfo: HostInfo | null = null; - private monitorService: MonitorService; - constructor(pluginService: PluginService, properties: Map, rdsUtils: RdsUtils = new RdsUtils(), monitorService?: MonitorServiceImpl) { + constructor(servicesContainer: FullServicesContainer, properties: Map, monitorService?: HostMonitorService) { super(); - this.pluginService = pluginService; + this.pluginService = servicesContainer.pluginService; this.properties = properties; - this.rdsUtils = rdsUtils; - this.monitorService = monitorService ?? new MonitorServiceImpl(pluginService); + this.rdsUtils = new RdsUtils(); + this.monitorService = monitorService ?? new HostMonitorServiceImpl(servicesContainer); } getSubscribedMethods(): Set { @@ -59,10 +64,13 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp ): Promise { const targetClient = await connectFunc(); if (targetClient != null) { - const type: RdsUrlType = this.rdsUtils.identifyRdsType(hostInfo.host); + const connectionHostInfo: HostInfo = this.pluginService.getRoutedHostInfo() ?? hostInfo; + const type: RdsUrlType = this.rdsUtils.identifyRdsType(connectionHostInfo.host); if (type.isRdsCluster) { - hostInfo.resetAliases(); - await this.pluginService.fillAliases(targetClient, hostInfo); + const identifiedHostInfo: HostInfo | null = await this.pluginService.identifyConnection(targetClient, connectionHostInfo); + if (identifiedHostInfo) { + this.pluginService.setRoutedHostInfo(identifiedHostInfo); + } } } return targetClient; @@ -75,20 +83,19 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp return methodFunc(); } - const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties) as number; - const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties) as number; - const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties) as number; + const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties); + const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties); + const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties); let result: T; - let monitorContext: MonitorConnectionContext | null = null; + let context: ConnectionContext | null = null; try { logger.debug(Messages.get("HostMonitoringConnectionPlugin.activatedMonitoring", methodName)); const monitoringHostInfo: HostInfo = await this.getMonitoringHostInfo(); - monitorContext = await this.monitorService.startMonitoring( + context = await this.monitorService.startMonitoring( this.pluginService.getCurrentClient().targetClient, - monitoringHostInfo.allAliases, monitoringHostInfo, this.properties, failureDetectionTimeMillis, @@ -96,24 +103,22 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp failureDetectionCount ); - result = await Promise.race([monitorContext.trackHealthStatus(), methodFunc()]) - .then((result: any) => { - return result; - }) - .catch((error: any) => { - throw error; + const methodPromise = methodFunc(); + const raceResult = await Promise.race([this.waitForUnhealthy(context), methodPromise]); + if (raceResult === HostMonitoringConnectionPlugin.UNHEALTHY_STATE) { + methodPromise.catch(() => { + // Attach a no-op rejection handler to prevent it from throwing an unhandled promise rejection error when waitForUnhealthy times out first. }); + throw new AwsWrapperError("Host monitoring detected unhealthy host"); + } + result = raceResult as T; } finally { - if (monitorContext != null) { - await this.monitorService.stopMonitoring(monitorContext); + if (context != null) { + this.monitorService.stopMonitoring(context); logger.debug(Messages.get("HostMonitoringConnectionPlugin.monitoringDeactivated", methodName)); - if (monitorContext.isHostUnhealthy) { + if (context.isHostUnhealthy()) { const monitoringHostInfo = await this.getMonitoringHostInfo(); - if (monitoringHostInfo) { - this.pluginService.setAvailability(monitoringHostInfo.allAliases, HostAvailability.NOT_AVAILABLE); - } - const targetClient = this.pluginService.getCurrentClient().targetClient; let isClientValid = false; if (targetClient) { @@ -125,7 +130,9 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp await this.pluginService.abortTargetClient(targetClient); } // eslint-disable-next-line no-unsafe-finally - throw new UnavailableHostError(Messages.get("HostMonitoringConnectionPlugin.unavailableHost", monitoringHostInfo.host)); + throw new UnavailableHostError( + Messages.get("HostMonitoringConnectionPlugin.unavailableHost", monitoringHostInfo?.host ?? "Unknown host") + ); } } } @@ -134,6 +141,13 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp return result; } + private async waitForUnhealthy(context: ConnectionContext): Promise { + while (!context.isHostUnhealthy() && context.isActiveContext()) { + await sleep(100); + } + return HostMonitoringConnectionPlugin.UNHEALTHY_STATE; + } + private throwUnableToIdentifyConnection(host: HostInfo | null): never { const provider: HostListProvider | null = this.pluginService.getHostListProvider(); throw new AwsWrapperError( @@ -149,7 +163,7 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp if (this.monitoringHostInfo) { return this.monitoringHostInfo; } - this.monitoringHostInfo = this.pluginService.getCurrentHostInfo(); + this.monitoringHostInfo = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getCurrentHostInfo(); if (this.monitoringHostInfo == null) { this.throwUnableToIdentifyConnection(null); } @@ -158,12 +172,17 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp try { if (rdsUrlType.isRdsCluster) { logger.debug(Messages.get("HostMonitoringConnectionPlugin.identifyClusterConnection")); - this.monitoringHostInfo = await this.pluginService.identifyConnection(this.pluginService.getCurrentClient().targetClient!); + this.monitoringHostInfo = await this.pluginService.identifyConnection( + this.pluginService.getCurrentClient().targetClient!, + this.pluginService.getCurrentHostInfo() + ); if (this.monitoringHostInfo == null) { const host: HostInfo | null = this.pluginService.getCurrentHostInfo(); this.throwUnableToIdentifyConnection(host); } - await this.pluginService.fillAliases(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); + + // Update identified HostInfo for the current connection + await this.pluginService.setCurrentClient(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); } } catch (error: any) { if (!(error instanceof AwsWrapperError)) { @@ -175,15 +194,9 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp } async notifyConnectionChanged(changes: Set): Promise { - if (changes.has(HostChangeOptions.WENT_DOWN) || changes.has(HostChangeOptions.HOST_DELETED)) { - const aliases = (await this.getMonitoringHostInfo())?.allAliases; - if (aliases && aliases.size > 0) { - this.monitorService.stopMonitoringForAllConnections(aliases); - } + if (changes.has(HostChangeOptions.HOSTNAME) || changes.has(HostChangeOptions.HOST_CHANGED)) { + this.monitoringHostInfo = null; } - - // Reset monitoring host info since the associated connection has changed. - this.monitoringHostInfo = null; return OldConnectionSuggestionAction.NO_OPINION; } diff --git a/common/lib/plugins/efm/host_monitoring_plugin_factory.ts b/common/lib/plugins/efm/v1/host_monitoring_plugin_factory.ts similarity index 69% rename from common/lib/plugins/efm/host_monitoring_plugin_factory.ts rename to common/lib/plugins/efm/v1/host_monitoring_plugin_factory.ts index 62b59cd6a..49f3d6284 100644 --- a/common/lib/plugins/efm/host_monitoring_plugin_factory.ts +++ b/common/lib/plugins/efm/v1/host_monitoring_plugin_factory.ts @@ -14,22 +14,21 @@ limitations under the License. */ -import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; -import { ConnectionPlugin } from "../../connection_plugin"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AwsWrapperError } from "../../utils/errors"; -import { Messages } from "../../utils/messages"; +import { ConnectionPluginFactory } from "../../../plugin_factory"; +import { ConnectionPlugin } from "../../../connection_plugin"; +import { AwsWrapperError } from "../../../utils/errors"; +import { Messages } from "../../../utils/messages"; +import { FullServicesContainer } from "../../../utils/full_services_container"; export class HostMonitoringPluginFactory extends ConnectionPluginFactory { private static hostMonitoringPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!HostMonitoringPluginFactory.hostMonitoringPlugin) { HostMonitoringPluginFactory.hostMonitoringPlugin = await import("./host_monitoring_connection_plugin"); } - return new HostMonitoringPluginFactory.hostMonitoringPlugin.HostMonitoringConnectionPlugin(pluginService, properties, new RdsUtils()); + return new HostMonitoringPluginFactory.hostMonitoringPlugin.HostMonitoringConnectionPlugin(servicesContainer, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "HostMonitoringPlugin")); } diff --git a/common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts b/common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts new file mode 100644 index 000000000..3ca1194dd --- /dev/null +++ b/common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts @@ -0,0 +1,70 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { WrapperProperties } from "../../../wrapper_property"; +import { ConnectionContext } from "../base/connection_context"; +import { HostMonitorService, HostMonitorServiceImpl } from "../base/host_monitor_service"; +import { HostMonitoringConnectionPlugin } from "../v1/host_monitoring_connection_plugin"; +import { logger, uniqueId } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { SubscribedMethodHelper } from "../../../utils/subscribed_method_helper"; +import { FullServicesContainer } from "../../../utils/full_services_container"; + +export class HostMonitoring2ConnectionPlugin extends HostMonitoringConnectionPlugin { + id: string = uniqueId("_efm2Plugin"); + + constructor(servicesContainer: FullServicesContainer, properties: Map, monitorService?: HostMonitorService) { + super(servicesContainer, properties, monitorService ?? new HostMonitorServiceImpl(servicesContainer)); + } + + async execute(methodName: string, methodFunc: () => Promise, methodArgs: any): Promise { + const isEnabled: boolean = WrapperProperties.FAILURE_DETECTION_ENABLED.get(this.properties); + + if (!isEnabled || !SubscribedMethodHelper.NETWORK_BOUND_METHODS.includes(methodName)) { + return methodFunc(); + } + + const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties); + const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties); + const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties); + + let result: T; + let context: ConnectionContext | null = null; + + try { + logger.debug(Messages.get("HostMonitoringConnectionPlugin.activatedMonitoring", methodName)); + const monitoringHostInfo = await this.getMonitoringHostInfo(); + + context = await this.monitorService.startMonitoring( + this.pluginService.getCurrentClient().targetClient, + monitoringHostInfo, + this.properties, + failureDetectionTimeMillis, + failureDetectionIntervalMillis, + failureDetectionCount + ); + + result = await methodFunc(); + } finally { + if (context != null) { + this.monitorService.stopMonitoring(context); + logger.debug(Messages.get("HostMonitoringConnectionPlugin.monitoringDeactivated", methodName)); + } + } + + return result; + } +} diff --git a/common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts b/common/lib/plugins/efm/v2/host_monitoring2_plugin_factory.ts similarity index 69% rename from common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts rename to common/lib/plugins/efm/v2/host_monitoring2_plugin_factory.ts index 6763b0285..e21e4c8be 100644 --- a/common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts +++ b/common/lib/plugins/efm/v2/host_monitoring2_plugin_factory.ts @@ -14,22 +14,21 @@ limitations under the License. */ -import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; -import { ConnectionPlugin } from "../../connection_plugin"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AwsWrapperError } from "../../utils/errors"; -import { Messages } from "../../utils/messages"; +import { ConnectionPluginFactory } from "../../../plugin_factory"; +import { ConnectionPlugin } from "../../../connection_plugin"; +import { AwsWrapperError } from "../../../utils/errors"; +import { Messages } from "../../../utils/messages"; +import { FullServicesContainer } from "../../../utils/full_services_container"; export class HostMonitoring2PluginFactory extends ConnectionPluginFactory { private static hostMonitoring2Plugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!HostMonitoring2PluginFactory.hostMonitoring2Plugin) { HostMonitoring2PluginFactory.hostMonitoring2Plugin = await import("./host_monitoring2_connection_plugin"); } - return new HostMonitoring2PluginFactory.hostMonitoring2Plugin.HostMonitoring2ConnectionPlugin(pluginService, properties, new RdsUtils()); + return new HostMonitoring2PluginFactory.hostMonitoring2Plugin.HostMonitoring2ConnectionPlugin(servicesContainer, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "HostMonitoringPlugin")); } diff --git a/common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts b/common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts deleted file mode 100644 index 036b6dcd3..000000000 --- a/common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts +++ /dev/null @@ -1,163 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { PluginService } from "../../plugin_service"; -import { HostChangeOptions } from "../../host_change_options"; -import { HostInfo } from "../../host_info"; -import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; -import { RdsUrlType } from "../../utils/rds_url_type"; -import { WrapperProperties } from "../../wrapper_property"; -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { logger, uniqueId } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { MonitorService, MonitorServiceImpl } from "./monitor_service"; -import { AwsWrapperError } from "../../"; -import { HostListProvider } from "../../host_list_provider/host_list_provider"; -import { CanReleaseResources } from "../../can_release_resources"; -import { SubscribedMethodHelper } from "../../utils/subscribed_method_helper"; -import { ClientWrapper } from "../../client_wrapper"; - -export class HostMonitoring2ConnectionPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - id: string = uniqueId("_efm2Plugin"); - private readonly properties: Map; - private pluginService: PluginService; - private rdsUtils: RdsUtils; - private monitoringHostInfo: HostInfo | null = null; - private monitorService: MonitorService; - - constructor(pluginService: PluginService, properties: Map, rdsUtils: RdsUtils = new RdsUtils(), monitorService?: MonitorServiceImpl) { - super(); - this.pluginService = pluginService; - this.properties = properties; - this.rdsUtils = rdsUtils; - this.monitorService = monitorService ?? new MonitorServiceImpl(pluginService); - } - - getSubscribedMethods(): Set { - return new Set(["*"]); - } - - async connect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - connectFunc: () => Promise - ): Promise { - const targetClient = await connectFunc(); - if (targetClient != null) { - const type: RdsUrlType = this.rdsUtils.identifyRdsType(hostInfo.host); - if (type.isRdsCluster) { - hostInfo.resetAliases(); - await this.pluginService.fillAliases(targetClient, hostInfo); - } - } - return targetClient; - } - - async execute(methodName: string, methodFunc: () => Promise, methodArgs: any): Promise { - const isEnabled: boolean = WrapperProperties.FAILURE_DETECTION_ENABLED.get(this.properties); - - if (!isEnabled || !SubscribedMethodHelper.NETWORK_BOUND_METHODS.includes(methodName)) { - return methodFunc(); - } - - const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties); - const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties); - const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties); - - let result: T; - let monitorContext: MonitorConnectionContext | null = null; - - try { - logger.debug(Messages.get("HostMonitoringConnectionPlugin.activatedMonitoring", methodName)); - const monitoringHostInfo: HostInfo = await this.getMonitoringHostInfo(); - - monitorContext = await this.monitorService.startMonitoring( - this.pluginService.getCurrentClient().targetClient, - monitoringHostInfo, - this.properties, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount - ); - - result = await methodFunc(); - } finally { - if (monitorContext != null) { - await this.monitorService.stopMonitoring(monitorContext, this.pluginService.getCurrentClient().targetClient); - - logger.debug(Messages.get("HostMonitoringConnectionPlugin.monitoringDeactivated", methodName)); - } - } - - return result; - } - - private throwUnableToIdentifyConnection(host: HostInfo | null): never { - const provider: HostListProvider | null = this.pluginService.getHostListProvider(); - throw new AwsWrapperError( - Messages.get( - "HostMonitoringConnectionPlugin.unableToIdentifyConnection", - host !== null ? host.host : "unknown host", - provider !== null ? provider.getHostProviderType() : "unknown provider" - ) - ); - } - - async getMonitoringHostInfo(): Promise { - if (this.monitoringHostInfo) { - return this.monitoringHostInfo; - } - this.monitoringHostInfo = this.pluginService.getCurrentHostInfo(); - if (this.monitoringHostInfo === null) { - this.throwUnableToIdentifyConnection(null); - } - const rdsUrlType: RdsUrlType = this.rdsUtils.identifyRdsType(this.monitoringHostInfo.url); - - try { - if (rdsUrlType.isRdsCluster) { - logger.debug(Messages.get("HostMonitoringConnectionPlugin.identifyClusterConnection")); - this.monitoringHostInfo = await this.pluginService.identifyConnection(this.pluginService.getCurrentClient().targetClient!); - if (this.monitoringHostInfo == null) { - const host: HostInfo | null = this.pluginService.getCurrentHostInfo(); - this.throwUnableToIdentifyConnection(host); - } - await this.pluginService.fillAliases(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); - } - } catch (error: any) { - if (!(error instanceof AwsWrapperError)) { - logger.debug(Messages.get("HostMonitoringConnectionPlugin.errorIdentifyingConnection", error.message)); - } - throw error; - } - - return this.monitoringHostInfo; - } - - async notifyConnectionChanged(changes: Set): Promise { - if (changes.has(HostChangeOptions.HOSTNAME) || changes.has(HostChangeOptions.HOST_CHANGED)) { - // Reset monitoring host info since the associated connection has changed. - this.monitoringHostInfo = null; - } - return OldConnectionSuggestionAction.NO_OPINION; - } - - async releaseResources(): Promise { - await this.monitorService.releaseResources(); - } -} diff --git a/common/lib/plugins/efm2/monitor.ts b/common/lib/plugins/efm2/monitor.ts deleted file mode 100644 index f1d72a61a..000000000 --- a/common/lib/plugins/efm2/monitor.ts +++ /dev/null @@ -1,319 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { PluginService } from "../../plugin_service"; -import { logger } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { ClientWrapper } from "../../client_wrapper"; -import { getCurrentTimeNano, sleep } from "../../utils/utils"; -import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; -import { HostAvailability } from "../../host_availability/host_availability"; -import { MapUtils } from "../../utils/map_utils"; -import { WrapperProperties } from "../../wrapper_property"; - -export interface Monitor { - startMonitoring(context: MonitorConnectionContext): void; - - run(): Promise; - - canDispose(): boolean; - - endMonitoringClient(): Promise; - - releaseResources(): Promise; -} - -export class MonitorImpl implements Monitor { - private static readonly TASK_SLEEP_MILLIS: number = 100; - private activeContexts: WeakRef[] = []; - static newContexts: Map>> = new Map>>(); - private readonly pluginService: PluginService; - private readonly telemetryFactory: TelemetryFactory; - private readonly properties: Map; - private readonly hostInfo: HostInfo; - private stopped: boolean = false; - - private monitoringClient: ClientWrapper | null = null; - - private readonly failureDetectionTimeNano: number; - private readonly failureDetectionIntervalNanos: number; - private readonly failureDetectionCount: number; - - private invalidHostStartTimeNano: number; - private failureCount: number; - private hostUnhealthy: boolean = false; - private readonly abortedConnectionsCounter: TelemetryCounter; - private delayMillisTimeoutId: any; - private sleepWhenHostHealthyTimeoutId: any; - - constructor( - pluginService: PluginService, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number, - abortedConnectionsCounter: TelemetryCounter - ) { - this.pluginService = pluginService; - this.telemetryFactory = this.pluginService.getTelemetryFactory(); - this.hostInfo = hostInfo; - this.properties = properties; - this.failureDetectionTimeNano = failureDetectionTimeMillis * 1000000; - this.failureDetectionIntervalNanos = failureDetectionIntervalMillis * 1000000; - this.failureDetectionCount = failureDetectionCount; - this.abortedConnectionsCounter = abortedConnectionsCounter; - - const hostId: string = this.hostInfo.hostId ?? this.hostInfo.host; - - this.telemetryFactory.createGauge(`efm2.newContexts.size.${hostId}`, () => MonitorImpl.newContexts.size === Number.MAX_SAFE_INTEGER); - - this.telemetryFactory.createGauge(`efm2.activeContexts.size.${hostId}`, () => this.activeContexts.length === Number.MAX_SAFE_INTEGER); - - this.telemetryFactory.createGauge(`efm2.hostHealthy.${hostId}`, () => (this.hostUnhealthy ? 0 : 1)); - - Promise.race([this.newContextRun(), this.run()]).finally(() => { - this.stopped = true; - }); - } - - canDispose(): boolean { - return this.activeContexts.length === 0 && MonitorImpl.newContexts.size === 0; - } - - startMonitoring(context: MonitorConnectionContext): void { - if (this.isStopped()) { - logger.warn(Messages.get("MonitorImpl.monitorIsStopped", this.hostInfo.host)); - } - - const startMonitorTimeNano = getCurrentTimeNano() + this.failureDetectionTimeNano; - const connectionQueue = MapUtils.computeIfAbsent( - MonitorImpl.newContexts, - startMonitorTimeNano, - () => new Array>() - ); - connectionQueue.push(new WeakRef(context)); - } - - async newContextRun(): Promise { - logger.debug(Messages.get("MonitorImpl.startMonitoringTaskNewContext", this.hostInfo.host)); - - try { - while (!this.isStopped()) { - const currentTimeNanos = getCurrentTimeNano(); - // Get entries with key (that is a time in nanos) less than current time. - const processedKeys: number[] = new Array(); - for (const [key, val] of MonitorImpl.newContexts.entries()) { - if (key < currentTimeNanos) { - const queue: Array> = val; - - processedKeys.push(key); - // Each value of found entry is a queue of monitoring contexts awaiting active monitoring. - // Add all contexts to an active monitoring contexts queue. - // Ignore disposed contexts. - let monitorContextRef: WeakRef | undefined; - - while ((monitorContextRef = queue?.shift()) != null) { - const monitorContext: MonitorConnectionContext = monitorContextRef?.deref() ?? null; - if (monitorContext && monitorContext.isActive()) { - this.activeContexts.push(monitorContextRef); - } - } - } - } - processedKeys.forEach((key) => { - MonitorImpl.newContexts.delete(key); - }); - await sleep(1000); - } - return; - } catch (err) { - // do nothing, exit task - } - logger.debug(Messages.get("MonitorImpl.stopMonitoringTaskNewContext", this.hostInfo.host)); - } - - async run(): Promise { - logger.debug(Messages.get("MonitorImpl.startMonitoring", this.hostInfo.host)); - - try { - while (!this.isStopped()) { - try { - if (this.activeContexts.length === 0 && !this.hostUnhealthy) { - await new Promise((resolve) => { - this.delayMillisTimeoutId = setTimeout(resolve, MonitorImpl.TASK_SLEEP_MILLIS); - }); - continue; - } - - const statusCheckStartTimeNanos: number = getCurrentTimeNano(); - const isValid = await this.checkConnectionStatus(); - const statusCheckEndTimeNanos: number = getCurrentTimeNano(); - - await this.updateHostHealthStatus(isValid, statusCheckStartTimeNanos, statusCheckEndTimeNanos); - - if (this.hostUnhealthy) { - this.pluginService.setAvailability(this.hostInfo.aliases, HostAvailability.NOT_AVAILABLE); - } - const tmpActiveContexts: WeakRef[] = []; - - let monitorContextRef: WeakRef | undefined; - - while ((monitorContextRef = this.activeContexts?.shift()) != null) { - if (this.isStopped()) { - break; - } - - const monitorContext: MonitorConnectionContext = monitorContextRef?.deref() ?? null; - - if (!monitorContext) { - continue; - } - - if (this.hostUnhealthy) { - // Kill connection - monitorContext.setHostUnhealthy(true); - const clientToAbort = monitorContext.getClient(); - - monitorContext.setInactive(); - if (clientToAbort != null) { - await this.endMonitoringClient(clientToAbort); - - this.abortedConnectionsCounter.inc(); - } - } else if (monitorContext && monitorContext.isActive()) { - tmpActiveContexts.push(monitorContextRef); - } - } - - // activeContexts is empty now and tmpActiveContexts contains all yet active contexts - // Add active contexts back to the queue. - this.activeContexts.push(...tmpActiveContexts); - - const delayMillis = (this.failureDetectionIntervalNanos - (statusCheckEndTimeNanos - statusCheckStartTimeNanos)) / 1000000; - - await new Promise((resolve) => { - this.delayMillisTimeoutId = setTimeout( - resolve, - delayMillis < MonitorImpl.TASK_SLEEP_MILLIS ? MonitorImpl.TASK_SLEEP_MILLIS : delayMillis - ); - }); - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringContinue", error.message)); - } - } - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringStop", error.message)); - } finally { - await this.endMonitoringClient(); - } - - logger.debug(Messages.get("MonitorImpl.stopMonitoring", this.hostInfo.host)); - } - - /** - * Check the status of the monitored server by sending a ping. - * - * @return whether the server is still alive and the elapsed time spent checking. - */ - async checkConnectionStatus(): Promise { - const connectContext = this.telemetryFactory.openTelemetryContext("Connection status check", TelemetryTraceLevel.FORCE_TOP_LEVEL); - connectContext.setAttribute("url", this.hostInfo.host); - try { - if (!(await this.pluginService.isClientValid(this.monitoringClient))) { - // Open a new connection. - const monitoringConnProperties: Map = new Map(this.properties); - for (const key of monitoringConnProperties.keys()) { - if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { - continue; - } - monitoringConnProperties.set(key.substring(WrapperProperties.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); - monitoringConnProperties.delete(key); - } - - logger.debug(`Opening a monitoring connection to ${this.hostInfo.url}`); - this.monitoringClient = await this.pluginService.forceConnect(this.hostInfo, monitoringConnProperties); - logger.debug(`Successfully opened monitoring connection to ${this.monitoringClient.id} - ${this.hostInfo.url}`); - return true; - } - return true; - } catch (error: any) { - return false; - } - } - - isStopped(): boolean { - return this.stopped; - } - - updateHostHealthStatus(connectionValid: boolean, statusCheckStartNano: number, statusCheckEndNano: number): Promise { - if (!connectionValid) { - this.failureCount++; - - if (this.invalidHostStartTimeNano === 0) { - this.invalidHostStartTimeNano = statusCheckStartNano; - } - - const invalidHostDurationNano = statusCheckEndNano - this.invalidHostStartTimeNano; - const maxInvalidHostDurationNano = this.failureDetectionIntervalNanos * Math.max(0, this.failureDetectionCount - 1); - - if (this.failureCount >= this.failureDetectionCount || invalidHostDurationNano >= maxInvalidHostDurationNano) { - logger.debug(Messages.get("MonitorConnectionContext.hostDead", this.hostInfo.host)); - this.hostUnhealthy = true; - return Promise.resolve(); - } - logger.debug(Messages.get("MonitorConnectionContext.hostNotResponding", this.hostInfo.host)); - return Promise.resolve(); - } - - if (this.failureCount > 0) { - // Host is back alive - logger.debug(Messages.get("MonitorConnectionContext.hostAlive", this.hostInfo.host)); - } - this.failureCount = 0; - this.invalidHostStartTimeNano = 0; - this.hostUnhealthy = false; - } - - async releaseResources() { - this.stopped = true; - clearTimeout(this.delayMillisTimeoutId); - clearTimeout(this.sleepWhenHostHealthyTimeoutId); - this.activeContexts = null; - await this.endMonitoringClient(); - // Allow time for monitor loop to close. - await sleep(500); - } - - async endMonitoringClient(clientToAbort?: ClientWrapper) { - try { - if (clientToAbort) { - await this.pluginService.abortTargetClient(clientToAbort); - } else if (this.monitoringClient) { - await this.pluginService.abortTargetClient(this.monitoringClient); - this.monitoringClient = null; - } - this.stopped = true; - } catch (error: any) { - // ignore - logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); - } - } -} diff --git a/common/lib/plugins/efm2/monitor_connection_context.ts b/common/lib/plugins/efm2/monitor_connection_context.ts deleted file mode 100644 index 156b41963..000000000 --- a/common/lib/plugins/efm2/monitor_connection_context.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { uniqueId } from "../../../logutils"; -import { ClientWrapper } from "../../client_wrapper"; - -/** - * Monitoring context for each connection. This contains each connection's criteria for whether a - * server should be considered unhealthy. The context is shared between the main task and the monitor task. - */ -export class MonitorConnectionContext { - private clientToAbortRef: WeakRef | undefined; - isHostUnhealthy: boolean = false; - id: string = uniqueId("_monitorContext"); - - /** - * Constructor. - * - * @param clientToAbort A reference to the connection associated with this context that will be aborted. - */ - constructor(clientToAbort: ClientWrapper) { - if (clientToAbort) { - this.clientToAbortRef = new WeakRef(clientToAbort); - } - } - - setHostUnhealthy(hostUnhealthy: boolean) { - this.isHostUnhealthy = hostUnhealthy; - } - - shouldAbort(): boolean { - return this.isHostUnhealthy && this.clientToAbortRef != null; - } - - setInactive(): void { - this.clientToAbortRef = null; - } - - getClient(): ClientWrapper | null { - return this.clientToAbortRef?.deref() ?? null; - } - - isActive() { - return !!this.clientToAbortRef?.deref(); - } -} diff --git a/common/lib/plugins/efm2/monitor_service.ts b/common/lib/plugins/efm2/monitor_service.ts deleted file mode 100644 index 40a788178..000000000 --- a/common/lib/plugins/efm2/monitor_service.ts +++ /dev/null @@ -1,157 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { AwsWrapperError } from "../../utils/errors"; -import { Monitor, MonitorImpl } from "./monitor"; -import { WrapperProperties } from "../../wrapper_property"; -import { PluginService } from "../../plugin_service"; -import { Messages } from "../../utils/messages"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; -import { ClientWrapper } from "../../client_wrapper"; -import { logger } from "../../../logutils"; -import { SlidingExpirationCacheWithCleanupTask } from "../../utils/sliding_expiration_cache_with_cleanup_task"; - -export interface MonitorService { - startMonitoring( - clientToAbort: ClientWrapper, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise; - - /** - * Stop monitoring for a connection represented by the given {@link MonitorConnectionContext}. - * Removes the context from the {@link MonitorImpl}. - * - * @param context The {@link MonitorConnectionContext} representing a connection. - * @param clientToAbort A reference to the connection associated with this context that will be aborted. - */ - stopMonitoring(context: MonitorConnectionContext, clientToAbort: ClientWrapper): Promise; - - releaseResources(): Promise; -} - -export class MonitorServiceImpl implements MonitorService { - private static readonly CACHE_CLEANUP_NANOS = BigInt(60_000_000_000); - - protected static readonly monitors: SlidingExpirationCacheWithCleanupTask = new SlidingExpirationCacheWithCleanupTask( - MonitorServiceImpl.CACHE_CLEANUP_NANOS, - (monitor: Monitor) => monitor.canDispose(), - async (monitor: Monitor) => { - { - try { - await monitor.releaseResources(); - } catch (error) { - // ignore - } - } - }, - "efm2/MonitorServiceImpl.monitors" - ); - private readonly pluginService: PluginService; - private telemetryFactory: TelemetryFactory; - private readonly abortedConnectionsCounter: TelemetryCounter; - monitorSupplier = ( - pluginService: PluginService, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number, - abortedConnectionsCounter: TelemetryCounter - ) => - new MonitorImpl( - pluginService, - hostInfo, - properties, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount, - abortedConnectionsCounter - ); - - constructor(pluginService: PluginService) { - this.pluginService = pluginService; - this.telemetryFactory = pluginService.getTelemetryFactory(); - this.abortedConnectionsCounter = this.telemetryFactory.createCounter("efm2.connections.aborted"); - } - - async startMonitoring( - clientToAbort: ClientWrapper, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise { - const monitor = await this.getMonitor(hostInfo, properties, failureDetectionTimeMillis, failureDetectionIntervalMillis, failureDetectionCount); - - if (monitor) { - const context = new MonitorConnectionContext(clientToAbort); - monitor.startMonitoring(context); - return context; - } - - throw new AwsWrapperError(Messages.get("MonitorService.startMonitoringNullMonitor", hostInfo.host)); - } - - async stopMonitoring(context: MonitorConnectionContext, clientToAbort: ClientWrapper): Promise { - context.setInactive(); - if (context.shouldAbort()) { - try { - await clientToAbort.abort(); - this.abortedConnectionsCounter.inc(); - } catch (error) { - // ignore - logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); - } - } - } - - async getMonitor( - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise { - const monitorKey: string = `${failureDetectionTimeMillis.toString()} ${failureDetectionIntervalMillis.toString()} ${failureDetectionCount.toString()} ${hostInfo.host}`; - const cacheExpirationNanos = BigInt(WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties) * 1_000_000); - return MonitorServiceImpl.monitors.computeIfAbsent( - monitorKey, - () => - this.monitorSupplier( - this.pluginService, - hostInfo, - properties, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount, - this.abortedConnectionsCounter - ), - cacheExpirationNanos - ); - } - - async releaseResources() { - await MonitorServiceImpl.monitors.clear(); - } -} diff --git a/common/lib/plugins/execute_time_plugin_factory.ts b/common/lib/plugins/execute_time_plugin_factory.ts index 36e2885e4..3087ad204 100644 --- a/common/lib/plugins/execute_time_plugin_factory.ts +++ b/common/lib/plugins/execute_time_plugin_factory.ts @@ -15,15 +15,15 @@ */ import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; import { ConnectionPlugin } from "../connection_plugin"; import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; +import { FullServicesContainer } from "../utils/full_services_container"; export class ExecuteTimePluginFactory extends ConnectionPluginFactory { private static executeTimePlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!ExecuteTimePluginFactory.executeTimePlugin) { ExecuteTimePluginFactory.executeTimePlugin = await import("./execute_time_plugin"); diff --git a/common/lib/plugins/failover/failover_mode.ts b/common/lib/plugins/failover/failover_mode.ts index e6522335d..42444f6cc 100644 --- a/common/lib/plugins/failover/failover_mode.ts +++ b/common/lib/plugins/failover/failover_mode.ts @@ -15,19 +15,16 @@ */ export enum FailoverMode { - STRICT_WRITER, - STRICT_READER, - READER_OR_WRITER, - UNKNOWN + STRICT_WRITER = "strict-writer", + STRICT_READER = "strict-reader", + READER_OR_WRITER = "reader-or-writer", + UNKNOWN = "unknown" } -const nameToValue = new Map([ - ["strict-writer", FailoverMode.STRICT_WRITER], - ["strict-reader", FailoverMode.STRICT_READER], - ["reader-or-writer", FailoverMode.READER_OR_WRITER], - ["unknown", FailoverMode.UNKNOWN] -]); - -export function failoverModeFromValue(name: string): FailoverMode { - return nameToValue.get(name.toLowerCase()) ?? FailoverMode.UNKNOWN; +export function failoverModeFromValue(value: string | null | undefined): FailoverMode { + if (!value) { + return FailoverMode.UNKNOWN; + } + const normalized = value.toLowerCase(); + return Object.values(FailoverMode).find((v) => v === normalized) ?? FailoverMode.UNKNOWN; } diff --git a/common/lib/plugins/failover/failover_plugin.ts b/common/lib/plugins/failover/failover_plugin.ts index e6d7f4d75..73014d2dc 100644 --- a/common/lib/plugins/failover/failover_plugin.ts +++ b/common/lib/plugins/failover/failover_plugin.ts @@ -17,14 +17,14 @@ import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; import { logger } from "../../../logutils"; import { - HostInfo, AwsWrapperError, FailoverFailedError, FailoverSuccessError, - TransactionResolutionUnknownError, - UnavailableHostError, + HostAvailability, + HostInfo, HostRole, - HostAvailability + TransactionResolutionUnknownError, + UnavailableHostError } from "../../"; import { PluginService } from "../../plugin_service"; import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; @@ -43,6 +43,7 @@ import { ClientWrapper } from "../../client_wrapper"; import { getWriter, logTopology } from "../../utils/utils"; import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class FailoverPlugin extends AbstractConnectionPlugin { private static readonly TELEMETRY_WRITER_FAILOVER = "failover to writer instance"; @@ -79,18 +80,19 @@ export class FailoverPlugin extends AbstractConnectionPlugin { private hostListProviderService?: HostListProviderService; private readonly pluginService: PluginService; + private readonly servicesContainer: FullServicesContainer; protected enableFailoverSetting: boolean = WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.defaultValue; - constructor(pluginService: PluginService, properties: Map, rdsHelper: RdsUtils); + constructor(servicesContainer: FullServicesContainer, properties: Map, rdsHelper: RdsUtils); constructor( - pluginService: PluginService, + servicesContainer: FullServicesContainer, properties: Map, rdsHelper: RdsUtils, readerFailoverHandler: ClusterAwareReaderFailoverHandler, writerFailoverHandler: ClusterAwareWriterFailoverHandler ); constructor( - pluginService: PluginService, + servicesContainer: FullServicesContainer, properties: Map, rdsHelper: RdsUtils, readerFailoverHandler?: ClusterAwareReaderFailoverHandler, @@ -98,7 +100,8 @@ export class FailoverPlugin extends AbstractConnectionPlugin { ) { super(); this._properties = properties; - this.pluginService = pluginService; + this.pluginService = servicesContainer.pluginService; + this.servicesContainer = servicesContainer; this._rdsHelper = rdsHelper; this.initSettings(); @@ -106,7 +109,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { this._readerFailoverHandler = readerFailoverHandler ? readerFailoverHandler : new ClusterAwareReaderFailoverHandler( - pluginService, + this.pluginService, properties, this.failoverTimeoutMsSetting, this.failoverReaderConnectTimeoutMsSetting, @@ -115,7 +118,8 @@ export class FailoverPlugin extends AbstractConnectionPlugin { this._writerFailoverHandler = writerFailoverHandler ? writerFailoverHandler : new ClusterAwareWriterFailoverHandler( - pluginService, + this.pluginService, + this.servicesContainer, this._readerFailoverHandler, properties, this.failoverTimeoutMsSetting, @@ -180,12 +184,6 @@ export class FailoverPlugin extends AbstractConnectionPlugin { if (this.isHostStillValid(url, changes)) { return Promise.resolve(); } - - for (const alias of currentHost.allAliases) { - if (this.isHostStillValid(alias + "/", changes)) { - return Promise.resolve(); - } - } } logger.info(Messages.get("Failover.invalidHost", currentHost?.host ?? "empty host")); return Promise.resolve(); @@ -205,6 +203,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { return ( this.enableFailoverSetting && this._rdsUrlType !== RdsUrlType.RDS_PROXY && + this._rdsUrlType !== RdsUrlType.RDS_PROXY_ENDPOINT && this.pluginService.getAllHosts() && this.pluginService.getAllHosts().length > 0 ); @@ -255,6 +254,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { // Verify there aren't any unexpected error emitted while the connection was idle. if (this.pluginService.hasNetworkError()) { // Throw the unexpected error directly to be handled. + throw this.pluginService.getUnexpectedError(); } @@ -273,7 +273,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { await this.invalidateCurrentClient(); const currentHostInfo = this.pluginService.getCurrentHostInfo(); if (currentHostInfo !== null) { - this.pluginService.setAvailability(currentHostInfo.allAliases ?? new Set(), HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(currentHostInfo, HostAvailability.NOT_AVAILABLE); } this._lastError = e; @@ -285,7 +285,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { } async failover(failedHost: HostInfo) { - this.pluginService.setAvailability(failedHost.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(failedHost, HostAvailability.NOT_AVAILABLE); if (this.failoverMode === FailoverMode.STRICT_WRITER) { await this.failoverWriter(); @@ -302,8 +302,6 @@ export class FailoverPlugin extends AbstractConnectionPlugin { const telemetryContext = telemetryFactory.openTelemetryContext(FailoverPlugin.TELEMETRY_READER_FAILOVER, TelemetryTraceLevel.NESTED); this.failoverReaderTriggeredCounter.inc(); - const oldAliases = this.pluginService.getCurrentHostInfo()?.allAliases ?? new Set(); - let failedHost = null; if (failedHostInfo && failedHostInfo.getRawAvailability() === HostAvailability.AVAILABLE) { failedHost = failedHostInfo; @@ -328,7 +326,6 @@ export class FailoverPlugin extends AbstractConnectionPlugin { await this.pluginService.abortCurrentClient(); await this.pluginService.setCurrentClient(result.client, result.newHost); - this.pluginService.getCurrentHostInfo()?.removeAlias(Array.from(oldAliases)); await this.updateTopology(true); this.failoverWriterSuccessCounter.inc(); } catch (error: any) { @@ -540,7 +537,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { this._readerFailoverHandler.setEnableFailoverStrictReader(this.failoverMode === FailoverMode.STRICT_READER); - logger.debug(Messages.get("Failover.parameterValue", "failoverMode", FailoverMode[this.failoverMode])); + logger.debug(Messages.get("Failover.parameterValue", "failoverMode", String(this.failoverMode))); } } } diff --git a/common/lib/plugins/failover/failover_plugin_factory.ts b/common/lib/plugins/failover/failover_plugin_factory.ts index 5f8a5c41c..82ac44835 100644 --- a/common/lib/plugins/failover/failover_plugin_factory.ts +++ b/common/lib/plugins/failover/failover_plugin_factory.ts @@ -15,21 +15,21 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { RdsUtils } from "../../utils/rds_utils"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class FailoverPluginFactory extends ConnectionPluginFactory { private static failoverPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!FailoverPluginFactory.failoverPlugin) { FailoverPluginFactory.failoverPlugin = await import("./failover_plugin"); } - return new FailoverPluginFactory.failoverPlugin.FailoverPlugin(pluginService, properties, new RdsUtils()); + return new FailoverPluginFactory.failoverPlugin.FailoverPlugin(servicesContainer, properties, new RdsUtils()); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "FailoverPlugin")); } diff --git a/common/lib/plugins/failover/reader_failover_handler.ts b/common/lib/plugins/failover/reader_failover_handler.ts index 21c82b1e5..fddf12a60 100644 --- a/common/lib/plugins/failover/reader_failover_handler.ts +++ b/common/lib/plugins/failover/reader_failover_handler.ts @@ -141,7 +141,7 @@ export class ClusterAwareReaderFailoverHandler implements ReaderFailoverHandler async failoverInternal(hosts: HostInfo[], currentHost: HostInfo | null): Promise { if (currentHost) { - this.pluginService.setAvailability(currentHost.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(currentHost, HostAvailability.NOT_AVAILABLE); } const hostsByPriority = this.getHostsByPriority(hosts); return this.getConnectionFromHostGroup(hostsByPriority); @@ -336,7 +336,7 @@ class ConnectionAttemptTask { try { this.targetClient = await this.pluginService.forceConnect(this.newHost, copy); - this.pluginService.setAvailability(this.newHost.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(this.newHost, HostAvailability.AVAILABLE); logger.info(Messages.get("ClusterAwareReaderFailoverHandler.successfulReaderConnection", this.newHost.host)); if (this.taskHandler.getSelectedConnectionAttemptTask(this.failoverTaskId) === -1) { this.taskHandler.setSelectedConnectionAttemptTask(this.failoverTaskId, this.taskId); @@ -344,7 +344,7 @@ class ConnectionAttemptTask { } throw new AwsWrapperError(Messages.get("ClusterAwareReaderFailoverHandler.selectedTaskChosen", this.newHost.host)); } catch (error) { - this.pluginService.setAvailability(this.newHost.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(this.newHost, HostAvailability.NOT_AVAILABLE); throw error; } finally { await this.performFinalCleanup(); diff --git a/common/lib/plugins/failover/writer_failover_handler.ts b/common/lib/plugins/failover/writer_failover_handler.ts index 70d46eab2..dbc2320d6 100644 --- a/common/lib/plugins/failover/writer_failover_handler.ts +++ b/common/lib/plugins/failover/writer_failover_handler.ts @@ -27,6 +27,9 @@ import { logger } from "../../../logutils"; import { WrapperProperties } from "../../wrapper_property"; import { ClientWrapper } from "../../client_wrapper"; import { FailoverRestriction } from "./failover_restriction"; +import { FullServicesContainer } from "../../utils/full_services_container"; +import { ServiceUtils } from "../../utils/service_utils"; +import { DatabaseDialect } from "../../database_dialect/database_dialect"; export interface WriterFailoverHandler { failover(currentTopology: HostInfo[]): Promise; @@ -34,12 +37,14 @@ export interface WriterFailoverHandler { function isCurrentHostWriter(topology: HostInfo[], originalWriterHost: HostInfo): boolean { const latestWriter = getWriter(topology); - const latestWriterAllAliases = latestWriter?.allAliases; - const currentAliases = originalWriterHost.allAliases; - if (currentAliases && latestWriterAllAliases) { - return [...currentAliases].filter((alias) => latestWriterAllAliases.has(alias)).length > 0; + if (!latestWriter || !originalWriterHost) { + return false; } - return false; + + return ( + (!!latestWriter.hostId && latestWriter.hostId === originalWriterHost.hostId) || + (!!latestWriter.hostAndPort && latestWriter.hostAndPort.toLowerCase() === originalWriterHost.hostAndPort.toLowerCase()) + ); } export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler { @@ -47,6 +52,7 @@ export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler static readonly RECONNECT_WRITER_TASK = "TaskA"; static readonly WAIT_NEW_WRITER_TASK = "TaskB"; private readonly pluginService: PluginService; + private readonly servicesContainer: FullServicesContainer; private readonly readerFailoverHandler: ClusterAwareReaderFailoverHandler; private readonly initialConnectionProps: Map; maxFailoverTimeoutMs = 60000; // 60 sec @@ -55,6 +61,7 @@ export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler constructor( pluginService: PluginService, + servicesContainer: FullServicesContainer, readerFailoverHandler: ClusterAwareReaderFailoverHandler, initialConnectionProps: Map, failoverTimeoutMs?: number, @@ -62,6 +69,7 @@ export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler reconnectWriterIntervalMs?: number ) { this.pluginService = pluginService; + this.servicesContainer = servicesContainer; this.readerFailoverHandler = readerFailoverHandler; this.initialConnectionProps = initialConnectionProps; this.maxFailoverTimeoutMs = failoverTimeoutMs ?? this.maxFailoverTimeoutMs; @@ -69,16 +77,29 @@ export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler this.reconnectionWriterIntervalMs = reconnectWriterIntervalMs ?? this.reconnectionWriterIntervalMs; } + protected async newServicesContainer(): Promise { + const container = ServiceUtils.instance.createMinimalServiceContainerFrom(this.servicesContainer, this.initialConnectionProps); + await container.pluginManager.init(); + const initialHostInfo = this.pluginService.getInitialConnectionHostInfo(); + if (initialHostInfo) { + container.hostListProviderService.setInitialConnectionHostInfo(initialHostInfo); + } + return container; + } + async failover(currentTopology: HostInfo[]): Promise { if (!currentTopology || currentTopology.length == 0) { logger.error(Messages.get("ClusterAwareWriterFailoverHandler.failoverCalledWithInvalidTopology")); return ClusterAwareWriterFailoverHandler.DEFAULT_RESULT; } + const taskAContainer = await this.newServicesContainer(); + const taskBContainer = await this.newServicesContainer(); + const reconnectToWriterHandlerTask = new ReconnectToWriterHandlerTask( currentTopology, getWriter(currentTopology), - this.pluginService, + taskAContainer.pluginService, this.initialConnectionProps, this.reconnectionWriterIntervalMs, Date.now() + this.maxFailoverTimeoutMs @@ -88,7 +109,7 @@ export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler currentTopology, getWriter(currentTopology), this.readerFailoverHandler, - this.pluginService, + taskBContainer.pluginService, this.initialConnectionProps, this.readTopologyIntervalMs, Date.now() + this.maxFailoverTimeoutMs @@ -236,7 +257,7 @@ class ReconnectToWriterHandlerTask { const props = new Map(this.initialConnectionProps); props.set(WrapperProperties.HOST.name, this.originalWriterHost.host); this.currentClient = await this.pluginService.forceConnect(this.originalWriterHost, props); - await this.pluginService.forceRefreshHostList(this.currentClient); + await this.pluginService.forceRefreshHostList(); latestTopology = this.pluginService.getAllHosts(); } catch (error) { // Propagate errors that are not caused by network errors. @@ -254,7 +275,7 @@ class ReconnectToWriterHandlerTask { } success = isCurrentHostWriter(latestTopology, this.originalWriterHost); - this.pluginService.setAvailability(this.originalWriterHost.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(this.originalWriterHost, HostAvailability.AVAILABLE); return new WriterFailoverResult( success, false, @@ -379,10 +400,10 @@ class WaitForNewWriterHandlerTask { async refreshTopologyAndConnectToNewWriter(): Promise { const allowOldWriter: boolean = this.pluginService.getDialect().getFailoverRestrictions().includes(FailoverRestriction.ENABLE_WRITER_IN_TASK_B); - while (this.pluginService.getCurrentClient() && Date.now() < this.endTime && !this.failoverCompleted) { + while (Date.now() < this.endTime && !this.failoverCompleted) { try { if (this.currentReaderTargetClient) { - await this.pluginService.forceRefreshHostList(this.currentReaderTargetClient); + await this.pluginService.forceRefreshHostList(); } const topology = this.pluginService.getAllHosts(); @@ -435,13 +456,13 @@ class WaitForNewWriterHandlerTask { let targetClient = null; try { targetClient = await this.pluginService.forceConnect(writerCandidate, props); - this.pluginService.setAvailability(writerCandidate.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(writerCandidate, HostAvailability.AVAILABLE); await this.callCloseClient(this.currentReaderTargetClient); await this.callCloseClient(this.currentClient); this.currentClient = targetClient; return true; } catch (error) { - this.pluginService.setAvailability(writerCandidate.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(writerCandidate, HostAvailability.NOT_AVAILABLE); await this.pluginService.abortTargetClient(targetClient); return false; } diff --git a/common/lib/plugins/failover2/failover2_plugin.ts b/common/lib/plugins/failover2/failover2_plugin.ts index 5d7060d03..85288c657 100644 --- a/common/lib/plugins/failover2/failover2_plugin.ts +++ b/common/lib/plugins/failover2/failover2_plugin.ts @@ -38,46 +38,47 @@ import { ClientWrapper } from "../../client_wrapper"; import { HostAvailability } from "../../host_availability/host_availability"; import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; import { HostRole } from "../../host_role"; -import { CanReleaseResources } from "../../can_release_resources"; import { ReaderFailoverResult } from "../failover/reader_failover_result"; -import { BlockingHostListProvider, HostListProvider } from "../../host_list_provider/host_list_provider"; import { logTopology } from "../../utils/utils"; +import { FullServicesContainer } from "../../utils/full_services_container"; -export class Failover2Plugin extends AbstractConnectionPlugin implements CanReleaseResources { +export class Failover2Plugin extends AbstractConnectionPlugin { private static readonly TELEMETRY_WRITER_FAILOVER = "failover to writer instance"; private static readonly TELEMETRY_READER_FAILOVER = "failover to reader"; private static readonly METHOD_END = "end"; private static readonly SUBSCRIBED_METHODS: Set = new Set(["initHostProvider", "connect", "query"]); private readonly _staleDnsHelper: StaleDnsHelper; - private readonly _properties: Map; - private readonly pluginService: PluginService; - private readonly _rdsHelper: RdsUtils; - private readonly failoverWriterTriggeredCounter: TelemetryCounter; - private readonly failoverWriterSuccessCounter: TelemetryCounter; - private readonly failoverWriterFailedCounter: TelemetryCounter; - private readonly failoverReaderTriggeredCounter: TelemetryCounter; - private readonly failoverReaderSuccessCounter: TelemetryCounter; - private readonly failoverReaderFailedCounter: TelemetryCounter; - private telemetryFailoverAdditionalTopTraceSetting: boolean = false; - private _rdsUrlType: RdsUrlType | null = null; - private _isInTransaction: boolean = false; + protected readonly properties: Map; + private readonly servicesContainer: FullServicesContainer; + protected readonly pluginService: PluginService; + protected readonly rdsHelper: RdsUtils; + protected readonly failoverWriterTriggeredCounter: TelemetryCounter; + protected readonly failoverWriterSuccessCounter: TelemetryCounter; + protected readonly failoverWriterFailedCounter: TelemetryCounter; + protected readonly failoverReaderTriggeredCounter: TelemetryCounter; + protected readonly failoverReaderSuccessCounter: TelemetryCounter; + protected readonly failoverReaderFailedCounter: TelemetryCounter; + protected telemetryFailoverAdditionalTopTraceSetting: boolean = false; + protected rdsUrlType: RdsUrlType | null = null; + protected _isInTransaction: boolean = false; private _lastError: any; failoverMode: FailoverMode = FailoverMode.UNKNOWN; - private hostListProviderService?: HostListProviderService; + protected hostListProviderService?: HostListProviderService; protected enableFailoverSetting: boolean = WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.defaultValue; - private readonly failoverTimeoutSettingMs: number = WrapperProperties.FAILOVER_TIMEOUT_MS.defaultValue; - private readonly failoverReaderHostSelectorStrategy: string = WrapperProperties.FAILOVER_READER_HOST_SELECTOR_STRATEGY.defaultValue; + protected readonly failoverTimeoutSettingMs: number = WrapperProperties.FAILOVER_TIMEOUT_MS.defaultValue; + protected readonly failoverReaderHostSelectorStrategy: string = WrapperProperties.FAILOVER_READER_HOST_SELECTOR_STRATEGY.defaultValue; - constructor(pluginService: PluginService, properties: Map, rdsHelper: RdsUtils) { + constructor(servicesContainer: FullServicesContainer, properties: Map, rdsHelper: RdsUtils) { super(); - this._properties = properties; - this.pluginService = pluginService; - this._rdsHelper = rdsHelper; + this.properties = properties; + this.servicesContainer = servicesContainer; + this.pluginService = servicesContainer.pluginService; + this.rdsHelper = rdsHelper; this._staleDnsHelper = new StaleDnsHelper(this.pluginService); - this.enableFailoverSetting = WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.get(this._properties); - this.failoverTimeoutSettingMs = WrapperProperties.FAILOVER_TIMEOUT_MS.get(this._properties); - this.failoverReaderHostSelectorStrategy = WrapperProperties.FAILOVER_READER_HOST_SELECTOR_STRATEGY.get(this._properties); + this.enableFailoverSetting = WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.get(this.properties); + this.failoverTimeoutSettingMs = WrapperProperties.FAILOVER_TIMEOUT_MS.get(this.properties); + this.failoverReaderHostSelectorStrategy = WrapperProperties.FAILOVER_READER_HOST_SELECTOR_STRATEGY.get(this.properties); const telemetryFactory = this.pluginService.getTelemetryFactory(); this.failoverWriterTriggeredCounter = telemetryFactory.createCounter("writerFailover.triggered.count"); @@ -104,21 +105,13 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele } initHostProviderFunc(); - - this.failoverMode = failoverModeFromValue(WrapperProperties.FAILOVER_MODE.get(props)); - this._rdsUrlType = this._rdsHelper.identifyRdsType(hostInfo.host); - - if (this.failoverMode === FailoverMode.UNKNOWN) { - this.failoverMode = this._rdsUrlType === RdsUrlType.RDS_READER_CLUSTER ? FailoverMode.READER_OR_WRITER : FailoverMode.STRICT_WRITER; - } - - logger.debug(Messages.get("Failover.parameterValue", "failoverMode", FailoverMode[this.failoverMode])); } - private isFailoverEnabled(): boolean { + protected isFailoverEnabled(): boolean { return ( this.enableFailoverSetting && - this._rdsUrlType !== RdsUrlType.RDS_PROXY && + this.rdsUrlType !== RdsUrlType.RDS_PROXY && + this.rdsUrlType !== RdsUrlType.RDS_PROXY_ENDPOINT && this.pluginService.getAllHosts() && this.pluginService.getAllHosts().length > 0 ); @@ -130,6 +123,8 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele isInitialConnection: boolean, connectFunc: () => Promise ): Promise { + this.initFailoverMode(); + if ( // Failover is not enabled, does not require additional processing. !this.enableFailoverSetting || @@ -138,7 +133,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele return await this._staleDnsHelper.getVerifiedConnection(hostInfo.host, isInitialConnection, this.hostListProviderService!, props, connectFunc); } - const hostInfoWithAvailability: HostInfo = this.pluginService.getHosts().find((x) => x.getHostAndPort() === hostInfo.getHostAndPort()); + const hostInfoWithAvailability: HostInfo = this.pluginService.getHosts().find((x) => x.hostAndPort === hostInfo.hostAndPort); let client: ClientWrapper = null; if (!hostInfoWithAvailability || hostInfoWithAvailability.getAvailability() != HostAvailability.NOT_AVAILABLE) { @@ -155,7 +150,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele throw error; } - this.pluginService.setAvailability(hostInfo.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(hostInfo, HostAvailability.NOT_AVAILABLE); try { // Unable to directly connect, attempt failover. @@ -188,7 +183,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele } if (isInitialConnection) { - await this.pluginService.refreshHostList(client); + await this.pluginService.refreshHostList(); } return client; @@ -214,7 +209,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele await this.invalidateCurrentClient(); const currentHostInfo: HostInfo = this.pluginService.getCurrentHostInfo(); if (currentHostInfo !== null) { - this.pluginService.setAvailability(currentHostInfo.allAliases ?? new Set(), HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(currentHostInfo, HostAvailability.NOT_AVAILABLE); } await this.failover(); this._lastError = error; @@ -231,6 +226,10 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele await this.failoverReader(); } + this.throwFailoverSuccessException(); + } + + protected throwFailoverSuccessException(): void { if (this._isInTransaction || this.pluginService.isInTransaction()) { // "Transaction resolution unknown. Please re-configure session state if required and try // restarting transaction." @@ -243,12 +242,11 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele } } - async failoverReader() { + private async failoverReader() { const telemetryFactory = this.pluginService.getTelemetryFactory(); const telemetryContext = telemetryFactory.openTelemetryContext(Failover2Plugin.TELEMETRY_READER_FAILOVER, TelemetryTraceLevel.NESTED); this.failoverReaderTriggeredCounter.inc(); - const oldAliases = this.pluginService.getCurrentHostInfo()?.allAliases ?? new Set(); const failoverEndTimeMs = Date.now() + this.failoverTimeoutSettingMs; try { @@ -266,7 +264,6 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele this.failoverReaderSuccessCounter.inc(); await this.pluginService.abortCurrentClient(); await this.pluginService.setCurrentClient(result.client, result.newHost); - this.pluginService.getCurrentHostInfo()?.removeAlias(Array.from(oldAliases)); await this.pluginService.forceRefreshHostList(); } catch (error) { this.failoverReaderFailedCounter.inc(); @@ -367,7 +364,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele throw new InternalQueryTimeoutError(Messages.get("Failover.timeoutError")); } - async failoverWriter() { + private async failoverWriter() { const telemetryFactory = this.pluginService.getTelemetryFactory(); const telemetryContext = telemetryFactory.openTelemetryContext(Failover2Plugin.TELEMETRY_WRITER_FAILOVER, TelemetryTraceLevel.NESTED); this.failoverWriterTriggeredCounter.inc(); @@ -430,7 +427,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele } private async createConnectionForHost(hostInfo: HostInfo): Promise { - const copyProps = new Map(this._properties); + const copyProps = new Map(this.properties); copyProps.set(WrapperProperties.HOST.name, hostInfo.host); return await this.pluginService.connect(hostInfo, copyProps, this); } @@ -464,6 +461,22 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele return methodName === Failover2Plugin.METHOD_END; } + protected initFailoverMode(): void { + if (this.rdsUrlType) { + return; + } + + this.failoverMode = failoverModeFromValue(WrapperProperties.FAILOVER_MODE.get(this.properties)); + const initialHostInfo: HostInfo | undefined | null = this.hostListProviderService?.getInitialConnectionHostInfo(); + this.rdsUrlType = this.rdsHelper.identifyRdsType(initialHostInfo?.host); + + if (this.failoverMode === FailoverMode.UNKNOWN) { + this.failoverMode = this.rdsUrlType === RdsUrlType.RDS_READER_CLUSTER ? FailoverMode.READER_OR_WRITER : FailoverMode.STRICT_WRITER; + } + + logger.debug(Messages.get("Failover.parameterValue", "failoverMode", String(this.failoverMode))); + } + private shouldErrorTriggerClientSwitch(error: any): boolean { if (!this.isFailoverEnabled()) { logger.debug(Messages.get("Failover.failoverDisabled")); @@ -486,11 +499,4 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele this.failoverWriterFailedCounter.inc(); throw new FailoverFailedError(errorMessage); } - - async releaseResources(): Promise { - const hostListProvider: HostListProvider = this.pluginService.getHostListProvider(); - if (this.hostListProviderService.isBlockingHostListProvider(hostListProvider)) { - await (hostListProvider as BlockingHostListProvider).clearAll(); - } - } } diff --git a/common/lib/plugins/failover2/failover2_plugin_factory.ts b/common/lib/plugins/failover2/failover2_plugin_factory.ts index 64b2bcdbf..d60687601 100644 --- a/common/lib/plugins/failover2/failover2_plugin_factory.ts +++ b/common/lib/plugins/failover2/failover2_plugin_factory.ts @@ -15,21 +15,21 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { RdsUtils } from "../../utils/rds_utils"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class Failover2PluginFactory extends ConnectionPluginFactory { private static failover2Plugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!Failover2PluginFactory.failover2Plugin) { Failover2PluginFactory.failover2Plugin = await import("./failover2_plugin"); } - return new Failover2PluginFactory.failover2Plugin.Failover2Plugin(pluginService, properties, new RdsUtils()); + return new Failover2PluginFactory.failover2Plugin.Failover2Plugin(servicesContainer, properties, new RdsUtils()); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "Failover2Plugin")); } diff --git a/common/lib/plugins/federated_auth/credentials_provider_factory.ts b/common/lib/plugins/federated_auth/credentials_provider_factory.ts index e879fd8db..edca130fc 100644 --- a/common/lib/plugins/federated_auth/credentials_provider_factory.ts +++ b/common/lib/plugins/federated_auth/credentials_provider_factory.ts @@ -17,5 +17,9 @@ import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from "@smithy/types/dist-types/identity/awsCredentialIdentity"; export interface CredentialsProviderFactory { - getAwsCredentialsProvider(host: string, region: string, props: Map): Promise; + getAwsCredentialsProvider( + host: string, + region: string | null, + props: Map + ): Promise; } diff --git a/common/lib/plugins/federated_auth/federated_auth_plugin.ts b/common/lib/plugins/federated_auth/federated_auth_plugin.ts index 2a440dd54..406720379 100644 --- a/common/lib/plugins/federated_auth/federated_auth_plugin.ts +++ b/common/lib/plugins/federated_auth/federated_auth_plugin.ts @@ -14,118 +14,13 @@ limitations under the License. */ -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; import { PluginService } from "../../plugin_service"; -import { RdsUtils } from "../../utils/rds_utils"; -import { HostInfo } from "../../host_info"; -import { IamAuthUtils, TokenInfo } from "../../utils/iam_auth_utils"; -import { WrapperProperties } from "../../wrapper_property"; -import { logger } from "../../../logutils"; -import { AwsWrapperError } from "../../utils/errors"; -import { Messages } from "../../utils/messages"; import { CredentialsProviderFactory } from "./credentials_provider_factory"; -import { SamlUtils } from "../../utils/saml_utils"; -import { ClientWrapper } from "../../client_wrapper"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { RegionUtils } from "../../utils/region_utils"; -import { CanReleaseResources } from "../../can_release_resources"; +import { BaseSamlAuthPlugin } from "./saml_auth_plugin"; +import { IamAuthUtils } from "../../utils/iam_auth_utils"; -export class FederatedAuthPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - protected static readonly tokenCache = new Map(); - protected rdsUtils: RdsUtils = new RdsUtils(); - protected pluginService: PluginService; - private static readonly subscribedMethods = new Set(["connect", "forceConnect"]); - private readonly credentialsProviderFactory: CredentialsProviderFactory; - private readonly fetchTokenCounter: TelemetryCounter; - - public getSubscribedMethods(): Set { - return FederatedAuthPlugin.subscribedMethods; - } - - constructor(pluginService: PluginService, credentialsProviderFactory: CredentialsProviderFactory) { - super(); - this.credentialsProviderFactory = credentialsProviderFactory; - this.pluginService = pluginService; - this.fetchTokenCounter = this.pluginService.getTelemetryFactory().createCounter("federatedAuth.fetchToken.count"); - } - - connect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - connectFunc: () => Promise - ): Promise { - return this.connectInternal(hostInfo, props, connectFunc); - } - - forceConnect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - forceConnectFunc: () => Promise - ): Promise { - return this.connectInternal(hostInfo, props, forceConnectFunc); - } - - async connectInternal(hostInfo: HostInfo, props: Map, connectFunc: () => Promise): Promise { - SamlUtils.checkIdpCredentialsWithFallback(props); - - const host = IamAuthUtils.getIamHost(props, hostInfo); - const port = IamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); - const region: string = RegionUtils.getRegion(props.get(WrapperProperties.IAM_REGION.name), host); - - const cacheKey = IamAuthUtils.getCacheKey(port, WrapperProperties.DB_USER.get(props), host, region); - const tokenInfo = FederatedAuthPlugin.tokenCache.get(cacheKey); - - const isCachedToken: boolean = tokenInfo !== undefined && !tokenInfo.isExpired(); - - if (isCachedToken && tokenInfo) { - logger.debug(Messages.get("AuthenticationToken.useCachedToken", tokenInfo.token)); - WrapperProperties.PASSWORD.set(props, tokenInfo.token); - } else { - await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host); - } - WrapperProperties.USER.set(props, WrapperProperties.DB_USER.get(props)); - this.pluginService.updateConfigWithProperties(props); - - try { - return await connectFunc(); - } catch (e) { - if (!this.pluginService.isLoginError(e as Error) || !isCachedToken) { - throw e; - } - try { - await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host); - return await connectFunc(); - } catch (e: any) { - throw new AwsWrapperError(Messages.get("SamlAuthPlugin.unhandledError", e.message)); - } - } - } - - public async updateAuthenticationToken(hostInfo: HostInfo, props: Map, region: string, cacheKey: string, iamHost: string) { - const tokenExpirationSec = WrapperProperties.IAM_TOKEN_EXPIRATION.get(props); - if (tokenExpirationSec < 0) { - throw new AwsWrapperError(Messages.get("AuthenticationToken.tokenExpirationLessThanZero")); - } - const tokenExpiry: number = Date.now() + tokenExpirationSec * 1000; - const port = IamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); - const token = await IamAuthUtils.generateAuthenticationToken( - iamHost, - port, - region, - WrapperProperties.DB_USER.get(props), - await this.credentialsProviderFactory.getAwsCredentialsProvider(hostInfo.host, region, props), - this.pluginService - ); - this.fetchTokenCounter.inc(); - logger.debug(Messages.get("AuthenticationToken.generatedNewToken", token)); - WrapperProperties.PASSWORD.set(props, token); - FederatedAuthPlugin.tokenCache.set(cacheKey, new TokenInfo(token, tokenExpiry)); - } - - releaseResources(): Promise { - FederatedAuthPlugin.tokenCache.clear(); - return; +export class FederatedAuthPlugin extends BaseSamlAuthPlugin { + constructor(pluginService: PluginService, credentialsProviderFactory: CredentialsProviderFactory, iamAuthUtils: IamAuthUtils = new IamAuthUtils()) { + super(pluginService, credentialsProviderFactory, "federatedAuth.fetchToken.count", iamAuthUtils); } } diff --git a/common/lib/plugins/federated_auth/federated_auth_plugin_factory.ts b/common/lib/plugins/federated_auth/federated_auth_plugin_factory.ts index c77895020..ac391d642 100644 --- a/common/lib/plugins/federated_auth/federated_auth_plugin_factory.ts +++ b/common/lib/plugins/federated_auth/federated_auth_plugin_factory.ts @@ -15,16 +15,16 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class FederatedAuthPluginFactory extends ConnectionPluginFactory { private static federatedAuthPlugin: any; private static adfsCredentialsProvider: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!FederatedAuthPluginFactory.federatedAuthPlugin) { FederatedAuthPluginFactory.federatedAuthPlugin = await import("./federated_auth_plugin"); @@ -34,6 +34,7 @@ export class FederatedAuthPluginFactory extends ConnectionPluginFactory { FederatedAuthPluginFactory.adfsCredentialsProvider = await import("./adfs_credentials_provider_factory"); } + const pluginService = servicesContainer.pluginService; const adfsCredentialsProviderFactory = new FederatedAuthPluginFactory.adfsCredentialsProvider.AdfsCredentialsProviderFactory(pluginService); return new FederatedAuthPluginFactory.federatedAuthPlugin.FederatedAuthPlugin(pluginService, adfsCredentialsProviderFactory); } catch (error: any) { diff --git a/common/lib/plugins/federated_auth/okta_auth_plugin.ts b/common/lib/plugins/federated_auth/okta_auth_plugin.ts index 9de79bd15..31b6144f8 100644 --- a/common/lib/plugins/federated_auth/okta_auth_plugin.ts +++ b/common/lib/plugins/federated_auth/okta_auth_plugin.ts @@ -14,120 +14,13 @@ limitations under the License. */ -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; -import { HostInfo } from "../../host_info"; -import { SamlUtils } from "../../utils/saml_utils"; -import { IamAuthUtils, TokenInfo } from "../../utils/iam_auth_utils"; import { PluginService } from "../../plugin_service"; import { CredentialsProviderFactory } from "./credentials_provider_factory"; -import { RdsUtils } from "../../utils/rds_utils"; -import { WrapperProperties } from "../../wrapper_property"; -import { logger } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { AwsWrapperError } from "../../utils/errors"; -import { ClientWrapper } from "../../client_wrapper"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { RegionUtils } from "../../utils/region_utils"; -import { CanReleaseResources } from "../../can_release_resources"; +import { BaseSamlAuthPlugin } from "./saml_auth_plugin"; +import { IamAuthUtils } from "../../utils/iam_auth_utils"; -export class OktaAuthPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - protected static readonly tokenCache = new Map(); - private static readonly subscribedMethods = new Set(["connect", "forceConnect"]); - protected pluginService: PluginService; - protected rdsUtils = new RdsUtils(); - private readonly credentialsProviderFactory: CredentialsProviderFactory; - private readonly fetchTokenCounter: TelemetryCounter; - - constructor(pluginService: PluginService, credentialsProviderFactory: CredentialsProviderFactory) { - super(); - this.pluginService = pluginService; - this.credentialsProviderFactory = credentialsProviderFactory; - this.fetchTokenCounter = this.pluginService.getTelemetryFactory().createCounter("oktaAuth.fetchToken.count"); - } - - public getSubscribedMethods(): Set { - return OktaAuthPlugin.subscribedMethods; - } - - connect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - connectFunc: () => Promise - ): Promise { - return this.connectInternal(hostInfo, props, connectFunc); - } - - forceConnect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - connectFunc: () => Promise - ): Promise { - return this.connectInternal(hostInfo, props, connectFunc); - } - - async connectInternal(hostInfo: HostInfo, props: Map, connectFunc: () => Promise): Promise { - SamlUtils.checkIdpCredentialsWithFallback(props); - - const host = IamAuthUtils.getIamHost(props, hostInfo); - const port = IamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); - const region = RegionUtils.getRegion(props.get(WrapperProperties.IAM_REGION.name), host); - - const cacheKey = IamAuthUtils.getCacheKey(port, WrapperProperties.DB_USER.get(props), host, region); - const tokenInfo = OktaAuthPlugin.tokenCache.get(cacheKey); - - const isCachedToken = tokenInfo !== undefined && !tokenInfo.isExpired(); - - if (isCachedToken) { - logger.debug(Messages.get("AuthenticationToken.useCachedToken", tokenInfo.token)); - WrapperProperties.PASSWORD.set(props, tokenInfo.token); - } else { - await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host); - } - WrapperProperties.USER.set(props, WrapperProperties.DB_USER.get(props)); - this.pluginService.updateConfigWithProperties(props); - - try { - return await connectFunc(); - } catch (e: any) { - if (!this.pluginService.isLoginError(e as Error) || !isCachedToken) { - logger.debug(Messages.get("Authentication.connectError", e.message)); - throw e; - } - try { - await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host); - return await connectFunc(); - } catch (e: any) { - throw new AwsWrapperError(Messages.get("SamlAuthPlugin.unhandledError", e.message)); - } - } - } - - public async updateAuthenticationToken(hostInfo: HostInfo, props: Map, region: string, cacheKey: string, iamHost): Promise { - const tokenExpirationSec = WrapperProperties.IAM_TOKEN_EXPIRATION.get(props); - if (tokenExpirationSec < 0) { - throw new AwsWrapperError(Messages.get("AuthenticationToken.tokenExpirationLessThanZero")); - } - const tokenExpiry = Date.now() + tokenExpirationSec * 1000; - const port = IamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); - this.fetchTokenCounter.inc(); - const token = await IamAuthUtils.generateAuthenticationToken( - iamHost, - port, - region, - WrapperProperties.DB_USER.get(props), - await this.credentialsProviderFactory.getAwsCredentialsProvider(hostInfo.host, region, props), - this.pluginService - ); - logger.debug(Messages.get("AuthenticationToken.generatedNewToken", token)); - WrapperProperties.PASSWORD.set(props, token); - this.pluginService.updateConfigWithProperties(props); - OktaAuthPlugin.tokenCache.set(cacheKey, new TokenInfo(token, tokenExpiry)); - } - - releaseResources(): Promise { - OktaAuthPlugin.tokenCache.clear(); - return; +export class OktaAuthPlugin extends BaseSamlAuthPlugin { + constructor(pluginService: PluginService, credentialsProviderFactory: CredentialsProviderFactory, iamAuthUtils: IamAuthUtils = new IamAuthUtils()) { + super(pluginService, credentialsProviderFactory, "oktaAuth.fetchToken.count", iamAuthUtils); } } diff --git a/common/lib/plugins/federated_auth/okta_auth_plugin_factory.ts b/common/lib/plugins/federated_auth/okta_auth_plugin_factory.ts index c4b80147b..b3fc53329 100644 --- a/common/lib/plugins/federated_auth/okta_auth_plugin_factory.ts +++ b/common/lib/plugins/federated_auth/okta_auth_plugin_factory.ts @@ -15,16 +15,16 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class OktaAuthPluginFactory extends ConnectionPluginFactory { private static oktaAuthPlugin: any; private static oktaCredentialsProviderFactory: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!OktaAuthPluginFactory.oktaAuthPlugin) { OktaAuthPluginFactory.oktaAuthPlugin = await import("./okta_auth_plugin"); @@ -33,6 +33,7 @@ export class OktaAuthPluginFactory extends ConnectionPluginFactory { OktaAuthPluginFactory.oktaCredentialsProviderFactory = await import("./okta_credentials_provider_factory"); } + const pluginService = servicesContainer.pluginService; const oktaCredentialsProviderFactory = new OktaAuthPluginFactory.oktaCredentialsProviderFactory.OktaCredentialsProviderFactory(pluginService); return new OktaAuthPluginFactory.oktaAuthPlugin.OktaAuthPlugin(pluginService, oktaCredentialsProviderFactory); } catch (error: any) { diff --git a/common/lib/plugins/federated_auth/saml_auth_plugin.ts b/common/lib/plugins/federated_auth/saml_auth_plugin.ts new file mode 100644 index 000000000..7aa48db07 --- /dev/null +++ b/common/lib/plugins/federated_auth/saml_auth_plugin.ts @@ -0,0 +1,167 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; +import { PluginService } from "../../plugin_service"; +import { RdsUtils } from "../../utils/rds_utils"; +import { HostInfo } from "../../host_info"; +import { IamAuthUtils, TokenInfo } from "../../utils/iam_auth_utils"; +import { WrapperProperties } from "../../wrapper_property"; +import { logger } from "../../../logutils"; +import { AwsWrapperError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { CredentialsProviderFactory } from "./credentials_provider_factory"; +import { SamlUtils } from "../../utils/saml_utils"; +import { ClientWrapper } from "../../client_wrapper"; +import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; +import { RegionUtils } from "../../utils/region_utils"; +import { RdsUrlType } from "../../utils/rds_url_type"; +import { GDBRegionUtils } from "../../utils/gdb_region_utils"; +import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from "@smithy/types/dist-types/identity/awsCredentialIdentity"; + +export class BaseSamlAuthPlugin extends AbstractConnectionPlugin { + protected static readonly tokenCache = new Map(); + protected rdsUtils: RdsUtils = new RdsUtils(); + protected pluginService: PluginService; + private static readonly subscribedMethods = new Set(["connect", "forceConnect"]); + protected readonly credentialsProviderFactory: CredentialsProviderFactory; + protected readonly fetchTokenCounter: TelemetryCounter; + protected regionUtils: RegionUtils; + protected readonly tokenCacheInstance: Map; + + private readonly iamAuthUtils: IamAuthUtils; + + public getSubscribedMethods(): Set { + return BaseSamlAuthPlugin.subscribedMethods; + } + + protected constructor( + pluginService: PluginService, + credentialsProviderFactory: CredentialsProviderFactory, + telemetryCounterName: string, + iamAuthUtils: IamAuthUtils = new IamAuthUtils() + ) { + super(); + this.credentialsProviderFactory = credentialsProviderFactory; + this.pluginService = pluginService; + this.fetchTokenCounter = this.pluginService.getTelemetryFactory().createCounter(telemetryCounterName); + this.tokenCacheInstance = BaseSamlAuthPlugin.tokenCache; + this.iamAuthUtils = iamAuthUtils; + } + + connect( + hostInfo: HostInfo, + props: Map, + isInitialConnection: boolean, + connectFunc: () => Promise + ): Promise { + return this.connectInternal(hostInfo, props, connectFunc); + } + + forceConnect( + hostInfo: HostInfo, + props: Map, + isInitialConnection: boolean, + forceConnectFunc: () => Promise + ): Promise { + return this.connectInternal(hostInfo, props, forceConnectFunc); + } + + async connectInternal(hostInfo: HostInfo, props: Map, connectFunc: () => Promise): Promise { + SamlUtils.checkIdpCredentialsWithFallback(props); + + const host = this.iamAuthUtils.getIamHost(props, hostInfo); + const port = this.iamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); + + const type: RdsUrlType = this.rdsUtils.identifyRdsType(host.host); + + let credentialsProvider: AwsCredentialIdentity | AwsCredentialIdentityProvider | undefined = undefined; + if (type === RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER) { + credentialsProvider = await this.credentialsProviderFactory.getAwsCredentialsProvider(hostInfo.host, null, props); + } + + this.regionUtils = type == RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER ? new GDBRegionUtils(credentialsProvider) : new RegionUtils(); + const region: string | null = await this.regionUtils.getRegion(WrapperProperties.IAM_REGION.name, host, props); + + if (!region) { + throw new AwsWrapperError(Messages.get("SamlAuthPlugin.unableToDetermineRegion", WrapperProperties.IAM_REGION.name)); + } + + const cacheKey = this.iamAuthUtils.getCacheKey(port, WrapperProperties.DB_USER.get(props), host.host, region); + const tokenInfo = this.tokenCacheInstance.get(cacheKey); + + const isCachedToken: boolean = tokenInfo !== undefined && !tokenInfo.isExpired(); + + if (isCachedToken && tokenInfo) { + logger.debug(Messages.get("AuthenticationToken.useCachedToken", tokenInfo.token)); + WrapperProperties.PASSWORD.set(props, tokenInfo.token); + } else { + await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host.host, credentialsProvider); + } + WrapperProperties.USER.set(props, WrapperProperties.DB_USER.get(props)); + this.pluginService.updateConfigWithProperties(props); + + try { + return await connectFunc(); + } catch (e: any) { + if (!this.pluginService.isLoginError(e as Error) || !isCachedToken) { + throw e; + } + try { + await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host.host, credentialsProvider); + return await connectFunc(); + } catch (e: any) { + throw new AwsWrapperError(Messages.get("SamlAuthPlugin.unhandledError", e.message)); + } + } + } + + public async updateAuthenticationToken( + hostInfo: HostInfo, + props: Map, + region: string, + cacheKey: string, + iamHost: string, + credentials?: AwsCredentialIdentity | AwsCredentialIdentityProvider + ): Promise { + const tokenExpirationSec = WrapperProperties.IAM_TOKEN_EXPIRATION.get(props); + if (tokenExpirationSec < 0) { + throw new AwsWrapperError(Messages.get("AuthenticationToken.tokenExpirationLessThanZero")); + } + const tokenExpiry: number = Date.now() + tokenExpirationSec * 1000; + const port = this.iamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); + + this.fetchTokenCounter.inc(); + + const token = await this.iamAuthUtils.generateAuthenticationToken( + iamHost, + port, + region, + WrapperProperties.DB_USER.get(props), + credentials ?? (await this.credentialsProviderFactory.getAwsCredentialsProvider(hostInfo.host, region, props)), + this.pluginService + ); + + logger.debug(Messages.get("AuthenticationToken.generatedNewToken", token)); + WrapperProperties.PASSWORD.set(props, token); + this.pluginService.updateConfigWithProperties(props); + this.tokenCacheInstance.set(cacheKey, new TokenInfo(token, tokenExpiry)); + } + + static releaseResources(): void { + BaseSamlAuthPlugin.tokenCache.clear(); + } +} diff --git a/common/lib/plugins/federated_auth/saml_credentials_provider_factory.ts b/common/lib/plugins/federated_auth/saml_credentials_provider_factory.ts index bfdfe7f6d..e6dfc397c 100644 --- a/common/lib/plugins/federated_auth/saml_credentials_provider_factory.ts +++ b/common/lib/plugins/federated_auth/saml_credentials_provider_factory.ts @@ -17,7 +17,6 @@ import { CredentialsProviderFactory } from "./credentials_provider_factory"; import { AssumeRoleWithSAMLCommand, STSClient } from "@aws-sdk/client-sts"; import { WrapperProperties } from "../../wrapper_property"; - import { AwsWrapperError } from "../../utils/errors"; import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from "@smithy/types/dist-types/identity/awsCredentialIdentity"; import { decode } from "entities"; @@ -25,7 +24,7 @@ import { decode } from "entities"; export abstract class SamlCredentialsProviderFactory implements CredentialsProviderFactory { async getAwsCredentialsProvider( host: string, - region: string, + region: string | null, props: Map ): Promise { const samlAssertion = await this.getSamlAssertion(props); @@ -35,9 +34,7 @@ export abstract class SamlCredentialsProviderFactory implements CredentialsProvi PrincipalArn: WrapperProperties.IAM_IDP_ARN.get(props) }); - const stsClient = new STSClient({ - region: region - }); + const stsClient = region !== null ? new STSClient({ region }) : new STSClient(); const results = await stsClient.send(assumeRoleWithSamlRequest); const credentials = results["Credentials"]; diff --git a/common/lib/plugins/gdb_failover/global_db_failover_mode.ts b/common/lib/plugins/gdb_failover/global_db_failover_mode.ts new file mode 100644 index 000000000..975372eca --- /dev/null +++ b/common/lib/plugins/gdb_failover/global_db_failover_mode.ts @@ -0,0 +1,34 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +export enum GlobalDbFailoverMode { + STRICT_WRITER = "strict-writer", + STRICT_HOME_READER = "strict-home-reader", + STRICT_OUT_OF_HOME_READER = "strict-out-of-home-reader", + STRICT_ANY_READER = "strict-any-reader", + HOME_READER_OR_WRITER = "home-reader-or-writer", + OUT_OF_HOME_READER_OR_WRITER = "out-of-home-reader-or-writer", + ANY_READER_OR_WRITER = "any-reader-or-writer", + UNKNOWN = "unknown" +} + +export function globalDbFailoverModeFromValue(value: string | null | undefined): GlobalDbFailoverMode { + if (!value) { + return GlobalDbFailoverMode.UNKNOWN; + } + const normalized = value.toLowerCase(); + return Object.values(GlobalDbFailoverMode).find((v) => v === normalized) ?? GlobalDbFailoverMode.UNKNOWN; +} diff --git a/common/lib/plugins/gdb_failover/global_db_failover_plugin.ts b/common/lib/plugins/gdb_failover/global_db_failover_plugin.ts new file mode 100644 index 000000000..dd366c1c5 --- /dev/null +++ b/common/lib/plugins/gdb_failover/global_db_failover_plugin.ts @@ -0,0 +1,367 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { RdsUtils } from "../../utils/rds_utils"; +import { GlobalDbFailoverMode, globalDbFailoverModeFromValue } from "./global_db_failover_mode"; +import { HostInfo } from "../../host_info"; +import { WrapperProperties } from "../../wrapper_property"; +import { RdsUrlType } from "../../utils/rds_url_type"; +import { logger } from "../../../logutils"; +import { Messages } from "../../utils/messages"; +import { AwsTimeoutError, AwsWrapperError, FailoverFailedError, FailoverSuccessError, UnsupportedMethodError } from "../../utils/errors"; +import { ClientWrapper } from "../../client_wrapper"; +import { HostAvailability } from "../../host_availability/host_availability"; +import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; +import { HostRole } from "../../host_role"; +import { ReaderFailoverResult } from "../failover/reader_failover_result"; +import { containsHostAndPort, convertNanosToMs, equalsIgnoreCase, getTimeInNanos, getWriter, logTopology, sleep } from "../../utils/utils"; +import { Failover2Plugin } from "../failover2/failover2_plugin"; +import { FullServicesContainer } from "../../utils/full_services_container"; + +export class GlobalDbFailoverPlugin extends Failover2Plugin { + private static readonly TELEMETRY_FAILOVER = "failover"; + + protected activeHomeFailoverMode: GlobalDbFailoverMode = GlobalDbFailoverMode.UNKNOWN; + protected inactiveHomeFailoverMode: GlobalDbFailoverMode = GlobalDbFailoverMode.UNKNOWN; + protected homeRegion: string | null = null; + + constructor(servicesContainer: FullServicesContainer, properties: Map, rdsHelper: RdsUtils) { + super(servicesContainer, properties, rdsHelper); + } + + protected initFailoverMode(): void { + if (this.rdsUrlType !== null) { + return; + } + + const initialHostInfo = this.hostListProviderService?.getInitialConnectionHostInfo(); + if (!initialHostInfo) { + throw new AwsWrapperError(Messages.get("GlobalDbFailoverPlugin.missingInitialHost")); + } + + this.rdsUrlType = this.rdsHelper.identifyRdsType(initialHostInfo.host); + + this.homeRegion = WrapperProperties.FAILOVER_HOME_REGION.get(this.properties) ?? null; + if (!this.homeRegion) { + if (!this.rdsUrlType.hasRegion) { + throw new AwsWrapperError(Messages.get("GlobalDbFailoverPlugin.missingHomeRegion")); + } + this.homeRegion = this.rdsHelper.getRdsRegion(initialHostInfo.host); + if (!this.homeRegion) { + throw new AwsWrapperError(Messages.get("GlobalDbFailoverPlugin.missingHomeRegion")); + } + } + + logger.debug(Messages.get("Failover.parameterValue", "failoverHomeRegion", this.homeRegion)); + + const activeHomeMode = WrapperProperties.ACTIVE_HOME_FAILOVER_MODE.get(this.properties); + const inactiveHomeMode = WrapperProperties.INACTIVE_HOME_FAILOVER_MODE.get(this.properties); + + this.activeHomeFailoverMode = globalDbFailoverModeFromValue(activeHomeMode); + this.inactiveHomeFailoverMode = globalDbFailoverModeFromValue(inactiveHomeMode); + + if (this.activeHomeFailoverMode === GlobalDbFailoverMode.UNKNOWN) { + switch (this.rdsUrlType) { + case RdsUrlType.RDS_WRITER_CLUSTER: + case RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER: + this.activeHomeFailoverMode = GlobalDbFailoverMode.STRICT_WRITER; + break; + default: + this.activeHomeFailoverMode = GlobalDbFailoverMode.HOME_READER_OR_WRITER; + } + } + + if (this.inactiveHomeFailoverMode === GlobalDbFailoverMode.UNKNOWN) { + switch (this.rdsUrlType) { + case RdsUrlType.RDS_WRITER_CLUSTER: + case RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER: + this.inactiveHomeFailoverMode = GlobalDbFailoverMode.STRICT_WRITER; + break; + default: + this.inactiveHomeFailoverMode = GlobalDbFailoverMode.HOME_READER_OR_WRITER; + } + } + + logger.debug(Messages.get("Failover.parameterValue", "activeHomeFailoverMode", this.activeHomeFailoverMode)); + logger.debug(Messages.get("Failover.parameterValue", "inactiveHomeFailoverMode", this.inactiveHomeFailoverMode)); + } + + override async failover(): Promise { + const telemetryFactory = this.pluginService.getTelemetryFactory(); + const telemetryContext = telemetryFactory.openTelemetryContext(GlobalDbFailoverPlugin.TELEMETRY_FAILOVER, TelemetryTraceLevel.NESTED); + + const failoverStartTimeNs = getTimeInNanos(); + const failoverEndTimeNs = failoverStartTimeNs + BigInt(this.failoverTimeoutSettingMs) * BigInt(1_000_000); + + try { + await telemetryContext.start(async () => { + logger.info(Messages.get("GlobalDbFailoverPlugin.startFailover")); + + // Force refresh host list and wait for topology to stabilize + const refreshResult = await this.pluginService.forceMonitoringRefresh(true, this.failoverTimeoutSettingMs); + if (!refreshResult) { + this.failoverWriterTriggeredCounter.inc(); + this.failoverWriterFailedCounter.inc(); + logger.error(Messages.get("Failover.unableToRefreshHostList")); + throw new FailoverFailedError(Messages.get("Failover.unableToRefreshHostList")); + } + + const updatedHosts = this.pluginService.getAllHosts(); + const writerCandidate = getWriter(updatedHosts); + + if (!writerCandidate) { + this.failoverWriterTriggeredCounter.inc(); + this.failoverWriterFailedCounter.inc(); + const message = logTopology(updatedHosts, Messages.get("Failover.unableToDetermineWriter")); + logger.error(message); + throw new FailoverFailedError(message); + } + + // Check writer region to determine failover mode + const writerRegion = this.rdsHelper.getRdsRegion(writerCandidate.host); + const isHomeRegion = equalsIgnoreCase(this.homeRegion, writerRegion); + logger.debug(Messages.get("GlobalDbFailoverPlugin.isHomeRegion", String(isHomeRegion))); + + const currentFailoverMode = isHomeRegion ? this.activeHomeFailoverMode : this.inactiveHomeFailoverMode; + logger.debug(Messages.get("GlobalDbFailoverPlugin.currentFailoverMode", String(currentFailoverMode))); + + switch (currentFailoverMode) { + case GlobalDbFailoverMode.STRICT_WRITER: + await this.failoverToWriter(writerCandidate); + break; + case GlobalDbFailoverMode.STRICT_HOME_READER: + await this.failoverToAllowedHost( + () => this.pluginService.getHosts().filter((x) => x.role === HostRole.READER && this.isHostInHomeRegion(x)), + HostRole.READER, + failoverEndTimeNs + ); + break; + case GlobalDbFailoverMode.STRICT_OUT_OF_HOME_READER: + await this.failoverToAllowedHost( + () => this.pluginService.getHosts().filter((x) => x.role === HostRole.READER && !this.isHostInHomeRegion(x)), + HostRole.READER, + failoverEndTimeNs + ); + break; + case GlobalDbFailoverMode.STRICT_ANY_READER: + await this.failoverToAllowedHost( + () => this.pluginService.getHosts().filter((x) => x.role === HostRole.READER), + HostRole.READER, + failoverEndTimeNs + ); + break; + case GlobalDbFailoverMode.HOME_READER_OR_WRITER: + await this.failoverToAllowedHost( + () => + this.pluginService.getHosts().filter((x) => x.role === HostRole.WRITER || (x.role === HostRole.READER && this.isHostInHomeRegion(x))), + null, + failoverEndTimeNs + ); + break; + case GlobalDbFailoverMode.OUT_OF_HOME_READER_OR_WRITER: + await this.failoverToAllowedHost( + () => + this.pluginService + .getHosts() + .filter((x) => x.role === HostRole.WRITER || (x.role === HostRole.READER && !this.isHostInHomeRegion(x))), + null, + failoverEndTimeNs + ); + break; + case GlobalDbFailoverMode.ANY_READER_OR_WRITER: + await this.failoverToAllowedHost(() => [...this.pluginService.getHosts()], null, failoverEndTimeNs); + break; + case GlobalDbFailoverMode.UNKNOWN: + default: + throw new UnsupportedMethodError(`Unsupported failover mode: ${currentFailoverMode}`); + } + + logger.debug(Messages.get("Failover.establishedConnection", this.pluginService.getCurrentHostInfo()?.host ?? "unknown")); + this.throwFailoverSuccessException(); + }); + } finally { + logger.debug(Messages.get("GlobalDbFailoverPlugin.failoverElapsed", String(convertNanosToMs(getTimeInNanos() - failoverStartTimeNs)))); + + if (this.telemetryFailoverAdditionalTopTraceSetting && telemetryContext) { + await telemetryFactory.postCopy(telemetryContext, TelemetryTraceLevel.FORCE_TOP_LEVEL); + } + } + } + + private isHostInHomeRegion(host: HostInfo): boolean { + const hostRegion = this.rdsHelper.getRdsRegion(host.host); + return equalsIgnoreCase(hostRegion, this.homeRegion); + } + + protected async failoverToWriter(writerCandidate: HostInfo): Promise { + this.failoverWriterTriggeredCounter.inc(); + let writerCandidateConn: ClientWrapper | null = null; + + try { + const allowedHosts = this.pluginService.getHosts(); + if (!containsHostAndPort(allowedHosts, writerCandidate.hostAndPort)) { + this.failoverWriterFailedCounter.inc(); + const topologyString = logTopology(allowedHosts, ""); + logger.error(Messages.get("Failover.newWriterNotAllowed", writerCandidate.url, topologyString)); + throw new FailoverFailedError(Messages.get("Failover.newWriterNotAllowed", writerCandidate.url, topologyString)); + } + + try { + writerCandidateConn = await this.pluginService.connect(writerCandidate, this.properties, this); + } catch (error) { + this.failoverWriterFailedCounter.inc(); + logger.error(Messages.get("Failover.unableToConnectToWriterDueToError", writerCandidate.host, error.message)); + throw new FailoverFailedError(Messages.get("Failover.unableToConnectToWriterDueToError", writerCandidate.host, error.message)); + } + + const role = await this.pluginService.getHostRole(writerCandidateConn); + if (role !== HostRole.WRITER) { + await writerCandidateConn?.abort(); + writerCandidateConn = null; + this.failoverWriterFailedCounter.inc(); + logger.error(Messages.get("Failover.unexpectedReaderRole", writerCandidate.host)); + throw new FailoverFailedError(Messages.get("Failover.unexpectedReaderRole", writerCandidate.host)); + } + + await this.pluginService.setCurrentClient(writerCandidateConn, writerCandidate); + writerCandidateConn = null; // Prevent connection from being closed in finally block + + this.failoverWriterSuccessCounter.inc(); + } catch (ex) { + if (!(ex instanceof FailoverFailedError)) { + // Counter has already been incremented in Failover2Plugin before throwing the FailoverFailedError. + // So no need to increment again here. + this.failoverWriterFailedCounter.inc(); + } + throw ex; + } finally { + if (writerCandidateConn && this.pluginService.getCurrentClient().targetClient !== writerCandidateConn) { + await writerCandidateConn.abort(); + } + } + } + + protected async failoverToAllowedHost(getAllowedHosts: () => HostInfo[], verifyRole: HostRole | null, failoverEndTimeNs: bigint): Promise { + this.failoverReaderTriggeredCounter.inc(); + + let result: ReaderFailoverResult | null = null; + try { + try { + result = await this.getAllowedFailoverConnection(getAllowedHosts, verifyRole, failoverEndTimeNs); + await this.pluginService.setCurrentClient(result.client!, result.newHost!); + result = null; + } catch (e) { + if (e instanceof AwsTimeoutError) { + logger.error(Messages.get("Failover.unableToConnectToReader")); + throw new FailoverFailedError(Messages.get("Failover.unableToConnectToReader")); + } + throw e; + } + + logger.info(Messages.get("Failover.establishedConnection", this.pluginService.getCurrentHostInfo()?.host ?? "unknown")); + this.throwFailoverSuccessException(); + } catch (ex) { + if (ex instanceof FailoverSuccessError) { + this.failoverReaderSuccessCounter.inc(); + } else { + this.failoverReaderFailedCounter.inc(); + } + throw ex; + } finally { + if (result?.client && result?.client !== this.pluginService.getCurrentClient().targetClient) { + await result?.client.abort(); + } + } + } + + protected async getAllowedFailoverConnection( + getAllowedHosts: () => HostInfo[], + verifyRole: HostRole | null, + failoverEndTimeNs: bigint + ): Promise { + do { + await this.pluginService.refreshHostList(); + let updatedAllowedHosts = getAllowedHosts(); + + // Make a copy of hosts and set their availability + updatedAllowedHosts = updatedAllowedHosts.map((x) => + this.pluginService.getHostInfoBuilder().copyFrom(x).withAvailability(HostAvailability.AVAILABLE).build() + ); + + const remainingAllowedHosts = [...updatedAllowedHosts]; + + if (remainingAllowedHosts.length === 0) { + await sleep(100); + continue; + } + + while (remainingAllowedHosts.length > 0 && getTimeInNanos() < failoverEndTimeNs) { + let candidateHost: HostInfo | undefined; + try { + candidateHost = this.pluginService.getHostInfoByStrategy(verifyRole, this.failoverReaderHostSelectorStrategy, remainingAllowedHosts); + } catch { + // Strategy can't get a host according to requested conditions. + // Do nothing + } + + if (!candidateHost) { + logger.debug( + logTopology( + remainingAllowedHosts, + `${Messages.get("GlobalDbFailoverPlugin.unableToFindCandidateWithMatchingRole", String(verifyRole), this.failoverReaderHostSelectorStrategy)}` + ) + ); + await sleep(100); + break; + } + + let candidateConn: ClientWrapper | null = null; + try { + candidateConn = await this.pluginService.connect(candidateHost, this.properties, this); + // Since the roles in the host list might not be accurate, we execute a query to check the instance's role + const role = verifyRole === null ? null : await this.pluginService.getHostRole(candidateConn); + + if (verifyRole === null || verifyRole === role) { + const updatedHostSpec = this.pluginService + .getHostInfoBuilder() + .copyFrom(candidateHost) + .withRole(role ?? candidateHost.role) + .build(); + return new ReaderFailoverResult(candidateConn, updatedHostSpec, true); + } + + // The role is not as expected, so the connection is not valid + const index = remainingAllowedHosts.findIndex((h) => h.hostAndPort === candidateHost!.hostAndPort); + if (index !== -1) { + remainingAllowedHosts.splice(index, 1); + } + await candidateConn.abort(); + candidateConn = null; + } catch { + const index = remainingAllowedHosts.findIndex((h) => h.hostAndPort === candidateHost!.hostAndPort); + if (index !== -1) { + remainingAllowedHosts.splice(index, 1); + } + if (candidateConn) { + await candidateConn.abort(); + } + } + } + } while (getTimeInNanos() < failoverEndTimeNs); // All hosts failed. Keep trying until we hit the timeout. + + throw new AwsTimeoutError(Messages.get("Failover.failoverReaderTimeout")); + } +} diff --git a/common/lib/plugins/gdb_failover/global_db_failover_plugin_factory.ts b/common/lib/plugins/gdb_failover/global_db_failover_plugin_factory.ts new file mode 100644 index 000000000..259505096 --- /dev/null +++ b/common/lib/plugins/gdb_failover/global_db_failover_plugin_factory.ts @@ -0,0 +1,38 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ConnectionPluginFactory } from "../../plugin_factory"; +import { PluginService } from "../../plugin_service"; +import { ConnectionPlugin } from "../../connection_plugin"; +import { RdsUtils } from "../../utils/rds_utils"; +import { AwsWrapperError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; + +export class GlobalDbFailoverPluginFactory extends ConnectionPluginFactory { + private static globalDbFailoverPlugin: any; + + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { + try { + if (!GlobalDbFailoverPluginFactory.globalDbFailoverPlugin) { + GlobalDbFailoverPluginFactory.globalDbFailoverPlugin = await import("./global_db_failover_plugin"); + } + return new GlobalDbFailoverPluginFactory.globalDbFailoverPlugin.GlobalDbFailoverPlugin(servicesContainer, properties, new RdsUtils()); + } catch (error: any) { + throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "GlobalDbFailoverPlugin")); + } + } +} diff --git a/common/lib/plugins/limitless/limitless_connection_context.ts b/common/lib/plugins/limitless/limitless_connection_context.ts index ccc244c9b..5891c2da6 100644 --- a/common/lib/plugins/limitless/limitless_connection_context.ts +++ b/common/lib/plugins/limitless/limitless_connection_context.ts @@ -17,6 +17,7 @@ import { HostInfo } from "../../host_info"; import { ClientWrapper } from "../../client_wrapper"; import { ConnectionPlugin } from "../../connection_plugin"; +import Map from "@arrows/array/src/map"; export class LimitlessConnectionContext { private readonly hostInfo: HostInfo; @@ -25,6 +26,7 @@ export class LimitlessConnectionContext { private readonly connectFunc: () => Promise; private routers: HostInfo[] | null; private plugin: ConnectionPlugin; + private connectionHostInfo: HostInfo | null; constructor( hostInfo: HostInfo, @@ -40,6 +42,7 @@ export class LimitlessConnectionContext { this.connectFunc = connectFunc; this.routers = routers; this.plugin = plugin; + this.connectionHostInfo = null; } public getHostInfo(): HostInfo { @@ -58,6 +61,14 @@ export class LimitlessConnectionContext { return this.connectFunc; } + getConnectionHostInfo(): HostInfo | null { + return this.connectionHostInfo; + } + + setConnectionHostInfo(value: HostInfo | null) { + this.connectionHostInfo = value; + } + public getRouters(): HostInfo[] | null { return this.routers; } diff --git a/common/lib/plugins/limitless/limitless_connection_plugin.ts b/common/lib/plugins/limitless/limitless_connection_plugin.ts index 775c6cbd6..768674ae3 100644 --- a/common/lib/plugins/limitless/limitless_connection_plugin.ts +++ b/common/lib/plugins/limitless/limitless_connection_plugin.ts @@ -65,6 +65,9 @@ export class LimitlessConnectionPlugin extends AbstractConnectionPlugin { await this.limitlessRouterService.establishConnection(context); if (context.getConnection() != null) { + if (context.getConnectionHostInfo() != null) { + this.pluginService.setRoutedHostInfo(context.getConnectionHostInfo()); + } return context.getConnection(); } throw new AwsWrapperError(Messages.get("LimitlessConnectionPlugin.failedToConnectToHost", hostInfo.host)); diff --git a/common/lib/plugins/limitless/limitless_connection_plugin_factory.ts b/common/lib/plugins/limitless/limitless_connection_plugin_factory.ts index 330c02e63..b32677664 100644 --- a/common/lib/plugins/limitless/limitless_connection_plugin_factory.ts +++ b/common/lib/plugins/limitless/limitless_connection_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class LimitlessConnectionPluginFactory implements ConnectionPluginFactory { private static limitlessPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!LimitlessConnectionPluginFactory.limitlessPlugin) { LimitlessConnectionPluginFactory.limitlessPlugin = await import("./limitless_connection_plugin"); } - return new LimitlessConnectionPluginFactory.limitlessPlugin.LimitlessConnectionPlugin(pluginService, properties); + return new LimitlessConnectionPluginFactory.limitlessPlugin.LimitlessConnectionPlugin(servicesContainer.pluginService, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "LimitlessConnectionPlugin")); } diff --git a/common/lib/plugins/limitless/limitless_router_service.ts b/common/lib/plugins/limitless/limitless_router_service.ts index b33569c2f..46850838d 100644 --- a/common/lib/plugins/limitless/limitless_router_service.ts +++ b/common/lib/plugins/limitless/limitless_router_service.ts @@ -118,6 +118,7 @@ export class LimitlessRouterServiceImpl implements LimitlessRouterService { try { context.setConnection(await this.pluginService.connect(selectedHostSpec, context.getProperties(), context.getPlugin())); + context.setConnectionHostInfo(selectedHostSpec); } catch (e) { logger.debug(Messages.get("LimitlessRouterServiceImpl.failedToConnectToHost", selectedHostSpec.host)); selectedHostSpec.setAvailability(HostAvailability.NOT_AVAILABLE); @@ -161,12 +162,12 @@ export class LimitlessRouterServiceImpl implements LimitlessRouterService { } } - let selectedHostSpec: HostInfo | undefined = undefined; + let selectedHostInfo: HostInfo | undefined = undefined; try { // Select healthiest router for best chance of connection over load-balancing with round-robin - selectedHostSpec = this.pluginService.getHostInfoByStrategy(HostRole.WRITER, HighestWeightHostSelector.STRATEGY_NAME, context.getRouters()); - logger.debug(Messages.get("LimitlessRouterServiceImpl.selectedHostForRetry", selectedHostSpec ? selectedHostSpec.host : "undefined")); - if (!selectedHostSpec) { + selectedHostInfo = this.pluginService.getHostInfoByStrategy(HostRole.WRITER, HighestWeightHostSelector.STRATEGY_NAME, context.getRouters()); + logger.debug(Messages.get("LimitlessRouterServiceImpl.selectedHostForRetry", selectedHostInfo ? selectedHostInfo.host : "undefined")); + if (!selectedHostInfo) { continue; } } catch (e) { @@ -179,13 +180,14 @@ export class LimitlessRouterServiceImpl implements LimitlessRouterService { } try { - context.setConnection(await this.pluginService.connect(selectedHostSpec, context.getProperties(), context.getPlugin())); + context.setConnection(await this.pluginService.connect(selectedHostInfo, context.getProperties(), context.getPlugin())); if (context.getConnection()) { + context.setConnectionHostInfo(selectedHostInfo); return; } } catch (error) { - selectedHostSpec.setAvailability(HostAvailability.NOT_AVAILABLE); - logger.debug(Messages.get("LimitlessRouterServiceImpl.failedToConnectToHost", selectedHostSpec.host)); + selectedHostInfo.setAvailability(HostAvailability.NOT_AVAILABLE); + logger.debug(Messages.get("LimitlessRouterServiceImpl.failedToConnectToHost", selectedHostInfo.host)); } } throw new AwsWrapperError(Messages.get("LimitlessRouterServiceImpl.maxRetriesExceeded")); diff --git a/common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts new file mode 100644 index 000000000..14008e4aa --- /dev/null +++ b/common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts @@ -0,0 +1,310 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { CanReleaseResources } from "../../can_release_resources"; +import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; +import { ClientWrapper } from "../../client_wrapper"; +import { PluginService } from "../../plugin_service"; +import { HostListProviderService } from "../../host_list_provider_service"; +import { HostInfo } from "../../host_info"; +import { HostChangeOptions } from "../../host_change_options"; +import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; +import { Messages } from "../../utils/messages"; +import { logger } from "../../../logutils"; +import { HostRole } from "../../host_role"; +import { SqlMethodUtils } from "../../utils/sql_method_utils"; +import { FailoverError } from "../../utils/errors"; +import { WrapperProperties } from "../../wrapper_property"; +import { convertMsToNanos, getTimeInNanos, logAndThrowError } from "../../utils/utils"; +import { CacheItem } from "../../utils/cache_map"; + +export abstract class AbstractReadWriteSplittingPlugin extends AbstractConnectionPlugin implements CanReleaseResources { + private static readonly subscribedMethods: Set = new Set(["initHostProvider", "connect", "notifyConnectionChanged", "query"]); + + protected _hostListProviderService: HostListProviderService | undefined; + protected pluginService: PluginService; + protected readonly _properties: Map; + protected readerHostInfo?: HostInfo = undefined; + protected writerHostInfo?: HostInfo = undefined; + protected isReaderClientFromInternalPool: boolean = false; + protected isWriterClientFromInternalPool: boolean = false; + + protected writerTargetClient?: ClientWrapper = undefined; + protected readerCacheItem?: CacheItem = undefined; + protected readonly readerSelectorStrategy: string = ""; + + private _inReadWriteSplit = false; + + protected constructor(pluginService: PluginService, properties: Map) { + super(); + this.pluginService = pluginService; + this._properties = properties; + this.readerSelectorStrategy = WrapperProperties.READER_HOST_SELECTOR_STRATEGY.get(properties); + } + + override getSubscribedMethods(): Set { + return AbstractReadWriteSplittingPlugin.subscribedMethods; + } + + override initHostProvider( + hostInfo: HostInfo, + props: Map, + hostListProviderService: HostListProviderService, + initHostProviderFunc: () => void + ) { + this._hostListProviderService = hostListProviderService; + initHostProviderFunc(); + } + + override async notifyConnectionChanged(changes: Set): Promise { + try { + await this.updateInternalClientInfo(); + } catch (e) { + // pass + } + if (this._inReadWriteSplit) { + return Promise.resolve(OldConnectionSuggestionAction.PRESERVE); + } + return Promise.resolve(OldConnectionSuggestionAction.NO_OPINION); + } + + async updateInternalClientInfo(): Promise { + const currentTargetClient = this.pluginService.getCurrentClient().targetClient; + const currentHost = this.pluginService.getCurrentHostInfo(); + if (currentHost === null || currentTargetClient === null) { + return; + } + + if (this.shouldUpdateWriterClient(currentTargetClient, currentHost)) { + this.setWriterClient(currentTargetClient, currentHost); + } else if (this.shouldUpdateReaderClient(currentTargetClient, currentHost)) { + await this.setReaderClient(currentTargetClient, currentHost); + } + } + + setWriterClient(writerTargetClient: ClientWrapper | undefined, writerHostInfo: HostInfo): void { + this.writerTargetClient = writerTargetClient; + this.writerHostInfo = writerHostInfo; + logger.debug(Messages.get("ReadWriteSplittingPlugin.setWriterClient", writerHostInfo.hostAndPort)); + } + + async setReaderClient(readerTargetClient: ClientWrapper | undefined, readerHost: HostInfo): Promise { + await this.closeReaderClientIfIdle(); + this.readerCacheItem = new CacheItem(readerTargetClient, this.getKeepAliveTimeout(this.isReaderClientFromInternalPool)); + this.readerHostInfo = readerHost; + logger.debug(Messages.get("ReadWriteSplittingPlugin.setReaderClient", readerHost.hostAndPort)); + } + + async switchClientIfRequired(readOnly: boolean) { + const currentClient = this.pluginService.getCurrentClient(); + if (!(await currentClient.isValid())) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.setReadOnlyOnClosedClient", currentClient.targetClient?.id ?? "undefined client")); + } + + await this.refreshAndStoreTopology(currentClient.targetClient); + + const currentHost = this.pluginService.getCurrentHostInfo(); + if (currentHost == null) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.unavailableHostInfo")); + } else if (readOnly) { + if (!this.pluginService.isInTransaction() && currentHost.role != HostRole.READER) { + try { + await this.switchToReaderTargetClient(); + } catch (error: any) { + if (!(await currentClient.isValid())) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorSwitchingToReader", error.message)); + } + logger.warn(Messages.get("ReadWriteSplittingPlugin.fallbackToWriter", currentHost.url)); + } + } + } else if (currentHost.role != HostRole.WRITER) { + if (this.pluginService.isInTransaction()) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.setReadOnlyFalseInTransaction")); + } + try { + await this.switchToWriterTargetClient(); + } catch (error: any) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorSwitchingToWriter", error.message)); + } + } + } + + override async execute(methodName: string, executeFunc: () => Promise, methodArgs: any): Promise { + const statement = SqlMethodUtils.parseMethodArgs(methodArgs, this.pluginService.getDriverDialect()); + const statements = SqlMethodUtils.parseMultiStatementQueries(statement); + + const updateReadOnly: boolean | undefined = SqlMethodUtils.doesSetReadOnly(statements, this.pluginService.getDialect()); + if (updateReadOnly !== undefined) { + try { + await this.switchClientIfRequired(updateReadOnly); + } catch (error) { + await this.closeIdleClients(); + throw error; + } + } + + try { + return await executeFunc(); + } catch (error: any) { + if (error instanceof FailoverError) { + logger.debug(Messages.get("ReadWriteSplittingPlugin.failoverErrorWhileExecutingCommand", methodName)); + await this.closeIdleClients(); + } else { + logger.debug(Messages.get("ReadWriteSplittingPlugin.errorWhileExecutingCommand", methodName, error.message)); + } + + throw error; + } + } + + async switchCurrentTargetClientTo(newTargetClient: ClientWrapper | undefined, newClientHost: HostInfo | undefined) { + if (!newTargetClient || !newClientHost) { + return; + } + + const currentTargetClient = this.pluginService.getCurrentClient().targetClient; + + if (currentTargetClient === newTargetClient) { + return; + } + + try { + await this.pluginService.setCurrentClient(newTargetClient, newClientHost); + logger.debug(Messages.get("ReadWriteSplittingPlugin.settingCurrentClient", newTargetClient.id, newClientHost.hostAndPort)); + } catch (error) { + // pass + } + } + + async switchToWriterTargetClient() { + const currentHost = this.pluginService.getCurrentHostInfo(); + const currentClient = this.pluginService.getCurrentClient(); + if (this.isWriter(currentHost) && (await this.isTargetClientUsable(currentClient.targetClient))) { + // Already connected to writer. + return; + } + this._inReadWriteSplit = true; + if (!(await this.isTargetClientUsable(this.writerTargetClient))) { + await this.initializeWriterClient(); + } else { + await this.switchCurrentTargetClientTo(this.writerTargetClient, this.writerHostInfo); + } + + if (this.isReaderClientFromInternalPool) { + await this.closeReaderClientIfIdle(); + } + + logger.debug(Messages.get("ReadWriteSplittingPlugin.switchedFromReaderToWriter", this.writerHostInfo?.hostAndPort ?? "unknown")); + } + + async switchToReaderTargetClient() { + const currentHost = this.pluginService.getCurrentHostInfo(); + const currentClient = this.pluginService.getCurrentClient(); + if (this.isReader(currentHost) && currentClient) { + // Already connected to reader. + return; + } + + await this.closeReaderIfNecessary(); + + this._inReadWriteSplit = true; + if (this.readerCacheItem == null || !(await this.isTargetClientUsable(this.readerCacheItem.get()))) { + await this.initializeReaderClient(); + } else { + try { + await this.switchCurrentTargetClientTo(this.readerCacheItem.get(), this.readerHostInfo); + logger.debug(Messages.get("ReadWriteSplittingPlugin.switchedFromWriterToReader", this.readerHostInfo?.hostAndPort ?? "unknown")); + } catch (error: any) { + logger.debug( + Messages.get( + "ReadWriteSplittingPlugin.errorSwitchingToCachedReader", + this.readerHostInfo?.hostAndPort ?? "undefined reader host", + error.message + ) + ); + await this.closeReaderClientIfIdle(); + await this.initializeReaderClient(); + } + } + if (this.isWriterClientFromInternalPool) { + await this.closeWriterClientIfIdle(); + } + } + + async isTargetClientUsable(targetClient: ClientWrapper | undefined): Promise { + if (!targetClient) { + return Promise.resolve(false); + } + return await this.pluginService.isClientValid(targetClient); + } + + async closeWriterClientIfIdle() { + const currentTargetClient = this.pluginService.getCurrentClient().targetClient; + if (this.writerTargetClient == null || this.writerTargetClient === currentTargetClient) { + return; + } + + try { + await this.pluginService.abortTargetClient(this.writerTargetClient); + } catch (error) { + // ignore + } + this.writerTargetClient = undefined; + } + + async closeReaderClientIfIdle(): Promise { + const currentTargetClient = this.pluginService.getCurrentClient().targetClient; + const readerClient = this.readerCacheItem?.get(true); + if (readerClient == null || readerClient === currentTargetClient) { + return; + } + + try { + await this.pluginService.abortTargetClient(readerClient); + } catch (error) { + // ignore + } + this.readerCacheItem = undefined; + this.readerHostInfo = undefined; + } + + async closeIdleClients() { + await this.closeReaderClientIfIdle(); + await this.closeWriterClientIfIdle(); + } + + protected getKeepAliveTimeout(isPooledClient: boolean): bigint { + if (isPooledClient) { + return BigInt(0); + } + const keepAliveMs = WrapperProperties.CACHED_READER_KEEP_ALIVE_TIMEOUT.get(this._properties); + + return keepAliveMs > 0 ? getTimeInNanos() + convertMsToNanos(keepAliveMs) : BigInt(0); + } + + async releaseResources() { + await this.closeIdleClients(); + } + + protected abstract shouldUpdateReaderClient(currentClient: ClientWrapper | undefined, host: HostInfo): boolean; + protected abstract shouldUpdateWriterClient(currentClient: ClientWrapper | undefined, host: HostInfo): boolean; + protected abstract isWriter(currentHost: HostInfo): boolean; + protected abstract isReader(currentHost: HostInfo): boolean; + protected abstract refreshAndStoreTopology(currentClient: ClientWrapper | undefined): Promise; + protected abstract initializeWriterClient(): Promise; + protected abstract initializeReaderClient(): Promise; + protected abstract closeReaderIfNecessary(): Promise; +} diff --git a/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin.ts new file mode 100644 index 000000000..b9f4d94f5 --- /dev/null +++ b/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin.ts @@ -0,0 +1,96 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ReadWriteSplittingPlugin } from "./read_write_splitting_plugin"; +import { WrapperProperties } from "../../wrapper_property"; +import { HostInfo } from "../../host_info"; +import { RdsUtils } from "../../utils/rds_utils"; +import { ReadWriteSplittingError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { logger } from "../../../logutils"; +import { ClientWrapper } from "../../client_wrapper"; +import { equalsIgnoreCase } from "../../utils/utils"; + +export class GdbReadWriteSplittingPlugin extends ReadWriteSplittingPlugin { + protected readonly rdsUtils: RdsUtils = new RdsUtils(); + + protected restrictWriterToHomeRegion: boolean; + protected restrictReaderToHomeRegion: boolean; + + protected isInitialized: boolean = false; + protected homeRegion: string; + + protected initSettings(initHostInfo: HostInfo, properties: Map): void { + if (this.isInitialized) { + return; + } + this.restrictWriterToHomeRegion = WrapperProperties.GDB_RW_RESTRICT_WRITER_TO_HOME_REGION.get(properties); + this.restrictReaderToHomeRegion = WrapperProperties.GDB_RW_RESTRICT_READER_TO_HOME_REGION.get(properties); + + this.homeRegion = WrapperProperties.GDB_RW_HOME_REGION.get(properties); + if (!this.homeRegion) { + const rdsUrlType = this.rdsUtils.identifyRdsType(initHostInfo.host); + if (rdsUrlType.hasRegion) { + this.homeRegion = this.rdsUtils.getRdsRegion(initHostInfo.host); + } + } + + if (!this.homeRegion) { + throw new ReadWriteSplittingError(Messages.get("GdbReadWriteSplittingPlugin.missingHomeRegion", initHostInfo.host)); + } + + logger.debug(Messages.get("GdbReadWriteSplittingPlugin.parameterValue", "gdbRwHomeRegion", this.homeRegion)); + + this.isInitialized = true; + } + + override async connect( + hostInfo: HostInfo, + props: Map, + isInitialConnection: boolean, + connectFunc: () => Promise + ): Promise { + this.initSettings(hostInfo, props); + return super.connect(hostInfo, props, isInitialConnection, connectFunc); + } + + override setWriterClient(writerTargetClient: ClientWrapper | undefined, writerHostInfo: HostInfo) { + if ( + this.restrictWriterToHomeRegion && + writerHostInfo != null && + !equalsIgnoreCase(this.rdsUtils.getRdsRegion(writerHostInfo.host), this.homeRegion) + ) { + throw new ReadWriteSplittingError( + Messages.get("GdbReadWriteSplittingPlugin.cantConnectWriterOutOfHomeRegion", writerHostInfo.host, this.homeRegion) + ); + } + super.setWriterClient(writerTargetClient, writerHostInfo); + } + + protected getReaderHostCandidates(): HostInfo[] { + if (this.restrictReaderToHomeRegion) { + const hostsInRegion: HostInfo[] = this.pluginService + .getHosts() + .filter((x) => equalsIgnoreCase(this.rdsUtils.getRdsRegion(x.host), this.homeRegion)); + + if (hostsInRegion.length === 0) { + throw new ReadWriteSplittingError(Messages.get("GdbReadWriteSplittingPlugin.noAvailableReadersInHomeRegion", this.homeRegion)); + } + return hostsInRegion; + } + return super.getReaderHostCandidates(); + } +} diff --git a/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin_factory.ts b/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin_factory.ts new file mode 100644 index 000000000..7aa6d1ad5 --- /dev/null +++ b/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin_factory.ts @@ -0,0 +1,39 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ConnectionPluginFactory } from "../../plugin_factory"; +import { ConnectionPlugin } from "../../connection_plugin"; +import { AwsWrapperError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; + +export class GdbReadWriteSplittingPluginFactory extends ConnectionPluginFactory { + private static gdbReadWriteSplittingPlugin: any; + + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { + try { + if (!GdbReadWriteSplittingPluginFactory.gdbReadWriteSplittingPlugin) { + GdbReadWriteSplittingPluginFactory.gdbReadWriteSplittingPlugin = await import("./gdb_read_write_splitting_plugin"); + } + return new GdbReadWriteSplittingPluginFactory.gdbReadWriteSplittingPlugin.GdbReadWriteSplittingPlugin( + servicesContainer.pluginService, + properties + ); + } catch (error: any) { + throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "gdbReadWriteSplittingPlugin")); + } + } +} diff --git a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts new file mode 100644 index 000000000..2d70b8936 --- /dev/null +++ b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts @@ -0,0 +1,187 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { HostInfo, HostRole } from "../../index"; +import { PluginService } from "../../plugin_service"; +import { HostListProviderService } from "../../host_list_provider_service"; +import { Messages } from "../../utils/messages"; +import { ClientWrapper } from "../../client_wrapper"; +import { containsHostAndPort, getWriter, logAndThrowError, logTopology } from "../../utils/utils"; +import { AbstractReadWriteSplittingPlugin } from "./abstract_read_write_splitting_plugin"; +import { WrapperProperties } from "../../wrapper_property"; +import { logger } from "../../../logutils"; +import { CacheItem } from "../../utils/cache_map"; + +export class ReadWriteSplittingPlugin extends AbstractReadWriteSplittingPlugin { + protected hosts: HostInfo[] = []; + + constructor(pluginService: PluginService, properties: Map); + constructor( + pluginService: PluginService, + properties: Map, + hostListProviderService: HostListProviderService, + writerClient: ClientWrapper, + readerClient: ClientWrapper + ); + constructor( + pluginService: PluginService, + properties: Map, + hostListProviderService?: HostListProviderService, + writerClient?: ClientWrapper, + readerClient?: ClientWrapper + ) { + super(pluginService, properties); + this._hostListProviderService = hostListProviderService; + this.writerTargetClient = writerClient; + this.readerCacheItem = new CacheItem(readerClient, BigInt(0)); + } + + override async connect( + hostInfo: HostInfo, + props: Map, + isInitialConnection: boolean, + connectFunc: () => Promise + ): Promise { + if (!this.pluginService.acceptsStrategy(hostInfo.role, this.readerSelectorStrategy)) { + const message: string = Messages.get("ReadWriteSplittingPlugin.unsupportedHostSelectorStrategy", this.readerSelectorStrategy); + logAndThrowError(message); + } + + const result = await connectFunc(); + if (!isInitialConnection || !this._hostListProviderService?.isDynamicHostListProvider()) { + return result; + } + const currentRole = await this.pluginService.getHostRole(result); + + if (currentRole == HostRole.UNKNOWN) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorVerifyingInitialHostRole")); + } + const currentHost: HostInfo | null = this.pluginService.getInitialConnectionHostInfo(); + if (currentHost !== null) { + if (currentRole === currentHost.role) { + return result; + } + const updatedHost: HostInfo = Object.assign({}, currentHost); + updatedHost.role = currentRole ?? HostRole.UNKNOWN; + this._hostListProviderService?.setInitialConnectionHostInfo(updatedHost); + } + return result; + } + + protected override isWriter(currentHost?: HostInfo): boolean { + return currentHost?.role === HostRole.WRITER; + } + + protected override isReader(currentHost?: HostInfo): boolean { + return currentHost?.role === HostRole.READER; + } + + protected override async refreshAndStoreTopology(currentClient: ClientWrapper | undefined): Promise { + if (await this.pluginService.isClientValid(currentClient)) { + try { + await this.pluginService.refreshHostList(); + } catch { + // pass + } + } + + this.hosts = this.pluginService.getHosts(); + if (!this.hosts?.length) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.emptyHostList")); + } + + this.writerHostInfo = getWriter(this.hosts, Messages.get("ReadWriteSplittingPlugin.noWriterFound")); + } + + protected override async initializeWriterClient(): Promise { + const client: ClientWrapper = await this.connectToWriter(); + this.isWriterClientFromInternalPool = this.pluginService.isPooledClient(); + this.setWriterClient(client, this.writerHostInfo); + await this.switchCurrentTargetClientTo(this.writerTargetClient, this.writerHostInfo); + } + + private async connectToWriter() { + if (!this.writerHostInfo) { + throw new Error(Messages.get("ReadWriteSplittingPlugin.noWriterFound")); + } + const copyProps = new Map(this._properties); + copyProps.set(WrapperProperties.HOST.name, this.writerHostInfo.host); + return await this.pluginService.connect(this.writerHostInfo, copyProps, this); + } + + protected override async initializeReaderClient() { + if (this.hosts?.length === 1) { + if (!(await this.isTargetClientUsable(this.writerTargetClient))) { + await this.initializeWriterClient(); + } + logger.warn(Messages.get("ReadWriteSplittingPlugin.noReadersFound", this.writerHostInfo?.hostAndPort ?? "unknown")); + } else { + await this.getNewReaderClient(); + logger.debug(Messages.get("ReadWriteSplittingPlugin.switchedFromWriterToReader", this.readerHostInfo?.hostAndPort ?? "unknown")); + } + } + + protected override shouldUpdateReaderClient(currentClient: ClientWrapper | undefined, host: HostInfo): boolean { + return this.isReader(host); + } + + protected override shouldUpdateWriterClient(currentClient: ClientWrapper | undefined, host: HostInfo): boolean { + return this.isWriter(host); + } + + protected async getNewReaderClient() { + let targetClient = undefined; + let readerHost: HostInfo | undefined = undefined; + + const hostCandidates: HostInfo[] = this.getReaderHostCandidates(); + + const connectAttempts = hostCandidates.length * 2; + + for (let i = 0; i < connectAttempts; i++) { + const host = this.pluginService.getHostInfoByStrategy(HostRole.READER, this.readerSelectorStrategy, hostCandidates); + if (host) { + try { + const copyProps = new Map(this._properties); + copyProps.set(WrapperProperties.HOST.name, host.host); + targetClient = await this.pluginService.connect(host, copyProps, this); + this.isReaderClientFromInternalPool = this.pluginService.isPooledClient(); + readerHost = host; + break; + } catch (any) { + logger.warn(Messages.get("ReadWriteSplittingPlugin.failedToConnectToReader", host.hostAndPort)); + } + } + } + if (targetClient == undefined || readerHost === undefined) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.noReadersAvailable")); + return; + } + logger.debug(Messages.get("ReadWriteSplittingPlugin.successfullyConnectedToReader", readerHost.hostAndPort)); + await this.setReaderClient(targetClient, readerHost); + await this.switchCurrentTargetClientTo(this.readerCacheItem?.get(), this.readerHostInfo); + } + + protected override async closeReaderIfNecessary(): Promise { + if (this.readerHostInfo != null && !containsHostAndPort(this.hosts, this.readerHostInfo.hostAndPort)) { + logger.debug(Messages.get("ReadWriteSplittingPlugin.previousReaderNotAllowed", this.readerHostInfo.toString(), logTopology(this.hosts, ""))); + await this.closeReaderClientIfIdle(); + } + } + + protected getReaderHostCandidates(): HostInfo[] { + return this.pluginService.getHosts(); + } +} diff --git a/common/lib/plugins/read_write_splitting_plugin_factory.ts b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts similarity index 70% rename from common/lib/plugins/read_write_splitting_plugin_factory.ts rename to common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts index 18fd230b7..ac7fd53bd 100644 --- a/common/lib/plugins/read_write_splitting_plugin_factory.ts +++ b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts @@ -14,21 +14,21 @@ limitations under the License. */ -import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; -import { ConnectionPlugin } from "../connection_plugin"; -import { AwsWrapperError } from "../utils/errors"; -import { Messages } from "../utils/messages"; +import { ConnectionPluginFactory } from "../../plugin_factory"; +import { ConnectionPlugin } from "../../connection_plugin"; +import { AwsWrapperError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class ReadWriteSplittingPluginFactory extends ConnectionPluginFactory { private static readWriteSplittingPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!ReadWriteSplittingPluginFactory.readWriteSplittingPlugin) { ReadWriteSplittingPluginFactory.readWriteSplittingPlugin = await import("./read_write_splitting_plugin"); } - return new ReadWriteSplittingPluginFactory.readWriteSplittingPlugin.ReadWriteSplittingPlugin(pluginService, properties); + return new ReadWriteSplittingPluginFactory.readWriteSplittingPlugin.ReadWriteSplittingPlugin(servicesContainer.pluginService, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "readWriteSplittingPlugin")); } diff --git a/common/lib/plugins/read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting_plugin.ts deleted file mode 100644 index bb30b232c..000000000 --- a/common/lib/plugins/read_write_splitting_plugin.ts +++ /dev/null @@ -1,389 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { AbstractConnectionPlugin } from "../abstract_connection_plugin"; -import { HostInfo, FailoverError, HostRole } from "../"; -import { PluginService } from "../plugin_service"; -import { HostListProviderService } from "../host_list_provider_service"; -import { OldConnectionSuggestionAction } from "../old_connection_suggestion_action"; -import { HostChangeOptions } from "../host_change_options"; -import { WrapperProperties } from "../wrapper_property"; -import { Messages } from "../utils/messages"; -import { logger } from "../../logutils"; -import { SqlMethodUtils } from "../utils/sql_method_utils"; -import { ClientWrapper } from "../client_wrapper"; -import { getWriter, logAndThrowError } from "../utils/utils"; -import { CanReleaseResources } from "../can_release_resources"; -import { PoolClientWrapper } from "../pool_client_wrapper"; - -export class ReadWriteSplittingPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - private static readonly subscribedMethods: Set = new Set(["initHostProvider", "connect", "notifyConnectionChanged", "query"]); - private readonly readerSelectorStrategy: string = ""; - - private _hostListProviderService: HostListProviderService | undefined; - private pluginService: PluginService; - private readonly _properties: Map; - private _readerHostInfo?: HostInfo = undefined; - private isReaderClientFromInternalPool: boolean = false; - private isWriterClientFromInternalPool: boolean = false; - private _inReadWriteSplit = false; - writerTargetClient: ClientWrapper | undefined; - readerTargetClient: ClientWrapper | undefined; - - constructor(pluginService: PluginService, properties: Map); - constructor( - pluginService: PluginService, - properties: Map, - hostListProviderService: HostListProviderService, - writerClient: ClientWrapper, - readerClient: ClientWrapper - ); - constructor( - pluginService: PluginService, - properties: Map, - hostListProviderService?: HostListProviderService, - writerClient?: ClientWrapper, - readerClient?: ClientWrapper - ) { - super(); - this.pluginService = pluginService; - this._properties = properties; - this.readerSelectorStrategy = WrapperProperties.READER_HOST_SELECTOR_STRATEGY.get(properties); - this._hostListProviderService = hostListProviderService; - this.writerTargetClient = writerClient; - this.readerTargetClient = readerClient; - } - - override getSubscribedMethods(): Set { - return ReadWriteSplittingPlugin.subscribedMethods; - } - - override initHostProvider( - hostInfo: HostInfo, - props: Map, - hostListProviderService: HostListProviderService, - initHostProviderFunc: () => void - ) { - this._hostListProviderService = hostListProviderService; - initHostProviderFunc(); - } - - override notifyConnectionChanged(changes: Set): Promise { - try { - this.updateInternalClientInfo(); - } catch (e) { - // pass - } - if (this._inReadWriteSplit) { - return Promise.resolve(OldConnectionSuggestionAction.PRESERVE); - } - return Promise.resolve(OldConnectionSuggestionAction.NO_OPINION); - } - - updateInternalClientInfo(): void { - const currentTargetClient = this.pluginService.getCurrentClient().targetClient; - const currentHost = this.pluginService.getCurrentHostInfo(); - if (currentHost === null || currentTargetClient === null) { - return; - } - - if (currentHost.role === HostRole.WRITER) { - this.setWriterClient(currentTargetClient, currentHost); - } else { - this.setReaderClient(currentTargetClient, currentHost); - } - } - - override async connect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - connectFunc: () => Promise - ): Promise { - if (!this.pluginService.acceptsStrategy(hostInfo.role, this.readerSelectorStrategy)) { - const message: string = Messages.get("ReadWriteSplittingPlugin.unsupportedHostSelectorStrategy", this.readerSelectorStrategy); - logAndThrowError(message); - } - - const result = await connectFunc(); - if (!isInitialConnection || this._hostListProviderService?.isStaticHostListProvider()) { - return result; - } - const currentRole = this.pluginService.getCurrentHostInfo()?.role; - - if (currentRole == HostRole.UNKNOWN) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorVerifyingInitialHostRole")); - } - const currentHost: HostInfo | null = this.pluginService.getInitialConnectionHostInfo(); - if (currentHost !== null) { - if (currentRole === currentHost.role) { - return result; - } - const updatedHost: HostInfo = Object.assign({}, currentHost); - updatedHost.role = currentRole ?? HostRole.UNKNOWN; - this._hostListProviderService?.setInitialConnectionHostInfo(updatedHost); - } - return result; - } - - override async execute(methodName: string, executeFunc: () => Promise, methodArgs: any): Promise { - const statement = SqlMethodUtils.parseMethodArgs(methodArgs, this.pluginService.getDriverDialect()); - const statements = SqlMethodUtils.parseMultiStatementQueries(statement); - - const updateReadOnly: boolean | undefined = SqlMethodUtils.doesSetReadOnly(statements, this.pluginService.getDialect()); - if (updateReadOnly !== undefined) { - try { - await this.switchClientIfRequired(updateReadOnly); - } catch (error) { - await this.closeIdleClients(); - throw error; - } - } - - try { - return await executeFunc(); - } catch (error: any) { - if (error instanceof FailoverError) { - logger.debug(Messages.get("ReadWriteSplittingPlugin.failoverErrorWhileExecutingCommand", methodName)); - await this.closeIdleClients(); - } else { - logger.debug(Messages.get("ReadWriteSplittingPlugin.errorWhileExecutingCommand", methodName, error.message)); - } - - throw error; - } - } - - setWriterClient(writerTargetClient: ClientWrapper | undefined, writerHostInfo: HostInfo): void { - this.writerTargetClient = writerTargetClient; - logger.debug(Messages.get("ReadWriteSplittingPlugin.setWriterClient", writerHostInfo.url)); - } - - setReaderClient(readerTargetClient: ClientWrapper | undefined, readerHost: HostInfo): void { - this.readerTargetClient = readerTargetClient; - this._readerHostInfo = readerHost; - logger.debug(Messages.get("ReadWriteSplittingPlugin.setReaderClient", readerHost.url)); - } - - async getNewWriterClient(writerHost: HostInfo) { - const props = new Map(this._properties); - props.set(WrapperProperties.HOST.name, writerHost.host); - try { - const copyProps = new Map(props); - WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.set(copyProps, false); - const targetClient = await this.pluginService.connect(writerHost, copyProps, this); - this.isWriterClientFromInternalPool = targetClient instanceof PoolClientWrapper; - this.setWriterClient(targetClient, writerHost); - await this.switchCurrentTargetClientTo(this.writerTargetClient, writerHost); - } catch (any) { - logger.warn(Messages.get("ReadWriteSplittingPlugin.failedToConnectToWriter", writerHost.url)); - } - } - - async switchClientIfRequired(readOnly: boolean) { - const currentClient = this.pluginService.getCurrentClient(); - if (!(await currentClient.isValid())) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.setReadOnlyOnClosedClient", currentClient.targetClient?.id ?? "undefined client")); - } - try { - await this.pluginService.refreshHostList(); - } catch { - // pass - } - - const hosts: HostInfo[] = this.pluginService.getHosts(); - if (hosts == null || hosts.length === 0) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.emptyHostList")); - } - const currentHost = this.pluginService.getCurrentHostInfo(); - if (currentHost == null) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.unavailableHostInfo")); - } else if (readOnly) { - if (!this.pluginService.isInTransaction() && currentHost.role != HostRole.READER) { - try { - await this.switchToReaderTargetClient(hosts); - } catch (error: any) { - if (!(await currentClient.isValid())) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorSwitchingToReader", error.message)); - } - logger.warn(Messages.get("ReadWriteSplittingPlugin.fallbackToWriter", currentHost.url)); - } - } - } else if (currentHost.role != HostRole.WRITER) { - if (this.pluginService.isInTransaction()) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.setReadOnlyFalseInTransaction")); - } - try { - await this.switchToWriterTargetClient(hosts); - } catch (error: any) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorSwitchingToWriter", error.message)); - } - } - } - - async switchCurrentTargetClientTo(newTargetClient: ClientWrapper | undefined, newClientHost: HostInfo | undefined) { - const currentTargetClient = this.pluginService.getCurrentClient().targetClient; - - if (currentTargetClient === newTargetClient) { - return; - } - if (newClientHost && newTargetClient) { - try { - await this.pluginService.setCurrentClient(newTargetClient, newClientHost); - logger.debug(Messages.get("ReadWriteSplittingPlugin.settingCurrentClient", newTargetClient.id, newClientHost.url)); - } catch (error) { - // pass - } - } - } - - async initializeReaderClient(hosts: HostInfo[]) { - if (hosts.length === 1) { - const writerHost = getWriter(hosts, Messages.get("ReadWriteSplittingPlugin.noWriterFound")); - if (writerHost) { - if (!(await this.isTargetClientUsable(this.writerTargetClient))) { - await this.getNewWriterClient(writerHost); - } - logger.warn(Messages.get("ReadWriteSplittingPlugin.noReadersFound", writerHost.url)); - } - } else { - await this.getNewReaderClient(); - } - } - - async getNewReaderClient() { - let targetClient = undefined; - let readerHost: HostInfo | undefined = undefined; - const connectAttempts = this.pluginService.getHosts().length; - - for (let i = 0; i < connectAttempts; i++) { - const host = this.pluginService.getHostInfoByStrategy(HostRole.READER, this.readerSelectorStrategy); - if (host) { - const props = new Map(this._properties); - props.set(WrapperProperties.HOST.name, host.host); - - try { - const copyProps = new Map(props); - WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.set(copyProps, false); - targetClient = await this.pluginService.connect(host, copyProps, this); - this.isReaderClientFromInternalPool = targetClient instanceof PoolClientWrapper; - readerHost = host; - break; - } catch (any) { - logger.warn(Messages.get("ReadWriteSplittingPlugin.failedToConnectToReader", host.url)); - } - } - } - if (targetClient == undefined || readerHost === undefined) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.noReadersAvailable")); - return; - } - logger.debug(Messages.get("ReadWriteSplittingPlugin.successfullyConnectedToReader", readerHost.url)); - this.setReaderClient(targetClient, readerHost); - await this.switchCurrentTargetClientTo(this.readerTargetClient, this._readerHostInfo); - } - - async switchToWriterTargetClient(hosts: HostInfo[]) { - const currentHost = this.pluginService.getCurrentHostInfo(); - const currentClient = this.pluginService.getCurrentClient(); - if (currentHost !== null && currentHost?.role === HostRole.WRITER && (await currentClient.isValid())) { - return; - } - this._inReadWriteSplit = true; - const writerHost = getWriter(hosts, Messages.get("ReadWriteSplittingPlugin.noWriterFound")); - if (!writerHost) { - return; - } - if (!(await this.isTargetClientUsable(this.writerTargetClient))) { - await this.getNewWriterClient(writerHost); - } else if (this.writerTargetClient) { - await this.switchCurrentTargetClientTo(this.writerTargetClient, writerHost); - } - - logger.debug(Messages.get("ReadWriteSplittingPlugin.switchedFromReaderToWriter", writerHost.url)); - if (this.isReaderClientFromInternalPool) { - await this.closeTargetClientIfIdle(this.readerTargetClient); - } - } - - async switchToReaderTargetClient(hosts: HostInfo[]) { - const currentHost = this.pluginService.getCurrentHostInfo(); - const currentClient = this.pluginService.getCurrentClient(); - if (currentHost !== null && currentHost?.role === HostRole.READER && currentClient) { - return; - } - - if (this._readerHostInfo && !hosts.some((hostInfo: HostInfo) => hostInfo.host === this._readerHostInfo?.host)) { - // The old reader cannot be used anymore because it is no longer in the list of allowed hosts. - await this.closeTargetClientIfIdle(this.readerTargetClient); - } - - this._inReadWriteSplit = true; - if (!(await this.isTargetClientUsable(this.readerTargetClient))) { - await this.initializeReaderClient(hosts); - } else if (this.readerTargetClient != null && this._readerHostInfo != null) { - try { - await this.switchCurrentTargetClientTo(this.readerTargetClient, this._readerHostInfo); - logger.debug(Messages.get("ReadWriteSplittingPlugin.switchedFromWriterToReader", this._readerHostInfo.url)); - } catch (error: any) { - logger.debug(Messages.get("ReadWriteSplittingPlugin.errorSwitchingToCachedReader", this._readerHostInfo.url)); - await this.pluginService.abortTargetClient(this.readerTargetClient); - this.readerTargetClient = undefined; - this._readerHostInfo = undefined; - await this.initializeReaderClient(hosts); - } - } - if (this.isWriterClientFromInternalPool) { - await this.closeTargetClientIfIdle(this.writerTargetClient); - } - } - - async isTargetClientUsable(targetClient: ClientWrapper | undefined): Promise { - if (!targetClient) { - return Promise.resolve(false); - } - return await this.pluginService.isClientValid(targetClient); - } - - async closeTargetClientIfIdle(internalTargetClient: ClientWrapper | undefined) { - const currentTargetClient = this.pluginService.getCurrentClient().targetClient; - try { - if (internalTargetClient != null && internalTargetClient !== currentTargetClient && (await this.isTargetClientUsable(internalTargetClient))) { - await this.pluginService.abortTargetClient(internalTargetClient); - - if (internalTargetClient === this.writerTargetClient) { - this.writerTargetClient = undefined; - } - if (internalTargetClient === this.readerTargetClient) { - this.readerTargetClient = undefined; - this._readerHostInfo = undefined; - } - } - } catch (error) { - // ignore - } - } - - async closeIdleClients() { - logger.debug(Messages.get("ReadWriteSplittingPlugin.closingInternalClients")); - await this.closeTargetClientIfIdle(this.readerTargetClient); - await this.closeTargetClientIfIdle(this.writerTargetClient); - } - - async releaseResources() { - await this.closeIdleClients(); - } -} diff --git a/common/lib/plugins/stale_dns/stale_dns_helper.ts b/common/lib/plugins/stale_dns/stale_dns_helper.ts index 501fd5bfc..d1b715350 100644 --- a/common/lib/plugins/stale_dns/stale_dns_helper.ts +++ b/common/lib/plugins/stale_dns/stale_dns_helper.ts @@ -14,28 +14,26 @@ limitations under the License. */ -import { logger } from "../../../logutils"; +import { levels, logger } from "../../../logutils"; import { HostInfo } from "../../host_info"; import { HostListProviderService } from "../../host_list_provider_service"; import { HostRole } from "../../host_role"; import { PluginService } from "../../plugin_service"; import { Messages } from "../../utils/messages"; import { RdsUtils } from "../../utils/rds_utils"; -import { lookup, LookupAddress } from "dns"; -import { promisify } from "util"; -import { AwsWrapperError } from "../../utils/errors"; import { HostChangeOptions } from "../../host_change_options"; import { WrapperProperties } from "../../wrapper_property"; import { ClientWrapper } from "../../client_wrapper"; -import { getWriter, logTopology } from "../../utils/utils"; +import { containsHostAndPort, getWriter, logTopology } from "../../utils/utils"; import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; +import { RdsUrlType } from "../../utils/rds_url_type"; +import { AwsWrapperError } from "../../utils/errors"; export class StaleDnsHelper { private readonly pluginService: PluginService; private readonly rdsUtils: RdsUtils = new RdsUtils(); private writerHostInfo: HostInfo | null = null; - private writerHostAddress: string = ""; private readonly telemetryFactory: TelemetryFactory; private readonly staleDNSDetectedCounter: TelemetryCounter; @@ -53,39 +51,44 @@ export class StaleDnsHelper { props: Map, connectFunc: () => Promise ): Promise { - if (!this.rdsUtils.isWriterClusterDns(host)) { - return connectFunc(); - } + const type: RdsUrlType = this.rdsUtils.identifyRdsType(host); - const currentTargetClient = await connectFunc(); - - let clusterInetAddress = ""; - try { - const lookupResult = await this.lookupResult(host); - clusterInetAddress = lookupResult.address; - } catch (error) { - // ignore + if (type !== RdsUrlType.RDS_WRITER_CLUSTER && type !== RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER) { + return connectFunc(); } - const hostInetAddress = clusterInetAddress; - logger.debug(Messages.get("StaleDnsHelper.clusterEndpointDns", hostInetAddress)); - - if (!clusterInetAddress) { - return currentTargetClient; + if (type === RdsUrlType.RDS_WRITER_CLUSTER) { + const writer = getWriter(this.pluginService.getAllHosts()); + if (writer != null && this.rdsUtils.isRdsInstance(writer.host)) { + if ( + isInitialConnection && + WrapperProperties.SKIP_INACTIVE_WRITER_CLUSTER_CHECK.get(props) && + !this.rdsUtils.isSameRegion(writer.host, host) + ) { + // The cluster writer endpoint belongs to a different region than the current writer region. + // It means that the cluster is Aurora Global Database and cluster writer endpoint is in secondary region. + // In this case the cluster writer endpoint is in inactive state and doesn't represent the current writer + // so any connection check should be skipped. + // Continue with a normal workflow. + return connectFunc(); + } + } else { + // No writer is available. It could be the case with the first connection when topology isn't yet available. + // Continue with a normal workflow. + return connectFunc(); + } } - const currentHostInfo = this.pluginService.getCurrentHostInfo(); - if (!currentHostInfo) { - throw new AwsWrapperError("Stale DNS Helper: Current hostInfo was null."); - } + const currentTargetClient = await connectFunc(); - if (currentHostInfo && currentHostInfo.role === HostRole.READER) { + const isConnectedToReader: boolean = (await this.pluginService.getHostRole(currentTargetClient)) === HostRole.READER; + if (isConnectedToReader) { // This is if-statement is only reached if the connection url is a writer cluster endpoint. // If the new connection resolves to a reader instance, this means the topology is outdated. // Force refresh to update the topology. - await this.pluginService.forceRefreshHostList(currentTargetClient); + await this.pluginService.forceRefreshHostList(); } else { - await this.pluginService.refreshHostList(currentTargetClient); + await this.pluginService.refreshHostList(); } logger.debug(logTopology(this.pluginService.getAllHosts(), "[StaleDnsHelper.getVerifiedConnection] ")); @@ -104,27 +107,18 @@ export class StaleDnsHelper { return currentTargetClient; } - if (!this.writerHostAddress) { - try { - const lookupResult = await this.lookupResult(this.writerHostInfo.host); - this.writerHostAddress = lookupResult.address; - } catch (error) { - // ignore - } - } - - logger.debug(Messages.get("StaleDnsHelper.writerInetAddress", this.writerHostAddress)); - - if (!this.writerHostAddress) { - return currentTargetClient; - } + if (isConnectedToReader) { + // Reconnect to writer host if current connection is reader. - if (this.writerHostAddress !== clusterInetAddress) { - // DNS resolves a cluster endpoint to a wrong writer - // opens a connection to a proper writer host logger.debug(Messages.get("StaleDnsHelper.staleDnsDetected", this.writerHostInfo.host)); this.staleDNSDetectedCounter.inc(); + const allowedHosts: HostInfo[] = this.pluginService.getHosts(); + + if (!containsHostAndPort(allowedHosts, this.writerHostInfo.hostAndPort)) { + throw new AwsWrapperError(Messages.get("StaleDnsHelper.currentWriterNotAllowed", this.writerHostInfo.host, logTopology(allowedHosts, ""))); + } + let targetClient = null; try { const newProps = new Map(props); @@ -149,7 +143,7 @@ export class StaleDnsHelper { } for (const [key, values] of changes.entries()) { - if (logger.level === "debug") { + if (levels[logger.level] <= levels.debug) { const valStr = Array.from(values) .map((x) => HostChangeOptions[x]) .join(", "); @@ -159,14 +153,9 @@ export class StaleDnsHelper { if (key === this.writerHostInfo.url && values.has(HostChangeOptions.PROMOTED_TO_READER)) { logger.debug(Messages.get("StaleDnsHelper.reset")); this.writerHostInfo = null; - this.writerHostAddress = ""; } } } return Promise.resolve(); } - - lookupResult(host: string): Promise { - return promisify(lookup)(host, {}); - } } diff --git a/common/lib/plugins/stale_dns/stale_dns_plugin_factory.ts b/common/lib/plugins/stale_dns/stale_dns_plugin_factory.ts index 8669e59d4..2c6d67403 100644 --- a/common/lib/plugins/stale_dns/stale_dns_plugin_factory.ts +++ b/common/lib/plugins/stale_dns/stale_dns_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class StaleDnsPluginFactory extends ConnectionPluginFactory { private static staleDnsPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!StaleDnsPluginFactory.staleDnsPlugin) { StaleDnsPluginFactory.staleDnsPlugin = await import("./stale_dns_plugin"); } - return new StaleDnsPluginFactory.staleDnsPlugin.StaleDnsPlugin(pluginService, properties); + return new StaleDnsPluginFactory.staleDnsPlugin.StaleDnsPlugin(servicesContainer.pluginService, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "StaleDnsPlugin")); } diff --git a/common/lib/plugins/strategy/fastest_response/fastest_response_strategy_plugin.ts b/common/lib/plugins/strategy/fastest_response/fastest_response_strategy_plugin.ts index e0d3a1748..54e10a7a9 100644 --- a/common/lib/plugins/strategy/fastest_response/fastest_response_strategy_plugin.ts +++ b/common/lib/plugins/strategy/fastest_response/fastest_response_strategy_plugin.ts @@ -26,6 +26,7 @@ import { HostChangeOptions } from "../../../host_change_options"; import { RandomHostSelector } from "../../../random_host_selector"; import { Messages } from "../../../utils/messages"; import { equalsIgnoreCase, logAndThrowError } from "../../../utils/utils"; +import { FullServicesContainer } from "../../../utils/full_services_container"; export class FastestResponseStrategyPlugin extends AbstractConnectionPlugin { static readonly FASTEST_RESPONSE_STRATEGY_NAME: string = "fastestResponse"; @@ -43,13 +44,13 @@ export class FastestResponseStrategyPlugin extends AbstractConnectionPlugin { private pluginService: PluginService; private randomHostSelector: RandomHostSelector = new RandomHostSelector(); - constructor(pluginService: PluginService, properties: Map, hostResponseTimeService?: HostResponseTimeService) { + constructor(servicesContainer: FullServicesContainer, properties: Map, hostResponseTimeService?: HostResponseTimeService) { super(); - this.pluginService = pluginService; + this.pluginService = servicesContainer.pluginService; this.properties = properties; this.hostResponseTimeService = hostResponseTimeService ?? - new HostResponseTimeServiceImpl(pluginService, properties, WrapperProperties.RESPONSE_MEASUREMENT_INTERVAL_MILLIS.get(this.properties)); + new HostResponseTimeServiceImpl(servicesContainer, properties, WrapperProperties.RESPONSE_MEASUREMENT_INTERVAL_MILLIS.get(this.properties)); this.cacheExpirationNanos = BigInt(WrapperProperties.RESPONSE_MEASUREMENT_INTERVAL_MILLIS.get(this.properties) * 1_000_000); } @@ -91,18 +92,13 @@ export class FastestResponseStrategyPlugin extends AbstractConnectionPlugin { if (!this.acceptsStrategy(role, strategy)) { logAndThrowError(Messages.get("FastestResponseStrategyPlugin.unsupportedHostSelectorStrategy", strategy)); } - // The cache holds a host with the fastest response time. - // If the cache doesn't have a host for a role, it's necessary to find the fastest host in the topology. const fastestResponseHost: HostInfo = FastestResponseStrategyPlugin.cachedFastestResponseHostByRole.get(role); if (fastestResponseHost) { - // Found the fastest host. Find the host in the latest topology. const foundHost = this.pluginService.getHosts().find((host) => host === fastestResponseHost); if (foundHost) { - // Found a host in the topology. return foundHost; } } - // Cached result isn't available. Need to find the fastest response time host. const calculatedFastestResponseHost: ResponseTimeTuple[] = this.pluginService .getHosts() .filter((host) => role === host.role) @@ -113,8 +109,6 @@ export class FastestResponseStrategyPlugin extends AbstractConnectionPlugin { const calculatedHost = calculatedFastestResponseHost.length === 0 ? null : calculatedFastestResponseHost[0]; if (!calculatedHost) { - // Unable to identify the fastest response host. - // As a last resort, let's use a random host selector. return this.randomHostSelector.getHost(hosts, role, this.properties); } FastestResponseStrategyPlugin.cachedFastestResponseHostByRole.put(role, calculatedHost.hostInfo, Number(this.cacheExpirationNanos)); diff --git a/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts b/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts index 05146d74b..340e6f720 100644 --- a/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts +++ b/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../../../plugin_factory"; -import { PluginService } from "../../../plugin_service"; import { ConnectionPlugin } from "../../../connection_plugin"; import { AwsWrapperError } from "../../../utils/errors"; import { Messages } from "../../../utils/messages"; +import { FullServicesContainer } from "../../../utils/full_services_container"; export class FastestResponseStrategyPluginFactory extends ConnectionPluginFactory { private static fastestResponseStrategyPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin) { FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin = await import("./fastest_response_strategy_plugin"); } - return new FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin.FastestResponseStrategyPlugin(pluginService, properties); + return new FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin.FastestResponseStrategyPlugin(servicesContainer, properties); } catch (error: any) { throw new AwsWrapperError( Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "FastestResponseStrategyPluginFactory") diff --git a/common/lib/plugins/strategy/fastest_response/host_response_time_monitor.ts b/common/lib/plugins/strategy/fastest_response/host_response_time_monitor.ts index 5e0a8b809..050747423 100644 --- a/common/lib/plugins/strategy/fastest_response/host_response_time_monitor.ts +++ b/common/lib/plugins/strategy/fastest_response/host_response_time_monitor.ts @@ -17,66 +17,95 @@ import { HostInfo } from "../../../host_info"; import { PluginService } from "../../../plugin_service"; import { TelemetryFactory } from "../../../utils/telemetry/telemetry_factory"; -import { sleep } from "../../../utils/utils"; +import { sleepWithAbort } from "../../../utils/utils"; import { logger } from "../../../../logutils"; import { Messages } from "../../../utils/messages"; import { TelemetryTraceLevel } from "../../../utils/telemetry/telemetry_trace_level"; import { ClientWrapper } from "../../../client_wrapper"; -import { TelemetryContext } from "../../../utils/telemetry/telemetry_context"; import { WrapperProperties } from "../../../wrapper_property"; +import { AbstractMonitor } from "../../../utils/monitoring/monitor"; +import { FullServicesContainer } from "../../../utils/full_services_container"; -export class HostResponseTimeMonitor { - static readonly MONITORING_PROPERTY_PREFIX = "frt_"; +export class ResponseTimeHolder { + private readonly url: string; + private readonly responseTime: number; + + constructor(url: string, responseTime: number) { + this.url = url; + this.responseTime = responseTime; + } + + getUrl(): string { + return this.url; + } + + getResponseTime(): number { + return this.responseTime; + } +} + +export class HostResponseTimeMonitor extends AbstractMonitor { static readonly NUM_OF_MEASURES = 5; + private static readonly TERMINATION_TIMEOUT_SEC = 5; private readonly intervalMs: number; private readonly hostInfo: HostInfo; - private stopped = false; private responseTimeMs = Number.MAX_SAFE_INTEGER; - private checkTimestamp = Date.now(); private readonly properties: Map; - private pluginService: PluginService; - private telemetryFactory: TelemetryFactory; + private readonly servicesContainer: FullServicesContainer; + private readonly pluginService: PluginService; + private readonly telemetryFactory: TelemetryFactory; protected monitoringClient: ClientWrapper | null = null; + private abortSleep?: () => void; - constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, intervalMs: number) { - this.pluginService = pluginService; + constructor(servicesContainer: FullServicesContainer, hostInfo: HostInfo, properties: Map, intervalMs: number) { + super(HostResponseTimeMonitor.TERMINATION_TIMEOUT_SEC); + this.servicesContainer = servicesContainer; + this.pluginService = servicesContainer.pluginService; this.hostInfo = hostInfo; this.properties = properties; this.intervalMs = intervalMs; this.telemetryFactory = this.pluginService.getTelemetryFactory(); - const hostId: string = this.hostInfo.hostId ?? this.getHostInfo().host; - /** - * Report current response time (in milliseconds) to telemetry engine. - * Report -1 if response time couldn't be measured. - */ - this.telemetryFactory.createGauge(`frt.response.time.${hostId}`, () => this.getResponseTime() == Number.MAX_SAFE_INTEGER); - this.run(); + const hostId: string = this.hostInfo.hostId ?? this.hostInfo.host; + this.telemetryFactory.createGauge(`frt.response.time.${hostId}`, () => + this.responseTimeMs === Number.MAX_SAFE_INTEGER ? -1 : this.responseTimeMs + ); } - getResponseTime() { + getResponseTime(): number { return this.responseTimeMs; } - getCheckTimeStamp() { - return this.checkTimestamp; - } - - getHostInfo() { + getHostInfo(): HostInfo { return this.hostInfo; } async close(): Promise { - this.stopped = true; - await sleep(500); - logger.debug(Messages.get("HostResponseTimeMonitor.stopped", this.hostInfo.host)); + if (this.abortSleep) { + try { + this.abortSleep(); + } catch (error) { + // ignore + } + this.abortSleep = undefined; + } + if (this.monitoringClient) { + try { + await this.monitoringClient.abort(); + } catch (error) { + // ignore + } + this.monitoringClient = null; + } } - async run(): Promise { - const telemetryContext: TelemetryContext = this.telemetryFactory.openTelemetryContext("host response time task", TelemetryTraceLevel.TOP_LEVEL); + async monitor(): Promise { + const telemetryContext = this.telemetryFactory.openTelemetryContext("host response time task", TelemetryTraceLevel.TOP_LEVEL); telemetryContext.setAttribute("url", this.hostInfo.host); - while (!this.stopped) { + + while (!this._stop) { + this.lastActivityTimestampNanos = BigInt(Date.now() * 1_000_000); await telemetryContext.start(async () => { try { await this.openConnection(); @@ -84,33 +113,29 @@ export class HostResponseTimeMonitor { let responseTimeSum = 0; let count = 0; for (let i = 0; i < HostResponseTimeMonitor.NUM_OF_MEASURES; i++) { - if (this.stopped) { + if (this._stop) { break; } const startTime = Date.now(); if (await this.pluginService.isClientValid(this.monitoringClient)) { - const responseTime = Date.now() - startTime; - responseTimeSum += responseTime; + responseTimeSum += Date.now() - startTime; count++; } } if (count > 0) { this.responseTimeMs = responseTimeSum / count; + this.servicesContainer.storageService.set(this.hostInfo.url, new ResponseTimeHolder(this.hostInfo.url, this.responseTimeMs)); } else { this.responseTimeMs = Number.MAX_SAFE_INTEGER; + this.servicesContainer.storageService.remove(ResponseTimeHolder, this.hostInfo.url); } - this.checkTimestamp = Date.now(); logger.debug(Messages.get("HostResponseTimeMonitor.responseTime", this.hostInfo.host, this.responseTimeMs.toString())); } - await sleep(this.intervalMs); + const [sleepPromise, abortFn] = sleepWithAbort(this.intervalMs); + this.abortSleep = abortFn as () => void; + await sleepPromise; } catch (error) { logger.debug(Messages.get("HostResponseTimeMonitor.interruptedErrorDuringMonitoring", this.hostInfo.host, error.message)); - } finally { - this.stopped = true; - if (this.monitoringClient) { - await this.monitoringClient.abort(); - } - this.monitoringClient = null; } }); } @@ -119,8 +144,7 @@ export class HostResponseTimeMonitor { async openConnection(): Promise { try { if (this.monitoringClient) { - const clientIsValid = await this.pluginService.isClientValid(this.monitoringClient); - if (clientIsValid) { + if (await this.pluginService.isClientValid(this.monitoringClient)) { return; } } diff --git a/common/lib/plugins/strategy/fastest_response/host_response_time_service.ts b/common/lib/plugins/strategy/fastest_response/host_response_time_service.ts index 627a69b88..be6025e14 100644 --- a/common/lib/plugins/strategy/fastest_response/host_response_time_service.ts +++ b/common/lib/plugins/strategy/fastest_response/host_response_time_service.ts @@ -15,80 +15,65 @@ */ import { HostInfo } from "../../../host_info"; -import { PluginService } from "../../../plugin_service"; -import { TelemetryFactory } from "../../../utils/telemetry/telemetry_factory"; -import { SlidingExpirationCache } from "../../../utils/sliding_expiration_cache"; -import { HostResponseTimeMonitor } from "./host_response_time_monitor"; +import { HostResponseTimeMonitor, ResponseTimeHolder } from "./host_response_time_monitor"; +import { FullServicesContainer } from "../../../utils/full_services_container"; +import { MonitorErrorResponse } from "../../../utils/monitoring/monitor"; export interface HostResponseTimeService { - /** - * Return a response time in milliseconds to the host. - * Return Number.MAX_SAFE_INTEGER if response time is not available. - * - * @param hostInfo the host details - * @return response time in milliseconds for a desired host. - */ getResponseTime(hostInfo: HostInfo): number; - - /** - * Provides an updated host list to a service. - */ setHosts(hosts: HostInfo[]): void; } export class HostResponseTimeServiceImpl implements HostResponseTimeService { - static readonly CACHE_EXPIRATION_NANOS: bigint = BigInt(10 * 60_000_000_000); // 10 minutes - static readonly CACHE_CLEANUP_NANOS: bigint = BigInt(60_000_000_000); // 1 minute + private static readonly MONITOR_DISPOSAL_TIME_NANOS: bigint = BigInt(10 * 60_000_000_000); // 10 minutes + private static readonly INACTIVE_TIMEOUT_NANOS: bigint = BigInt(3 * 60_000_000_000); // 3 minutes - private readonly pluginService: PluginService; - readonly properties: Map; - readonly intervalMs: number; - protected hosts: HostInfo[]; - private readonly telemetryFactory: TelemetryFactory; - protected static monitoringHosts: SlidingExpirationCache = new SlidingExpirationCache( - HostResponseTimeServiceImpl.CACHE_CLEANUP_NANOS, - undefined, - async (monitor: HostResponseTimeMonitor) => { - { - try { - await monitor.close(); - } catch (error) { - // ignore - } - } - } - ); + private readonly servicesContainer: FullServicesContainer; + private readonly properties: Map; + private readonly intervalMs: number; + private hosts: HostInfo[] = []; - constructor(pluginService: PluginService, properties: Map, intervalMs: number) { - this.pluginService = pluginService; + constructor(servicesContainer: FullServicesContainer, properties: Map, intervalMs: number) { + this.servicesContainer = servicesContainer; this.properties = properties; this.intervalMs = intervalMs; - this.telemetryFactory = this.pluginService.getTelemetryFactory(); - HostResponseTimeServiceImpl.monitoringHosts.cleanupIntervalNs = BigInt(intervalMs) ?? HostResponseTimeServiceImpl.CACHE_CLEANUP_NANOS; - this.telemetryFactory.createGauge("frt.hosts.count", () => HostResponseTimeServiceImpl.monitoringHosts.size); + + this.servicesContainer.storageService.registerItemClassIfAbsent( + ResponseTimeHolder, + true, + HostResponseTimeServiceImpl.MONITOR_DISPOSAL_TIME_NANOS, + null, + null + ); + + this.servicesContainer.monitorService.registerMonitorTypeIfAbsent( + HostResponseTimeMonitor, + HostResponseTimeServiceImpl.MONITOR_DISPOSAL_TIME_NANOS, + HostResponseTimeServiceImpl.INACTIVE_TIMEOUT_NANOS, + new Set([MonitorErrorResponse.RECREATE]), + ResponseTimeHolder + ); } getResponseTime(hostInfo: HostInfo): number { - const monitor: HostResponseTimeMonitor = HostResponseTimeServiceImpl.monitoringHosts.get( - hostInfo.url, - HostResponseTimeServiceImpl.CACHE_EXPIRATION_NANOS - ); - if (!monitor) { - return Number.MAX_SAFE_INTEGER; - } - return monitor.getResponseTime(); + const holder: ResponseTimeHolder | null = this.servicesContainer.storageService.get(ResponseTimeHolder, hostInfo.url); + return holder === null ? Number.MAX_SAFE_INTEGER : holder.getResponseTime(); } setHosts(hosts: HostInfo[]): void { - const oldHostMap: string[] = hosts.flatMap((host) => host.url); + const oldHostUrls: Set = new Set(this.hosts.map((host) => host.url)); + this.hosts = hosts; + + const servicesContainer = this.servicesContainer; + const properties = this.properties; + const intervalMs = this.intervalMs; + hosts - .filter((hostInfo: HostInfo) => !(hostInfo.url in oldHostMap)) - .forEach((hostInfo: HostInfo) => { - HostResponseTimeServiceImpl.monitoringHosts.computeIfAbsent( - hostInfo.url, - (key) => new HostResponseTimeMonitor(this.pluginService, hostInfo, this.properties, this.intervalMs), - HostResponseTimeServiceImpl.CACHE_EXPIRATION_NANOS - ); + .filter((hostInfo) => !oldHostUrls.has(hostInfo.url)) + .forEach((hostInfo) => { + servicesContainer.monitorService.runIfAbsent(HostResponseTimeMonitor, hostInfo.url, servicesContainer, properties, { + createMonitor: (sc: FullServicesContainer) => new HostResponseTimeMonitor(sc, hostInfo, properties, intervalMs) + }); }); } } diff --git a/common/lib/profile/driver_configuration_profiles.ts b/common/lib/profile/driver_configuration_profiles.ts index ee387b82f..a9e2ffb62 100644 --- a/common/lib/profile/driver_configuration_profiles.ts +++ b/common/lib/profile/driver_configuration_profiles.ts @@ -17,10 +17,10 @@ import { ConfigurationProfile } from "./configuration_profile"; import { ConfigurationProfilePresetCodes } from "./configuration_profile_codes"; import { WrapperProperties } from "../wrapper_property"; -import { HostMonitoringPluginFactory } from "../plugins/efm/host_monitoring_plugin_factory"; +import { HostMonitoringPluginFactory } from "../plugins/efm/v1/host_monitoring_plugin_factory"; import { AuroraInitialConnectionStrategyFactory } from "../plugins/aurora_initial_connection_strategy_plugin_factory"; import { AuroraConnectionTrackerPluginFactory } from "../plugins/connection_tracker/aurora_connection_tracker_plugin_factory"; -import { ReadWriteSplittingPluginFactory } from "../plugins/read_write_splitting_plugin_factory"; +import { ReadWriteSplittingPluginFactory } from "../plugins/read_write_splitting/read_write_splitting_plugin_factory"; import { FailoverPluginFactory } from "../plugins/failover/failover_plugin_factory"; import { InternalPooledConnectionProvider } from "../internal_pooled_connection_provider"; import { AwsPoolConfig } from "../aws_pool_config"; diff --git a/common/lib/random_host_selector.ts b/common/lib/random_host_selector.ts index d38d985c2..5d35f0ebf 100644 --- a/common/lib/random_host_selector.ts +++ b/common/lib/random_host_selector.ts @@ -25,7 +25,9 @@ export class RandomHostSelector implements HostSelector { public static STRATEGY_NAME = "random"; getHost(hosts: HostInfo[], role: HostRole, props?: Map): HostInfo { - const eligibleHosts = hosts.filter((hostInfo: HostInfo) => hostInfo.role === role && hostInfo.getAvailability() === HostAvailability.AVAILABLE); + const eligibleHosts = hosts.filter( + (hostInfo: HostInfo) => (role === null || hostInfo.role === role) && hostInfo.getAvailability() === HostAvailability.AVAILABLE + ); if (eligibleHosts.length === 0) { throw new AwsWrapperError(Messages.get("HostSelector.noHostsMatchingRole", role)); } diff --git a/common/lib/round_robin_host_selector.ts b/common/lib/round_robin_host_selector.ts index bc53a251c..d6fcd6216 100644 --- a/common/lib/round_robin_host_selector.ts +++ b/common/lib/round_robin_host_selector.ts @@ -31,7 +31,7 @@ export class RoundRobinHostSelector implements HostSelector { getHost(hosts: HostInfo[], role: HostRole, props?: Map): HostInfo { const eligibleHosts: HostInfo[] = hosts - .filter((host: HostInfo) => host.role === role && host.availability === HostAvailability.AVAILABLE) + .filter((host: HostInfo) => (role === null || host.role === role) && host.availability === HostAvailability.AVAILABLE) .sort((hostA: HostInfo, hostB: HostInfo) => { const hostAHostName = hostA.host.toLowerCase(); const hostBHostName = hostB.host.toLowerCase(); diff --git a/common/lib/session_state_client.ts b/common/lib/session_state_client.ts index 8c978f22e..cd660158e 100644 --- a/common/lib/session_state_client.ts +++ b/common/lib/session_state_client.ts @@ -19,11 +19,11 @@ import { TransactionIsolationLevel } from "./utils/transaction_isolation_level"; export interface SessionStateClient { setReadOnly(readOnly: boolean): Promise; - isReadOnly(): boolean; + isReadOnly(): boolean | undefined; setAutoCommit(autoCommit: boolean): Promise; - getAutoCommit(): boolean; + getAutoCommit(): boolean | undefined; setTransactionIsolation(level: TransactionIsolationLevel): Promise; @@ -31,9 +31,9 @@ export interface SessionStateClient { setSchema(schema: any): Promise; - getSchema(): string; + getSchema(): string | undefined; setCatalog(catalog: string): Promise; - getCatalog(): string; + getCatalog(): string | undefined; } diff --git a/common/lib/types.ts b/common/lib/types.ts new file mode 100644 index 000000000..a89e6af4b --- /dev/null +++ b/common/lib/types.ts @@ -0,0 +1,33 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/** + * Type representing a constructor for any class. + */ +export type Constructor = new (...args: unknown[]) => T; + +/** + * Function type that determines whether an item should be disposed when expired. + * @param item The item to check + * @returns true if the item should be disposed, false otherwise + */ +export type ShouldDisposeFunc = (item: V) => boolean; + +/** + * Function type that defines how to dispose of an item when it is removed. + * @param item The item to dispose + */ +export type ItemDisposalFunc = (item: V) => void; diff --git a/common/lib/utils/cache_map.ts b/common/lib/utils/cache_map.ts index 8728dcabf..69650202d 100644 --- a/common/lib/utils/cache_map.ts +++ b/common/lib/utils/cache_map.ts @@ -14,17 +14,40 @@ limitations under the License. */ -class CacheItem { +import { getTimeInNanos } from "./utils"; + +export class CacheItem { readonly item: V; - private readonly expirationTimeNanos: bigint; + private _expirationTimeNs: bigint; - constructor(item: V, expirationTimeNanos: bigint) { + constructor(item: V, expirationTime: bigint) { this.item = item; - this.expirationTimeNanos = expirationTimeNanos; + this._expirationTimeNs = expirationTime; } isExpired(): boolean { - return process.hrtime.bigint() > this.expirationTimeNanos; + if (this._expirationTimeNs <= 0) { + // No expiration time. + return false; + } + return getTimeInNanos() > this._expirationTimeNs; + } + + get(returnExpired: boolean = false): V | null { + return this.isExpired() && !returnExpired ? null : this.item; + } + + updateExpiration(expirationIntervalNanos: bigint): CacheItem { + this._expirationTimeNs = getTimeInNanos() + expirationIntervalNanos; + return this; + } + + get expirationTimeNs(): bigint { + return this._expirationTimeNs; + } + + toString(): string { + return `CacheItem [item=${this.item}, expirationTime=${this._expirationTimeNs}]`; } } @@ -38,7 +61,7 @@ export class CacheMap { get(key: K, defaultItemValue?: any, itemExpirationNano?: any): V | null { const cacheItem: CacheItem | undefined = this.cache.get(key); if (cacheItem && !cacheItem.isExpired()) { - return cacheItem.item; + return cacheItem.get(); } if (!defaultItemValue || !itemExpirationNano) { @@ -74,14 +97,6 @@ export class CacheMap { this.cache.clear(); } - getEntries(): Map { - const entries: Map = new Map(); - this.cache.forEach((v, k) => { - entries.set(k, v.item); - }); - return entries; - } - protected cleanUp() { if (this.cleanupTimeNanos < process.hrtime.bigint()) { this.cleanupTimeNanos = process.hrtime.bigint() + this.cleanupIntervalNanos; diff --git a/common/lib/utils/core_services_container.ts b/common/lib/utils/core_services_container.ts new file mode 100644 index 000000000..c3bed54d5 --- /dev/null +++ b/common/lib/utils/core_services_container.ts @@ -0,0 +1,53 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { StorageService, StorageServiceImpl } from "./storage/storage_service"; +import { MonitorService, MonitorServiceImpl } from "./monitoring/monitor_service"; +import { EventPublisher } from "./events/event"; +import { BatchingEventPublisher } from "./events/batching_event_publisher"; + +/** + * A singleton container object used to instantiate and access core universal services. This class should be used + * instead of directly instantiating core services so that only one instance of each service is instantiated. + * + * @see FullServicesContainer for a container that holds both connection-specific services and core universal + * services. + */ +export class CoreServicesContainer { + private static readonly INSTANCE = new CoreServicesContainer(); + + readonly monitorService: MonitorService; + readonly storageService: StorageService; + readonly eventPublisher: EventPublisher; + + private constructor() { + this.eventPublisher = new BatchingEventPublisher(); + this.storageService = new StorageServiceImpl(this.eventPublisher); + this.monitorService = new MonitorServiceImpl(this.eventPublisher); + } + + static getInstance(): CoreServicesContainer { + return CoreServicesContainer.INSTANCE; + } + + static async releaseResources(): Promise { + await CoreServicesContainer.INSTANCE.storageService.releaseResources(); + await CoreServicesContainer.INSTANCE.monitorService.releaseResources(); + if (CoreServicesContainer.INSTANCE.eventPublisher instanceof BatchingEventPublisher) { + CoreServicesContainer.INSTANCE.eventPublisher.releaseResources(); + } + } +} diff --git a/common/lib/utils/errors.ts b/common/lib/utils/errors.ts index 4d854c0c5..58e37aec1 100644 --- a/common/lib/utils/errors.ts +++ b/common/lib/utils/errors.ts @@ -48,8 +48,12 @@ export class FailoverFailedError extends FailoverError {} export class TransactionResolutionUnknownError extends FailoverError {} +export class ReadWriteSplittingError extends AwsWrapperError {} + export class LoginError extends AwsWrapperError {} -export class InternalQueryTimeoutError extends AwsWrapperError {} +export class AwsTimeoutError extends AwsWrapperError {} + +export class InternalQueryTimeoutError extends AwsTimeoutError {} export class UnavailableHostError extends AwsWrapperError {} diff --git a/common/lib/utils/events/batching_event_publisher.ts b/common/lib/utils/events/batching_event_publisher.ts new file mode 100644 index 000000000..750afdeef --- /dev/null +++ b/common/lib/utils/events/batching_event_publisher.ts @@ -0,0 +1,99 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Event, EventClass, EventPublisher, EventSubscriber } from "./event"; +import { Messages } from "../messages"; +import { logger } from "../../../logutils"; + +const DEFAULT_MESSAGE_INTERVAL_MS = 30_000; // 30 seconds + +/** + * An event publisher that periodically publishes a batch of all unique events + * encountered during the latest time interval. + */ +export class BatchingEventPublisher implements EventPublisher { + protected readonly subscribersMap = new Map>(); + protected readonly pendingEvents = new Set(); + protected publishingInterval?: ReturnType; + + constructor(messageIntervalMs: number = DEFAULT_MESSAGE_INTERVAL_MS) { + this.initPublishingInterval(messageIntervalMs); + } + + protected initPublishingInterval(messageIntervalMs: number): void { + this.publishingInterval = setInterval(() => this.sendMessages(), messageIntervalMs); + // Unref the timer to prevent this background task from blocking the application from gracefully exiting. + this.publishingInterval.unref(); + } + + protected async sendMessages(): Promise { + for (const event of this.pendingEvents) { + this.pendingEvents.delete(event); + await this.deliverEvent(event); + } + } + + protected async deliverEvent(event: Event): Promise { + const subscribers = this.subscribersMap.get(event.constructor as EventClass); + if (!subscribers) { + return; + } + + for (const subscriber of subscribers) { + await subscriber.processEvent(event); + } + } + + subscribe(subscriber: EventSubscriber, eventClasses: Set): void { + for (const eventClass of eventClasses) { + let subscribers = this.subscribersMap.get(eventClass); + if (!subscribers) { + subscribers = new Set(); + this.subscribersMap.set(eventClass, subscribers); + } + subscribers.add(subscriber); + } + } + + unsubscribe(subscriber: EventSubscriber, eventClasses: Set): void { + for (const eventClass of eventClasses) { + const subscribers = this.subscribersMap.get(eventClass); + if (subscribers) { + subscribers.delete(subscriber); + if (subscribers.size === 0) { + this.subscribersMap.delete(eventClass); + } + } + } + } + + publish(event: Event): void { + if (event.isImmediateDelivery) { + this.deliverEvent(event).catch((err) => { + logger.debug(Messages.get("BatchingEventPublisher.errorDeliveringImmediateEvent", err?.message ?? String(err))); + }); + } else { + this.pendingEvents.add(event); + } + } + + releaseResources(): void { + if (this.publishingInterval) { + clearInterval(this.publishingInterval); + this.publishingInterval = undefined; + } + } +} diff --git a/common/lib/utils/events/data_access_event.ts b/common/lib/utils/events/data_access_event.ts new file mode 100644 index 000000000..bb58a168d --- /dev/null +++ b/common/lib/utils/events/data_access_event.ts @@ -0,0 +1,35 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Event } from "./event"; + +/** + * A class defining a data access event. The class specifies the class of the data + * that was accessed and the key for the data. + * + * Used by StorageService to notify MonitorService when data is accessed, + * allowing monitors to extend their expiration time. + */ +export class DataAccessEvent implements Event { + readonly isImmediateDelivery = false; + readonly dataClass: new (...args: any[]) => any; + readonly key: unknown; + + constructor(dataClass: new (...args: any[]) => any, key: unknown) { + this.dataClass = dataClass; + this.key = key; + } +} diff --git a/common/lib/utils/events/event.ts b/common/lib/utils/events/event.ts new file mode 100644 index 000000000..8a69c643f --- /dev/null +++ b/common/lib/utils/events/event.ts @@ -0,0 +1,64 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Constructor } from "../../types"; + +export type EventClass = Constructor; + +/** + * An interface for events that need to be communicated between different components. + */ +export interface Event { + readonly isImmediateDelivery: boolean; +} + +/** + * An event subscriber. Subscribers can subscribe to a publisher's events. + */ +export interface EventSubscriber { + /** + * Processes an event. This method will only be called on this subscriber + * if it has subscribed to the event class. + * @param event the event to process. + */ + processEvent(event: Event): Promise; +} + +/** + * An event publisher that publishes events to subscribers. + * Subscribers can specify which types of events they would like to receive. + */ +export interface EventPublisher { + /** + * Registers the given subscriber for the given event classes. + * @param subscriber the subscriber to be notified when the given event classes occur. + * @param eventClasses the classes of events that the subscriber should be notified of. + */ + subscribe(subscriber: EventSubscriber, eventClasses: Set): void; + + /** + * Unsubscribes the given subscriber from the given event classes. + * @param subscriber the subscriber to unsubscribe from the given event classes. + * @param eventClasses the classes of events that the subscriber wants to unsubscribe from. + */ + unsubscribe(subscriber: EventSubscriber, eventClasses: Set): void; + + /** + * Publishes an event. All subscribers to the given event class will be notified of the event. + * @param event the event to publish. + */ + publish(event: Event): void; +} diff --git a/common/lib/utils/events/monitor_reset_event.ts b/common/lib/utils/events/monitor_reset_event.ts new file mode 100644 index 000000000..2acf3bf9d --- /dev/null +++ b/common/lib/utils/events/monitor_reset_event.ts @@ -0,0 +1,32 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Event } from "./event"; + +/** + * Event indicating that a monitor should be reset with new endpoints. + * Used by ClusterTopologyMonitorImpl to reset monitoring when cluster topology changes. + */ +export class MonitorResetEvent implements Event { + readonly isImmediateDelivery = true; + readonly clusterId: string; + readonly endpoints: Set; + + constructor(clusterId: string, endpoints: Set) { + this.clusterId = clusterId; + this.endpoints = endpoints; + } +} diff --git a/common/lib/utils/events/monitor_stop_event.ts b/common/lib/utils/events/monitor_stop_event.ts new file mode 100644 index 000000000..029fa35ed --- /dev/null +++ b/common/lib/utils/events/monitor_stop_event.ts @@ -0,0 +1,33 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Event } from "./event"; +import { Monitor } from "../monitoring/monitor"; + +/** + * Event indicating that a monitor should be stopped. + * Used by MonitorService to stop and remove monitors. + */ +export class MonitorStopEvent implements Event { + readonly isImmediateDelivery = true; + readonly monitorClass: new (...args: any[]) => Monitor; + readonly key: unknown; + + constructor(monitorClass: new (...args: any[]) => Monitor, key: unknown) { + this.monitorClass = monitorClass; + this.key = key; + } +} diff --git a/common/lib/utils/full_services_container.ts b/common/lib/utils/full_services_container.ts new file mode 100644 index 000000000..c1be9827e --- /dev/null +++ b/common/lib/utils/full_services_container.ts @@ -0,0 +1,70 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { PluginService } from "../plugin_service"; +import { HostListProviderService } from "../host_list_provider_service"; +import { PluginManager } from "../index"; +import { ConnectionProvider } from "../connection_provider"; +import { TelemetryFactory } from "./telemetry/telemetry_factory"; +import { StorageService } from "./storage/storage_service"; +import { MonitorService } from "./monitoring/monitor_service"; +import { EventPublisher } from "./events/event"; +import { ImportantEventService } from "./important_event_service"; +import { HostIdCacheService } from "./host_id_cache_service"; + +/** + * Container for services used throughout the wrapper. + */ +export interface FullServicesContainer { + storageService: StorageService; + monitorService: MonitorService; + eventPublisher: EventPublisher; + readonly defaultConnectionProvider: ConnectionProvider; + telemetryFactory: TelemetryFactory; + pluginManager: PluginManager; + hostListProviderService: HostListProviderService; + pluginService: PluginService; + importantEventService: ImportantEventService; + hostIdCacheService: HostIdCacheService; +} + +export class FullServicesContainerImpl implements FullServicesContainer { + storageService: StorageService; + monitorService: MonitorService; + eventPublisher: EventPublisher; + readonly defaultConnectionProvider: ConnectionProvider; + telemetryFactory: TelemetryFactory; + pluginManager!: PluginManager; + hostListProviderService!: HostListProviderService; + pluginService!: PluginService; + importantEventService: ImportantEventService; + hostIdCacheService: HostIdCacheService; + + constructor( + storageService: StorageService, + monitorService: MonitorService, + eventPublisher: EventPublisher, + defaultConnProvider: ConnectionProvider, + telemetryFactory: TelemetryFactory + ) { + this.storageService = storageService; + this.monitorService = monitorService; + this.eventPublisher = eventPublisher; + this.defaultConnectionProvider = defaultConnProvider; + this.telemetryFactory = telemetryFactory; + this.importantEventService = new ImportantEventService(); + } +} diff --git a/common/lib/utils/gdb_region_utils.ts b/common/lib/utils/gdb_region_utils.ts new file mode 100644 index 000000000..815cc64ee --- /dev/null +++ b/common/lib/utils/gdb_region_utils.ts @@ -0,0 +1,108 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { RegionUtils } from "./region_utils"; +import { HostInfo } from "../host_info"; +import { AwsCredentialsManager } from "../authentication/aws_credentials_manager"; +import { DescribeGlobalClustersCommand, GlobalCluster, GlobalClusterMember, RDSClient } from "@aws-sdk/client-rds"; +import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from "@smithy/types/dist-types/identity/awsCredentialIdentity"; +import { logger } from "../../logutils"; +import { Messages } from "./messages"; +import { AwsWrapperError } from "./errors"; + +export class GDBRegionUtils extends RegionUtils { + private static readonly GDB_CLUSTER_ARN_PATTERN = /^arn:aws[^:]*:rds:(?[^:\n]*):([^:\n]*):([^:/\n]*[:/])?(.*)$/; + private static readonly REGION_GROUP = "region"; + private credentialsProvider: AwsCredentialIdentity | AwsCredentialIdentityProvider | undefined; + + constructor(credentialsProvider?: AwsCredentialIdentity | AwsCredentialIdentityProvider) { + super(); + this.credentialsProvider = credentialsProvider; + } + + async getRegion(regionKey: string, hostInfo?: HostInfo, props?: Map): Promise { + if (!hostInfo || !props) { + return null; + } + + if (props.get(regionKey)) { + return this.getRegionFromRegionString(props.get(regionKey)); + } + + const clusterId = GDBRegionUtils.rdsUtils.getRdsClusterId(hostInfo.host); + if (!clusterId) { + return null; + } + + const writerClusterArn = await this.findWriterClusterArn(hostInfo, props, clusterId); + return writerClusterArn ? this.getRegionFromClusterArn(writerClusterArn) : null; + } + + private async findWriterClusterArn(hostInfo: HostInfo, props: Map, globalClusterIdentifier: string): Promise { + if (!this.credentialsProvider) { + this.credentialsProvider = AwsCredentialsManager.getProvider(hostInfo, props); + } + + const rdsClient = this.getRdsClient(); + + try { + const command = new DescribeGlobalClustersCommand({ + GlobalClusterIdentifier: globalClusterIdentifier + }); + + const response = await rdsClient.send(command); + return this.extractWriterClusterArn(response.GlobalClusters); + } catch (error) { + logger.debug(Messages.get("GDBRegionUtils.unableToRetrieveGlobalClusterARN")); + throw new AwsWrapperError(Messages.get("GDBRegionUtils.unableToRetrieveGlobalClusterARN")); + } finally { + rdsClient.destroy(); + } + } + + private extractWriterClusterArn(globalClusters?: GlobalCluster[]): string | null { + if (!globalClusters) { + return null; + } + + for (const cluster of globalClusters) { + const writerArn = this.findWriterMemberArn(cluster.GlobalClusterMembers); + if (writerArn) { + return writerArn; + } + } + + return null; + } + + getRegionFromClusterArn(clusterArn: string): string | null { + const match = clusterArn.match(GDBRegionUtils.GDB_CLUSTER_ARN_PATTERN); + return match?.groups?.[GDBRegionUtils.REGION_GROUP] ?? null; + } + + private findWriterMemberArn(members?: GlobalClusterMember[]): string | null { + if (!members) { + return null; + } + + const writerMember = members.find((member) => member.IsWriter); + return writerMember?.DBClusterArn ?? null; + } + + private getRdsClient(): RDSClient { + return new RDSClient({ credentials: this.credentialsProvider }); + } +} diff --git a/common/lib/utils/host_id_cache_service.ts b/common/lib/utils/host_id_cache_service.ts new file mode 100644 index 000000000..9ac7e32c1 --- /dev/null +++ b/common/lib/utils/host_id_cache_service.ts @@ -0,0 +1,137 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ClientWrapper } from "../client_wrapper"; +import { HostInfo } from "../host_info"; +import { PluginService } from "../plugin_service"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { TopologyAwareDatabaseDialect } from "../database_dialect/topology_aware_database_dialect"; +import { RdsUtils } from "./rds_utils"; +import { RdsUrlType } from "./rds_url_type"; +import { AwsWrapperError } from "./errors"; +import { Messages } from "./messages"; + +/** + * Tuple of (instanceId, instanceName) identifying a connection's underlying instance, as + * returned by a topology-aware dialect. Either value may be null when identification fails. + */ +export type InstanceIdAndName = [instanceId: string | null, instanceName: string | null]; + +/** + * Identifies the underlying instance a connection is established to. + * + * For static endpoints (IP addresses or custom domain names) the identification result is cached, + * keyed by the connection's host name, to avoid repeatedly querying the database. + */ +export interface HostIdCacheService { + /** + * Identify the connected host, using the cache where possible. + * + * @param targetClient the connection to be identified. + * @param connectionHostInfo the {@link HostInfo} the connection was established with. + * @param pluginService the plugin service instance. + * @returns the identified {@link HostInfo}, or null if it cannot be determined. + */ + identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo, pluginService: PluginService): Promise; +} + +function isTopologyAwareDialect(dialect: DatabaseDialect): dialect is DatabaseDialect & TopologyAwareDatabaseDialect { + return typeof (dialect as Partial).getInstanceId === "function"; +} + +export class HostIdCacheServiceImpl implements HostIdCacheService { + static readonly PROP_ENABLED = "AWS_NODEJS_HOST_CACHE_ENABLED"; + static readonly PROP_REGEXP = "AWS_NODEJS_HOST_CACHE_REGEXP"; + + private static readonly cache = new Map(); + private static readonly isEnabled = (process.env[HostIdCacheServiceImpl.PROP_ENABLED] ?? "true").toLowerCase() === "true"; + private static readonly hostRegexp = new RegExp(process.env[HostIdCacheServiceImpl.PROP_REGEXP] ?? ".*"); + private static readonly rdsHelper = new RdsUtils(); + + async identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo, pluginService: PluginService): Promise { + if (!targetClient || !connectionHostInfo || !pluginService) { + return null; + } + + const urlType: RdsUrlType = HostIdCacheServiceImpl.rdsHelper.identifyRdsType(connectionHostInfo.host); + switch (urlType) { + case RdsUrlType.RDS_INSTANCE: + return connectionHostInfo; + case RdsUrlType.IP_ADDRESS: + case RdsUrlType.OTHER: + // It might be a custom domain name. Cache the identification keyed by host name when allowed. + if (HostIdCacheServiceImpl.isEnabled && HostIdCacheServiceImpl.hostRegexp.test(connectionHostInfo.host)) { + return this.getCachedHostInfo(targetClient, connectionHostInfo, pluginService); + } + return pluginService.identifyConnection(targetClient); + default: + // Other hosts are dynamic and may change at any time, so they can't be cached. + return pluginService.identifyConnection(targetClient); + } + } + + protected async getCachedHostInfo( + targetClient: ClientWrapper, + connectionHostInfo: HostInfo, + pluginService: PluginService + ): Promise { + const host = connectionHostInfo.host; + + let instanceIdAndName = HostIdCacheServiceImpl.cache.get(host); + if (!instanceIdAndName) { + instanceIdAndName = await this.queryInstanceIdAndName(targetClient, pluginService); + HostIdCacheServiceImpl.cache.set(host, instanceIdAndName); + } + + const [instanceId, instanceName] = instanceIdAndName; + if (!instanceId && !instanceName) { + // We've already tried to identify the connection, but got nothing. + return null; + } + + let topology = pluginService.getAllHosts(); + if (!topology || topology.length === 0) { + const provider = pluginService.getHostListProvider(); + topology = provider ? await provider.forceRefresh() : null; + if (!topology || topology.length === 0) { + return null; + } + } + + return topology.find((candidate) => instanceId === candidate.hostId || instanceName === candidate.host) ?? null; + } + + private async queryInstanceIdAndName(targetClient: ClientWrapper, pluginService: PluginService): Promise { + const dialect = pluginService.getDialect(); + if (!isTopologyAwareDialect(dialect)) { + return [null, null]; + } + + try { + const [instanceId, instanceName] = await dialect.getInstanceId(targetClient); + return [instanceId ?? null, instanceName ?? null]; + } catch (error: any) { + throw new AwsWrapperError(Messages.get("HostIdCacheService.errorIdentifyConnection"), error); + } + } + + /** + * Clears the static host identification cache. Intended for test cleanup. + */ + static clearCache(): void { + HostIdCacheServiceImpl.cache.clear(); + } +} diff --git a/common/lib/utils/iam_auth_utils.ts b/common/lib/utils/iam_auth_utils.ts index 42525ddf8..7a8d29e5c 100644 --- a/common/lib/utils/iam_auth_utils.ts +++ b/common/lib/utils/iam_auth_utils.ts @@ -16,23 +16,26 @@ import { logger } from "../../logutils"; import { HostInfo } from "../host_info"; -import { WrapperProperties, WrapperProperty } from "../wrapper_property"; -import { AwsWrapperError } from "./errors"; +import { WrapperProperties } from "../wrapper_property"; import { Messages } from "./messages"; -import { RdsUtils } from "./rds_utils"; import { Signer } from "@aws-sdk/rds-signer"; import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from "@smithy/types/dist-types/identity/awsCredentialIdentity"; import { PluginService } from "../plugin_service"; import { TelemetryTraceLevel } from "./telemetry/telemetry_trace_level"; +import { HostInfoBuilder } from "../host_info_builder"; export class IamAuthUtils { private static readonly TELEMETRY_FETCH_TOKEN = "fetch IAM token"; - public static getIamHost(props: Map, hostInfo: HostInfo): string { - return WrapperProperties.IAM_HOST.get(props) ? WrapperProperties.IAM_HOST.get(props) : hostInfo.host; + public getIamHost(props: Map, hostInfo: HostInfo): HostInfo { + const iamHost: string | null = WrapperProperties.IAM_HOST.get(props); + + return iamHost + ? new HostInfoBuilder({ hostAvailabilityStrategy: hostInfo.hostAvailabilityStrategy }).copyFrom(hostInfo).withHost(iamHost).build() + : hostInfo; } - public static getIamPort(props: Map, hostInfo: HostInfo, defaultPort: number): number { + public getIamPort(props: Map, hostInfo: HostInfo, defaultPort: number): number { const port = WrapperProperties.IAM_DEFAULT_PORT.get(props); if (port) { if (isNaN(port) || port <= 0) { @@ -49,23 +52,11 @@ export class IamAuthUtils { } } - public static getRdsRegion(hostname: string, rdsUtils: RdsUtils, props: Map, wrapperProperty: WrapperProperty): string { - const rdsRegion = rdsUtils.getRdsRegion(hostname); - - if (!rdsRegion) { - const errorMessage = Messages.get("Authentication.unsupportedHostname", hostname); - logger.debug(errorMessage); - throw new AwsWrapperError(errorMessage); - } - - return wrapperProperty.get(props) ? wrapperProperty.get(props) : rdsRegion; - } - - public static getCacheKey(port: number, user?: string, hostname?: string, region?: string): string { + public getCacheKey(port: number, user?: string, hostname?: string, region?: string): string { return `${region}:${hostname}:${port}:${user}`; } - public static async generateAuthenticationToken( + public async generateAuthenticationToken( hostname: string, port: number, region: string, diff --git a/common/lib/utils/important_event_service.ts b/common/lib/utils/important_event_service.ts new file mode 100644 index 000000000..5ae19fd18 --- /dev/null +++ b/common/lib/utils/important_event_service.ts @@ -0,0 +1,88 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +export class ImportantEvent { + readonly timestamp: Date; + readonly description: string; + + constructor(timestamp: Date, description: string) { + this.timestamp = timestamp; + this.description = description; + } +} + +export class ImportantEventService { + private static readonly DEFAULT_EVENT_QUEUE_MS = 60000; + + private readonly events: ImportantEvent[] = []; + private readonly eventQueueMs: number; + private readonly isEnabled: boolean; + + constructor(isEnabled: boolean = true, eventQueueMs: number = ImportantEventService.DEFAULT_EVENT_QUEUE_MS) { + this.isEnabled = isEnabled; + this.eventQueueMs = eventQueueMs; + } + + clear(): void { + this.events.length = 0; + } + + registerEvent(descriptionSupplier: () => string): void { + if (!this.isEnabled) { + return; + } + + this.removeExpiredEvents(); + + this.events.push(new ImportantEvent(new Date(), descriptionSupplier())); + } + + getEvents(): ImportantEvent[] { + if (!this.isEnabled) { + return []; + } + + this.removeExpiredEvents(); + return [...this.events]; + } + + private removeExpiredEvents(): void { + if (!this.isEnabled || this.events.length === 0) { + return; + } + + const current = Date.now(); + const cutoffTime = current - this.eventQueueMs; + + while (this.events.length > 0 && this.events[0].timestamp.getTime() <= cutoffTime) { + this.events.shift(); + } + } +} + +export class DriverImportantEventService { + private static readonly INSTANCE = new ImportantEventService(true, 60000); + + private constructor() {} + + static getInstance(): ImportantEventService { + return DriverImportantEventService.INSTANCE; + } + + static clear(): void { + DriverImportantEventService.INSTANCE.clear(); + } +} diff --git a/common/lib/utils/messages.ts b/common/lib/utils/messages.ts index 5c0af3935..94e619b9b 100644 --- a/common/lib/utils/messages.ts +++ b/common/lib/utils/messages.ts @@ -56,9 +56,10 @@ const MESSAGES: Record = { "ClusterAwareReaderFailoverHandler.selectedTaskChosen": "Selected task has already been chosen. Abort client for host: %s", "Utils.topology": "Topology: %s", "RdsHostListProvider.incorrectDialect": "Dialect needs to be a topology aware dialect.", - "RdsHostListProvider.suggestedClusterId": "ClusterId '%s' is suggested for url '%s'.", + "RdsHostListProvider.noClusterId": "No clusterId found. Please ensure clusterId parameter is set to a non-empty string.", "ConnectionStringHostListProvider.parsedListEmpty": "Can't parse connection string: '%s'.", "ConnectionStringHostListProvider.errorIdentifyConnection": "An error occurred while obtaining the connection's host ID.", + "HostIdCacheService.errorIdentifyConnection": "An error occurred while identifying the connection's host ID.", "ExecuteTimePlugin.executeTime": "Executed method '%s' in %s milliseconds.", "ConnectTimePlugin.connectTime": "Connected to '%s' in %s milliseconds.", "ClusterAwareWriterFailoverHandler.failoverCalledWithInvalidTopology": "Failover was called with an invalid (null or empty) topology.", @@ -89,7 +90,11 @@ const MESSAGES: Record = { "Failover.unableToConnectToWriter": "Unable to establish SQL connection to the writer instance.", "Failover.unableToConnectToWriterDueToError": "Unable to establish SQL connection to the writer instance: %s due to error: %s.", "Failover.unableToConnectToReader": "Unable to establish SQL connection to the reader instance.", + "Failover.unableToRefreshHostList": "The request to discover the new topology timed out or was unsuccessful.", "Failover.unableToDetermineWriter": "Unable to determine the current writer instance.", + "Failover.unexpectedReaderRole": "The new writer was identified to be '%s', but querying the instance for its role returned a role of %s.", + "Failover.strictReaderUnknownHostRole": + "Unable to determine host role for '%s'. Since failover mode is set to STRICT_READER and the host may be a writer, it will not be selected for reader failover.", "Failover.detectedError": "[Failover] Detected an error while executing a command: %s", "Failover.failoverDisabled": "Cluster-aware failover is disabled.", "Failover.establishedConnection": "[Failover] Connected to %s", @@ -99,24 +104,23 @@ const MESSAGES: Record = { "Failover.noOperationsAfterConnectionClosed": "No operations allowed after client ended.", "Failover.transactionResolutionUnknownError": "Unknown transaction resolution error occurred during failover.", "Failover.connectionExplicitlyClosed": "Unable to failover on an explicitly closed connection.", + "Failover.failoverReaderTimeout": "The reader failover process was not able to establish a connection before timing out.", "Failover.timeoutError": "Internal failover task has timed out.", "Failover.newWriterNotAllowed": "The failover process identified the new writer but the host is not in the list of allowed hosts. New writer host: '%s'. Allowed hosts: '%s'.", + "GDBRegionUtils.unableToRetrieveGlobalClusterARN": "Unable to retrieve the primary global region for the provided global database cluster.", "StaleDnsHelper.clusterEndpointDns": "Cluster endpoint resolves to '%s'.", "StaleDnsHelper.writerHostInfo": "Writer host: '%s'.", "StaleDnsHelper.writerInetAddress": "Writer host address: '%s'", "StaleDnsHelper.staleDnsDetected": "Stale DNS data detected. Opening a connection to '%s'.", "StaleDnsHelper.reset": "Reset stored writer host.", "StaleDnsPlugin.requireDynamicProvider": "Dynamic host list provider is required.", + "StaleDnsHelper.currentWriterNotAllowed": "The current writer is not in the list of allowed hosts. Current host: '%s'. Allowed hosts: %s", "Client.methodNotSupported": "Method '%s' not supported.", "Client.invalidTransactionIsolationLevel": "An invalid transaction isolation level was provided: '%s'.", - "AuroraStaleDnsHelper.clusterEndpointDns": "Cluster endpoint resolves to '%s'.", - "AuroraStaleDnsHelper.writerHostSpec": "Writer host: '%s'.", - "AuroraStaleDnsHelper.writerInetAddress": "Writer host address: '%s'", - "AuroraStaleDnsHelper.staleDnsDetected": "Stale DNS data detected. Opening a connection to '%s'.", "ReadWriteSplittingPlugin.setReadOnlyOnClosedClient": "setReadOnly cannot be called on a closed client '%s'.", "ReadWriteSplittingPlugin.errorSwitchingToCachedReader": - "An error occurred while trying to switch to a cached reader client: '%s'. The driver will attempt to establish a new reader client.", + "An error occurred while trying to switch to a cached reader client: '%s'. Error message: '%s'. The driver will attempt to establish a new reader client.", "ReadWriteSplittingPlugin.errorSwitchingToReader": "An error occurred while trying to switch to a reader client: '%s'.", "ReadWriteSplittingPlugin.errorSwitchingToWriter": "An error occurred while trying to switch to a writer client: '%s'.", "ReadWriteSplittingPlugin.closingInternalClients": "Closing all internal clients except for the current one.", @@ -137,6 +141,8 @@ const MESSAGES: Record = { "ReadWriteSplittingPlugin.failoverErrorWhileExecutingCommand": "Detected a failover error while executing a command: '%s'", "ReadWriteSplittingPlugin.noReadersAvailable": "The plugin was unable to establish a reader client to any reader instance.", "ReadWriteSplittingPlugin.successfullyConnectedToReader": "Successfully connected to a new reader host: '%s'", + "ReadWriteSplittingPlugin.previousReaderNotAllowed": + "The previous reader connection cannot be used because it is no longer in the list of allowed hosts. Previous reader: %s. Allowed hosts: %s", "ReadWriteSplittingPlugin.failedToConnectToReader": "Failed to connect to reader host: '%s'", "ReadWriteSplittingPlugin.unsupportedHostSelectorStrategy": "Unsupported host selection strategy '%s' specified in plugin configuration parameter 'readerHostSelectorStrategy'. Please visit the Read/Write Splitting Plugin documentation for all supported strategies.", @@ -167,6 +173,8 @@ const MESSAGES: Record = { "Okta SAML Assertion request failed with HTTP status '%s', reason phrase '%s', and response '%s'", "SamlCredentialsProviderFactory.getSamlAssertionFailed": "Failed to get SAML Assertion due to error: '%s'", "SamlAuthPlugin.unhandledError": "Unhandled error: '%s'", + "SamlAuthPlugin.unableToDetermineRegion": + "Unable to determine connection region. If you are using a non-standard RDS URL, please set the '%s' property.", "HostAvailabilityStrategy.invalidMaxRetries": "Invalid value of '%s' for configuration parameter `hostAvailabilityStrategyMaxRetries`. It must be an integer greater or equal to 1.", "HostAvailabilityStrategy.invalidInitialBackoffTime": @@ -186,19 +194,31 @@ const MESSAGES: Record = { "MonitorImpl.stopMonitoringTaskNewContext": "Stop monitoring task for checking new contexts for '%s'", "MonitorService.startMonitoringNullMonitor": "Start monitoring called but could not find monitor for host: '%s'.", "MonitorService.emptyAliasSet": "Empty alias set passed for '%s'. Set should not be empty.", + "MonitorService.monitorClassMismatch": + "The monitor stored at '%s' did not have the expected type. The expected type was '%s', but the monitor '%s' had a type of '%s'.", + "MonitorService.monitorStuck": "Monitor '%s' has not been updated within the inactive timeout of %s milliseconds. The monitor will be stopped.", + "MonitorService.monitorTypeNotRegistered": + "The given monitor class '%s' is not registered. Please register the monitor class before running monitors of that class with the monitor service.", + "MonitorService.recreatingMonitor": "Recreating monitor: '%s'.", + "MonitorService.removedErrorMonitor": "Removed monitor in error state: '%s'.", + "MonitorService.removedExpiredMonitor": "Removed expired monitor: '%s'.", + "MonitorService.stopAndRemoveMissingMonitorType": + "The monitor service received a request to stop a monitor with type '%s' and key '%s', but the monitor service does not have any monitors registered under the given type. Please ensure monitors are registered under the correct type.", + "MonitorService.stopAndRemoveMonitorsMissingType": + "The monitor service received a request to stop all monitors with type '%s', but the monitor service does not have any monitors registered under the given type. Please ensure monitors are registered under the correct type.", + "MonitorService.cleanupTaskInterrupted": "Monitor service cleanup task interrupted.", "PluginService.hostListEmpty": "Current host list is empty.", "PluginService.releaseResources": "Releasing resources.", - "PluginService.hostsChangeListEmpty": "There are no changes in the hosts' availability.", "PluginService.failedToRetrieveHostPort": "Could not retrieve Host:Port for connection.", "PluginService.nonEmptyAliases": "fillAliases called when HostInfo already contains the following aliases: '%s'.", "PluginService.forceMonitoringRefreshTimeout": "A timeout error occurred after waiting '%s' ms for refreshed topology.", "PluginService.requiredBlockingHostListProvider": "The detected host list provider is not a BlockingHostListProvider. A BlockingHostListProvider is required to force refresh the host list. Detected host list provider: '%s'.", + "PluginService.requiredDynamicHostListProvider": + "The forceMonitoringRefresh method requires a DynamicHostListProvider. The current host list provider '%s' does not support this operation.", "PluginService.currentHostNotAllowed": "The current host is not in the list of allowed hosts. Current host: '%s'. Allowed hosts: '%s'.", "PluginService.currentHostNotDefined": "The current host is undefined.", - "MonitoringHostListProvider.requiresMonitor": - "The MonitoringRdsHostListProvider could not retrieve or initialize a ClusterTopologyMonitor for refreshing the topology.", - "MonitoringHostListProvider.errorForceRefresh": "The MonitoringRdsHostListProvider could not refresh the topology, caught error: '%s'", + "PartialPluginService.unexpectedMethodCall": "Unexpected method call: '%s'. This method is not supported by PartialPluginService.", "HostMonitoringConnectionPlugin.activatedMonitoring": "Executing method '%s', monitoring is activated.", "HostMonitoringConnectionPlugin.unableToIdentifyConnection": "Unable to identify the given connection: '%s', please ensure the correct host list provider is specified. The host list provider in use is: '%s'.", @@ -291,14 +311,20 @@ const MESSAGES: Record = { "An error occurred while attempting to obtain the writer id because the query was invalid. Please ensure you are connecting to an Aurora or RDS DB cluster. Error: '%s'", "ClusterTopologyMonitor.unableToConnect": "Could not connect to initial host: '%s'.", "ClusterTopologyMonitor.openedMonitoringConnection": "Opened monitoring connection to: '%s'.", - "ClusterTopologyMonitor.startMonitoring": "Start cluster monitoring task.", + "ClusterTopologyMonitor.startMonitoring": "[clusterId: '%s'] Start cluster topology monitoring for '%s'.", + "ClusterTopologyMonitor.startingHostMonitoringTasks": "Starting host monitoring tasks.", + "ClusterTopologyMonitor.stopHostMonitoringTask": "Stop cluster topology monitoring task for '%s'.", "ClusterTopologyMonitor.errorDuringMonitoring": "Error thrown during cluster topology monitoring: '%s'.", "ClusterTopologyMonitor.endMonitoring": "Stop cluster topology monitoring.", + "ClusterTopologyMonitor.matchingReaderTopologies": "Reader topologies have been consistent for '%s' ms. Updating topology cache.", + "ClusterTopologyMonitor.reset": "[clusterId: '%s'] Resetting cluster topology monitor for '%s'.", + "ClusterTopologyMonitor.resetEventReceived": "MonitorResetEvent received.", "HostMonitor.startMonitoring": "Host monitor '%s' started.", - "HostMonitor.detectedWriter": "Detected writer: '%s' - '%s'.", - "HostMonitor.endMonitoring": "Host monitor '%s' completed in '%s'.", + "HostMonitor.detectedWriter": "Detected writer: '%s'.", + "HostMonitor.endMonitoring": "Host monitor '%s' completed in '%s' ms.", "HostMonitor.writerHostChanged": "Writer host has changed from '%s' to '%s'.", "HostMonitor.writerIsStale": "Connected writer instance '%s' is stale.", + "HostMonitor.loginErrorDuringMonitoring": "Login error detected during monitoring.", "SlidingExpirationCacheWithCleanupTask.cleaningUp": "Cleanup interval of '%s' minutes has passed, cleaning up sliding expiration cache '%s'.", "SlidingExpirationCacheWithCleanupTask.cleanUpTaskInterrupted": "Sliding expiration cache '%s' cleanup task has been interrupted and is exiting.", "SlidingExpirationCacheWithCleanupTask.cleanUpTaskStopped": "Sliding expiration cache '%s' cleanup task has been stopped and is exiting.", @@ -374,7 +400,35 @@ const MESSAGES: Record = { "Blue/Green Deployment switchover is still in progress and a corresponding host for '%s' is not found after %s ms. Try to connect again later.", "Bgd.correspondingHostFoundContinueWithConnect": "A corresponding host for '%s' is found. Continue with connect call. The call was suspended for %s ms.", - "Bgd.completedContinueWithConnect": "Blue/Green Deployment status is completed. Continue with 'connect' call. The call was suspended for %s ms." + "Bgd.completedContinueWithConnect": "Blue/Green Deployment status is completed. Continue with 'connect' call. The call was suspended for %s ms.", + "StorageService.itemClassNotRegistered": "[StorageService] Item class not registered: %s", + "StorageService.unexpectedValueMismatch": "[StorageService] Unexpected value mismatch for %s: %s", + "TopologyUtils.instanceIdRequired": "InstanceId must not be en empty string.", + "TopologyUtils.errorGettingHostRole": "An error occurred while trying to get the host role.", + "GlobalTopologyUtils.missingRegion": "Host '%s' is missing region information in the topology query result.", + "GlobalTopologyUtils.missingTemplateForRegion": "No cluster instance template found for region '%s' when processing host '%s'.", + "Utils.globalClusterInstanceHostPatternsRequired": "The 'globalClusterInstanceHostPatterns' property is required for Global Aurora Databases.", + "Utils.invalidPatternFormat": + "Invalid pattern format '%s'. Expected format: 'region:host-pattern' (e.g., 'us-east-1:?.cluster-xyz.us-east-1.rds.amazonaws.com').", + "GlobalAuroraTopologyMonitor.cannotFindRegionTemplate": "Cannot find cluster instance template for region '%s'.", + "GlobalAuroraTopologyMonitor.invalidTopologyUtils": "TopologyUtils must implement GdbTopologyUtils for GlobalAuroraTopologyMonitor.", + "GlobalDbFailoverPlugin.missingHomeRegion": + "The 'failoverHomeRegion' property is required when connecting to a Global Aurora Database without a region in the URL.", + "GlobalDbFailoverPlugin.missingInitialHost": "Unable to determine the initial connection host.", + "GlobalDbFailoverPlugin.startFailover": "Starting Global DB failover procedure.", + "GlobalDbFailoverPlugin.isHomeRegion": "Is home region: %s", + "GlobalDbFailoverPlugin.currentFailoverMode": "Current Global DB failover mode: %s", + "GlobalDbFailoverPlugin.failoverElapsed": "Global DB failover elapsed time: %s ms", + "GlobalDbFailoverPlugin.unableToFindCandidateWithMatchingRole": + "Unable to find a candidate host with the expected role (%s) based on the given host selection strategy: %s", + "GlobalDbFailoverPlugin.unableToConnect": "Unable to establish a connection during Global DB failover.", + "GdbReadWriteSplittingPlugin.missingHomeRegion": + "Unable to parse home region from endpoint '%s'. Please ensure you have set the 'gdbRwHomeRegion' connection parameter.", + "GdbReadWriteSplittingPlugin.cantConnectWriterOutOfHomeRegion": "Writer connection to '%s' is not allowed since it is out of home region '%s'.", + "GdbReadWriteSplittingPlugin.noAvailableReadersInHomeRegion": "No available reader nodes in home region '%s'.", + "GdbReadWriteSplittingPlugin.parameterValue": "%s=%s", + "BatchingEventPublisher.errorDeliveringImmediateEvent": "Error delivering immediate event: %s", + "WrapperProperty.invalidValue": "Invalid value '%s' for property '%s'. Allowed values: %s" }; export class Messages { diff --git a/common/lib/utils/monitoring/monitor.ts b/common/lib/utils/monitoring/monitor.ts new file mode 100644 index 000000000..5bf93ffcd --- /dev/null +++ b/common/lib/utils/monitoring/monitor.ts @@ -0,0 +1,128 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { FullServicesContainer } from "../full_services_container"; + +const DEFAULT_CLEANUP_INTERVAL_NANOS = BigInt(60_000_000_000); // 1 minute + +export enum MonitorState { + RUNNING, + STOPPED, + ERROR +} + +export enum MonitorErrorResponse { + NO_ACTION, + RECREATE +} + +export class MonitorSettings { + expirationTimeoutNanos: bigint; + inactiveTimeoutNanos: bigint; + errorResponses: Set; + + constructor(expirationTimeoutNanos: bigint, inactiveTimeoutNanos: bigint, errorResponses: Set) { + this.expirationTimeoutNanos = expirationTimeoutNanos; + this.inactiveTimeoutNanos = inactiveTimeoutNanos; + this.errorResponses = errorResponses; + } +} + +export interface Monitor { + start(): Promise; + + monitor(): Promise; + + stop(): Promise; + + close(): Promise; + + getLastActivityTimestampNanos(): bigint; + + getState(): MonitorState; + + canDispose(): boolean; +} + +export interface MonitorInitializer { + createMonitor(servicesContainer: FullServicesContainer): Monitor; +} + +export abstract class AbstractMonitor implements Monitor { + protected _stop = false; + protected terminationTimeoutMs: number; + protected lastActivityTimestampNanos: bigint; + protected state: MonitorState; + protected monitorPromise?: Promise; + + protected constructor(terminationTimeoutSec: number) { + this.terminationTimeoutMs = terminationTimeoutSec * 1000; + this.lastActivityTimestampNanos = BigInt(Date.now() * 1_000_000); + this.state = MonitorState.STOPPED; + } + + async start(): Promise { + this.monitorPromise = this.run(); + } + + async run(): Promise { + try { + this.state = MonitorState.RUNNING; + this.lastActivityTimestampNanos = BigInt(Date.now() * 1_000_000); + await this.monitor(); + } catch (error) { + this.state = MonitorState.ERROR; + } finally { + await this.close(); + } + } + + abstract monitor(): Promise; + + async stop(): Promise { + this._stop = true; + + if (this.monitorPromise) { + let timeoutId: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timeoutId = setTimeout(resolve, this.terminationTimeoutMs); + }); + await Promise.race([this.monitorPromise, timeout]); + if (timeoutId !== undefined) { + clearTimeout(timeoutId); + } + } + + await this.close(); + this.state = MonitorState.STOPPED; + } + + async close(): Promise { + // Do nothing + } + + getLastActivityTimestampNanos(): bigint { + return this.lastActivityTimestampNanos; + } + + getState(): MonitorState { + return this.state; + } + + canDispose(): boolean { + return true; + } +} diff --git a/common/lib/utils/monitoring/monitor_service.ts b/common/lib/utils/monitoring/monitor_service.ts new file mode 100644 index 000000000..138b429b5 --- /dev/null +++ b/common/lib/utils/monitoring/monitor_service.ts @@ -0,0 +1,450 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Monitor, MonitorErrorResponse, MonitorInitializer, MonitorSettings, MonitorState } from "./monitor"; +import { Constructor } from "../../types"; +import { FullServicesContainer } from "../full_services_container"; +import { logger } from "../../../logutils"; +import { Messages } from "../messages"; +import { AwsWrapperError } from "../errors"; +import { ClusterTopologyMonitorImpl } from "../../host_list_provider/monitoring/cluster_topology_monitor"; +import { Topology } from "../../host_list_provider/topology"; +import { Event, EventPublisher, EventSubscriber } from "../events/event"; +import { DataAccessEvent } from "../events/data_access_event"; +import { MonitorStopEvent } from "../events/monitor_stop_event"; +import { convertNanosToMs, getTimeInNanos, sleepWithAbort } from "../utils"; +import { CacheItem } from "../cache_map"; + +const DEFAULT_CLEANUP_INTERVAL_NS = BigInt(60_000_000_000); // 1 minute +const FIFTEEN_MINUTES_NS = BigInt(15 * 60 * 1_000_000_000); +const THREE_MINUTES_NS = BigInt(3 * 60 * 1_000_000_000); + +export interface MonitorService { + registerMonitorTypeIfAbsent( + monitorClass: Constructor, + expirationTimeoutNanos: bigint, + inactiveTimeoutNanos: bigint, + errorResponses: Set, + producedDataClass?: Constructor + ): void; + + runIfAbsent( + monitorClass: Constructor, + key: unknown, + servicesContainer: FullServicesContainer, + originalProps: Map, + initializer: MonitorInitializer + ): Promise; + + get(monitorClass: Constructor, key: unknown): T | null; + + remove(monitorClass: Constructor, key: unknown): T | null; + + stopAndRemove(monitorClass: Constructor, key: unknown): Promise; + + stopAndRemoveMonitors(monitorClass: Constructor): Promise; + + stopAndRemoveAll(): Promise; + + releaseResources(): Promise; +} + +/** + * A container object that holds a monitor together with the supplier used to generate the monitor. + * The supplier can be used to recreate the monitor if it encounters an error or becomes stuck. + */ +class MonitorItem { + private readonly monitorSupplier: () => Monitor; + private readonly _monitor: Monitor; + + constructor(monitorSupplier: () => Monitor) { + this.monitorSupplier = monitorSupplier; + this._monitor = monitorSupplier(); + } + + getMonitorSupplier(): () => Monitor { + return this.monitorSupplier; + } + + getMonitor(): Monitor { + return this._monitor; + } +} + +/** + * A container that holds a cache of monitors of a given type with the related settings and info for that type. + */ +class CacheContainer { + private readonly settings: MonitorSettings; + private readonly cache: Map>; + private readonly producedDataClass: Constructor | null; + + constructor(settings: MonitorSettings, producedDataClass: Constructor | null) { + this.settings = settings; + this.producedDataClass = producedDataClass; + this.cache = new Map>(); + } + + getSettings(): MonitorSettings { + return this.settings; + } + + getCache(): Map> { + return this.cache; + } + + getProducedDataClass(): Constructor | null { + return this.producedDataClass; + } +} + +export class MonitorServiceImpl implements MonitorService, EventSubscriber { + private static defaultSuppliers: Map, () => CacheContainer> | null = null; + + // Lazy initialization for the default suppliers to avoid circular dependencies. + private static getDefaultSuppliers(): Map, () => CacheContainer> { + if (!MonitorServiceImpl.defaultSuppliers) { + const recreateOnError = new Set([MonitorErrorResponse.RECREATE]); + const defaultSettings = new MonitorSettings(FIFTEEN_MINUTES_NS, THREE_MINUTES_NS, recreateOnError); + + MonitorServiceImpl.defaultSuppliers = new Map([[ClusterTopologyMonitorImpl, () => new CacheContainer(defaultSettings, Topology)]]); + } + return MonitorServiceImpl.defaultSuppliers; + } + + protected readonly publisher: EventPublisher; + protected readonly monitorCaches = new Map, CacheContainer>(); + // Use a pending promise map to prevent race conditions when creating monitors. + private readonly pendingMonitors = new Map>(); + private cleanupTask: Promise | null = null; + private interruptCleanupTask: (() => void) | null = null; + private isInitialized: boolean = false; + + constructor(publisher: EventPublisher, cleanupIntervalNs: bigint = DEFAULT_CLEANUP_INTERVAL_NS) { + this.publisher = publisher; + this.publisher.subscribe(this, new Set([DataAccessEvent, MonitorStopEvent])); + this.initCleanupTask(cleanupIntervalNs); + } + + protected initCleanupTask(cleanupIntervalNs: bigint): void { + this.isInitialized = true; + this.cleanupTask = this.runCleanupLoop(cleanupIntervalNs); + } + + private async runCleanupLoop(cleanupIntervalNs: bigint): Promise { + while (this.isInitialized) { + const [sleepPromise, abortSleepFunc] = sleepWithAbort( + convertNanosToMs(cleanupIntervalNs), + Messages.get("MonitorService.cleanupTaskInterrupted") + ); + this.interruptCleanupTask = abortSleepFunc; + try { + await sleepPromise; + } catch { + // Sleep has been interrupted, exit cleanup task. + return; + } + + await this.checkMonitors(); + } + } + + protected async checkMonitors(): Promise { + for (const container of this.monitorCaches.values()) { + const cache = container.getCache(); + const keysToProcess = Array.from(cache.keys()); + + for (const key of keysToProcess) { + const cacheItem = cache.get(key); + if (!cacheItem) { + continue; + } + + const monitorItem = cacheItem.get(true); + if (!monitorItem) { + continue; + } + + const monitor = monitorItem.getMonitor(); + const monitorSettings = container.getSettings(); + + // Check for stopped monitors + if (monitor.getState() === MonitorState.STOPPED) { + cache.delete(key); + await monitor.stop(); + continue; + } + + // Check for error state monitors + if (monitor.getState() === MonitorState.ERROR) { + cache.delete(key); + logger.debug(Messages.get("MonitorService.removedErrorMonitor", JSON.stringify(monitor))); + await this.handleMonitorError(container, key, monitorItem); + continue; + } + + // Check for inactive/stuck monitors + const inactiveTimeoutNs = monitorSettings.inactiveTimeoutNanos; + if (getTimeInNanos() - monitor.getLastActivityTimestampNanos() > inactiveTimeoutNs) { + cache.delete(key); + logger.info(Messages.get("MonitorService.monitorStuck", JSON.stringify(monitor), convertNanosToMs(inactiveTimeoutNs).toString())); + await this.handleMonitorError(container, key, monitorItem); + continue; + } + + // Check for expired monitors that can be disposed + if (cacheItem.isExpired() && monitor.canDispose()) { + cache.delete(key); + logger.info(Messages.get("MonitorService.removedExpiredMonitor", JSON.stringify(monitor))); + await monitor.stop(); + } + } + } + } + + protected async handleMonitorError(cacheContainer: CacheContainer, key: unknown, errorMonitorItem: MonitorItem): Promise { + const monitor = errorMonitorItem.getMonitor(); + await monitor.stop(); + + const errorResponses = cacheContainer.getSettings().errorResponses; + if (errorResponses && errorResponses.has(MonitorErrorResponse.RECREATE)) { + if (!cacheContainer.getCache().has(key)) { + logger.info(Messages.get("MonitorService.recreatingMonitor", JSON.stringify(monitor))); + const newMonitorItem = new MonitorItem(errorMonitorItem.getMonitorSupplier()); + const expirationNs = cacheContainer.getSettings().expirationTimeoutNanos; + cacheContainer.getCache().set(key, new CacheItem(newMonitorItem, getTimeInNanos() + expirationNs)); + await newMonitorItem.getMonitor().start(); + } + } + } + + registerMonitorTypeIfAbsent( + monitorClass: Constructor, + expirationTimeoutNanos: bigint, + inactiveTimeoutNanos: bigint, + errorResponses: Set, + producedDataClass?: Constructor + ): void { + if (this.monitorCaches.has(monitorClass)) { + return; + } + + const settings = new MonitorSettings(expirationTimeoutNanos, inactiveTimeoutNanos, errorResponses); + const cacheContainer = new CacheContainer(settings, producedDataClass ?? null); + this.monitorCaches.set(monitorClass, cacheContainer); + } + + async runIfAbsent( + monitorClass: Constructor, + key: unknown, + servicesContainer: FullServicesContainer, + _originalProps: Map, + initializer: MonitorInitializer + ): Promise { + let cacheContainer = this.monitorCaches.get(monitorClass); + + if (!cacheContainer) { + const supplier = MonitorServiceImpl.getDefaultSuppliers().get(monitorClass as Constructor); + if (!supplier) { + throw new AwsWrapperError(Messages.get("MonitorService.monitorTypeNotRegistered", monitorClass.name)); + } + + cacheContainer = supplier(); + this.monitorCaches.set(monitorClass, cacheContainer); + } + + const cache = cacheContainer.getCache(); + const existingCacheItem = cache.get(key); + if (existingCacheItem) { + const existingMonitorItem = existingCacheItem.get(true); + if (existingMonitorItem) { + existingCacheItem.updateExpiration(cacheContainer.getSettings().expirationTimeoutNanos); + return existingMonitorItem.getMonitor() as T; + } + } + + const pendingKey = `${monitorClass.name}:${JSON.stringify(key)}`; + + // Check if the monitor is already being created by another async task. + const pendingPromise = this.pendingMonitors.get(pendingKey); + if (pendingPromise) { + return (await pendingPromise) as T; + } + + // Use the pending promise pattern to create monitors. This prevents race condition. + const createPromise = (async (): Promise => { + try { + const recheckCacheItem = cache.get(key); + if (recheckCacheItem) { + const recheckMonitorItem = recheckCacheItem.get(true); + if (recheckMonitorItem) { + recheckCacheItem.updateExpiration(cacheContainer.getSettings().expirationTimeoutNanos); + return recheckMonitorItem.getMonitor(); + } + } + + const monitorItem = new MonitorItem(() => initializer.createMonitor(servicesContainer)); + const expirationNs = cacheContainer.getSettings().expirationTimeoutNanos; + cache.set(key, new CacheItem(monitorItem, getTimeInNanos() + expirationNs)); + await monitorItem.getMonitor().start(); + + return monitorItem.getMonitor(); + } finally { + // Delete the key once monitor has been successfully created. + this.pendingMonitors.delete(pendingKey); + } + })(); + + this.pendingMonitors.set(pendingKey, createPromise); + return (await createPromise) as T; + } + + get(monitorClass: Constructor, key: unknown): T | null { + const cacheContainer = this.monitorCaches.get(monitorClass); + if (!cacheContainer) { + return null; + } + + const cacheItem = cacheContainer.getCache().get(key); + if (!cacheItem) { + return null; + } + + const monitorItem = cacheItem.get(true); + if (!monitorItem) { + return null; + } + + const monitor = monitorItem.getMonitor(); + if (monitor instanceof monitorClass) { + return monitor as T; + } + + logger.info(Messages.get("MonitorService.monitorClassMismatch", JSON.stringify(key), monitorClass.name, JSON.stringify(monitor))); + return null; + } + + remove(monitorClass: Constructor, key: unknown): T | null { + const cacheContainer = this.monitorCaches.get(monitorClass); + if (!cacheContainer) { + return null; + } + + const cache = cacheContainer.getCache(); + const cacheItem = cache.get(key); + if (!cacheItem) { + return null; + } + + const monitorItem = cacheItem.get(true); + if (!monitorItem) { + return null; + } + + const monitor = monitorItem.getMonitor(); + if (monitor instanceof monitorClass) { + cache.delete(key); + return monitor as T; + } + + return null; + } + + async stopAndRemove(monitorClass: Constructor, key: unknown): Promise { + const cacheContainer = this.monitorCaches.get(monitorClass); + if (!cacheContainer) { + logger.info(Messages.get("MonitorService.stopAndRemoveMissingMonitorType", monitorClass.name, String(key))); + return; + } + + const cache = cacheContainer.getCache(); + const cacheItem = cache.get(key); + if (cacheItem) { + cache.delete(key); + await cacheItem.get(true)?.getMonitor().stop(); + } + } + + async stopAndRemoveMonitors(monitorClass: Constructor): Promise { + const cacheContainer = this.monitorCaches.get(monitorClass); + if (!cacheContainer) { + logger.info(Messages.get("MonitorService.stopAndRemoveMonitorsMissingType", monitorClass.name)); + return; + } + + const cache = cacheContainer.getCache(); + for (const [key, cacheItem] of cache.entries()) { + cache.delete(key); + await cacheItem.get(true)?.getMonitor().stop(); + } + } + + async stopAndRemoveAll(): Promise { + for (const monitorClass of this.monitorCaches.keys()) { + await this.stopAndRemoveMonitors(monitorClass); + } + } + + async releaseResources(): Promise { + // Stop cleanup task + this.isInitialized = false; + this.interruptCleanupTask?.(); + if (this.cleanupTask) { + await this.cleanupTask; + } + + await this.stopAndRemoveAll(); + } + + async processEvent(event: Event): Promise { + if (event instanceof DataAccessEvent) { + for (const container of this.monitorCaches.values()) { + if (!container.getProducedDataClass() || event.dataClass !== container.getProducedDataClass()) { + continue; + } + + // The data produced by the monitor in this cache with this key has been accessed recently, + // so we extend the monitor's expiration. + container.getCache().get(event.key)?.updateExpiration(container.getSettings().expirationTimeoutNanos); + } + return; + } + + if (event instanceof MonitorStopEvent) { + await this.stopAndRemove(event.monitorClass, event.key); + return; + } + + // Other event types should be propagated to monitors + for (const container of this.monitorCaches.values()) { + for (const cacheItem of container.getCache().values()) { + const monitorItem = cacheItem.get(true); + if (!monitorItem) { + continue; + } + + const monitor = monitorItem.getMonitor(); + if (this.isEventSubscriber(monitor)) { + await (monitor as unknown as EventSubscriber).processEvent(event); + } + } + } + } + + private isEventSubscriber(obj: unknown): obj is EventSubscriber { + return typeof obj === "object" && obj !== null && "processEvent" in obj && typeof (obj as EventSubscriber).processEvent === "function"; + } +} diff --git a/common/lib/utils/rds_url_type.ts b/common/lib/utils/rds_url_type.ts index ebcac9b8c..64955089a 100644 --- a/common/lib/utils/rds_url_type.ts +++ b/common/lib/utils/rds_url_type.ts @@ -15,17 +15,20 @@ */ export class RdsUrlType { - public static readonly IP_ADDRESS = new RdsUrlType(false, false); - public static readonly RDS_WRITER_CLUSTER = new RdsUrlType(true, true); - public static readonly RDS_READER_CLUSTER = new RdsUrlType(true, true); - public static readonly RDS_CUSTOM_CLUSTER = new RdsUrlType(true, false); - public static readonly RDS_PROXY = new RdsUrlType(true, false); - public static readonly RDS_INSTANCE = new RdsUrlType(true, false); - public static readonly RDS_AURORA_LIMITLESS_DB_SHARD_GROUP = new RdsUrlType(true, false); - public static readonly OTHER = new RdsUrlType(false, false); + public static readonly IP_ADDRESS = new RdsUrlType(false, false, false); + public static readonly RDS_WRITER_CLUSTER = new RdsUrlType(true, true, true); + public static readonly RDS_READER_CLUSTER = new RdsUrlType(true, true, true); + public static readonly RDS_CUSTOM_CLUSTER = new RdsUrlType(true, false, true); + public static readonly RDS_PROXY = new RdsUrlType(true, false, true); + public static readonly RDS_PROXY_ENDPOINT = new RdsUrlType(true, false, true); + public static readonly RDS_INSTANCE = new RdsUrlType(true, false, true); + public static readonly RDS_AURORA_LIMITLESS_DB_SHARD_GROUP = new RdsUrlType(true, false, true); + public static readonly RDS_GLOBAL_WRITER_CLUSTER = new RdsUrlType(true, true, false); + public static readonly OTHER = new RdsUrlType(false, false, false); private constructor( public readonly isRds: boolean, - public readonly isRdsCluster: boolean + public readonly isRdsCluster: boolean, + public readonly hasRegion: boolean ) {} } diff --git a/common/lib/utils/rds_utils.ts b/common/lib/utils/rds_utils.ts index 866dfaf9c..95c894d71 100644 --- a/common/lib/utils/rds_utils.ts +++ b/common/lib/utils/rds_utils.ts @@ -22,12 +22,13 @@ export class RdsUtils { // can be found at // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.Overview.Endpoints.html // - // Details how to use RDS Proxy endpoints can be found at + // Details how to use RDS Proxy endpoints can be found at // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/rds-proxy-endpoints.html // - // Values like "<...>" depend on particular Aurora cluster. + // Values like "<...>" depend on particular Aurora cluster. // For example: "" // + // // Cluster (Writer) Endpoint: .cluster-..rds.amazonaws.com // Example: test-postgres.cluster-123456789012.us-east-2.rds.amazonaws.com // @@ -41,7 +42,10 @@ export class RdsUtils { // Example: test-postgres-instance-1.123456789012.us-east-2.rds.amazonaws.com // // + // // Similar endpoints for China regions have different structure and are presented below. + // https://docs.amazonaws.cn/en_us/aws/latest/userguide/endpoints-Ningxia.html + // https://docs.amazonaws.cn/en_us/aws/latest/userguide/endpoints-Beijing.html // // Cluster (Writer) Endpoint: .cluster-.rds..amazonaws.com.cn // Example: test-postgres.cluster-123456789012.rds.cn-northwest-1.amazonaws.com.cn @@ -59,48 +63,51 @@ export class RdsUtils { // Governmental endpoints // https://aws.amazon.com/compliance/fips/#FIPS_Endpoints_by_Service // https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/Region.html + // + // + // Aurora Global Database + // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Concepts.Aurora_Fea_Regions_DB-eng.Feature.GlobalDatabase.html + // Global Database Endpoint: .global-.global.rds.amazonaws.com + // Example: test-global-db-name.global-123456789012.global.rds.amazonaws.com + // + // + // RDS Proxy + // RDS Proxy Endpoint: .proxy-..rds.amazonaws.com + // Example: test-rds-proxy-name.proxy-123456789012.us-east-2.rds.amazonaws.com + // + // RDS Proxy Custom Endpoint: .endpoint.proxy-..rds.amazonaws.com + // Example: test-custom-endpoint-name.endpoint.proxy-123456789012.us-east-2.rds.amazonaws.com + + private static readonly AURORA_GLOBAL_WRITER_DNS_PATTERN = + /^(?.+)\.(?global-)?(?[a-zA-Z0-9]+\.global\.rds\.amazonaws\.com\.?)$/i; private static readonly AURORA_DNS_PATTERN = - /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; - private static readonly AURORA_INSTANCE_PATTERN = /^(?.+)\.(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; + /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.(rds|rds-fips)\.amazonaws\.(com|au|eu|uk)\.?)$/i; private static readonly AURORA_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; - private static readonly AURORA_CUSTOM_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-custom-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; + /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.(rds|rds-fips)\.amazonaws\.(com|au|eu|uk)\.?)$/i; private static readonly AURORA_LIMITLESS_CLUSTER_PATTERN = - /^(?.+)\.(?shardgrp-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.(amazonaws\.com(\.cn)?|sc2s\.sgov\.gov|c2s\.ic\.gov))$/i; - private static readonly AURORA_PROXY_DNS_PATTERN = - /^(?.+)\.(?proxy-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; + /^(?.+)\.(?shardgrp-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.(rds|rds-fips)\.(amazonaws\.com\.?|amazonaws\.eu\.?|amazonaws\.au\.?|amazonaws\.uk\.?|amazonaws\.com\.cn\.?|sc2s\.sgov\.gov\.?|c2s\.ic\.gov\.?))$/i; private static readonly AURORA_CHINA_DNS_PATTERN = - /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i; + /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(rds|rds-fips)\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn\.?)$/i; private static readonly AURORA_OLD_CHINA_DNS_PATTERN = - /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_CHINA_INSTANCE_PATTERN = - /^(?.+)\.(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_OLD_CHINA_INSTANCE_PATTERN = - /^(?.+)\.(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i; + /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.(rds|rds-fips)\.amazonaws\.com\.cn\.?)$/i; private static readonly AURORA_CHINA_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_CHINA_LIMITLESS_CLUSTER_PATTERN = - /^(?.+)\.(?shardgrp-)?(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i; + /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(rds|rds-fips)\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn\.?)$/i; private static readonly AURORA_OLD_CHINA_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_OLD_CHINA_LIMITLESS_CLUSTER_PATTERN = - /^(?.+)\.(?shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_CHINA_CUSTOM_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-custom-)+(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_OLD_CHINA_CUSTOM_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-custom-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_CHINA_PROXY_DNS_PATTERN = - /^(?.+)\.(?proxy-)+(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-])+\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_OLD_CHINA_PROXY_DNS_PATTERN = - /^(?.+)\.(?proxy-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-])+\.rds\.amazonaws\.com\.cn)$/i; - + /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.(rds|rds-fips)\.amazonaws\.com\.cn\.?)$/i; private static readonly AURORA_GOV_DNS_PATTERN = - /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.(amazonaws\.com|c2s\.ic\.gov|sc2s\.sgov\.gov))$/i; + /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(rds|rds-fips)\.(?[a-zA-Z0-9-]+)\.(amazonaws\.com\.?|c2s\.ic\.gov\.?|sc2s\.sgov\.gov\.?))$/i; private static readonly AURORA_GOV_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.(amazonaws\.com|c2s\.ic\.gov|sc2s\.sgov\.gov))$/i; + /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(rds|rds-fips)\.(?[a-zA-Z0-9-]+)\.(amazonaws\.com\.?|c2s\.ic\.gov\.?|sc2s\.sgov\.gov\.?))$/i; + + // RDS Proxy Custom Endpoint: .endpoint.proxy-..rds.amazonaws.com + private static readonly RDS_PROXY_ENDPOINT_DNS_PATTERN = + /^(?.+)\.endpoint\.(?proxy-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.?)$/i; + private static readonly RDS_PROXY_ENDPOINT_CHINA_DNS_PATTERN = + /^(?.+)\.endpoint\.(?proxy-)+(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn\.?)$/i; + private static readonly RDS_PROXY_ENDPOINT_OLD_CHINA_DNS_PATTERN = + /^(?.+)\.endpoint\.(?proxy-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn\.?)$/i; private static readonly ELB_PATTERN = /^(?.+)\.elb\.((?[a-zA-Z0-9-]+)\.amazonaws\.com)$/i; private static readonly IP_V4 = @@ -117,20 +124,24 @@ export class RdsUtils { private static readonly cachedPatterns = new Map(); private static readonly cachedDnsPatterns = new Map(); + private static prepareHostFunc?: (host: string) => string; public isRdsClusterDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return equalsIgnoreCase(dnsGroup, "cluster-") || equalsIgnoreCase(dnsGroup, "cluster-ro-"); } public isRdsCustomClusterDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return equalsIgnoreCase(dnsGroup, "cluster-custom-"); } public isRdsDns(host: string): boolean { + const preparedHost = RdsUtils.getPreparedHost(host); const matcher = this.cacheMatcher( - host, + preparedHost, RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, @@ -139,24 +150,46 @@ export class RdsUtils { const group = this.getRegexGroup(matcher, RdsUtils.DNS_GROUP); if (group) { - RdsUtils.cachedDnsPatterns.set(host, group); + RdsUtils.cachedDnsPatterns.set(preparedHost, group); } return matcher != null; } public isRdsInstance(host: string): boolean { - return !this.getDnsGroup(host) && this.isRdsDns(host); + const preparedHost = RdsUtils.getPreparedHost(host); + return !this.getDnsGroup(preparedHost) && this.isRdsDns(preparedHost); } isRdsProxyDns(host: string) { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return dnsGroup && dnsGroup.startsWith("proxy-"); } + isRdsProxyEndpointDns(host: string): boolean { + if (!host) { + return false; + } + + const preparedHost = RdsUtils.getPreparedHost(host); + const matcher = this.cacheMatcher( + preparedHost, + RdsUtils.RDS_PROXY_ENDPOINT_DNS_PATTERN, + RdsUtils.RDS_PROXY_ENDPOINT_CHINA_DNS_PATTERN, + RdsUtils.RDS_PROXY_ENDPOINT_OLD_CHINA_DNS_PATTERN + ); + if (this.getRegexGroup(matcher, RdsUtils.DNS_GROUP) !== null) { + return this.getRegexGroup(matcher, RdsUtils.INSTANCE_GROUP) !== null; + } + + return false; + } + getRdsClusterId(host: string): string | null { + const preparedHost = RdsUtils.getPreparedHost(host); const matcher = this.cacheMatcher( - host, + preparedHost, RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, @@ -175,8 +208,9 @@ export class RdsUtils { return null; } + const preparedHost = RdsUtils.getPreparedHost(host); const matcher = this.cacheMatcher( - host, + preparedHost, RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, @@ -194,8 +228,9 @@ export class RdsUtils { return "?"; } + const preparedHost = RdsUtils.getPreparedHost(host); const matcher = this.cacheMatcher( - host, + preparedHost, RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, @@ -210,8 +245,9 @@ export class RdsUtils { return null; } + const preparedHost = RdsUtils.getPreparedHost(host); const matcher = this.cacheMatcher( - host, + preparedHost, RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, @@ -223,7 +259,7 @@ export class RdsUtils { return group; } - const elbMatcher = host.match(RdsUtils.ELB_PATTERN); + const elbMatcher = preparedHost.match(RdsUtils.ELB_PATTERN); if (elbMatcher && elbMatcher.length > 0) { return this.getRegexGroup(elbMatcher, RdsUtils.REGION_GROUP); } @@ -231,18 +267,36 @@ export class RdsUtils { return null; } + public isSameRegion(host1: string | null, host2: string | null): boolean { + if (!host1 || !host2) { + return false; + } + const host1Region = this.getRdsRegion(host1); + const host2Region = this.getRdsRegion(host2); + return host1Region !== null && equalsIgnoreCase(host1Region, host2Region); + } + + public isGlobalDbWriterClusterDns(host: string): boolean { + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); + return equalsIgnoreCase(dnsGroup, "global-"); + } + public isWriterClusterDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return equalsIgnoreCase(dnsGroup, "cluster-"); } public isReaderClusterDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return equalsIgnoreCase(dnsGroup, "cluster-ro-"); } public isLimitlessDbShardGroupDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); if (!dnsGroup) { return false; } @@ -254,25 +308,26 @@ export class RdsUtils { return null; } - const matcher = host.match(RdsUtils.AURORA_CLUSTER_PATTERN); + const preparedHost = RdsUtils.getPreparedHost(host); + const matcher = preparedHost.match(RdsUtils.AURORA_CLUSTER_PATTERN); if (matcher) { - return host.replace(RdsUtils.AURORA_CLUSTER_PATTERN, "$.cluster-$"); + return preparedHost.replace(RdsUtils.AURORA_CLUSTER_PATTERN, "$.cluster-$"); } - const limitlessMatcher = host.match(RdsUtils.AURORA_LIMITLESS_CLUSTER_PATTERN); + const limitlessMatcher = preparedHost.match(RdsUtils.AURORA_LIMITLESS_CLUSTER_PATTERN); if (limitlessMatcher) { - return host.replace(RdsUtils.AURORA_LIMITLESS_CLUSTER_PATTERN, "$.cluster-$"); + return preparedHost.replace(RdsUtils.AURORA_LIMITLESS_CLUSTER_PATTERN, "$.cluster-$"); } - const chinaMatcher = host.match(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN); + const chinaMatcher = preparedHost.match(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN); if (chinaMatcher) { - return host.replace(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN, "$.cluster-$"); + return preparedHost.replace(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN, "$.cluster-$"); } - const oldChinaMatcher = host.match(RdsUtils.AURORA_OLD_CHINA_CLUSTER_PATTERN); + const oldChinaMatcher = preparedHost.match(RdsUtils.AURORA_OLD_CHINA_CLUSTER_PATTERN); if (oldChinaMatcher) { - return host.replace(RdsUtils.AURORA_OLD_CHINA_CLUSTER_PATTERN, "$.cluster-$"); + return preparedHost.replace(RdsUtils.AURORA_OLD_CHINA_CLUSTER_PATTERN, "$.cluster-$"); } - const govMatcher = host.match(RdsUtils.AURORA_GOV_CLUSTER_PATTERN); + const govMatcher = preparedHost.match(RdsUtils.AURORA_GOV_CLUSTER_PATTERN); if (govMatcher) { - return host.replace(RdsUtils.AURORA_GOV_CLUSTER_PATTERN, "$.cluster-$"); + return preparedHost.replace(RdsUtils.AURORA_GOV_CLUSTER_PATTERN, "$.cluster-$"); } return null; } @@ -298,19 +353,24 @@ export class RdsUtils { return RdsUrlType.OTHER; } - if (this.isIPv4(host) || this.isIPv6(host)) { + const preparedHost = RdsUtils.getPreparedHost(host); + if (this.isIPv4(preparedHost) || this.isIPv6(preparedHost)) { return RdsUrlType.IP_ADDRESS; - } else if (this.isWriterClusterDns(host)) { + } else if (this.isGlobalDbWriterClusterDns(preparedHost)) { + return RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER; + } else if (this.isWriterClusterDns(preparedHost)) { return RdsUrlType.RDS_WRITER_CLUSTER; - } else if (this.isReaderClusterDns(host)) { + } else if (this.isReaderClusterDns(preparedHost)) { return RdsUrlType.RDS_READER_CLUSTER; - } else if (this.isRdsCustomClusterDns(host)) { + } else if (this.isRdsCustomClusterDns(preparedHost)) { return RdsUrlType.RDS_CUSTOM_CLUSTER; - } else if (this.isLimitlessDbShardGroupDns(host)) { + } else if (this.isLimitlessDbShardGroupDns(preparedHost)) { return RdsUrlType.RDS_AURORA_LIMITLESS_DB_SHARD_GROUP; - } else if (this.isRdsProxyDns(host)) { + } else if (this.isRdsProxyDns(preparedHost)) { return RdsUrlType.RDS_PROXY; - } else if (this.isRdsDns(host)) { + } else if (this.isRdsProxyEndpointDns(preparedHost)) { + return RdsUrlType.RDS_PROXY_ENDPOINT; + } else if (this.isRdsDns(preparedHost)) { return RdsUrlType.RDS_INSTANCE; } else { // ELB URLs will also be classified as other @@ -319,23 +379,27 @@ export class RdsUtils { } public isGreenInstance(host: string) { - return host && RdsUtils.BG_GREEN_HOST_PATTERN.test(host); + const preparedHost = RdsUtils.getPreparedHost(host); + return preparedHost && RdsUtils.BG_GREEN_HOST_PATTERN.test(preparedHost); } public isOldInstance(host: string): boolean { - return !!host && RdsUtils.BG_OLD_HOST_PATTERN.test(host); + const preparedHost = RdsUtils.getPreparedHost(host); + return !!preparedHost && RdsUtils.BG_OLD_HOST_PATTERN.test(preparedHost); } public isNotOldInstance(host: string): boolean { if (!host) { return true; } - return !RdsUtils.BG_OLD_HOST_PATTERN.test(host); + const preparedHost = RdsUtils.getPreparedHost(host); + return !RdsUtils.BG_OLD_HOST_PATTERN.test(preparedHost); } // Verify that provided host is a blue host name and contains neither green prefix nor old prefix. public isNotGreenAndOldPrefixInstance(host: string): boolean { - return !!host && !RdsUtils.BG_GREEN_HOST_PATTERN.test(host) && !RdsUtils.BG_OLD_HOST_PATTERN.test(host); + const preparedHost = RdsUtils.getPreparedHost(host); + return !!preparedHost && !RdsUtils.BG_GREEN_HOST_PATTERN.test(preparedHost) && !RdsUtils.BG_OLD_HOST_PATTERN.test(preparedHost); } public removeGreenInstancePrefix(host: string): string { @@ -343,7 +407,8 @@ export class RdsUtils { return host; } - const matcher = host.match(RdsUtils.BG_GREEN_HOST_PATTERN); + const preparedHost = RdsUtils.getPreparedHost(host); + const matcher = preparedHost.match(RdsUtils.BG_GREEN_HOST_PATTERN); if (!matcher || matcher.length === 0) { return host; } @@ -382,7 +447,8 @@ export class RdsUtils { RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, - RdsUtils.AURORA_GOV_DNS_PATTERN + RdsUtils.AURORA_GOV_DNS_PATTERN, + RdsUtils.AURORA_GLOBAL_WRITER_DNS_PATTERN ); return this.getRegexGroup(matcher, RdsUtils.DNS_GROUP); } @@ -415,4 +481,20 @@ export class RdsUtils { RdsUtils.cachedPatterns.clear(); RdsUtils.cachedDnsPatterns.clear(); } + + static setPrepareHostFunc(func?: (host: string) => string) { + RdsUtils.prepareHostFunc = func; + } + + static resetPrepareHostFunc() { + RdsUtils.prepareHostFunc = undefined; + } + + private static getPreparedHost(host: string): string { + const func = RdsUtils.prepareHostFunc; + if (!func) { + return host; + } + return func(host) ?? host; + } } diff --git a/common/lib/utils/region_utils.ts b/common/lib/utils/region_utils.ts index 7b26a619c..4c5a008a3 100644 --- a/common/lib/utils/region_utils.ts +++ b/common/lib/utils/region_utils.ts @@ -17,6 +17,7 @@ import { RdsUtils } from "./rds_utils"; import { AwsWrapperError } from "./errors"; import { Messages } from "./messages"; +import { HostInfo } from "../host_info"; export class RegionUtils { static readonly REGIONS: string[] = [ @@ -67,21 +68,21 @@ export class RegionUtils { protected static readonly rdsUtils = new RdsUtils(); - static getRegion(regionString: string, host?: string): string | null { - const region = RegionUtils.getRegionFromRegionString(regionString); + async getRegion(regionKey: string, hostInfo?: HostInfo, props?: Map): Promise { + const region = this.getRegionFromRegionString(props?.get(regionKey)); if (region !== null) { return region; } - if (host) { - return RegionUtils.getRegionFromHost(host); + if (hostInfo) { + return this.getRegionFromHost(hostInfo.host); } return region; } - private static getRegionFromRegionString(regionString: string): string { + getRegionFromRegionString(regionString: string): string | null { if (!regionString) { return null; } @@ -94,12 +95,12 @@ export class RegionUtils { return region; } - private static getRegionFromHost(host: string): string | null { + getRegionFromHost(host: string): string | null { const regionString = RegionUtils.rdsUtils.getRdsRegion(host); if (!regionString) { throw new AwsWrapperError(Messages.get("AwsSdk.unsupportedRegion", regionString)); } - return RegionUtils.getRegionFromRegionString(regionString); + return this.getRegionFromRegionString(regionString); } } diff --git a/common/lib/utils/service_utils.ts b/common/lib/utils/service_utils.ts new file mode 100644 index 000000000..35c98b382 --- /dev/null +++ b/common/lib/utils/service_utils.ts @@ -0,0 +1,127 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { FullServicesContainer, FullServicesContainerImpl } from "./full_services_container"; +import { StorageService } from "./storage/storage_service"; +import { PluginServiceImpl } from "../plugin_service"; +import { PluginManager } from "../plugin_manager"; +import { ConnectionProviderManager } from "../connection_provider_manager"; +import { DriverConnectionProvider } from "../driver_connection_provider"; +import { WrapperProperties } from "../wrapper_property"; +import { ConnectionProvider } from "../connection_provider"; +import { AwsClient } from "../aws_client"; +import { DatabaseDialect, DatabaseType } from "../database_dialect/database_dialect"; +import { DatabaseDialectCodes } from "../database_dialect/database_dialect_codes"; +import { DriverDialect } from "../driver_dialect/driver_dialect"; +import { MonitorService } from "./monitoring/monitor_service"; +import { TelemetryFactory } from "./telemetry/telemetry_factory"; +import { EventPublisher } from "./events/event"; +import { PartialPluginService } from "../partial_plugin_service"; +import { ConnectionUrlParser } from "./connection_url_parser"; +import { HostIdCacheServiceImpl } from "./host_id_cache_service"; + +export class ServiceUtils { + private static readonly _instance: ServiceUtils = new ServiceUtils(); + + static get instance(): ServiceUtils { + return this._instance; + } + + createStandardServiceContainer( + storageService: StorageService, + monitorService: MonitorService, + eventPublisher: EventPublisher, + client: AwsClient, + props: Map, + dbType: DatabaseType, + knownDialectsByCode: Map, + driverDialect: DriverDialect, + telemetryFactory: TelemetryFactory, + connectionProvider: ConnectionProvider | null + ): FullServicesContainer { + const servicesContainer: FullServicesContainer = new FullServicesContainerImpl( + storageService, + monitorService, + eventPublisher, + connectionProvider, + telemetryFactory + ); + + const pluginService = new PluginServiceImpl(servicesContainer, client, dbType, knownDialectsByCode, props, driverDialect); + const pluginManager = new PluginManager( + servicesContainer, + props, + new ConnectionProviderManager(connectionProvider ?? new DriverConnectionProvider(), WrapperProperties.CONNECTION_PROVIDER.get(props)), + telemetryFactory + ); + + servicesContainer.pluginService = pluginService; + servicesContainer.pluginManager = pluginManager; + servicesContainer.hostListProviderService = pluginService; + servicesContainer.hostIdCacheService = new HostIdCacheServiceImpl(); + + return servicesContainer; + } + + createMinimalServiceContainer( + storageService: StorageService, + monitorService: MonitorService, + eventPublisher: EventPublisher, + props: Map, + dialect: DatabaseDialect, + driverDialect: DriverDialect, + telemetryFactory: TelemetryFactory, + connectionProvider: ConnectionProvider | null, + connectionUrlParser: ConnectionUrlParser + ): FullServicesContainer { + const servicesContainer: FullServicesContainer = new FullServicesContainerImpl( + storageService, + monitorService, + eventPublisher, + connectionProvider, + telemetryFactory + ); + + const pluginService = new PartialPluginService(servicesContainer, props, dialect, driverDialect, connectionUrlParser); + const pluginManager = new PluginManager( + servicesContainer, + props, + new ConnectionProviderManager(connectionProvider ?? new DriverConnectionProvider(), WrapperProperties.CONNECTION_PROVIDER.get(props)), + telemetryFactory + ); + + servicesContainer.pluginService = pluginService; + servicesContainer.pluginManager = pluginManager; + servicesContainer.hostListProviderService = pluginService; + servicesContainer.hostIdCacheService = new HostIdCacheServiceImpl(); + + return servicesContainer; + } + + createMinimalServiceContainerFrom(servicesContainer: FullServicesContainer, props: Map): FullServicesContainer { + return this.createMinimalServiceContainer( + servicesContainer.storageService, + servicesContainer.monitorService, + servicesContainer.eventPublisher, + props, + servicesContainer.pluginService.getDialect(), + servicesContainer.pluginService.getDriverDialect(), + servicesContainer.telemetryFactory, + servicesContainer.defaultConnectionProvider, + servicesContainer.pluginService.getConnectionUrlParser() + ); + } +} diff --git a/common/lib/utils/sliding_expiration_cache.ts b/common/lib/utils/sliding_expiration_cache.ts index b2d27abb6..904d9dc30 100644 --- a/common/lib/utils/sliding_expiration_cache.ts +++ b/common/lib/utils/sliding_expiration_cache.ts @@ -16,25 +16,7 @@ import { getTimeInNanos } from "./utils"; import { MapUtils } from "./map_utils"; - -class CacheItem { - readonly item: V; - private _expirationTimeNanos: bigint; - - constructor(item: V, expirationTimeNanos: bigint) { - this.item = item; - this._expirationTimeNanos = expirationTimeNanos; - } - - get expirationTimeNs(): bigint { - return this._expirationTimeNanos; - } - - updateExpiration(expirationIntervalNanos: bigint): CacheItem { - this._expirationTimeNanos = getTimeInNanos() + expirationIntervalNanos; - return this; - } -} +import { CacheItem } from "./cache_map"; export class SlidingExpirationCache { protected _cleanupIntervalNanos: bigint = BigInt(10 * 60_000_000_000); // 10 minutes @@ -69,29 +51,29 @@ export class SlidingExpirationCache { computeIfAbsent(key: K, mappingFunc: (key: K) => V, itemExpirationNanos: bigint): V | null { this.cleanUp(); const cacheItem = MapUtils.computeIfAbsent(this.map, key, (k) => new CacheItem(mappingFunc(k), getTimeInNanos() + itemExpirationNanos)); - return cacheItem?.updateExpiration(itemExpirationNanos).item ?? null; + return cacheItem?.updateExpiration(itemExpirationNanos).get() ?? null; } putIfAbsent(key: K, value: V, itemExpirationNanos: bigint): V | null { const cacheItem = MapUtils.putIfAbsent(this.map, key, new CacheItem(value, getTimeInNanos() + itemExpirationNanos)); - return cacheItem?.item ?? null; + return cacheItem?.get(true) ?? null; } get(key: K, itemExpirationNanos?: bigint): V | undefined { this.cleanUp(); const cacheItem = this.map.get(key); - if (cacheItem?.item && itemExpirationNanos) { + if (cacheItem?.get(true) && itemExpirationNanos) { cacheItem.updateExpiration(itemExpirationNanos); - return cacheItem.item; + return cacheItem.get(); } - return cacheItem?.item; + return cacheItem?.get(true); } put(key: K, value: V, itemExpirationNanos: bigint): V | null { this.cleanUp(); const cacheItem = new CacheItem(value, getTimeInNanos() + itemExpirationNanos); this.map.set(key, cacheItem); - return cacheItem.item; + return cacheItem.get(true); } remove(key: K): void { @@ -102,7 +84,7 @@ export class SlidingExpirationCache { removeAndDispose(key: K): void { const cacheItem = MapUtils.remove(this.map, key); if (cacheItem != null && this._itemDisposalFunc != null) { - this._itemDisposalFunc(cacheItem.item); + this._itemDisposalFunc(cacheItem.get(true)); } } @@ -110,7 +92,7 @@ export class SlidingExpirationCache { let item; MapUtils.computeIfPresent(this.map, key, (key, cacheItem) => { if (this.shouldCleanupItem(cacheItem)) { - item = cacheItem.item; + item = cacheItem.get(true); return null; } return cacheItem; @@ -123,15 +105,15 @@ export class SlidingExpirationCache { shouldCleanupItem(cacheItem: CacheItem): boolean { if (this._shouldDisposeFunc != null) { - return getTimeInNanos() > cacheItem.expirationTimeNs && this._shouldDisposeFunc(cacheItem.item); + return cacheItem.isExpired() && this._shouldDisposeFunc(cacheItem.get(true)); } - return getTimeInNanos() > cacheItem.expirationTimeNs; + return cacheItem.isExpired(); } clear(): void { for (const [key, val] of this.map.entries()) { if (val !== undefined && this._itemDisposalFunc !== undefined) { - this._itemDisposalFunc(val.item); + this._itemDisposalFunc(val.get()); } } this.map.clear(); diff --git a/common/lib/utils/status_cache_item.ts b/common/lib/utils/status_cache_item.ts new file mode 100644 index 000000000..6e7489654 --- /dev/null +++ b/common/lib/utils/status_cache_item.ts @@ -0,0 +1,23 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +export class StatusCacheItem { + readonly status: T; + + constructor(status: T) { + this.status = status; + } +} diff --git a/common/lib/utils/storage/expiration_cache.ts b/common/lib/utils/storage/expiration_cache.ts new file mode 100644 index 000000000..72e3b7d1c --- /dev/null +++ b/common/lib/utils/storage/expiration_cache.ts @@ -0,0 +1,160 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { getTimeInNanos } from "../utils"; +import { ItemDisposalFunc, ShouldDisposeFunc } from "../../types"; + +export class CacheEntry { + readonly value: V; + private _expirationTimeNanos: bigint; + + constructor(value: V, expirationTimeNanos: bigint) { + this.value = value; + this._expirationTimeNanos = expirationTimeNanos; + } + + get expirationTimeNanos(): bigint { + return this._expirationTimeNanos; + } + + renewExpiration(timeToLiveNanos: bigint): void { + this._expirationTimeNanos = getTimeInNanos() + timeToLiveNanos; + } + + isExpired(): boolean { + return getTimeInNanos() > this._expirationTimeNanos; + } +} + +export class ExpirationCache { + private readonly map: Map> = new Map(); + private readonly isRenewableExpiration: boolean; + private readonly timeToLiveNanos: bigint; + private readonly shouldDisposeFunc?: ShouldDisposeFunc | null; + private readonly itemDisposalFunc?: ItemDisposalFunc | null; + + constructor( + isRenewableExpiration: boolean = true, + timeToLiveNanos: bigint = BigInt(300_000_000_000), // 5 minutes default + shouldDisposeFunc?: ShouldDisposeFunc | null, + itemDisposalFunc?: ItemDisposalFunc | null + ) { + this.isRenewableExpiration = isRenewableExpiration; + this.timeToLiveNanos = timeToLiveNanos; + this.shouldDisposeFunc = shouldDisposeFunc; + this.itemDisposalFunc = itemDisposalFunc; + } + + put(key: K, value: V): void { + const expirationTime = getTimeInNanos() + this.timeToLiveNanos; + this.map.set(key, new CacheEntry(value, expirationTime)); + } + + get(key: K): V | null | undefined { + const entry = this.map.get(key); + if (!entry) { + return null; + } + + if (entry.isExpired()) { + if (this.shouldDispose(entry.value)) { + this.removeEntry(key, entry); + return null; + } + } + + if (this.isRenewableExpiration) { + entry.renewExpiration(this.timeToLiveNanos); + } + + return entry.value; + } + + exists(key: K): boolean { + const entry = this.map.get(key); + if (!entry) { + return false; + } + + if (entry.isExpired() && this.shouldDispose(entry.value)) { + this.removeEntry(key, entry); + return false; + } + + return true; + } + + remove(key: K): void { + const entry = this.map.get(key); + if (entry) { + this.removeEntry(key, entry); + } + } + + clear(): void { + for (const entry of this.map.values()) { + this.disposeItem(entry.value); + } + this.map.clear(); + } + + size(): number { + return this.map.size; + } + + getEntries(): Map { + const result = new Map(); + for (const [key, entry] of this.map.entries()) { + result.set(key, entry.value); + } + return result; + } + + removeExpiredEntries(): void { + const keysToRemove: K[] = []; + + for (const [key, entry] of this.map.entries()) { + if (entry.isExpired() && this.shouldDispose(entry.value)) { + keysToRemove.push(key); + } + } + + for (const key of keysToRemove) { + const entry = this.map.get(key); + if (entry) { + this.removeEntry(key, entry); + } + } + } + + private shouldDispose(value: V): boolean { + if (this.shouldDisposeFunc) { + return this.shouldDisposeFunc(value); + } + return true; + } + + private removeEntry(key: K, entry: CacheEntry): void { + this.map.delete(key); + this.disposeItem(entry.value); + } + + private disposeItem(value: V): void { + if (this.itemDisposalFunc) { + this.itemDisposalFunc(value); + } + } +} diff --git a/common/lib/utils/storage/storage_service.ts b/common/lib/utils/storage/storage_service.ts new file mode 100644 index 000000000..dc2bd71d0 --- /dev/null +++ b/common/lib/utils/storage/storage_service.ts @@ -0,0 +1,269 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Constructor, ItemDisposalFunc, ShouldDisposeFunc } from "../../types"; +import { ExpirationCache } from "./expiration_cache"; +import { Topology } from "../../host_list_provider/topology"; +import { AwsWrapperError } from "../errors"; +import { Messages } from "../messages"; +import { EventPublisher } from "../events/event"; +import { DataAccessEvent } from "../events/data_access_event"; +import { AllowedAndBlockedHosts } from "../../allowed_and_blocked_hosts"; +import { BlueGreenStatus } from "../../plugins/bluegreen/blue_green_status"; +import { HostAvailabilityCacheItem } from "../../host_availability/host_availability_cache_item"; +import { StatusCacheItem } from "../status_cache_item"; + +const DEFAULT_CLEANUP_INTERVAL_NANOS = 5 * 60 * 1_000_000_000; // 5 minutes +const SIXTY_MINUTES_NANOS = BigInt(60 * 60 * 1_000_000_000); // 60 minutes + +/** + * Interface for a storage service that manages items with expiration and disposal logic. + * Extends StateSnapshotProvider for state management capabilities. + */ +export interface StorageService { + /** + * Registers a new item class with the storage service. This method needs to be called before adding new classes of + * items to the service, so that the service knows when and how to dispose of the item. Expected item classes will be + * added automatically during driver initialization, but this method can be called to add new classes of items. + * + * @param itemClass The constructor/class of the item that will be stored + * @param isRenewableExpiration Controls whether the item's expiration should be renewed if the item is fetched, + * regardless of whether it is already expired or not + * @param timeToLiveNanos How long an item should be stored before being considered expired, in nanoseconds + * @param shouldDisposeFunc A function defining whether an item should be disposed if expired. If null/undefined, + * the item will always be disposed if expired + * @param itemDisposalFunc A function defining how to dispose of an item when it is removed. If null/undefined, + * the item will be removed without performing any additional operations + */ + registerItemClassIfAbsent( + itemClass: Constructor, + isRenewableExpiration: boolean, + timeToLiveNanos: bigint, + shouldDisposeFunc?: ShouldDisposeFunc | null, + itemDisposalFunc?: ItemDisposalFunc | null + ): void; + + /** + * Stores an item in the storage service under the given item class. + * + * @param key The key for the item, e.g., "custom-endpoint.cluster-custom-XYZ.us-east-2.rds.amazonaws.com:5432" + * @param item The item to store + */ + set(key: unknown, item: V): void; + + /** + * Gets an item stored in the storage service. + * + * @param itemClass The expected constructor/class of the item being retrieved + * @param key The key for the item, e.g., "custom-endpoint.cluster-custom-XYZ.us-east-2.rds.amazonaws.com:5432" + * @param registerDataAccess Whether to register a data access event. Defaults to true. + * @returns The item stored at the given key for the given item class, or null/undefined if not found + */ + get(itemClass: Constructor, key?: unknown, registerDataAccess?: boolean): V | null; + + /** + * Indicates whether an item exists under the given item class and key. + * + * @param itemClass The constructor/class of the item + * @param key The key for the item, e.g., "custom-endpoint.cluster-custom-XYZ.us-east-2.rds.amazonaws.com:5432" + * @returns true if the item exists under the given item class and key, otherwise returns false + */ + exists(itemClass: Constructor, key: unknown): boolean; + + /** + * Removes an item stored under the given item class. + * + * @param itemClass The constructor/class of the item + * @param key The key for the item, e.g., "custom-endpoint.cluster-custom-XYZ.us-east-2.rds.amazonaws.com:5432" + */ + remove(itemClass: Constructor, key: unknown): void; + + /** + * Clears all items of the given item class. For example, storageService.clear(AllowedAndBlockedHosts) will + * remove all AllowedAndBlockedHost items from the storage service. + * + * @param itemClass The constructor/class of the items to clear + */ + clear(itemClass: Constructor): void; + + /** + * Clears all items from the storage service. + */ + clearAll(): void; + + /** + * Returns the number of items stored for the given item class. + * + * @param itemClass The constructor/class of the items to count + * @returns The number of items stored for the given item class + */ + size(itemClass: Constructor): number; + + /** + * Cleanup method to stop the cleanup interval timer. + * Should be called when the service is no longer needed. + */ + releaseResources(): Promise; +} + +type CacheSupplier = () => ExpirationCache; + +export class StorageServiceImpl implements StorageService { + private static readonly DEFAULT_HOST_AVAILABILITY_CACHE_EXPIRE_NANO = BigInt(5 * 60_000_000_000); // 5 minutes + + private static readonly defaultCacheSuppliers: Map = (() => { + const suppliers = new Map(); + suppliers.set(Topology, () => new ExpirationCache()); + suppliers.set(AllowedAndBlockedHosts, () => new ExpirationCache()); + suppliers.set(BlueGreenStatus, () => new ExpirationCache(false, SIXTY_MINUTES_NANOS, null, null)); + suppliers.set( + HostAvailabilityCacheItem, + () => new ExpirationCache(true, StorageServiceImpl.DEFAULT_HOST_AVAILABILITY_CACHE_EXPIRE_NANO, null, null) + ); + suppliers.set(StatusCacheItem, () => new ExpirationCache(false, SIXTY_MINUTES_NANOS, null, null)); + return suppliers; + })(); + + protected readonly caches: Map> = new Map(); + protected cleanupIntervalHandle?: NodeJS.Timeout; + protected readonly publisher: EventPublisher; + + constructor(publisher: EventPublisher, cleanupIntervalNanos: number = DEFAULT_CLEANUP_INTERVAL_NANOS) { + this.publisher = publisher; + this.initCleanupTask(cleanupIntervalNanos); + } + + protected initCleanupTask(cleanupIntervalNanos: number): void { + const intervalMs = cleanupIntervalNanos / 1_000_000; + this.cleanupIntervalHandle = setInterval(() => { + this.removeExpiredItems(); + }, intervalMs); + + // Unref the timer to prevent this background cleanup task from blocking the application from gracefully exiting. + this.cleanupIntervalHandle.unref(); + } + + protected removeExpiredItems(): void { + for (const cache of this.caches.values()) { + cache.removeExpiredEntries(); + } + } + + registerItemClassIfAbsent( + itemClass: Constructor, + isRenewableExpiration: boolean, + timeToLiveNanos: bigint, + shouldDisposeFunc?: ShouldDisposeFunc | null, + itemDisposalFunc?: ItemDisposalFunc | null + ): void { + if (!this.caches.has(itemClass)) { + const cache = new ExpirationCache( + isRenewableExpiration, + timeToLiveNanos, + shouldDisposeFunc ?? undefined, + itemDisposalFunc ?? undefined + ); + this.caches.set(itemClass, cache); + } + } + + set(key: unknown, item: V): void { + const itemClass = item.constructor as Constructor; + let cache = this.caches.get(itemClass); + + if (!cache) { + const supplier = StorageServiceImpl.defaultCacheSuppliers.get(itemClass); + if (!supplier) { + throw new AwsWrapperError(Messages.get("StorageService.itemClassNotRegistered", itemClass.name)); + } + cache = supplier(); + this.caches.set(itemClass, cache); + } + + try { + cache.put(key, item); + } catch (error) { + throw new AwsWrapperError(Messages.get("StorageService.unexpectedValueMismatch", itemClass.name, String(error))); + } + } + + get(itemClass: Constructor, key?: unknown, registerDataAccess: boolean = true): V | null { + const cache = this.caches.get(itemClass); + if (!cache) { + return null; + } + + const value = cache.get(key); + if (value === null || value === undefined) { + return null; + } + + if (value instanceof itemClass) { + if (registerDataAccess) { + const event = new DataAccessEvent(itemClass, key); + this.publisher.publish(event); + } + return value as V; + } + + return null; + } + + exists(itemClass: Constructor, key: unknown): boolean { + const cache = this.caches.get(itemClass); + if (!cache) { + return false; + } + return cache.exists(key); + } + + remove(itemClass: Constructor, key: unknown): void { + const cache = this.caches.get(itemClass); + if (cache) { + cache.remove(key); + } + } + + clear(itemClass: Constructor): void { + const cache = this.caches.get(itemClass); + if (cache) { + cache.clear(); + } + } + + clearAll(): void { + for (const cache of this.caches.values()) { + cache.clear(); + } + this.caches.clear(); + } + + size(itemClass: Constructor): number { + const cache = this.caches.get(itemClass); + if (!cache) { + return 0; + } + return cache.size(); + } + + async releaseResources(): Promise { + if (this.cleanupIntervalHandle) { + clearInterval(this.cleanupIntervalHandle); + this.cleanupIntervalHandle = undefined; + } + this.clearAll(); + } +} diff --git a/common/lib/utils/utils.ts b/common/lib/utils/utils.ts index 32dd0a476..4091b6a66 100644 --- a/common/lib/utils/utils.ts +++ b/common/lib/utils/utils.ts @@ -20,16 +20,26 @@ import { WrapperProperties } from "../wrapper_property"; import { HostRole } from "../host_role"; import { logger } from "../../logutils"; import { AwsWrapperError, InternalQueryTimeoutError } from "./errors"; -import { TopologyAwareDatabaseDialect } from "../topology_aware_database_dialect"; export function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } +/** + * Creates a sleep promise that can be aborted before completion. + * + * @param ms - Duration to sleep in milliseconds + * @param message - Error message when aborted + * @returns A tuple of [sleepPromise, abortFunction] + * - sleepPromise: Resolves after ms milliseconds, or rejects if aborted + * - abortFunction: Call to cancel the sleep and reject the promise + */ export function sleepWithAbort(ms: number, message?: string) { let abortSleep; const promise = new Promise((resolve, reject) => { const timeout = setTimeout(resolve, ms); + // Unref the timer to prevent this background task from blocking the application from gracefully exiting. + timeout.unref(); abortSleep = () => { clearTimeout(timeout); reject(new AwsWrapperError(message)); @@ -65,7 +75,7 @@ export function logTopology(hosts: HostInfo[], msgPrefix: string) { return `${msgPrefix}${Messages.get("Utils.topology", msg)}`; } -export function getTimeInNanos() { +export function getTimeInNanos(): bigint { return process.hrtime.bigint(); } @@ -113,24 +123,47 @@ export function equalsIgnoreCase(value1: string | null, value2: string | null): return value1 != null && value2 != null && value1.localeCompare(value2, undefined, { sensitivity: "accent" }) === 0; } -export function isDialectTopologyAware(dialect: any): dialect is TopologyAwareDatabaseDialect { - return dialect; -} +export function containsHostAndPort(hosts: HostInfo[] | null | undefined, hostAndPort: string): boolean { + if (!hosts || hosts.length === 0) { + return false; + } -export class Pair { - private readonly _left: K; - private readonly _right: V; + return hosts.some((host) => host.hostAndPort === hostAndPort); +} - constructor(value1: K, value2: V) { - this._left = value1; - this._right = value2; +export function parseInstanceTemplates( + instanceTemplatesString: string | null, + hostValidator: (hostPattern: string) => void, + hostInfoBuilderFunc: () => { withHost(host: string): { build(): HostInfo } } +): Map { + if (!instanceTemplatesString) { + throw new AwsWrapperError(Messages.get("Utils.globalClusterInstanceHostPatternsRequired")); } - get left(): K { - return this._left; - } + const instanceTemplates = new Map(); + const patterns = instanceTemplatesString.split(","); + + for (const pattern of patterns) { + const trimmedPattern = pattern.trim(); + const colonIndex = trimmedPattern.indexOf(":"); + if (colonIndex === -1) { + throw new AwsWrapperError(Messages.get("Utils.invalidPatternFormat", trimmedPattern)); + } - get right(): V { - return this._right; + const region = trimmedPattern.substring(0, colonIndex).trim(); + const hostPattern = trimmedPattern.substring(colonIndex + 1).trim(); + + if (!region || !hostPattern) { + throw new AwsWrapperError(Messages.get("Utils.invalidPatternFormat", trimmedPattern)); + } + + hostValidator(hostPattern); + + const hostInfo = hostInfoBuilderFunc().withHost(hostPattern).build(); + instanceTemplates.set(region, hostInfo); } + + logger.debug(`Detected Global Database patterns: ${JSON.stringify(Array.from(instanceTemplates.entries()))}`); + + return instanceTemplates; } diff --git a/common/lib/wrapper_property.ts b/common/lib/wrapper_property.ts index c0ee775c9..efb65a777 100644 --- a/common/lib/wrapper_property.ts +++ b/common/lib/wrapper_property.ts @@ -16,18 +16,20 @@ import { ConnectionProvider } from "./connection_provider"; import { DatabaseDialect } from "./database_dialect/database_dialect"; -import { ClusterTopologyMonitorImpl } from "./host_list_provider/monitoring/cluster_topology_monitor"; -import { BlueGreenStatusProvider } from "./plugins/bluegreen/blue_green_status_provider"; +import { AwsWrapperError } from "./utils/errors"; +import { Messages } from "./utils/messages"; export class WrapperProperty { name: string; description: string; defaultValue: any; + allowedValues?: T[]; - constructor(name: string, description: string, defaultValue?: any) { + constructor(name: string, description: string, defaultValue?: any, allowedValues?: T[]) { this.name = name; this.description = description; this.defaultValue = defaultValue; + this.allowedValues = allowedValues; } get(props: Map): T { @@ -36,16 +38,29 @@ export class WrapperProperty { return this.defaultValue; } + if (val != null && this.allowedValues?.length > 0) { + if (!this.allowedValues.includes(val)) { + throw new AwsWrapperError(Messages.get("WrapperProperty.invalidValue", String(val), this.name, this.allowedValues.join(", "))); + } + } + return val; } set(props: Map, val: T) { + if (val != null && this.allowedValues?.length > 0) { + if (!this.allowedValues.includes(val)) { + throw new AwsWrapperError(Messages.get("WrapperProperty.invalidValue", String(val), this.name, this.allowedValues.join(", "))); + } + } props.set(this.name, val); } } export class WrapperProperties { static readonly MONITORING_PROPERTY_PREFIX: string = "monitoring_"; + static readonly TOPOLOGY_MONITORING_PROPERTY_PREFIX: string = "topology_monitoring_"; + static readonly BG_MONITORING_PROPERTY_PREFIX: string = "blue_green_monitoring_"; static readonly DEFAULT_PLUGINS = "initialConnection,auroraConnectionTracker,failover2,efm2"; static readonly DEFAULT_TOKEN_EXPIRATION_SEC = 15 * 60; @@ -210,6 +225,38 @@ export class WrapperProperties { ); static readonly FAILOVER_MODE = new WrapperProperty("failoverMode", "Set host role to follow during failover.", ""); + static readonly FAILOVER_HOME_REGION = new WrapperProperty("failoverHomeRegion", "Set home region for GDB failover.", null); + + static readonly ACTIVE_HOME_FAILOVER_MODE = new WrapperProperty( + "activeHomeFailoverMode", + "Set host role to follow during failover when GDB primary region is in home region.", + null, + [ + "strict-writer", + "strict-home-reader", + "strict-out-of-home-reader", + "strict-any-reader", + "home-reader-or-writer", + "out-of-home-reader-or-writer", + "any-reader-or-writer" + ] + ); + + static readonly INACTIVE_HOME_FAILOVER_MODE = new WrapperProperty( + "inactiveHomeFailoverMode", + "Set host role to follow during failover when GDB primary region is not in home region.", + null, + [ + "strict-writer", + "strict-home-reader", + "strict-out-of-home-reader", + "strict-any-reader", + "home-reader-or-writer", + "out-of-home-reader-or-writer", + "any-reader-or-writer" + ] + ); + static readonly FAILOVER_READER_HOST_SELECTOR_STRATEGY = new WrapperProperty( "failoverReaderHostSelectorStrategy", "The strategy that should be used to select a new reader host while opening a new connection.", @@ -232,10 +279,8 @@ export class WrapperProperties { static readonly CLUSTER_ID = new WrapperProperty( "clusterId", - "A unique identifier for the cluster. " + - "Connections with the same cluster id share a cluster topology cache. " + - "If unspecified, a cluster id is automatically created for AWS RDS clusters.", - null + "A unique identifier for the cluster. Connections with the same cluster id share a cluster topology cache. If unspecified, cluster id will be '1'.", + "1" ); static readonly CLUSTER_INSTANCE_HOST_PATTERN = new WrapperProperty( @@ -246,6 +291,16 @@ export class WrapperProperties { "clusters. Otherwise, if unspecified, the pattern will be automatically created for AWS RDS clusters." ); + static readonly GLOBAL_CLUSTER_INSTANCE_HOST_PATTERNS = new WrapperProperty( + "globalClusterInstanceHostPatterns", + "Comma-separated list of the cluster instance DNS patterns that will be used to " + + "build complete instance endpoints. " + + 'A "?" character in these patterns should be used as a placeholder for cluster instance names. ' + + "This parameter is required for Global Aurora Databases. " + + "Each region in the Global Aurora Database should be specified in the list. " + + "Format: region1:pattern1,region2:pattern2" + ); + static readonly SINGLE_WRITER_CONNECTION_STRING = new WrapperProperty( "singleWriterConnectionString", "Set to true if you are providing a connection string with multiple comma-delimited hosts and your cluster has only one writer. The writer must be the first host in the connection string", @@ -473,10 +528,57 @@ export class WrapperProperties { false ); + static readonly CACHED_READER_KEEP_ALIVE_TIMEOUT = new WrapperProperty( + "cachedReaderKeepAliveTimeoutMs", + "The time in milliseconds to keep a reader connection alive in the cache. " + + "Default value 0 means the Wrapper will keep reusing the same cached reader connection.", + 0 + ); + static readonly SKIP_INACTIVE_WRITER_CLUSTER_CHECK = new WrapperProperty( + "skipInactiveWriterClusterEndpointCheck", + "Allows to avoid connection check for inactive cluster writer endpoint.", + false + ); + + static readonly INACTIVE_CLUSTER_WRITER_SUBSTITUTION_ROLE = new WrapperProperty( + "inactiveClusterWriterEndpointSubstitutionRole", + "Defines whether or not the inactive cluster writer endpoint in the initial connection URL should be replaced with a writer instance URL from the topology info when available.", + "writer", + ["writer", "none"] + ); + + static readonly VERIFY_OPENED_CONNECTION_ROLE = new WrapperProperty( + "verifyOpenedConnectionType", + "Defines whether an opened connection should be verified to be a writer or reader, or if no role verification should be performed.", + null, + ["writer", "reader", "none"] + ); + + static readonly VERIFY_INACTIVE_CLUSTER_WRITER_CONNECTION_ROLE = new WrapperProperty( + "verifyInactiveClusterWriterEndpointConnectionType", + "Defines whether inactive cluster writer connection should be verified to be a writer, or if no role verification should be performed.", + "writer", + ["writer", "none"] + ); + + static readonly GDB_RW_HOME_REGION = new WrapperProperty("gdbRwHomeRegion", "Specifies the home region for read/write splitting.", null); + + static readonly GDB_RW_RESTRICT_WRITER_TO_HOME_REGION = new WrapperProperty( + "gdbRwRestrictWriterToHomeRegion", + "Prevents connections to a writer node outside of the defined home region.", + true + ); + + static readonly GDB_RW_RESTRICT_READER_TO_HOME_REGION = new WrapperProperty( + "gdbRwRestrictReaderToHomeRegion", + "Prevents connections to a reader node outside of the defined home region.", + true + ); + private static readonly PREFIXES = [ WrapperProperties.MONITORING_PROPERTY_PREFIX, - ClusterTopologyMonitorImpl.MONITORING_PROPERTY_PREFIX, - BlueGreenStatusProvider.MONITORING_PROPERTY_PREFIX + WrapperProperties.TOPOLOGY_MONITORING_PROPERTY_PREFIX, + WrapperProperties.BG_MONITORING_PROPERTY_PREFIX ]; private static startsWithPrefix(key: string): boolean { diff --git a/docs/Documentation.md b/docs/README.md similarity index 93% rename from docs/Documentation.md rename to docs/README.md index 056c05bd3..cbb466a7c 100644 --- a/docs/Documentation.md +++ b/docs/README.md @@ -6,6 +6,7 @@ - [Telemetry](../docs/using-the-nodejs-wrapper/Telemetry.md) - [Database Dialects](../docs/using-the-nodejs-wrapper/DatabaseDialects.md) - [Driver Dialects](../docs/using-the-nodejs-wrapper/DriverDialects.md) + - [Cluster ID](../docs/using-the-nodejs-wrapper/ClusterId.md) - [Configuration Presets](../docs/using-the-nodejs-wrapper/ConfigurationPresets.md) - [Session State](../docs/using-the-nodejs-wrapper/SessionState.md) - [Support for RDS Multi-AZ DB Cluster](../docs/using-the-nodejs-wrapper/SupportForRDSMultiAzDBCluster.md) @@ -15,8 +16,10 @@ - [Failover Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheFailoverPlugin.md) - [Failover Configuration Guide](./using-the-nodejs-wrapper/FailoverConfigurationGuide.md) - [Failover2 Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheFailover2Plugin.md) + - [GDB Failover Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheGdbFailoverPlugin.md) - [Host Monitoring Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheHostMonitoringPlugin.md) - [Read-Write Splitting Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheReadWriteSplittingPlugin.md) + - [GDB Read-Write Splitting Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheGdbReadWriteSplittingPlugin.md) - [Fastest Response Strategy Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheFastestResponseStrategyPlugin.md) - [Okta Authentication Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheOktaAuthPlugin.md) - [Federated Authentication Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheFederatedAuthPlugin.md) diff --git a/docs/images/cluster_id_one_cluster_example.png b/docs/images/cluster_id_one_cluster_example.png new file mode 100644 index 000000000..aeae773b2 Binary files /dev/null and b/docs/images/cluster_id_one_cluster_example.png differ diff --git a/docs/images/cluster_id_two_cluster_example.png b/docs/images/cluster_id_two_cluster_example.png new file mode 100644 index 000000000..ddb4ba28d Binary files /dev/null and b/docs/images/cluster_id_two_cluster_example.png differ diff --git a/docs/using-the-nodejs-wrapper/ClusterId.md b/docs/using-the-nodejs-wrapper/ClusterId.md new file mode 100644 index 000000000..8e310f00c --- /dev/null +++ b/docs/using-the-nodejs-wrapper/ClusterId.md @@ -0,0 +1,239 @@ +# Understanding the clusterId Parameter + +## Overview + +The `clusterId` parameter is a critical configuration setting when using the AWS Advanced NodeJS Wrapper to **connect to multiple database clusters within a single application**. This parameter serves as a unique identifier that enables the driver to maintain separate caches and state for each distinct database cluster your application connects to. + +## What is a Cluster? + +Understanding what constitutes a cluster is crucial for correctly setting the `clusterId` parameter. In the context of the AWS Advanced NodeJS Wrapper, a **cluster** is a logical grouping of database instances that should share the same topology cache and monitoring services. + +A cluster represents one writer instance (primary) and zero or more reader instances (replicas). These make up shared topology that the driver needs to track, and are the group of instances the driver can reconnect to when a failover is detected. + +### Examples of Clusters + +- Aurora DB Cluster (one writer + multiple readers) +- RDS Multi-AZ DB Cluster (one writer + two readers) +- Aurora Global Database (when supplying a global db endpoint, the driver considers them as a single cluster) + +> **Rule of thumb:** If the driver should track separate topology information and perform independent failover operations, use different `clusterId` values. If instances share the same topology and failover domain, use the same `clusterId`. + +## Why clusterId is Important + +The AWS Advanced NodeJS Wrapper uses the `clusterId` as a **key for internal caching mechanisms** to optimize performance and maintain cluster-specific state. Without proper `clusterId` configuration, your application may experience: + +- Cache collisions between different clusters +- Incorrect topology information +- Degraded performance due to cache invalidation + +## Why Not Use AWS DB Cluster Identifiers? + +Host information can take many forms: + +- **IP Address Connections:** `10.0.1.50` ← No cluster info! +- **Custom Domain Names:** `db.mycompany.com` ← Custom domain +- **Custom Endpoints:** `my-custom-endpoint.cluster-custom-abc.us-east-1.rds.amazonaws.com` ← Custom endpoint +- **Proxy Connections:** `my-proxy.proxy-abc.us-east-1.rds.amazonaws.com` ← Proxy, not actual cluster + +In fact, all of these could reference the exact same cluster. Therefore, because the driver cannot reliably parse cluster information from all connection types, **it is up to the user to explicitly provide the `clusterId`**. + +## How clusterId is Used Internally + +The driver uses `clusterId` as a cache key for topology information and monitoring services. This enables multiple connections to the same cluster to share cached data and avoid redundant db meta-data. + +### Example: Single Cluster with Multiple Connections + +The following diagram shows how connections with the same `clusterId` share cached resources: + +![Single Cluster Example](../images/cluster_id_one_cluster_example.png) + +**Key Points:** + +- Three connections use different connection strings (custom endpoint, IP address, cluster endpoint) but all specify **`clusterId: "foo"`** +- All three connections share the same Topology Cache and Monitor Threads in the driver +- The Topology Cache stores a key-value mapping where `"foo"` maps to `["instance-1", "instance-2", "instance-3"]` +- Despite different connection URLs, all connections monitor and query the same physical database cluster + +**The Impact:** +Shared resources eliminate redundant topology queries and reduce monitoring overhead. + +### Example: Multiple Clusters with Separate Cache Isolation + +The following diagram shows how different `clusterId` values maintain separate caches for different clusters. + +![Two Cluster Example](../images/cluster_id_two_cluster_example.png) + +**Key Points:** + +- Connection 1 and 3 use **`clusterId: "foo"`** and share the same cache entries +- Connection 2 uses **`clusterId: "bar"`** and has completely separate cache entries +- Each `clusterId` acts as a key in the cache Map structure: `Map` +- Topology Cache maintains separate entries: `"foo"` → `[instance-1, instance-2, instance-3]` and `"bar"` → `[instance-4, instance-5]` +- Monitor Cache maintains separate monitor threads for each cluster +- Monitors poll their respective database clusters and update the corresponding topology cache entries + +**The Impact:** +This isolation prevents cache collisions and ensures correct failover behavior for each cluster. + +## When to Specify clusterId + +### Required: Multiple Clusters in One Application + +You **must** specify a unique `clusterId` for every DB cluster when your application connects to multiple database clusters: + +```typescript +// Sample data migration app +const sourceParams = { + host: "source-db.us-east-1.rds.amazonaws.com", + port: 5432, + user: "admin", + password: "***", + database: "mydb", + clusterId: "source-cluster" +}; +const sourceClient = new AwsPGClient(sourceParams); +await sourceClient.connect(); + +const destParams = { + host: "dest-db.us-west-2.rds.amazonaws.com", + port: 5432, + user: "admin", + password: "***", + database: "mydb", + clusterId: "destination-cluster" // Different clusterId! +}; +const destClient = new AwsPGClient(destParams); +await destClient.connect(); + +// Read from source, write to destination +const result = await sourceClient.query("SELECT * FROM users"); +// ... migration logic + +// If you are connecting `source-db` with a different host later on, use the same clusterId +const sourceIpParams = { + host: "10.0.0.1", + port: 5432, + user: "admin", + password: "***", + database: "mydb", + clusterId: "source-cluster" // Same ID as sourceClient +}; +const sourceIpClient = new AwsPGClient(sourceIpParams); +await sourceIpClient.connect(); +``` + +### Optional: Single Cluster Applications + +If your application only connects to one cluster, you can omit `clusterId` (defaults to `"1"`): + +```typescript +// Single cluster - clusterId defaults to "1" +const params = { + host: "my-cluster.us-east-1.rds.amazonaws.com", + port: 5432, + user: "admin", + password: "***", + database: "mydb" +}; +const client = new AwsPGClient(params); +await client.connect(); +``` + +This also includes if you have multiple connections using different host information: + +```typescript +const params = { + host: "my-cluster.us-east-1.rds.amazonaws.com", + port: 5432, + user: "admin", + password: "***", + database: "mydb" + // clusterId defaults to "1" +}; +const urlClient = new AwsPGClient(params); +await urlClient.connect(); + +// "10.0.0.1" -> IP address of source-db. So it is the same cluster. +const ipParams = { + host: "10.0.0.1", + port: 5432, + user: "admin", + password: "***", + database: "mydb" + // clusterId defaults to "1" +}; +const ipClient = new AwsPGClient(ipParams); +await ipClient.connect(); +``` + +## Critical Warnings + +### NEVER Share clusterId Between Different Clusters + +Using the same `clusterId` for different database clusters will cause serious issues: + +```typescript +// ❌ WRONG - Same clusterId for different clusters +const sourceParams = { + host: "source-db.us-east-1.rds.amazonaws.com", + clusterId: "shared-id" // ← BAD! + // ... +}; + +const destParams = { + host: "dest-db.us-west-2.rds.amazonaws.com", + clusterId: "shared-id" // ← BAD! Same ID for different cluster + // ... +}; +``` + +**Problems this causes:** + +- Topology cache collision (dest-db's topology could overwrite source-db's) +- Incorrect failover behavior (driver may try to failover to wrong cluster) +- Monitor conflicts (Only one monitor instance for both clusters will lead to undefined results) + +**Correct approach:** + +```typescript +// ✅ CORRECT - Unique clusterId for each cluster +const sourceParams = { clusterId: "source-cluster" /* ... */ }; +const destParams = { clusterId: "destination-cluster" /* ... */ }; +``` + +### Always Use Same clusterId for Same Cluster + +Using different `clusterId` values for the same cluster reduces efficiency: + +```typescript +// ⚠️ SUBOPTIMAL - Different clusterIds for same cluster +const params1 = { + host: "my-cluster.us-east-1.rds.amazonaws.com", + clusterId: "my-cluster-1" + // ... +}; + +const params2 = { + host: "my-cluster.us-east-1.rds.amazonaws.com", + clusterId: "my-cluster-2" // Different ID for same cluster + // ... +}; +``` + +**Problems this causes:** + +- Duplication of caches +- Multiple monitoring threads for the same cluster + +**Best practice:** + +```typescript +// ✅ BEST - Same clusterId for same cluster +const CLUSTER_ID = "my-cluster"; +const params1 = { clusterId: CLUSTER_ID /* ... */ }; +const params2 = { clusterId: CLUSTER_ID /* ... */ }; // Shared cache and resources +``` + +## Summary + +The `clusterId` parameter is essential for applications connecting to multiple database clusters. It serves as a cache key for topology information and monitoring services. Always use unique `clusterId` values for different clusters, and consistent values for the same cluster to maximize performance and avoid conflicts. diff --git a/docs/using-the-nodejs-wrapper/DatabaseDialects.md b/docs/using-the-nodejs-wrapper/DatabaseDialects.md index 04aa92f57..d6834ec38 100644 --- a/docs/using-the-nodejs-wrapper/DatabaseDialects.md +++ b/docs/using-the-nodejs-wrapper/DatabaseDialects.md @@ -33,7 +33,7 @@ Dialect codes specify what kind of database any connections will be made to. If you are interested in using the AWS Advanced NodeJS Wrapper but your desired database type is not currently supported, it is possible to create a custom dialect. -To create a custom dialect, implement the [`DatabaseDialect`](../../common/lib/database_dialect/database_dialect.ts) interface. For databases clusters that are aware of their topology, the [`TopologyAwareDatabaseDialect`](../../common/lib/topology_aware_database_dialect.ts) interface should also be implemented. For database clusters that use an [Aurora Limitless Database](../../docs/using-the-nodejs-wrapper/using-plugins/UsingTheLimitlessConnectionPlugin.md#what-is-amazon-aurora-limitless-database) then [`LimitlessDatabaseDialect`](../../common/lib/database_dialect/limitless_database_dialect.ts) should be implemented. +To create a custom dialect, implement the [`DatabaseDialect`](../../common/lib/database_dialect/database_dialect.ts) interface. For databases clusters that are aware of their topology, the [`TopologyAwareDatabaseDialect`](../../common/lib/database_dialect/topology_aware_database_dialect.ts) interface should also be implemented. For database clusters that use an [Aurora Limitless Database](../../docs/using-the-nodejs-wrapper/using-plugins/UsingTheLimitlessConnectionPlugin.md#what-is-amazon-aurora-limitless-database) then [`LimitlessDatabaseDialect`](../../common/lib/database_dialect/limitless_database_dialect.ts) should be implemented. See the following classes for examples: diff --git a/docs/using-the-nodejs-wrapper/UsingTheConnectionPool.md b/docs/using-the-nodejs-wrapper/UsingTheConnectionPool.md index 339fb3123..579ead16f 100644 --- a/docs/using-the-nodejs-wrapper/UsingTheConnectionPool.md +++ b/docs/using-the-nodejs-wrapper/UsingTheConnectionPool.md @@ -407,7 +407,7 @@ const result = await pool.query("SELECT NOW()"); ### Resources Cleanup -Throughout the application lifetime, some plugins like the Aurora Connection Tracker Plugin or the Host Monitoring Connection Plugin may create background threads shared by all connections. +Throughout the application lifetime, some plugins like the Aurora Connection Tracker Plugin or the Host Monitoring Connection Plugin may create background tasks shared by all connections. At the end of your application, call `PluginManager.releaseResources()` to clean up these shared resources. diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFailover2Plugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFailover2Plugin.md index e943fe2d1..37789e599 100644 --- a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFailover2Plugin.md +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFailover2Plugin.md @@ -62,14 +62,15 @@ Please refer to the [failover configuration guide](../FailoverConfigurationGuide In addition to the parameters that you can configure for the underlying driver, you can pass the following parameters to the AWS Advanced NodeJS Wrapper through the connection URL to specify additional failover behavior. -| Parameter | Value | Required | Description | Default Value | -| ------------------------------------ | :-----: | :------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| `failoverMode` | String | No | Defines a mode for failover process. Failover process may prioritize nodes with different roles and connect to them. Possible values:

- `strict-writer` - Failover process follows writer instance and connects to a new writer when it changes.
- `reader-or-writer` - During failover, the wrapper tries to connect to any available/accessible reader instance. If no reader is available, the wrapper will connect to a writer instance. This logic mimics the logic of the Aurora read-only cluster endpoint.
- `strict-reader` - During failover, the wrapper tries to connect to any available reader instance. If no reader is available, the wrapper raises an error. Reader failover to a writer instance will only be allowed for single-instance clusters. This logic mimics the logic of the Aurora read-only cluster endpoint. | Default value depends on connection url. For Aurora read-only cluster endpoint, it's set to `reader-or-writer`. Otherwise, it's `strict-writer`. | -| `enableClusterAwareFailover` | Boolean | No | Set to `true` to enable the fast failover behavior offered by the AWS Advanced NodeJS Wrapper. Set to `false` for simple connections that do not require fast failover functionality. | `true` | -| `clusterTopologyRefreshRateMs` | Number | No | Cluster topology refresh rate in milliseconds when a cluster is not in failover. | `30000` | -| `clusterTopologyHighRefreshRateMs` | Number | No | Interval of time in milliseconds to wait between attempts to update cluster topology after the writer has come back online following a failover event. Usually, the topology monitoring component uses this increased monitoring rate for 30s after a new writer is detected. | `100` | -| `failoverTimeoutMs` | Number | No | Maximum allowed time in milliseconds to attempt reconnecting to a new writer or reader instance after a cluster failover is initiated. | `300000` | -| `failoverReaderHostSelectorStrategy` | String | No | The strategy that should be used to select a new reader host while opening a new connection. See: [Reader Selection Strategies](./../ReaderSelectionStrategies.md) for options. | `random` | +| Parameter | Value | Required | Description | Default Value | +| ------------------------------------ | :-----: | :-----------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `failoverMode` | String | No | Defines a mode for failover process. Failover process may prioritize nodes with different roles and connect to them. Possible values:

- `strict-writer` - Failover process follows writer instance and connects to a new writer when it changes.
- `reader-or-writer` - During failover, the wrapper tries to connect to any available/accessible reader instance. If no reader is available, the wrapper will connect to a writer instance. This logic mimics the logic of the Aurora read-only cluster endpoint.
- `strict-reader` - During failover, the wrapper tries to connect to any available reader instance. If no reader is available, the wrapper raises an error. Reader failover to a writer instance will only be allowed for single-instance clusters. This logic mimics the logic of the Aurora read-only cluster endpoint. | Default value depends on connection url. For Aurora read-only cluster endpoint, it's set to `reader-or-writer`. Otherwise, it's `strict-writer`. | +| `enableClusterAwareFailover` | Boolean | No | Set to `true` to enable the fast failover behavior offered by the AWS Advanced NodeJS Wrapper. Set to `false` for simple connections that do not require fast failover functionality. | `true` | +| `clusterTopologyRefreshRateMs` | Number | No | Cluster topology refresh rate in milliseconds when a cluster is not in failover. | `30000` | +| `clusterTopologyHighRefreshRateMs` | Number | No | Interval of time in milliseconds to wait between attempts to update cluster topology after the writer has come back online following a failover event. Usually, the topology monitoring component uses this increased monitoring rate for 30s after a new writer is detected. | `100` | +| `failoverTimeoutMs` | Number | No | Maximum allowed time in milliseconds to attempt reconnecting to a new writer or reader instance after a cluster failover is initiated. | `300000` | +| `clusterId` | String | If using multiple database clusters, yes; otherwise, no | A unique identifier for the cluster. Connections with the same cluster id share a cluster topology cache. This parameter is optional and defaults to `1`. When supporting multiple database clusters, this parameter becomes mandatory. Each connection string must include the `clusterId` parameter with a value that can be any number or string. However, all connection strings associated with the same database cluster must use identical `clusterId` values, while connection strings belonging to different database clusters must specify distinct values. Examples of value: `1`, `2`, `1234`, `abc-1`, `abc-2`. | `1` | +| `failoverReaderHostSelectorStrategy` | String | No | The strategy that should be used to select a new reader host while opening a new connection. See: [Reader Selection Strategies](./../ReaderSelectionStrategies.md) for options. | `random` | Please refer to the original [Failover Plugin](./UsingTheFailoverPlugin.md) for more details about error codes, configurations, connection pooling and sample codes. diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFederatedAuthPlugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFederatedAuthPlugin.md index ba2130625..f34de5488 100644 --- a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFederatedAuthPlugin.md +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFederatedAuthPlugin.md @@ -48,3 +48,22 @@ Note: AWS IAM database authentication is needed to use the Federated Authenticat | `iamDefaultPort` | `Number` | No | This property overrides the default port that is used to generate the IAM token. The default port is determined based on the underlying driver protocol. Target drivers with different protocols will require users to provide a default port. | `null` | `1234` | | `iamTokenExpiration` | `Number` | No | Overrides the default IAM token cache expiration in seconds. | `900` | `123` | | `httpsAgentOptions` | `Object` | No | This property adds parameters to the httpsAgent that connects to the hosting URL.
For more information on the parameters, see this [documentation](https://nodejs.org/api/https.html#class-httpsagent). | `null` | `{ timeout: 5000 }` | + +## Using Federated Authentication with Global Databases + +When using Federated authentication with [Amazon Aurora Global Databases](https://aws.amazon.com/rds/aurora/global-database/), the IAM user or role requires the additional `rds:DescribeGlobalClusters` permission. This permission allows the driver to resolve the Global Database endpoint to the appropriate regional cluster for IAM token generation. + +Example IAM policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["rds-db:connect", "rds:DescribeGlobalClusters"], + "Resource": "*" + } + ] +} +``` diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbFailoverPlugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbFailoverPlugin.md new file mode 100644 index 000000000..91e1b1f16 --- /dev/null +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbFailoverPlugin.md @@ -0,0 +1,122 @@ +# Global Database (GDB) Failover Plugin + +The AWS Advanced NodeJS Wrapper uses the GDB Failover Plugin to provide minimal downtime in the event of a DB instance failure. The plugin is based on [Failover Plugin v2](./UsingTheFailover2Plugin.md) and unless explicitly stated otherwise, most of the information and suggestions for the [Failover Plugin](./UsingTheFailoverPlugin.md) and [Failover Plugin v2](./UsingTheFailover2Plugin.md) are applicable to the GDB Failover Plugin. + +## Plugin Availability + +The plugin is available since version 3.0.0. + +## Differences between the GDB Failover Plugin and the Failover Plugin v2 + +The GDB Failover Plugin introduces the notion of home region and extends configuration parameters to allow setting different failover logic for **in-home** and **out-of-home** cases. + +Driver configuration for the GDB failover plugin should include a home region defined with an AWS region name. This introduces two cases: + +- **in-home**: when the primary region of the Global Database is the same as the specified home region +- **out-of-home**: when GDB switches over to another region and the primary region is not the same as the specified home region. + +Users are allowed to specify different failover logic individually for **in-home** and **out-of-home** cases. The following are two examples where in-home and out-of-home cases impact the failover logic. + +**Example 1** + +When a user application needs a writer connection, it makes sense to define the failover mode to follow the writer (`strict-writer`). However, some applications may choose not to follow a writer node when cross-region failover occurs (see [Configuration Example 3](#configuration-example-3) below). + +- **in-home**: when in-region failover occurs, the driver reconnects to a new writer node and continues serving the user application with a write connection. +- **out-of-home**: application turns to inactive mode and stops performing writes, this prioritizes performance due to reduced connection latency over being connected to a writer node in another region. + +**Example 2** + +When a user application needs a reader connection, prioritize reader connections in the home region to reduce connection latency (see [Configuration Example 2](#configuration-example-2) below). + +- **in-home**: the primary region of the Global Database is the same as the specified home region, connect to any such reader. +- **out-of-home**: connect to readers that are in the specified home region, these readers are no longer part of the primary region of the Global Database. + +The mentioned scenarios can be configured with the GDB Failover plugin. [Failover2 Plugin](./UsingTheFailover2Plugin.md) and [Failover Plugin](./UsingTheFailoverPlugin.md) don't support the home region and can't be used for the cases mentioned above. + +## Using the GDB Failover Plugin + +The GDB Failover Plugin will not be enabled by default if the [`plugins`](../UsingTheNodejsWrapper.md#connection-plugin-manager-parameters) value is not specified. If you would like to override the default plugins, you can explicitly include the GDB Failover Plugin in your list of plugins by adding the plugin code `gdbFailover` to the [`plugins`](../UsingTheNodejsWrapper.md#aws-advanced-nodejs-wrapper-parameters) value, or by adding it to the current [driver profile](../UsingTheNodejsWrapper.md#connection-plugin-manager-parameters). After you load the plugin, the failover feature will be enabled. + +Please refer to the [failover configuration guide](../FailoverConfigurationGuide.md) for tips to keep in mind when using the failover plugin. + +> [!WARNING] +> Do not use the `gdbFailover`, `failover` and/or `failover2` plugins in any combination at the same time for the same connection! + +Verify plugin compatibility within your driver configuration using the [compatibility guide](../Compatibility.md). + +### GDB Failover Plugin Configuration Parameters + +In addition to the parameters that you can configure for the underlying driver, you can pass the following parameters for the AWS Advanced NodeJS Wrapper through the connection configuration to specify additional failover behavior. + +| Parameter | Value | Required | Description | Default Value | +| ---------------------------------------- | :-------: | :-----------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `failoverHomeRegion` | `String` | If connecting using an IP address, a custom domain URL, Global Database endpoint or other endpoint with no region: Yes

Otherwise: No | Defines a home region.

Examples: `us-west-2`, `us-east-1`.

If this parameter is omitted, the value is parsed from connection url. For regional cluster endpoints and instance endpoints, it's set to the region of the provided endpoint. If the provided endpoint has no region (for example, a Global Database endpoint or IP address) the configuration parameter is mandatory. | | +| `activeHomeFailoverMode` | `String` | No | Defines a mode for failover process when GDB primary region **is home region**. Failover process may prioritize nodes with different roles and connect to them. Possible values:

- `strict-writer` - Failover process follows writer node and connects to a new writer (in any region) when it changes.
- `home-reader-or-writer` - During failover, the driver tries to connect to any available/accessible reader node in home region. If no reader is available, the driver will connect to a writer node (in any region).
- `strict-home-reader` - During failover, the driver tries to connect to any available reader node in home region. If no reader is available, the driver raises an error.
- `strict-out-of-home-reader`, `strict-any-reader`, `out-of-home-reader-or-writer`, `any-reader-or-writer` - other possible values with prioritizing different writer or reader nodes in home, out of home, or in any region.

If this parameter is omitted, default value depends on connection url. For Aurora writer cluster endpoint or Global Database endpoint, it's set to `strict-writer`. Otherwise, it's `home-reader-or-writer`. | Default value depends on connection url. For Aurora writer cluster endpoint or Global Database endpoint, it's set to `strict-writer`. Otherwise, it's `home-reader-or-writer`. | +| `inactiveHomeFailoverMode` | `String` | No | Defines a mode for failover process when GDB primary region **is not a home region**. Possible values are the same as for `activeHomeFailoverMode` parameter: `strict-writer`, `home-reader-or-writer`, `strict-home-reader`, `strict-out-of-home-reader`, `strict-any-reader`, `out-of-home-reader-or-writer`, `any-reader-or-writer`.
If this parameter is omitted, default value depends on connection url. For Aurora writer cluster endpoint or Global Database endpoint, it's set to `strict-writer`. Otherwise, it's `home-reader-or-writer`. | Default value depends on connection url. For Aurora writer cluster endpoint or Global Database endpoint, it's set to `strict-writer`. Otherwise, it's `home-reader-or-writer`. | +| `clusterInstanceHostPattern` | `String` | If connecting using an IP address or custom domain URL: Yes

Otherwise: No | This parameter is not required unless connecting to an AWS RDS cluster via an IP address or custom domain URL. In those cases, this parameter specifies the cluster instance DNS pattern that will be used to build a complete instance endpoint. A "?" character in this pattern should be used as a placeholder for the DB instance identifiers of the instances in the cluster. See [here](#host-pattern) for more information.

Example: `?.my-domain.com`, `any-subdomain.?.my-domain.com:9999`

Use case Example: If your cluster instance endpoints follow this pattern:`instanceIdentifier1.customHost`, `instanceIdentifier2.customHost`, etc. and you want your initial connection to be to `customHost:1234`, then your configuration should include `clusterInstanceHostPattern: "?.customHost"` with host `customHost` and port `1234`. | If the provided connection string is not an IP address or custom domain, the NodeJS Wrapper will automatically acquire the cluster instance host pattern from the customer-provided connection string. | +| `globalClusterInstanceHostPatterns` | `String` | For Global Databases: Yes

Otherwise: No | This parameter is similar to the `clusterInstanceHostPattern` parameter but it provides a comma-separated list of instance host patterns. This parameter is required for Aurora Global Databases. The list should contain host patterns for each region of the global database. Each host pattern can be based on an RDS instance endpoint or a custom user domain name. If a custom domain name is used, the instance template pattern should be prefixed with the AWS region name in square brackets (`[]`).

The parameter is ignored for other types of databases (Aurora Clusters, RDS Clusters, plain RDS databases, etc.).

Example: for an Aurora Global Database with two AWS regions `us-east-2` and `us-west-2`, the parameter value should be set to `?.XYZ1.us-east-2.rds.amazonaws.com,?.XYZ2.us-west-2.rds.amazonaws.com`. Please note that user identifiers are different for different AWS regions (`XYZ1` and `XYZ2` in the example above).

Example: if using custom domain names, the parameter value should be similar to `[us-east-2]?.customHost,[us-west-2]?.anotherCustomHost`. The port can also be provided: `[us-east-2]?.customHost:8888,[us-west-2]?.anotherCustomHost:9999`

For complete Aurora Global Database configuration, see [Aurora Global Databases](../GlobalDatabases.md). | | +| `clusterTopologyRefreshRateMs` | `Number` | No | Cluster topology refresh rate in milliseconds when a cluster is not in failover. It refers to the regular slow monitoring rate described in [Failover Plugin v2](./UsingTheFailover2Plugin.md). | `30000` | +| `failoverTimeoutMs` | `Number` | No | Maximum allowed time in milliseconds to attempt reconnecting to a new writer or reader instance after a cluster failover is initiated. | `300000` | +| `clusterTopologyHighRefreshRateMs` | `Number` | No | Interval of time in milliseconds to wait between attempts to update cluster topology after the writer has come back online following a failover event. It corresponds to the increased monitoring rate described in [Failover Plugin v2](./UsingTheFailover2Plugin.md). Usually, the topology monitoring component uses this increased monitoring rate for 30s after a new writer was detected. | `100` | +| `failoverReaderHostSelectorStrategy` | `String` | No | Strategy used to select a reader host during failover. For more information on the available reader selection strategies, see this [table](../ReaderSelectionStrategies.md). | `random` | +| `clusterId` | `String` | If connecting to multiple database clusters within a single application:: Yes

Otherwise: No

| A unique identifier for the cluster. Connections with the same cluster id share a cluster topology cache. This parameter is optional and defaults to `1`. When supporting multiple database clusters, this parameter becomes mandatory. Each connection string must include the `clusterId` parameter with a value that can be any number or string. However, all connection strings associated with the same database cluster must use identical `clusterId` values, while connection strings belonging to different database clusters must specify distinct values. Examples of value: `1`, `2`, `1234`, `abc-1`, `abc-2`. | `1` | +| `telemetryFailoverAdditionalTopTrace` | `Boolean` | No | Allows the driver to produce an additional telemetry span associated with failover. Such span helps to facilitate telemetry analysis in AWS CloudWatch. | `false` | +| `skipFailoverOnInterruptedThread` | `Boolean` | No | Enable to skip failover if the current thread is interrupted. This may leave the Connection in an invalid state so the Connection should be disposed. | `false` | +| `skipInactiveWriterClusterEndpointCheck` | `Boolean` | No | Enable to skip a connection role verification when connection is opened with inactive cluster writer endpoint. This parameter is applicable to Aurora Global Databases. Region-bound cluster writer endpoints are available in Global Databases. However, depending on the GDB primary region, they may be inactive and may act differently. The parameter is aimed at supporting such inactive cluster endpoints and helps configure the desired behavior for them. | `false` | + +Please refer to the original [Failover Plugin](./UsingTheFailoverPlugin.md) and [Failover2 Plugin](./UsingTheFailover2Plugin.md) for more details about error codes, configurations, connection pooling and sample codes. + +### Sample Code + +[PostgreSQL Failover Sample Code](./../../../examples/aws_driver_example/aws_failover_postgresql_example.ts) + +This sample code uses the original `failover` plugin, but it can also be used with the `gdbFailover` plugin. Configuration parameters should be adjusted in accordance with the table above. + +### Failover Configuration Examples + +#### Configuration Example 1 + +**Goal:** Provide a user application with a writer connection. The application is deployed in `us-west-1` region and connects to Global Database with `us-east-1`, `us-east-2` and `us-west-1` regions. + +**Solution:** Use the following configuration parameters to configure the driver: + +- `failoverHomeRegion=us-west-1` +- `activeHomeFailoverMode=strict-writer` +- `inactiveHomeFailoverMode=strict-writer` +- `globalClusterInstanceHostPatterns=?.XYZ1.us-east-1.rds.amazonaws.com,?.XYZ2.us-east-2.rds.amazonaws.com,?.XYZ3.us-west-1.rds.amazonaws.com` (make sure you replace `XYZ1`, `XYZ2`, `XYZ3` with actual values that correspond to your database) +- `wrapperDialect=global-aurora-mysql` (or `global-aurora-pg`) +- `plugins=initialConnection,gdbFailover,efm2` +- use Global Database endpoint in your connection string + +#### Configuration Example 2 + +**Goal:** Provide a user application with a reader connection in `us-west-1`. The application is deployed in the `us-west-1` region and connects to a Global Database with `us-east-1`, `us-east-2` and `us-west-1` regions. If the GDB primary region switches over, prioritize reader connections from the `us-west-1` region. + +**Solution:** Use the following configuration parameters to configure the driver: + +- `failoverHomeRegion=us-west-1` +- `activeHomeFailoverMode=strict-home-reader` +- `inactiveHomeFailoverMode=strict-home-reader` +- `globalClusterInstanceHostPatterns=?.XYZ1.us-east-1.rds.amazonaws.com,?.XYZ2.us-east-2.rds.amazonaws.com,?.XYZ3.us-west-1.rds.amazonaws.com` (make sure you replace `XYZ1`, `XYZ2`, `XYZ3` with actual values that correspond to your database) +- `wrapperDialect=global-aurora-mysql` (or `global-aurora-pg`) +- `plugins=initialConnection,gdbFailover,efm2` +- use cluster reader endpoint in region `us-west-1` in your connection string (`.cluster-ro-XYZ3.us-west-1.rds.amazonaws.com`) + +#### Configuration Example 3 + +**Goal:** Provide a user application with a writer connection when the primary region is the closest to the application deployed region. When the primary region switches over to another region and network latency becomes unacceptable, the application deployment in region `us-west-1` becomes inactive and lets applications in other regions act and process business transactions. There are application deployments in other regions. The Global Database has `us-east-1`, `us-east-2` and `us-west-1` regions. + +**Solution:** Use the following configuration parameters to configure the driver for application deployment in the `us-west-1` region. Other application deployments need similar driver configuration with different home regions. + +- `failoverHomeRegion=us-west-1` +- `activeHomeFailoverMode=strict-writer` +- `inactiveHomeFailoverMode=strict-any-reader` +- `globalClusterInstanceHostPatterns=?.XYZ1.us-east-1.rds.amazonaws.com,?.XYZ2.us-east-2.rds.amazonaws.com,?.XYZ3.us-west-1.rds.amazonaws.com` (make sure you replace `XYZ1`, `XYZ2`, `XYZ3` with actual values that correspond to your database) +- `wrapperDialect=global-aurora-mysql` (or `global-aurora-pg`) +- `plugins=initialConnection,gdbFailover,efm2` +- use cluster writer endpoint in region `us-west-1` in your connection string (`.cluster-XYZ3.us-west-1.rds.amazonaws.com`) + +**Explanation:** While the primary region of GDB is `us-west-1`, the application deployed in this region needs writable connections. By using the cluster writer endpoint, the driver opens connections to a writer node. If in-region failover occurs, the driver reconnects to a new writer node in the same `us-west-1` region. Since after GDB failover the writer node is in the same region, the driver considers this case as in-home and effectively uses the `activeHomeFailoverMode=strict-writer` setting. Following the writer node for in-region failovers helps the application deployment in the `us-west-1` region to process business transactions with minimal latency. When a cross-region failover event occurs, the driver detects a new writer node, let's assume it's in the us-east-1 region, and uses the `inactiveHomeFailoverMode=strict-any-reader` setting. The driver serves a reader connection from any available region `us-east-1`, `us-east-2` and `us-west-1`. + +> [!WARNING] +> This example covers failover and failover-related settings only. Complete driver configuration may require settings for other plugins. For instance, `initialConnection` may require `endpointSubstitutionRole` and `verifyOpenedConnectionType` parameters to be set properly. diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbReadWriteSplittingPlugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbReadWriteSplittingPlugin.md new file mode 100644 index 000000000..22932ddd1 --- /dev/null +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbReadWriteSplittingPlugin.md @@ -0,0 +1,57 @@ +# Global Database (GDB) Read/Write Splitting Plugin + +The GDB Read/Write Splitting plugin extends the functionality of the [read/write splitting plugin](./UsingTheReadWriteSplittingPlugin.md) and adopts some additional settings to improve support for Global Databases. + +The GDB Read/Write Splitting plugin adds the notion of a home region and allows users to constrain new connections to this region. Such restrictions may be helpful to prevent opening new connections in environments where remote AWS regions add substantial latency that cannot be tolerated. + +Unless otherwise stated, all recommendations, configurations and code examples made for the [read/write splitting plugin](./UsingTheReadWriteSplittingPlugin.md) are applicable to the current GDB Read/Write Splitting plugin. + +## Plugin Availability + +The plugin is available since version 3.0.0. + +## Loading the Global Database Read/Write Splitting Plugin + +The GDB Read/Write Splitting plugin is not loaded by default. To load the plugin, include it in the `plugins` connection parameter. See the example below to properly load the GDB read/write splitting plugin with these plugins. + +```typescript +const params = { + plugins: "gdbReadWriteSplitting,failover2,efm2" + // Add other connection properties below... +}; + +// If using MySQL: +const client = new AwsMySQLClient(params); +await client.connect(); + +// If using Postgres: +const client = new AwsPGClient(params); +await client.connect(); +``` + +If you would like to use the GDB read/write splitting plugin without the failover plugin, make sure you have the `gdbReadWriteSplitting` plugin in the `plugins` property, and that the failover plugin is not part of it. + +```typescript +const params = { + plugins: "gdbReadWriteSplitting" + // Add other connection properties below... +}; +``` + +> [!WARNING] +> Do not use the `readWriteSplitting` and `gdbReadWriteSplitting` plugins at the same time for the same connection! + +## Using the GDB Read/Write Splitting Plugin against non-GDB clusters + +The GDB Read/Write Splitting plugin can be used against Aurora clusters and RDS clusters. However, since these cluster types are single-region clusters, setting a home region does not make much sense. In these cases, use the original [Read/Write Splitting](./UsingTheReadWriteSplittingPlugin.md) plugin instead. + +## Configuration Parameters + +| Parameter | Value | Required | Description | Default Value | +| --------------------------------- | :----: | :-----------------------------------------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `readerHostSelectorStrategy` | `str` | No | The name of the strategy that should be used to select a new reader host. For more information on the available reader selection strategies, see this [table](../ReaderSelectionStrategies.md). | `random` | +| `gdbRwHomeRegion` | `str` | If connecting using an IP address, a custom domain URL, Global Database endpoint or other endpoint with no region: Yes

Otherwise: No | Defines a home region.

Examples: `us-west-2`, `us-east-1`.

If this parameter is omitted, the value is parsed from the connection configuration. For regional cluster endpoints and instance endpoints, it's set to the region of the provided endpoint. If the provided endpoint has no region (for example, a Global Database endpoint or IP address), the configuration parameter is mandatory. | For regional cluster endpoints and instance endpoints, it's set to the region of the provided endpoint.

Otherwise: `null` | +| `gdbRwRestrictWriterToHomeRegion` | `bool` | No | If set to `true`, prevents following and connecting to a writer node outside the defined home region. An exception will be raised when such a connection to a writer outside the home region is requested. | `true` | +| `gdbRwRestrictReaderToHomeRegion` | `bool` | No | If set to `true`, prevents connecting to a reader node outside the defined home region. If no reader nodes in the home region are available, an exception will be raised. | `true` | + +Please refer to the original [Read/Write Splitting plugin](./UsingTheReadWriteSplittingPlugin.md) for more details about error codes, configurations, connection pooling and sample codes. diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheIamAuthenticationPlugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheIamAuthenticationPlugin.md index 055862b22..ee6dafe3f 100644 --- a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheIamAuthenticationPlugin.md +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheIamAuthenticationPlugin.md @@ -76,3 +76,22 @@ You can learn more about `clusterInstanceHostPattern` [here](../UsingTheNodejsWr [IAM Authentication Plugin example for PostgreSQL](../../../examples/aws_driver_example/aws_iam_authentication_postgresql_example.ts)
[IAM Authentication Plugin example for MySQL](../../../examples/aws_driver_example/aws_iam_authentication_mysql_example.ts) + +## Using IAM Authentication with Global Databases + +When using IAM authentication with [Amazon Aurora Global Databases](https://aws.amazon.com/rds/aurora/global-database/), the IAM user or role requires the additional `rds:DescribeGlobalClusters` permission. This permission allows the driver to resolve the Global Database endpoint to the appropriate regional cluster for IAM token generation. + +Example IAM policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["rds-db:connect", "rds:DescribeGlobalClusters"], + "Resource": "*" + } + ] +} +``` diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheOktaAuthPlugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheOktaAuthPlugin.md index 7ece8ba0c..9b810ac58 100644 --- a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheOktaAuthPlugin.md +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheOktaAuthPlugin.md @@ -49,3 +49,22 @@ In the case of AD FS, the user signs into the AD FS sign in page. This generates | `iamDefaultPort` | `Number` | No | This property overrides the default port that is used to generate the IAM token. The default port is determined based on the underlying driver protocol. Target drivers with different protocols will require users to provide a default port. | `null` | `1234` | | `iamTokenExpiration` | `Number` | No | Overrides the default IAM token cache expiration in seconds. | `900` | `123` | | `httpsAgentOptions` | `Object` | No | This property adds parameters to the httpsAgent that connects to the hosting URL.
For more information on the parameters, see this [documentation](https://nodejs.org/api/https.html#new-agentoptions). | `null` | `{ timeout: 5000 }` | + +## Using Okta Authentication with Global Databases + +When using Okta authentication with [Amazon Aurora Global Databases](https://aws.amazon.com/rds/aurora/global-database/), the IAM user or role requires the additional `rds:DescribeGlobalClusters` permission. This permission allows the driver to resolve the Global Database endpoint to the appropriate regional cluster for IAM token generation. + +Example IAM policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["rds-db:connect", "rds:DescribeGlobalClusters"], + "Resource": "*" + } + ] +} +``` diff --git a/eslint.config.js b/eslint.config.js index b02eb491d..85a4dfb9a 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -35,7 +35,7 @@ const compat = new FlatCompat({ }); export default defineConfig([ - globalIgnores(["**/node_modules", "**/gradle", "**/dist", "**/coverage"]), + globalIgnores(["**/node_modules", "**/gradle", "**/dist", "**/coverage", "**/*.js"]), { extends: compat.extends( "eslint:recommended", diff --git a/index.ts b/index.ts index 067cf6e05..74a840bed 100644 --- a/index.ts +++ b/index.ts @@ -27,11 +27,11 @@ export { ConnectTimePlugin } from "./common/lib/plugins/connect_time_plugin"; export { ExecuteTimePlugin } from "./common/lib/plugins/execute_time_plugin"; export { AuroraInitialConnectionStrategyPlugin } from "./common/lib/plugins/aurora_initial_connection_strategy_plugin"; export { DefaultPlugin } from "./common/lib/plugins/default_plugin"; -export { ReadWriteSplittingPlugin } from "./common/lib/plugins/read_write_splitting_plugin"; +export { ReadWriteSplittingPlugin } from "./common/lib/plugins/read_write_splitting/read_write_splitting_plugin"; export { FailoverPlugin } from "./common/lib/plugins/failover/failover_plugin"; export { Failover2Plugin } from "./common/lib/plugins/failover2/failover2_plugin"; -export { HostMonitoringConnectionPlugin } from "./common/lib/plugins/efm/host_monitoring_connection_plugin"; -export { HostMonitoring2ConnectionPlugin } from "./common/lib/plugins/efm2/host_monitoring2_connection_plugin"; +export { HostMonitoringConnectionPlugin } from "./common/lib/plugins/efm/v1/host_monitoring_connection_plugin"; +export { HostMonitoring2ConnectionPlugin } from "./common/lib/plugins/efm/v2/host_monitoring2_connection_plugin"; export { DeveloperConnectionPlugin } from "./common/lib/plugins/dev/developer_connection_plugin"; export { BlueGreenPlugin } from "./common/lib/plugins/bluegreen/blue_green_plugin"; export { AuroraConnectionTrackerPlugin } from "./common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin"; @@ -44,7 +44,7 @@ export type { ConnectionProvider } from "./common/lib/connection_provider"; export type { HostSelector } from "./common/lib/host_selector"; export type { DatabaseDialect } from "./common/lib/database_dialect/database_dialect"; export type { PluginService } from "./common/lib/plugin_service"; -export type { HostListProvider, BlockingHostListProvider } from "./common/lib/host_list_provider/host_list_provider"; +export type { HostListProvider } from "./common/lib/host_list_provider/host_list_provider"; export type { ErrorHandler } from "./common/lib/error_handler"; export type { SessionStateService } from "./common/lib/session_state_service"; export type { DriverDialect } from "./common/lib/driver_dialect/driver_dialect"; diff --git a/mysql/lib/client.ts b/mysql/lib/client.ts index 185a68753..2048848c5 100644 --- a/mysql/lib/client.ts +++ b/mysql/lib/client.ts @@ -28,6 +28,7 @@ import { AwsWrapperError, ConnectionProvider, FailoverSuccessError, + HostInfo, InternalPooledConnectionProvider, TransactionIsolationLevel, UndefinedClientError, @@ -39,15 +40,17 @@ import { ClientUtils } from "../../common/lib/utils/client_utils"; import { RdsMultiAZClusterMySQLDatabaseDialect } from "./dialect/rds_multi_az_mysql_database_dialect"; import { TelemetryTraceLevel } from "../../common/lib/utils/telemetry/telemetry_trace_level"; import { MySQL2DriverDialect } from "./dialect/mysql2_driver_dialect"; -import { isDialectTopologyAware } from "../../common/lib/utils/utils"; +import { isDialectTopologyAware } from "../../common/lib/database_dialect/topology_aware_database_dialect"; import { MySQLClient, MySQLPoolClient } from "./mysql_client"; import { DriverConnectionProvider } from "../../common/lib/driver_connection_provider"; +import { GlobalAuroraMySQLDatabaseDialect } from "./dialect/global_aurora_mysql_database_dialect"; class BaseAwsMySQLClient extends AwsClient implements MySQLClient { private static readonly knownDialectsByCode: Map = new Map([ [DatabaseDialectCodes.MYSQL, new MySQLDatabaseDialect()], [DatabaseDialectCodes.RDS_MYSQL, new RdsMySQLDatabaseDialect()], [DatabaseDialectCodes.AURORA_MYSQL, new AuroraMySQLDatabaseDialect()], + [DatabaseDialectCodes.GLOBAL_AURORA_MYSQL, new GlobalAuroraMySQLDatabaseDialect()], [DatabaseDialectCodes.RDS_MULTI_AZ_MYSQL, new RdsMultiAZClusterMySQLDatabaseDialect()] ]); @@ -84,7 +87,9 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { // Ignore } } - await this.pluginService.setCurrentClient(result, result.hostInfo); + const connectedHostInfo: HostInfo | null = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getInitialConnectionHostInfo(); + await this.pluginService.setCurrentClient(result, connectedHostInfo); + this.pluginService.setRoutedHostInfo(null); await this.internalPostConnect(); }); } @@ -113,7 +118,7 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { return result; } - isReadOnly(): boolean { + isReadOnly(): boolean | undefined { return this.pluginService.getSessionStateService().getReadOnly(); } @@ -129,7 +134,7 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { return result; } - getAutoCommit(): boolean { + getAutoCommit(): boolean | undefined { return this.pluginService.getSessionStateService().getAutoCommit(); } @@ -142,7 +147,7 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { this.pluginService.getSessionStateService().setCatalog(catalog); } - getCatalog(): string { + getCatalog(): string | undefined { return this.pluginService.getSessionStateService().getCatalog(); } @@ -150,7 +155,7 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { throw new UnsupportedMethodError(Messages.get("Client.methodNotSupported", "setSchema")); } - getSchema(): string { + getSchema(): string | undefined { throw new UnsupportedMethodError(Messages.get("Client.methodNotSupported", "getSchema")); } @@ -181,7 +186,7 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { this.pluginService.getSessionStateService().setTransactionIsolation(level); } - getTransactionIsolation(): TransactionIsolationLevel { + getTransactionIsolation(): TransactionIsolationLevel | undefined { return this.pluginService.getSessionStateService().getTransactionIsolation(); } @@ -197,6 +202,10 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { this.properties, "end", () => { + if (!this.targetClient) { + return Promise.resolve(undefined); + } + this.pluginService.removeErrorListener(this.targetClient); const res = ClientUtils.queryWithTimeout(this.targetClient.end(), this.properties); this.targetClient = undefined; diff --git a/mysql/lib/dialect/aurora_mysql_database_dialect.ts b/mysql/lib/dialect/aurora_mysql_database_dialect.ts index c2843edc5..a4f3c69a5 100644 --- a/mysql/lib/dialect/aurora_mysql_database_dialect.ts +++ b/mysql/lib/dialect/aurora_mysql_database_dialect.ts @@ -15,18 +15,16 @@ */ import { MySQLDatabaseDialect } from "./mysql_database_dialect"; -import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; -import { HostInfo } from "../../../common/lib/host_info"; -import { TopologyAwareDatabaseDialect } from "../../../common/lib/topology_aware_database_dialect"; +import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { HostRole } from "../../../common/lib/host_role"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { DatabaseDialectCodes } from "../../../common/lib/database_dialect/database_dialect_codes"; -import { WrapperProperties } from "../../../common/lib/wrapper_property"; -import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; -import { PluginService } from "../../../common/lib/plugin_service"; import { BlueGreenDialect, BlueGreenResult } from "../../../common/lib/database_dialect/blue_green_dialect"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { AuroraTopologyUtils } from "../../../common/lib/host_list_provider/aurora_topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements TopologyAwareDatabaseDialect, BlueGreenDialect { private static readonly TOPOLOGY_QUERY: string = @@ -41,22 +39,21 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements "SELECT server_id " + "FROM information_schema.replica_host_status " + "WHERE SESSION_ID = 'MASTER_SESSION_ID' AND SERVER_ID = @@aurora_server_id"; + protected static readonly INSTANCE_ID_QUERY: string = "SELECT @@aurora_server_id as instance_id, @@aurora_server_id as instance_name"; private static readonly AURORA_VERSION_QUERY = "SHOW VARIABLES LIKE 'aurora_version'"; private static readonly BG_STATUS_QUERY: string = "SELECT * FROM mysql.rds_topology"; private static readonly TOPOLOGY_TABLE_EXIST_QUERY: string = "SELECT 1 AS tmp FROM information_schema.tables WHERE table_schema = 'mysql' AND table_name = 'rds_topology'"; - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider(props, originalUrl, hostListProviderService, (hostListProviderService)); - } - return new RdsHostListProvider(props, originalUrl, hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + const topologyUtils = new AuroraTopologyUtils(this, servicesContainer.hostListProviderService.getHostInfoBuilder()); + return new RdsHostListProvider(props, originalUrl, topologyUtils, servicesContainer); } - async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + async queryForTopology(targetClient: ClientWrapper): Promise { const res = await targetClient.query(AuroraMySQLDatabaseDialect.TOPOLOGY_QUERY); - const hosts: HostInfo[] = []; + const results: TopologyQueryResult[] = []; const rows: any[] = res[0]; rows.forEach((row) => { // According to the topology query the result set @@ -66,10 +63,15 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements const cpuUtilization: number = row["cpu"]; const hostLag: number = row["lag"]; const lastUpdateTime: number = row["last_update_timestamp"] ? Date.parse(row["last_update_timestamp"]) : Date.now(); - const host: HostInfo = hostListProvider.createHost(hostName, isWriter, Math.round(hostLag) * 100 + Math.round(cpuUtilization), lastUpdateTime); - hosts.push(host); + const result: TopologyQueryResult = new TopologyQueryResult({ + host: hostName, + isWriter: isWriter, + weight: Math.round(hostLag) * 100 + Math.round(cpuUtilization), + lastUpdateTime: lastUpdateTime + }); + results.push(result); }); - return hosts; + return results; } async identifyConnection(targetClient: ClientWrapper): Promise { @@ -87,7 +89,7 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements try { const writerId: string = res[0][0]["server_id"]; return writerId ? writerId : null; - } catch (e) { + } catch (e: any) { if (e.message.includes("Cannot read properties of undefined")) { // Query returned no result, targetClient is not connected to a writer. return null; @@ -96,6 +98,21 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements } } + async getInstanceId(targetClient: ClientWrapper): Promise<[string, string]> { + const res = await targetClient.query(AuroraMySQLDatabaseDialect.INSTANCE_ID_QUERY); + try { + const instance_id: string = res[0][0]["instance_id"]; + const instance_name: string = res[0][0]["instance_name"]; + return [instance_id, instance_name]; + } catch (e: any) { + if (e.message.includes("Cannot read properties of undefined")) { + // Query returned no result, targetClient is not connected to a writer. + return ["", ""]; + } + throw e; + } + } + async isDialect(targetClient: ClientWrapper): Promise { return targetClient .query(AuroraMySQLDatabaseDialect.AURORA_VERSION_QUERY) @@ -112,7 +129,7 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements } getDialectUpdateCandidates(): string[] { - return [DatabaseDialectCodes.RDS_MULTI_AZ_MYSQL]; + return [DatabaseDialectCodes.GLOBAL_AURORA_MYSQL, DatabaseDialectCodes.RDS_MULTI_AZ_MYSQL]; } async isBlueGreenStatusAvailable(clientWrapper: ClientWrapper): Promise { diff --git a/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts b/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts new file mode 100644 index 000000000..faf89a3ca --- /dev/null +++ b/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts @@ -0,0 +1,116 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { AuroraMySQLDatabaseDialect } from "./aurora_mysql_database_dialect"; +import { GlobalAuroraTopologyDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; +import { ClientWrapper } from "../../../common/lib/client_wrapper"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; +import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; +import { GlobalAuroraHostListProvider } from "../../../common/lib/host_list_provider/global_aurora_host_list_provider"; +import { GlobalTopologyUtils } from "../../../common/lib/host_list_provider/global_topology_utils"; + +export class GlobalAuroraMySQLDatabaseDialect extends AuroraMySQLDatabaseDialect implements GlobalAuroraTopologyDialect { + private static readonly GLOBAL_STATUS_TABLE_EXISTS_QUERY = + "SELECT 1 AS tmp FROM information_schema.tables WHERE" + + " upper(table_schema) = 'INFORMATION_SCHEMA' AND upper(table_name) = 'AURORA_GLOBAL_DB_STATUS'"; + + private static readonly GLOBAL_INSTANCE_STATUS_EXISTS_QUERY = + "SELECT 1 AS tmp FROM information_schema.tables WHERE" + + " upper(table_schema) = 'INFORMATION_SCHEMA' AND upper(table_name) = 'AURORA_GLOBAL_DB_INSTANCE_STATUS'"; + + private static readonly GLOBAL_TOPOLOGY_QUERY = + "SELECT server_id, CASE WHEN SESSION_ID = 'MASTER_SESSION_ID' THEN TRUE ELSE FALSE END AS is_writer, " + + "visibility_lag_in_msec, aws_region " + + "FROM information_schema.aurora_global_db_instance_status"; + + private static readonly REGION_COUNT_QUERY = "SELECT count(1) FROM information_schema.aurora_global_db_status"; + + private static readonly REGION_BY_INSTANCE_ID_QUERY = + "SELECT AWS_REGION FROM information_schema.aurora_global_db_instance_status WHERE SERVER_ID = ?"; + + async isDialect(targetClient: ClientWrapper): Promise { + try { + // Check if both global status tables exist + const [statusRows] = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.GLOBAL_STATUS_TABLE_EXISTS_QUERY); + if (!statusRows?.[0]) { + return false; + } + + const [instanceStatusRows] = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.GLOBAL_INSTANCE_STATUS_EXISTS_QUERY); + if (!instanceStatusRows?.[0]) { + return false; + } + + // Check if there are multiple regions + const [regionCountRows] = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.REGION_COUNT_QUERY); + if (!regionCountRows?.[0]) { + return false; + } + + const awsRegionCount = regionCountRows[0]["count(1)"]; + return awsRegionCount > 1; + } catch { + return false; + } + } + + getDialectUpdateCandidates(): string[] { + return []; + } + + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + return new GlobalAuroraHostListProvider( + props, + originalUrl, + new GlobalTopologyUtils(this, servicesContainer.pluginService.getHostInfoBuilder()), + servicesContainer + ); + } + + async queryForTopology(targetClient: ClientWrapper): Promise { + const res = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.GLOBAL_TOPOLOGY_QUERY); + const results: TopologyQueryResult[] = []; + const rows: any[] = res[0]; + rows.forEach((row) => { + const hostName: string = row["server_id"]; + const isWriter: boolean = row["is_writer"]; + const hostLag: number = row["visibility_lag_in_msec"] ?? 0; // visibility_lag_in_msec is nullable. + const awsRegion: string = row["aws_region"]; + + const host: TopologyQueryResult = new TopologyQueryResult({ + host: hostName, + isWriter: isWriter, + weight: Math.round(hostLag) * 100, + awsRegion: awsRegion + }); + results.push(host); + }); + return results; + } + + async getRegionByInstanceId(targetClient: ClientWrapper, instanceId: string): Promise { + try { + const [rows] = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.REGION_BY_INSTANCE_ID_QUERY, [instanceId]); + if (!rows?.[0]) { + return null; + } + return rows[0]["aws_region"] ?? null; + } catch { + return null; + } + } +} diff --git a/mysql/lib/dialect/mysql2_driver_dialect.ts b/mysql/lib/dialect/mysql2_driver_dialect.ts index c9e3e5841..3e7b9f3af 100644 --- a/mysql/lib/dialect/mysql2_driver_dialect.ts +++ b/mysql/lib/dialect/mysql2_driver_dialect.ts @@ -47,6 +47,8 @@ export class MySQL2DriverDialect implements DriverDialect { preparePoolClientProperties(props: Map, poolConfig: AwsPoolConfig | undefined): any { const finalPoolConfig: PoolOptions = {}; const finalClientProps = WrapperProperties.removeWrapperProperties(props); + this.setKeepAliveProperties(finalClientProps, props.get(WrapperProperties.KEEPALIVE_PROPERTIES.name)); + this.setConnectTimeout(finalClientProps, props.get(WrapperProperties.WRAPPER_CONNECT_TIMEOUT.name)); Object.assign(finalPoolConfig, Object.fromEntries(finalClientProps.entries())); finalPoolConfig.connectionLimit = poolConfig?.maxConnections; @@ -69,6 +71,9 @@ export class MySQL2DriverDialect implements DriverDialect { } setQueryTimeout(props: Map, sql?: any, wrapperQueryTimeout?: any) { + if (!sql) { + return; + } const timeout = wrapperQueryTimeout ?? props.get(WrapperProperties.WRAPPER_QUERY_TIMEOUT.name); if (timeout && !sql[MySQL2DriverDialect.QUERY_TIMEOUT_PROPERTY_NAME]) { sql[MySQL2DriverDialect.QUERY_TIMEOUT_PROPERTY_NAME] = Number(timeout); diff --git a/mysql/lib/dialect/mysql_database_dialect.ts b/mysql/lib/dialect/mysql_database_dialect.ts index 084ea6b21..f0fe20ad6 100644 --- a/mysql/lib/dialect/mysql_database_dialect.ts +++ b/mysql/lib/dialect/mysql_database_dialect.ts @@ -28,6 +28,7 @@ import { ErrorHandler } from "../../../common/lib/error_handler"; import { MySQLErrorHandler } from "../mysql_error_handler"; import { Messages } from "../../../common/lib/utils/messages"; import { HostRole } from "../../../common/lib/host_role"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class MySQLDatabaseDialect implements DatabaseDialect { protected dialectName: string = this.constructor.name; @@ -108,8 +109,8 @@ export class MySQLDatabaseDialect implements DatabaseDialect { }); } - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - return new ConnectionStringHostListProvider(props, originalUrl, this.getDefaultPort(), hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + return new ConnectionStringHostListProvider(props, originalUrl, this.getDefaultPort(), servicesContainer.hostListProviderService); } getErrorHandler(): ErrorHandler { diff --git a/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts b/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts index da6d26a74..33ac21693 100644 --- a/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts +++ b/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts @@ -15,20 +15,17 @@ */ import { MySQLDatabaseDialect } from "./mysql_database_dialect"; -import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; -import { HostInfo } from "../../../common/lib/host_info"; import { HostRole } from "../../../common/lib/host_role"; import { Messages } from "../../../common/lib/utils/messages"; -import { logger } from "../../../common/logutils"; import { AwsWrapperError } from "../../../common/lib/utils/errors"; -import { TopologyAwareDatabaseDialect } from "../../../common/lib/topology_aware_database_dialect"; +import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; import { FailoverRestriction } from "../../../common/lib/plugins/failover/failover_restriction"; -import { WrapperProperties } from "../../../common/lib/wrapper_property"; -import { PluginService } from "../../../common/lib/plugin_service"; -import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { AuroraTopologyUtils } from "../../../common/lib/host_list_provider/aurora_topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect implements TopologyAwareDatabaseDialect { private static readonly TOPOLOGY_QUERY: string = "SELECT id, endpoint, port FROM mysql.rds_topology"; @@ -41,6 +38,8 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect private static readonly HOST_ID_QUERY_COLUMN_NAME: string = "host"; private static readonly IS_READER_QUERY: string = "SELECT @@read_only AS is_reader"; private static readonly IS_READER_QUERY_COLUMN_NAME: string = "is_reader"; + protected static readonly INSTANCE_ID_QUERY: string = + "SELECT id as instance_id, SUBSTRING_INDEX(endpoint, '.', 1) as instance_name FROM mysql.rds_topology WHERE id = @@server_id"; async isDialect(targetClient: ClientWrapper): Promise { let res = await targetClient.query(RdsMultiAZClusterMySQLDatabaseDialect.TOPOLOGY_TABLE_EXIST_QUERY).catch(() => false); @@ -70,14 +69,12 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect .catch(() => false); } - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider(props, originalUrl, hostListProviderService, (hostListProviderService)); - } - return new RdsHostListProvider(props, originalUrl, hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + const topologyUtils = new AuroraTopologyUtils(this, servicesContainer.hostListProviderService.getHostInfoBuilder()); + return new RdsHostListProvider(props, originalUrl, topologyUtils, servicesContainer); } - async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + async queryForTopology(targetClient: ClientWrapper): Promise { try { let writerHostId: string = await this.executeTopologyRelatedQuery( targetClient, @@ -90,7 +87,7 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect const res = await targetClient.query(RdsMultiAZClusterMySQLDatabaseDialect.TOPOLOGY_QUERY); const rows: any[] = res[0]; - return this.processTopologyQueryResults(hostListProvider, writerHostId, rows); + return this.processTopologyQueryResults(writerHostId, rows); } catch (error: any) { throw new AwsWrapperError(Messages.get("RdsMultiAZMySQLDatabaseDialect.invalidQuery", error.message)); } @@ -105,8 +102,8 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect return ""; } - private async processTopologyQueryResults(hostListProvider: HostListProvider, writerHostId: string, rows: any[]): Promise { - const hostMap: Map = new Map(); + private async processTopologyQueryResults(writerHostId: string, rows: any[]): Promise { + const hosts: TopologyQueryResult[] = []; rows.forEach((row) => { // According to the topology query the result set // should contain 3 columns: endpoint, id, and port @@ -114,39 +111,23 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect const id: string = row["id"]; const port: number = row["port"]; const isWriter: boolean = id === writerHostId; - - const host: HostInfo = hostListProvider.createHost(endpoint.substring(0, endpoint.indexOf(".")), isWriter, 0, Date.now(), port); - host.hostId = id; - host.addAlias(endpoint); - hostMap.set(host.host, host); + const host: TopologyQueryResult = new TopologyQueryResult({ + host: endpoint.substring(0, endpoint.indexOf(".")), + isWriter: isWriter, + weight: 0, + lastUpdateTime: Date.now(), + port: port, + id: id + }); + hosts.push(host); }); - - let hosts: HostInfo[] = []; - const writers: HostInfo[] = []; - - for (const [key, info] of hostMap.entries()) { - if (info.role !== HostRole.WRITER) { - hosts.push(info); - } else { - writers.push(info); - } - } - - const writerCount: number = writers.length; - if (writerCount === 0) { - logger.error(Messages.get("RdsMultiAzDatabaseDialect.invalidTopology", this.dialectName)); - hosts = []; - } else { - hosts.push(writers[0]); - } - return hosts; } async getHostRole(client: ClientWrapper): Promise { return (await this.executeTopologyRelatedQuery( client, - RdsMultiAZClusterMySQLDatabaseDialect.IS_READER_QUERY, + RdsMultiAZClusterMySQLDatabaseDialect.INSTANCE_ID_QUERY, RdsMultiAZClusterMySQLDatabaseDialect.IS_READER_QUERY_COLUMN_NAME )) == "0" ? HostRole.WRITER @@ -171,6 +152,17 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect } } + async getInstanceId(targetClient: ClientWrapper): Promise<[string, string]> { + try { + const res = await targetClient.query(RdsMultiAZClusterMySQLDatabaseDialect.INSTANCE_ID_QUERY); + const instance_id = res[0][0]["instance_id"]; + const instance_name = res[0][0]["instance_name"]; + return [instance_id, instance_name]; + } catch (error: any) { + throw new AwsWrapperError(Messages.get("RdsMultiAZMySQLDatabaseDialect.invalidQuery", error.message)); + } + } + async identifyConnection(client: ClientWrapper): Promise { return await this.executeTopologyRelatedQuery( client, diff --git a/package.json b/package.json index 7977242a7..ad0de7f97 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "format": "prettier . --write --config .prettierrc", "integration": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" npx jest --config=jest.integration.config.json --runInBand --verbose", "debug-integration": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --inspect-brk=0.0.0.0:5005\" npx jest --config=jest.integration.config.json --runInBand", - "lint": "eslint --fix --ext .ts .", + "lint": "eslint --ext .ts .", "test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" npx jest --config=jest.unit.config.json", "fix-imports": "babel --extensions \".js\" dist -d dist", "build": "tsc", diff --git a/pg/lib/client.ts b/pg/lib/client.ts index 13e0bd710..bb0b3d9fd 100644 --- a/pg/lib/client.ts +++ b/pg/lib/client.ts @@ -38,15 +38,17 @@ import { ClientWrapper } from "../../common/lib/client_wrapper"; import { RdsMultiAZClusterPgDatabaseDialect } from "./dialect/rds_multi_az_pg_database_dialect"; import { TelemetryTraceLevel } from "../../common/lib/utils/telemetry/telemetry_trace_level"; import { NodePostgresDriverDialect } from "./dialect/node_postgres_driver_dialect"; -import { isDialectTopologyAware } from "../../common/lib/utils/utils"; +import { isDialectTopologyAware } from "../../common/lib/database_dialect/topology_aware_database_dialect"; import { PGClient, PGPoolClient } from "./pg_client"; import { DriverConnectionProvider } from "../../common/lib/driver_connection_provider"; +import { GlobalAuroraPgDatabaseDialect } from "./dialect/global_aurora_pg_database_dialect"; class BaseAwsPgClient extends AwsClient implements PGClient { private static readonly knownDialectsByCode: Map = new Map([ [DatabaseDialectCodes.PG, new PgDatabaseDialect()], [DatabaseDialectCodes.RDS_PG, new RdsPgDatabaseDialect()], [DatabaseDialectCodes.AURORA_PG, new AuroraPgDatabaseDialect()], + [DatabaseDialectCodes.GLOBAL_AURORA_PG, new GlobalAuroraPgDatabaseDialect()], [DatabaseDialectCodes.RDS_MULTI_AZ_PG, new RdsMultiAZClusterPgDatabaseDialect()] ]); @@ -82,7 +84,7 @@ class BaseAwsPgClient extends AwsClient implements PGClient { return result; } - isReadOnly(): boolean { + isReadOnly(): boolean | undefined { return this.pluginService.getSessionStateService().getReadOnly(); } @@ -120,7 +122,7 @@ class BaseAwsPgClient extends AwsClient implements PGClient { this.pluginService.getSessionStateService().setTransactionIsolation(level); } - getTransactionIsolation(): TransactionIsolationLevel { + getTransactionIsolation(): TransactionIsolationLevel | undefined { return this.pluginService.getSessionStateService().getTransactionIsolation(); } @@ -147,7 +149,7 @@ class BaseAwsPgClient extends AwsClient implements PGClient { return result; } - getSchema(): string { + getSchema(): string | undefined { return this.pluginService.getSessionStateService().getSchema(); } @@ -213,7 +215,10 @@ class BaseAwsPgClient extends AwsClient implements PGClient { // Ignore } } - await this.pluginService.setCurrentClient(result, result.hostInfo); + + const connectedHostInfo: HostInfo | null = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getInitialConnectionHostInfo(); + await this.pluginService.setCurrentClient(result, connectedHostInfo); + this.pluginService.setRoutedHostInfo(null); await this.internalPostConnect(); }); } @@ -399,7 +404,7 @@ export class AwsPgPoolClient implements PGPoolClient { await awsPGPooledConnection.connect(); const res = await awsPGPooledConnection.query(queryTextOrConfig as any, values); await awsPGPooledConnection.end(); - return res; + return res as any; } catch (error: any) { if (!(error instanceof FailoverSuccessError)) { // Release pooled connection. diff --git a/pg/lib/dialect/aurora_pg_database_dialect.ts b/pg/lib/dialect/aurora_pg_database_dialect.ts index 9009ddbb4..baf8e14cd 100644 --- a/pg/lib/dialect/aurora_pg_database_dialect.ts +++ b/pg/lib/dialect/aurora_pg_database_dialect.ts @@ -15,18 +15,17 @@ */ import { PgDatabaseDialect } from "./pg_database_dialect"; -import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; -import { TopologyAwareDatabaseDialect } from "../../../common/lib/topology_aware_database_dialect"; -import { HostInfo, HostRole } from "../../../common/lib"; +import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; +import { HostRole } from "../../../common/lib"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { DatabaseDialectCodes } from "../../../common/lib/database_dialect/database_dialect_codes"; import { LimitlessDatabaseDialect } from "../../../common/lib/database_dialect/limitless_database_dialect"; -import { WrapperProperties } from "../../../common/lib/wrapper_property"; -import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; -import { PluginService } from "../../../common/lib/plugin_service"; import { BlueGreenDialect, BlueGreenResult } from "../../../common/lib/database_dialect/blue_green_dialect"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { AuroraTopologyUtils } from "../../../common/lib/host_list_provider/aurora_topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements TopologyAwareDatabaseDialect, LimitlessDatabaseDialect, BlueGreenDialect { private static readonly VERSION = process.env.npm_package_version; @@ -40,6 +39,8 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo private static readonly EXTENSIONS_SQL: string = "SELECT (setting LIKE '%aurora_stat_utils%') AS aurora_stat_utils FROM pg_catalog.pg_settings WHERE name OPERATOR(pg_catalog.=) 'rds.extensions'"; private static readonly HOST_ID_QUERY: string = "SELECT pg_catalog.aurora_db_instance_identifier() as host"; + protected static readonly INSTANCE_ID_QUERY: string = + "SELECT pg_catalog.aurora_db_instance_identifier() as instance_id, pg_catalog.aurora_db_instance_identifier() as instance_name"; private static readonly IS_READER_QUERY: string = "SELECT pg_catalog.pg_is_in_recovery() as is_reader"; private static readonly IS_WRITER_QUERY: string = "SELECT server_id " + @@ -50,16 +51,14 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo private static readonly TOPOLOGY_TABLE_EXIST_QUERY: string = "SELECT pg_catalog.'get_blue_green_fast_switchover_metadata'::regproc"; - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider(props, originalUrl, hostListProviderService, (hostListProviderService)); - } - return new RdsHostListProvider(props, originalUrl, hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + const topologyUtils = new AuroraTopologyUtils(this, servicesContainer.hostListProviderService.getHostInfoBuilder()); + return new RdsHostListProvider(props, originalUrl, topologyUtils, servicesContainer); } - async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { - const res = await targetClient.query(AuroraPgDatabaseDialect.TOPOLOGY_QUERY); - const hosts: HostInfo[] = []; + async queryForTopology(targetClient: ClientWrapper): Promise { + const res = await targetClient.queryWithTimeout(AuroraPgDatabaseDialect.TOPOLOGY_QUERY); + const results: TopologyQueryResult[] = []; const rows: any[] = res.rows; rows.forEach((row) => { // According to the topology query the result set @@ -69,10 +68,15 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo const cpuUtilization: number = row["cpu"]; const hostLag: number = row["lag"]; const lastUpdateTime: number = row["last_update_timestamp"] ? Date.parse(row["last_update_timestamp"]) : Date.now(); - const host: HostInfo = hostListProvider.createHost(hostName, isWriter, Math.round(hostLag) * 100 + Math.round(cpuUtilization), lastUpdateTime); - hosts.push(host); + const host: TopologyQueryResult = new TopologyQueryResult({ + host: hostName, + isWriter, + weight: Math.round(hostLag) * 100 + Math.round(cpuUtilization), + lastUpdateTime + }); + results.push(host); }); - return hosts; + return results; } async identifyConnection(targetClient: ClientWrapper): Promise { @@ -90,7 +94,7 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo try { const writerId: string = res.rows[0]["server_id"]; return writerId ? writerId : null; - } catch (e) { + } catch (e: any) { if (e.message.includes("Cannot read properties of undefined")) { // Query returned no result, targetClient is not connected to a writer. return null; @@ -99,6 +103,21 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo } } + async getInstanceId(targetClient: ClientWrapper): Promise<[string, string]> { + const res = await targetClient.query(AuroraPgDatabaseDialect.INSTANCE_ID_QUERY); + try { + const instance_id: string = res.rows[0]["instance_id"]; + const instance_name: string = res.rows[0]["instance_name"]; + return [instance_id, instance_name]; + } catch (e: any) { + if (e.message.includes("Cannot read properties of undefined")) { + // Query returned no result, targetClient is not connected to a writer. + return ["", ""]; + } + throw e; + } + } + async isDialect(targetClient: ClientWrapper): Promise { if (!(await super.isDialect(targetClient))) { return false; @@ -119,7 +138,7 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo } getDialectUpdateCandidates(): string[] { - return [DatabaseDialectCodes.RDS_MULTI_AZ_PG]; + return [DatabaseDialectCodes.GLOBAL_AURORA_PG, DatabaseDialectCodes.RDS_MULTI_AZ_PG]; } getLimitlessRoutersQuery(): string { diff --git a/pg/lib/dialect/global_aurora_pg_database_dialect.ts b/pg/lib/dialect/global_aurora_pg_database_dialect.ts new file mode 100644 index 000000000..9bc886118 --- /dev/null +++ b/pg/lib/dialect/global_aurora_pg_database_dialect.ts @@ -0,0 +1,124 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { AuroraPgDatabaseDialect } from "./aurora_pg_database_dialect"; +import { GlobalAuroraTopologyDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; +import { ClientWrapper } from "../../../common/lib/client_wrapper"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; +import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; +import { GlobalAuroraHostListProvider } from "../../../common/lib/host_list_provider/global_aurora_host_list_provider"; +import { GlobalTopologyUtils } from "../../../common/lib/host_list_provider/global_topology_utils"; + +export class GlobalAuroraPgDatabaseDialect extends AuroraPgDatabaseDialect implements GlobalAuroraTopologyDialect { + private static readonly GLOBAL_STATUS_FUNC_EXISTS_QUERY = "select 'pg_catalog.aurora_global_db_status'::regproc"; + + private static readonly GLOBAL_INSTANCE_STATUS_FUNC_EXISTS_QUERY = "select 'pg_catalog.aurora_global_db_instance_status'::regproc"; + + private static readonly GLOBAL_TOPOLOGY_QUERY = + "SELECT SERVER_ID, CASE WHEN SESSION_ID = 'MASTER_SESSION_ID' THEN TRUE ELSE FALSE END AS IS_WRITER, " + + "VISIBILITY_LAG_IN_MSEC, AWS_REGION " + + "FROM pg_catalog.aurora_global_db_instance_status()"; + + private static readonly REGION_COUNT_QUERY = "SELECT count(1) FROM pg_catalog.aurora_global_db_status()"; + + private static readonly REGION_BY_INSTANCE_ID_QUERY = "SELECT AWS_REGION FROM pg_catalog.aurora_global_db_instance_status() WHERE SERVER_ID = $1"; + + async isDialect(targetClient: ClientWrapper): Promise { + try { + // First check if aurora_stat_utils extension is available + const extensionsResult = await targetClient.query( + "SELECT (setting LIKE '%aurora_stat_utils%') AS aurora_stat_utils FROM pg_catalog.pg_settings WHERE name OPERATOR(pg_catalog.=) 'rds.extensions'" + ); + + if (!extensionsResult.rows?.[0]) { + return false; + } + + const auroraUtils = extensionsResult.rows[0]["aurora_stat_utils"]; + if (!auroraUtils) { + return false; + } + + // Check if both global status functions exist + const statusResult = await targetClient.query(GlobalAuroraPgDatabaseDialect.GLOBAL_STATUS_FUNC_EXISTS_QUERY); + if (!statusResult.rows?.[0]) { + return false; + } + + const instanceStatusResult = await targetClient.query(GlobalAuroraPgDatabaseDialect.GLOBAL_INSTANCE_STATUS_FUNC_EXISTS_QUERY); + if (!instanceStatusResult.rows?.[0]) { + return false; + } + + // Check if there are multiple regions + const regionCountResult = await targetClient.query(GlobalAuroraPgDatabaseDialect.REGION_COUNT_QUERY); + if (!regionCountResult.rows?.[0]) { + return false; + } + + const awsRegionCount = regionCountResult.rows[0]["count"]; + return awsRegionCount > 1; + } catch { + return false; + } + } + + getDialectUpdateCandidates(): string[] { + return []; + } + + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + return new GlobalAuroraHostListProvider( + props, + originalUrl, + new GlobalTopologyUtils(this, servicesContainer.pluginService.getHostInfoBuilder()), + servicesContainer + ); + } + + async queryForTopology(targetClient: ClientWrapper): Promise { + const res = await targetClient.queryWithTimeout(GlobalAuroraPgDatabaseDialect.GLOBAL_TOPOLOGY_QUERY); + const hosts: TopologyQueryResult[] = []; + const rows: any[] = res.rows; + rows.forEach((row) => { + const hostName: string = row["server_id"]; + const isWriter: boolean = row["is_writer"]; + const hostLag: number = row["visibility_lag_in_msec"] ?? 0; + const awsRegion: string = row["aws_region"]; + const host: TopologyQueryResult = new TopologyQueryResult({ + host: hostName, + isWriter: isWriter, + weight: Math.round(hostLag) * 100, + awsRegion: awsRegion + }); + hosts.push(host); + }); + return hosts; + } + + async getRegionByInstanceId(targetClient: ClientWrapper, instanceId: string): Promise { + try { + const result = await targetClient.query(GlobalAuroraPgDatabaseDialect.REGION_BY_INSTANCE_ID_QUERY, [instanceId]); + if (!result.rows?.[0]) { + return null; + } + return result.rows[0]["aws_region"] ?? null; + } catch { + return null; + } + } +} diff --git a/pg/lib/dialect/node_postgres_driver_dialect.ts b/pg/lib/dialect/node_postgres_driver_dialect.ts index 7cf659392..40d1ba381 100644 --- a/pg/lib/dialect/node_postgres_driver_dialect.ts +++ b/pg/lib/dialect/node_postgres_driver_dialect.ts @@ -49,8 +49,12 @@ export class NodePostgresDriverDialect implements DriverDialect { preparePoolClientProperties(props: Map, poolConfig: AwsPoolConfig | undefined): any { const finalPoolConfig: pkgPg.PoolConfig = {}; const finalClientProps = WrapperProperties.removeWrapperProperties(props); + this.setKeepAliveProperties(finalClientProps, props.get(WrapperProperties.KEEPALIVE_PROPERTIES.name)); + this.setConnectTimeout(finalClientProps, props.get(WrapperProperties.WRAPPER_CONNECT_TIMEOUT.name)); + this.setQueryTimeout(finalClientProps, undefined, props.get(WrapperProperties.WRAPPER_QUERY_TIMEOUT.name)); Object.assign(finalPoolConfig, Object.fromEntries(finalClientProps.entries())); + finalPoolConfig.max = poolConfig?.maxConnections; finalPoolConfig.idleTimeoutMillis = poolConfig?.idleTimeoutMillis; finalPoolConfig.allowExitOnIdle = poolConfig?.allowExitOnIdle; diff --git a/pg/lib/dialect/pg_database_dialect.ts b/pg/lib/dialect/pg_database_dialect.ts index 3b03ef24a..0afa54f29 100644 --- a/pg/lib/dialect/pg_database_dialect.ts +++ b/pg/lib/dialect/pg_database_dialect.ts @@ -25,6 +25,7 @@ import { FailoverRestriction } from "../../../common/lib/plugins/failover/failov import { ErrorHandler } from "../../../common/lib/error_handler"; import { PgErrorHandler } from "../pg_error_handler"; import { Messages } from "../../../common/lib/utils/messages"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class PgDatabaseDialect implements DatabaseDialect { protected dialectName: string = this.constructor.name; @@ -105,8 +106,8 @@ export class PgDatabaseDialect implements DatabaseDialect { }); } - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - return new ConnectionStringHostListProvider(props, originalUrl, this.getDefaultPort(), hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + return new ConnectionStringHostListProvider(props, originalUrl, this.getDefaultPort(), servicesContainer.hostListProviderService); } getErrorHandler(): ErrorHandler { diff --git a/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts b/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts index ee517c130..5b4b79b2a 100644 --- a/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts +++ b/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts @@ -14,20 +14,18 @@ limitations under the License. */ -import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; -import { AwsWrapperError, HostInfo, HostRole } from "../../../common/lib"; +import { AwsWrapperError, HostRole } from "../../../common/lib"; import { Messages } from "../../../common/lib/utils/messages"; -import { logger } from "../../../common/logutils"; -import { TopologyAwareDatabaseDialect } from "../../../common/lib/topology_aware_database_dialect"; +import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; import { PgDatabaseDialect } from "./pg_database_dialect"; import { ErrorHandler } from "../../../common/lib/error_handler"; import { MultiAzPgErrorHandler } from "../multi_az_pg_error_handler"; -import { WrapperProperties } from "../../../common/lib/wrapper_property"; -import { PluginService } from "../../../common/lib/plugin_service"; -import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { AuroraTopologyUtils } from "../../../common/lib/host_list_provider/aurora_topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implements TopologyAwareDatabaseDialect { constructor() { @@ -44,6 +42,10 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem private static readonly HOST_ID_QUERY_COLUMN_NAME: string = "dbi_resource_id"; private static readonly IS_READER_QUERY: string = "SELECT pg_catalog.pg_is_in_recovery() AS is_reader"; private static readonly IS_READER_QUERY_COLUMN_NAME: string = "is_reader"; + protected static readonly INSTANCE_ID_QUERY: string = + "SELECT id as instance_id, SUBSTRING(endpoint FROM 0 FOR POSITION('.' IN endpoint)) as instance_name " + + "FROM rds_tools.show_topology() " + + "WHERE id OPERATOR(pg_catalog.=) rds_tools.dbi_resource_id()"; async isDialect(targetClient: ClientWrapper): Promise { const res = await targetClient.query(RdsMultiAZClusterPgDatabaseDialect.WRITER_HOST_FUNC_EXIST_QUERY).catch(() => false); @@ -60,14 +62,12 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem } } - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider(props, originalUrl, hostListProviderService, (hostListProviderService)); - } - return new RdsHostListProvider(props, originalUrl, hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + const topologyUtils = new AuroraTopologyUtils(this, servicesContainer.hostListProviderService.getHostInfoBuilder()); + return new RdsHostListProvider(props, originalUrl, topologyUtils, servicesContainer); } - async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + async queryForTopology(targetClient: ClientWrapper): Promise { try { let writerHostId: string = await this.executeTopologyRelatedQuery( targetClient, @@ -80,7 +80,7 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem const res = await targetClient.query(RdsMultiAZClusterPgDatabaseDialect.TOPOLOGY_QUERY); const rows: any[] = res.rows; - return this.processTopologyQueryResults(hostListProvider, writerHostId, rows); + return this.processTopologyQueryResults(writerHostId, rows); } catch (error: any) { throw new AwsWrapperError(Messages.get("RdsMultiAZPgDatabaseDialect.invalidQuery", error.message)); } @@ -95,8 +95,8 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem return ""; } - private async processTopologyQueryResults(hostListProvider: HostListProvider, writerHostId: string, rows: any[]): Promise { - const hostMap: Map = new Map(); + private async processTopologyQueryResults(writerHostId: string, rows: any[]): Promise { + const hosts: TopologyQueryResult[] = []; rows.forEach((row) => { // According to the topology query the result set // should contain 3 columns: endpoint, id, and port @@ -104,32 +104,16 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem const id: string = row["id"]; const port: number = row["port"]; const isWriter: boolean = id === writerHostId; - - const host: HostInfo = hostListProvider.createHost(endpoint.substring(0, endpoint.indexOf(".")), isWriter, 0, Date.now(), port); - host.hostId = id; - host.addAlias(endpoint); - hostMap.set(host.host, host); + const host: TopologyQueryResult = new TopologyQueryResult({ + host: endpoint.substring(0, endpoint.indexOf(".")), + isWriter: isWriter, + weight: 0, + lastUpdateTime: Date.now(), + port: port, + id: id + }); + hosts.push(host); }); - - let hosts: HostInfo[] = []; - const writers: HostInfo[] = []; - - for (const [key, info] of hostMap.entries()) { - if (info.role !== HostRole.WRITER) { - hosts.push(info); - } else { - writers.push(info); - } - } - - const writerCount: number = writers.length; - if (writerCount === 0) { - logger.error(Messages.get("RdsMultiAzDatabaseDialect.invalidTopology", this.dialectName)); - hosts = []; - } else { - hosts.push(writers[0]); - } - return hosts; } @@ -143,7 +127,7 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem : HostRole.READER; } - async getWriterId(targetClient: ClientWrapper): Promise { + async getWriterId(targetClient: ClientWrapper): Promise { try { const writerHostId: string = await this.executeTopologyRelatedQuery( targetClient, @@ -158,6 +142,17 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem } } + async getInstanceId(targetClient: ClientWrapper): Promise<[string, string]> { + try { + const res = await targetClient.query(RdsMultiAZClusterPgDatabaseDialect.INSTANCE_ID_QUERY); + const instance_id = res.rows[0]["instance_id"]; + const instance_name = res.rows[0]["instance_name"]; + return [instance_id, instance_name]; + } catch (error: any) { + throw new AwsWrapperError(Messages.get("RdsMultiAZPgDatabaseDialect.invalidQuery", error.message)); + } + } + getErrorHandler(): ErrorHandler { return new MultiAzPgErrorHandler(); } diff --git a/tests/integration/container/tests/aurora_failover.test.ts b/tests/integration/container/tests/aurora_failover.test.ts deleted file mode 100644 index f580bc1f3..000000000 --- a/tests/integration/container/tests/aurora_failover.test.ts +++ /dev/null @@ -1,315 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { TestEnvironment } from "./utils/test_environment"; -import { DriverHelper } from "./utils/driver_helper"; -import { AuroraTestUtility } from "./utils/aurora_test_utility"; -import { FailoverSuccessError, PluginManager, TransactionIsolationLevel, TransactionResolutionUnknownError } from "../../../../index"; -import { DatabaseEngine } from "./utils/database_engine"; -import { QueryResult } from "pg"; -import { ProxyHelper } from "./utils/proxy_helper"; -import { logger } from "../../../../common/logutils"; -import { features, instanceCount } from "./config"; -import { TestEnvironmentFeatures } from "./utils/test_environment_features"; -import { RdsUtils } from "../../../../common/lib/utils/rds_utils"; - -const itIf = - features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && - !features.includes(TestEnvironmentFeatures.PERFORMANCE) && - !features.includes(TestEnvironmentFeatures.RUN_AUTOSCALING_TESTS_ONLY) && - instanceCount >= 2 - ? it - : it.skip; -const itIfTwoInstance = instanceCount == 2 ? itIf : it.skip; -const itIfThreeInstanceAuroraCluster = instanceCount == 3 && !features.includes(TestEnvironmentFeatures.RDS_MULTI_AZ_SUPPORTED) ? it : it.skip; - -let env: TestEnvironment; -let driver; -let client: any; -let secondaryClient: any; -let initClientFunc: (props: any) => any; - -let auroraTestUtility: AuroraTestUtility; - -async function initDefaultConfig(host: string, port: number, connectToProxy: boolean): Promise { - let config: any = { - user: env.databaseInfo.username, - host: host, - database: env.databaseInfo.defaultDbName, - password: env.databaseInfo.password, - port: port, - plugins: "failover", - failoverTimeoutMs: 250000, - enableTelemetry: true, - telemetryTracesBackend: "OTLP", - telemetryMetricsBackend: "OTLP" - }; - if (connectToProxy) { - config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; - } - config = DriverHelper.addDriverSpecificConfiguration(config, env.engine); - return config; -} - -async function initConfigWithEFM2(host: string, port: number, connectToProxy: boolean): Promise { - const config: any = await initDefaultConfig(host, port, connectToProxy); - config["plugins"] = "failover,efm2"; - config["failoverTimeoutMs"] = 20000; - config["failureDetectionCount"] = 2; - config["failureDetectionInterval"] = 1000; - config["failureDetectionTime"] = 2000; - config["connectTimeout"] = 10000; - config["wrapperQueryTimeout"] = 20000; - config["monitoring_wrapperQueryTimeout"] = 3000; - config["monitoring_wrapperConnectTimeout"] = 3000; - return config; -} - -describe("aurora failover", () => { - beforeEach(async () => { - logger.info(`Test started: ${expect.getState().currentTestName}`); - env = await TestEnvironment.getCurrent(); - - auroraTestUtility = new AuroraTestUtility(env.region); - driver = DriverHelper.getDriverForDatabaseEngine(env.engine); - initClientFunc = DriverHelper.getClient(driver); - await ProxyHelper.enableAllConnectivity(); - await TestEnvironment.verifyClusterStatus(); - - client = null; - secondaryClient = null; - }, 1320000); - - afterEach(async () => { - if (client !== null) { - try { - await client.end(); - } catch (error) { - // pass - } - } - - if (secondaryClient !== null) { - try { - await secondaryClient.end(); - } catch (error) { - // pass - } - } - await PluginManager.releaseResources(); - logger.info(`Test finished: ${expect.getState().currentTestName}`); - }, 1320000); - - itIfThreeInstanceAuroraCluster( - "writer failover efm", - async () => { - // Connect to writer instance. - const writerConfig = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - writerConfig["failoverMode"] = "reader-or-writer"; - - client = initClientFunc(writerConfig); - await client.connect(); - - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - const instances = env.databaseInfo.instances; - const readerInstance = instances[1].instanceId; - await ProxyHelper.disableAllConnectivity(env.engine); - - try { - await ProxyHelper.enableConnectivity(initialWriterId); - - // Sleep query activates monitoring connection after monitoring_wrapperQueryTimeout time is reached. - await auroraTestUtility.queryInstanceIdWithSleep(client); - - await ProxyHelper.enableConnectivity(readerInstance); - await ProxyHelper.disableConnectivity(env.engine, initialWriterId); - } catch (error) { - fail("The disable connectivity task was unexpectedly interrupted."); - } - // Failure occurs on connection invocation. - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(false); - expect(currentConnectionId).not.toBe(initialWriterId); - }, - 1320000 - ); - - itIf( - "fails from writer to new writer on connection invocation", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Crash instance 1 and nominate a new writer - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - // Assert that we are connected to the new writer after failover happens - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).not.toBe(initialWriterId); - }, - 1320000 - ); - - itIf( - "writer fails within transaction", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - await DriverHelper.executeQuery(env.engine, client, "CREATE TABLE test3_3 (id int not null primary key, test3_3_field varchar(255) not null)"); - - await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION"); // start transaction - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (1, 'test field string 1')"); - - // Crash instance 1 and nominate a new writer - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (2, 'test field string 2')"); - }).rejects.toThrow(TransactionResolutionUnknownError); - - // Attempt to query the instance id. - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - - // Assert that we are connected to the new writer after failover happens. - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - - const nextClusterWriterId = await auroraTestUtility.getClusterWriterInstanceId(); - expect(currentConnectionId).toBe(nextClusterWriterId); - expect(initialWriterId).not.toBe(nextClusterWriterId); - - // Assert that NO row has been inserted to the table. - const result = await DriverHelper.executeQuery(env.engine, client, "SELECT count(*) from test3_3"); - if (env.engine === DatabaseEngine.PG) { - expect((result as QueryResult).rows[0]["count"]).toBe("0"); - } else if (env.engine === DatabaseEngine.MYSQL) { - expect(JSON.parse(JSON.stringify(result))[0][0]["count(*)"]).toBe(0); - } - - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - }, - 2000000 - ); - - itIf( - "fails from writer and transfers session state", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toBe(true); - - await client.setReadOnly(true); - await client.setTransactionIsolation(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); - - if (driver === DatabaseEngine.PG) { - await client.setSchema(env.databaseInfo.defaultDbName); - } else if (driver === DatabaseEngine.MYSQL) { - await client.setAutoCommit(false); - await client.setCatalog(env.databaseInfo.defaultDbName); - } - - // Failover cluster and nominate a new writer - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - // Assert that we are connected to the new writer after failover happens - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).not.toBe(initialWriterId); - expect(client.isReadOnly()).toBe(true); - expect(client.getTransactionIsolation()).toBe(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); - if (driver === DatabaseEngine.PG) { - expect(client.getSchema()).toBe(env.databaseInfo.defaultDbName); - } else if (driver === DatabaseEngine.MYSQL) { - expect(client.getAutoCommit()).toBe(false); - expect(client.getCatalog()).toBe(env.databaseInfo.defaultDbName); - } - }, - 1320000 - ); - - itIfTwoInstance( - "fails from reader to writer", - async () => { - // Connect to writer instance - const writerConfig = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - client = initClientFunc(writerConfig); - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Get a reader instance - let readerInstanceHost; - for (const host of env.proxyDatabaseInfo.instances) { - if (host.instanceId && host.instanceId !== initialWriterId) { - readerInstanceHost = host.host; - } - } - if (!readerInstanceHost) { - throw new Error("Could not find a reader instance"); - } - const readerConfig = await initDefaultConfig(readerInstanceHost, env.proxyDatabaseInfo.instanceEndpointPort, true); - - secondaryClient = initClientFunc(readerConfig); - await secondaryClient.connect(); - - // Crash the reader instance - const rdsUtils = new RdsUtils(); - const readerInstanceId = rdsUtils.getRdsInstanceId(readerInstanceHost); - if (readerInstanceId) { - await ProxyHelper.disableConnectivity(env.engine, readerInstanceId); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(secondaryClient); - }).rejects.toThrow(FailoverSuccessError); - - await ProxyHelper.enableConnectivity(readerInstanceId); - - // Assert that we are currently connected to the writer instance - const currentConnectionId = await auroraTestUtility.queryInstanceId(secondaryClient); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).toBe(initialWriterId); - } - }, - 1320000 - ); -}); diff --git a/tests/integration/container/tests/aurora_failover2.test.ts b/tests/integration/container/tests/aurora_failover2.test.ts deleted file mode 100644 index 61ebf3621..000000000 --- a/tests/integration/container/tests/aurora_failover2.test.ts +++ /dev/null @@ -1,259 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { TestEnvironment } from "./utils/test_environment"; -import { DriverHelper } from "./utils/driver_helper"; -import { AuroraTestUtility } from "./utils/aurora_test_utility"; -import { FailoverSuccessError, PluginManager, TransactionIsolationLevel, TransactionResolutionUnknownError } from "../../../../index"; -import { DatabaseEngine } from "./utils/database_engine"; -import { QueryResult } from "pg"; -import { ProxyHelper } from "./utils/proxy_helper"; -import { logger } from "../../../../common/logutils"; -import { features, instanceCount } from "./config"; -import { TestEnvironmentFeatures } from "./utils/test_environment_features"; -import { RdsUtils } from "../../../../common/lib/utils/rds_utils"; - -const itIf = - features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && - !features.includes(TestEnvironmentFeatures.PERFORMANCE) && - !features.includes(TestEnvironmentFeatures.RUN_AUTOSCALING_TESTS_ONLY) && - instanceCount >= 2 - ? it - : it.skip; -const itIfTwoInstance = instanceCount == 2 ? itIf : it.skip; - -let env: TestEnvironment; -let driver; -let client: any; -let secondaryClient: any; -let initClientFunc: (props: any) => any; - -let auroraTestUtility: AuroraTestUtility; - -async function initDefaultConfig(host: string, port: number, connectToProxy: boolean): Promise { - let config: any = { - user: env.databaseInfo.username, - host: host, - database: env.databaseInfo.defaultDbName, - password: env.databaseInfo.password, - port: port, - plugins: "failover2", - failoverTimeoutMs: 250000, - enableTelemetry: true, - telemetryTracesBackend: "OTLP", - telemetryMetricsBackend: "OTLP" - }; - if (connectToProxy) { - config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; - } - config = DriverHelper.addDriverSpecificConfiguration(config, env.engine); - return config; -} - -describe("aurora failover2", () => { - beforeEach(async () => { - logger.info(`Test started: ${expect.getState().currentTestName}`); - env = await TestEnvironment.getCurrent(); - - auroraTestUtility = new AuroraTestUtility(env.region); - driver = DriverHelper.getDriverForDatabaseEngine(env.engine); - initClientFunc = DriverHelper.getClient(driver); - await ProxyHelper.enableAllConnectivity(); - await TestEnvironment.verifyClusterStatus(); - - client = null; - secondaryClient = null; - }, 1320000); - - afterEach(async () => { - if (client !== null) { - try { - await client.end(); - } catch (error) { - // pass - } - } - - if (secondaryClient !== null) { - try { - await secondaryClient.end(); - } catch (error) { - // pass - } - } - await PluginManager.releaseResources(); - logger.info(`Test finished: ${expect.getState().currentTestName}`); - }, 1320000); - - itIf( - "fails from writer to new writer on connection invocation", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Crash instance 1 and nominate a new writer. - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - // Assert that we are connected to the new writer after failover happens. - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).not.toBe(initialWriterId); - }, - 1320000 - ); - - itIf( - "writer fails within transaction", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - await DriverHelper.executeQuery(env.engine, client, "CREATE TABLE test3_3 (id int not null primary key, test3_3_field varchar(255) not null)"); - - await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION"); // start transaction - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (1, 'test field string 1')"); - - // Crash instance 1 and nominate a new writer. - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (2, 'test field string 2')"); - }).rejects.toThrow(TransactionResolutionUnknownError); - - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - // Assert that we are connected to the new writer after failover happens. - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - - const nextClusterWriterId = await auroraTestUtility.getClusterWriterInstanceId(); - expect(currentConnectionId).toBe(nextClusterWriterId); - expect(initialWriterId).not.toBe(nextClusterWriterId); - - // Assert that NO row has been inserted to the table. - const result = await DriverHelper.executeQuery(env.engine, client, "SELECT count(*) from test3_3"); - if (env.engine === DatabaseEngine.PG) { - expect((result as QueryResult).rows[0]["count"]).toBe("0"); - } else if (env.engine === DatabaseEngine.MYSQL) { - expect(JSON.parse(JSON.stringify(result))[0][0]["count(*)"]).toBe(0); - } - - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - }, - 2000000 - ); - - itIf( - "fails from writer and transfers session state", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toBe(true); - - await client.setReadOnly(true); - await client.setTransactionIsolation(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); - - if (driver === DatabaseEngine.PG) { - await client.setSchema(env.databaseInfo.defaultDbName); - } else if (driver === DatabaseEngine.MYSQL) { - await client.setAutoCommit(false); - await client.setCatalog(env.databaseInfo.defaultDbName); - } - - // Failover cluster and nominate a new writer. - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - // Assert that we are connected to the new writer after failover happens. - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).not.toBe(initialWriterId); - expect(client.isReadOnly()).toBe(true); - expect(client.getTransactionIsolation()).toBe(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); - if (driver === DatabaseEngine.PG) { - expect(client.getSchema()).toBe(env.databaseInfo.defaultDbName); - } else if (driver === DatabaseEngine.MYSQL) { - expect(client.getAutoCommit()).toBe(false); - expect(client.getCatalog()).toBe(env.databaseInfo.defaultDbName); - } - }, - 1320000 - ); - - itIfTwoInstance( - "fails from reader to writer", - async () => { - // Connect to writer instance. - const writerConfig = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - client = initClientFunc(writerConfig); - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Get a reader instance. - let readerInstanceHost; - for (const host of env.proxyDatabaseInfo.instances) { - if (host.instanceId && host.instanceId !== initialWriterId) { - readerInstanceHost = host.host; - } - } - if (!readerInstanceHost) { - throw new Error("Could not find a reader instance"); - } - const readerConfig = await initDefaultConfig(readerInstanceHost, env.proxyDatabaseInfo.instanceEndpointPort, true); - - secondaryClient = initClientFunc(readerConfig); - await secondaryClient.connect(); - - // Crash the reader instance. - const rdsUtils = new RdsUtils(); - const readerInstanceId = rdsUtils.getRdsInstanceId(readerInstanceHost); - if (readerInstanceId) { - await ProxyHelper.disableConnectivity(env.engine, readerInstanceId); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(secondaryClient); - }).rejects.toThrow(FailoverSuccessError); - - await ProxyHelper.enableConnectivity(readerInstanceId); - - // Assert that we are currently connected to the writer instance. - const currentConnectionId = await auroraTestUtility.queryInstanceId(secondaryClient); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).toBe(initialWriterId); - } - }, - 1320000 - ); -}); diff --git a/tests/integration/container/tests/config.ts b/tests/integration/container/tests/config.ts index b50208cfa..221b397dc 100644 --- a/tests/integration/container/tests/config.ts +++ b/tests/integration/container/tests/config.ts @@ -15,6 +15,7 @@ */ import { CustomConsole, LogMessage, LogType } from "@jest/console"; +import { TestEnvironment } from "./utils/test_environment"; function simpleFormatter(type: LogType, message: LogMessage): string { return message @@ -34,3 +35,7 @@ const testInfo = JSON.parse(infoJson); const request = testInfo.request; export const features = request.features; export const instanceCount = request.numOfInstances; + +afterAll(async () => { + await TestEnvironment.shutdownTelemetry(); +}); diff --git a/tests/integration/container/tests/connect_execute_time_plugin.test.ts b/tests/integration/container/tests/connect_execute_time_plugin.test.ts index 3ff336add..b8091d264 100644 --- a/tests/integration/container/tests/connect_execute_time_plugin.test.ts +++ b/tests/integration/container/tests/connect_execute_time_plugin.test.ts @@ -70,8 +70,7 @@ describe("aurora connect and execute time plugin", () => { await TestEnvironment.verifyAllInstancesHasRightState("available"); await TestEnvironment.verifyAllInstancesUp(); - RdsHostListProvider.clearAll(); - PluginServiceImpl.clearHostAvailabilityCache(); + await PluginManager.releaseResources(); }, 1320000); afterEach(async () => { diff --git a/tests/integration/container/tests/failover/aurora_failover.test.ts b/tests/integration/container/tests/failover/aurora_failover.test.ts new file mode 100644 index 000000000..7d91ee451 --- /dev/null +++ b/tests/integration/container/tests/failover/aurora_failover.test.ts @@ -0,0 +1,125 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TestEnvironment } from "../utils/test_environment"; +import { DriverHelper } from "../utils/driver_helper"; +import { AuroraTestUtility } from "../utils/aurora_test_utility"; +import { FailoverSuccessError, PluginManager } from "../../../../../index"; +import { ProxyHelper } from "../utils/proxy_helper"; +import { logger } from "../../../../../common/logutils"; +import { features, instanceCount } from "../config"; +import { TestEnvironmentFeatures } from "../utils/test_environment_features"; +import { createFailoverTests } from "./failover_tests"; + +const itIfThreeInstanceAuroraCluster = instanceCount == 3 && !features.includes(TestEnvironmentFeatures.RDS_MULTI_AZ_SUPPORTED) ? it : it.skip; + +describe("aurora failover", createFailoverTests({ plugins: "failover" })); + +describe("aurora failover - efm specific", () => { + let env: TestEnvironment; + let client: any; + let initClientFunc: (props: any) => any; + let auroraTestUtility: AuroraTestUtility; + + async function initConfigWithEFM2(host: string, port: number, connectToProxy: boolean): Promise { + let config: any = { + user: env.databaseInfo.username, + host: host, + database: env.databaseInfo.defaultDbName, + password: env.databaseInfo.password, + port: port, + plugins: "failover,efm2", + failoverTimeoutMs: 20000, + failureDetectionCount: 2, + failureDetectionInterval: 1000, + failureDetectionTime: 2000, + connectTimeout: 10000, + wrapperQueryTimeout: 20000, + monitoring_wrapperQueryTimeout: 3000, + monitoring_wrapperConnectTimeout: 3000, + enableTelemetry: true, + telemetryTracesBackend: "OTLP", + telemetryMetricsBackend: "OTLP" + }; + if (connectToProxy) { + config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; + } + config = DriverHelper.addDriverSpecificConfiguration(config, env.engine); + return config; + } + + beforeEach(async () => { + logger.info(`Test started: ${expect.getState().currentTestName}`); + env = await TestEnvironment.getCurrent(); + auroraTestUtility = new AuroraTestUtility(env.region); + const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + initClientFunc = DriverHelper.getClient(driver); + await ProxyHelper.enableAllConnectivity(); + await TestEnvironment.verifyClusterStatus(); + client = null; + }, 1320000); + + afterEach(async () => { + if (client !== null) { + try { + await client.end(); + } catch (error) { + // pass + } + } + await PluginManager.releaseResources(); + logger.info(`Test finished: ${expect.getState().currentTestName}`); + }, 1320000); + + itIfThreeInstanceAuroraCluster( + "writer failover efm", + async () => { + // Connect to writer instance + const writerConfig = await initConfigWithEFM2(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); + writerConfig["failoverMode"] = "reader-or-writer"; + + client = initClientFunc(writerConfig); + await client.connect(); + + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + const instances = env.databaseInfo.instances; + const readerInstance = instances[1].instanceId; + await ProxyHelper.disableAllConnectivity(env.engine); + + try { + await ProxyHelper.enableConnectivity(initialWriterId); + + // Sleep query activates monitoring connection after monitoring_wrapperQueryTimeout time is reached + await auroraTestUtility.queryInstanceIdWithSleep(client); + + await ProxyHelper.enableConnectivity(readerInstance); + await ProxyHelper.disableConnectivity(env.engine, initialWriterId); + } catch (error) { + fail("The disable connectivity task was unexpectedly interrupted."); + } + // Failure occurs on connection invocation + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(false); + expect(currentConnectionId).not.toBe(initialWriterId); + }, + 1320000 + ); +}); diff --git a/tests/integration/container/tests/failover/aurora_failover2.test.ts b/tests/integration/container/tests/failover/aurora_failover2.test.ts new file mode 100644 index 000000000..aab1564e0 --- /dev/null +++ b/tests/integration/container/tests/failover/aurora_failover2.test.ts @@ -0,0 +1,19 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { createFailoverTests } from "./failover_tests"; + +describe("aurora failover2", createFailoverTests({ plugins: "failover2" })); diff --git a/tests/integration/container/tests/failover/failover_tests.ts b/tests/integration/container/tests/failover/failover_tests.ts new file mode 100644 index 000000000..ce430464a --- /dev/null +++ b/tests/integration/container/tests/failover/failover_tests.ts @@ -0,0 +1,271 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TestEnvironment } from "../utils/test_environment"; +import { DriverHelper } from "../utils/driver_helper"; +import { AuroraTestUtility } from "../utils/aurora_test_utility"; +import { FailoverSuccessError, PluginManager, TransactionIsolationLevel, TransactionResolutionUnknownError } from "../../../../../index"; +import { DatabaseEngine } from "../utils/database_engine"; +import { QueryResult } from "pg"; +import { ProxyHelper } from "../utils/proxy_helper"; +import { logger } from "../../../../../common/logutils"; +import { features, instanceCount } from "../config"; +import { TestEnvironmentFeatures } from "../utils/test_environment_features"; +import { RdsUtils } from "../../../../../common/lib/utils/rds_utils"; + +export interface FailoverTestOptions { + plugins: string; + getExtraConfig?: () => Record; +} + +export function createFailoverTests(options: FailoverTestOptions) { + const itIf = + features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && + !features.includes(TestEnvironmentFeatures.PERFORMANCE) && + !features.includes(TestEnvironmentFeatures.RUN_AUTOSCALING_TESTS_ONLY) && + instanceCount >= 2 + ? it + : it.skip; + const itIfTwoInstance = instanceCount == 2 ? itIf : it.skip; + + return () => { + let env: TestEnvironment; + let driver: any; + let client: any; + let secondaryClient: any; + let initClientFunc: (props: any) => any; + let auroraTestUtility: AuroraTestUtility; + + async function initDefaultConfig(host: string, port: number, connectToProxy: boolean): Promise { + let config: any = { + user: env.databaseInfo.username, + host: host, + database: env.databaseInfo.defaultDbName, + password: env.databaseInfo.password, + port: port, + plugins: options.plugins, + failoverTimeoutMs: 250000, + enableTelemetry: true, + telemetryTracesBackend: "OTLP", + telemetryMetricsBackend: "OTLP", + ...options.getExtraConfig?.() + }; + if (connectToProxy) { + config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; + } + config = DriverHelper.addDriverSpecificConfiguration(config, env.engine); + return config; + } + + beforeEach(async () => { + logger.info(`Test started: ${expect.getState().currentTestName}`); + env = await TestEnvironment.getCurrent(); + auroraTestUtility = new AuroraTestUtility(env.region); + driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + initClientFunc = DriverHelper.getClient(driver); + await ProxyHelper.enableAllConnectivity(); + await TestEnvironment.verifyClusterStatus(); + client = null; + secondaryClient = null; + }, 1320000); + + afterEach(async () => { + await ProxyHelper.enableAllConnectivity(); + + if (client !== null) { + try { + await client.end(); + } catch (error) { + // pass + } + } + if (secondaryClient !== null) { + try { + await secondaryClient.end(); + } catch (error) { + // pass + } + } + await PluginManager.releaseResources(); + logger.info(`Test finished: ${expect.getState().currentTestName}`); + }, 1320000); + + itIf( + "fails from writer to new writer on connection invocation", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); + + await client.connect(); + + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + // Crash instance 1 and nominate a new writer + await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + + // Assert that we are connected to the new writer after failover happens + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + expect(currentConnectionId).not.toBe(initialWriterId); + }, + 1320000 + ); + + itIf( + "writer fails within transaction", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); + + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); + await DriverHelper.executeQuery( + env.engine, + client, + "CREATE TABLE test3_3 (id int not null primary key, test3_3_field varchar(255) not null)" + ); + + await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION"); + await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (1, 'test field string 1')"); + + // Crash instance 1 and nominate a new writer + await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); + + await expect(async () => { + await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (2, 'test field string 2')"); + }).rejects.toThrow(TransactionResolutionUnknownError); + + // Attempt to query the instance id + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + + // Assert that we are connected to the new writer after failover happens + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + + const nextClusterWriterId = await auroraTestUtility.getClusterWriterInstanceId(); + expect(currentConnectionId).toBe(nextClusterWriterId); + expect(initialWriterId).not.toBe(nextClusterWriterId); + + // Assert that NO row has been inserted to the table + const result = await DriverHelper.executeQuery(env.engine, client, "SELECT count(*) from test3_3"); + if (env.engine === DatabaseEngine.PG) { + expect((result as QueryResult).rows[0]["count"]).toBe("0"); + } else if (env.engine === DatabaseEngine.MYSQL) { + expect(JSON.parse(JSON.stringify(result))[0][0]["count(*)"]).toBe(0); + } + + await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); + }, + 2000000 + ); + + itIf( + "fails from writer and transfers session state", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); + + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toBe(true); + + await client.setReadOnly(true); + await client.setTransactionIsolation(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); + + if (driver === DatabaseEngine.PG) { + await client.setSchema(env.databaseInfo.defaultDbName); + } else if (driver === DatabaseEngine.MYSQL) { + await client.setAutoCommit(false); + await client.setCatalog(env.databaseInfo.defaultDbName); + } + + // Failover cluster and nominate a new writer + await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + + // Assert that we are connected to the new writer after failover happens + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + expect(currentConnectionId).not.toBe(initialWriterId); + expect(client.isReadOnly()).toBe(true); + expect(client.getTransactionIsolation()).toBe(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); + if (driver === DatabaseEngine.PG) { + expect(client.getSchema()).toBe(env.databaseInfo.defaultDbName); + } else if (driver === DatabaseEngine.MYSQL) { + expect(client.getAutoCommit()).toBe(false); + expect(client.getCatalog()).toBe(env.databaseInfo.defaultDbName); + } + }, + 1320000 + ); + + itIfTwoInstance( + "fails from reader to writer", + async () => { + // Connect to writer instance + const writerConfig = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); + client = initClientFunc(writerConfig); + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + // Get a reader instance + let readerInstanceHost; + for (const host of env.proxyDatabaseInfo.instances) { + if (host.instanceId && host.instanceId !== initialWriterId) { + readerInstanceHost = host.host; + } + } + if (!readerInstanceHost) { + throw new Error("Could not find a reader instance"); + } + const readerConfig = await initDefaultConfig(readerInstanceHost, env.proxyDatabaseInfo.instanceEndpointPort, true); + + secondaryClient = initClientFunc(readerConfig); + await secondaryClient.connect(); + + // Crash the reader instance + const rdsUtils = new RdsUtils(); + const readerInstanceId = rdsUtils.getRdsInstanceId(readerInstanceHost); + if (readerInstanceId) { + await ProxyHelper.disableConnectivity(env.engine, readerInstanceId); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(secondaryClient); + }).rejects.toThrow(FailoverSuccessError); + + await ProxyHelper.enableConnectivity(readerInstanceId); + + // Assert that we are currently connected to the writer instance + const currentConnectionId = await auroraTestUtility.queryInstanceId(secondaryClient); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + expect(currentConnectionId).toBe(initialWriterId); + } + }, + 1320000 + ); + }; +} diff --git a/tests/integration/container/tests/failover/gdb_failover.test.ts b/tests/integration/container/tests/failover/gdb_failover.test.ts new file mode 100644 index 000000000..2d5348a39 --- /dev/null +++ b/tests/integration/container/tests/failover/gdb_failover.test.ts @@ -0,0 +1,185 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TestEnvironment } from "../utils/test_environment"; +import { DriverHelper } from "../utils/driver_helper"; +import { AuroraTestUtility } from "../utils/aurora_test_utility"; +import { FailoverSuccessError, PluginManager } from "../../../../../index"; +import { ProxyHelper } from "../utils/proxy_helper"; +import { logger } from "../../../../../common/logutils"; +import { features, instanceCount } from "../config"; +import { TestEnvironmentFeatures } from "../utils/test_environment_features"; +import { createFailoverTests } from "./failover_tests"; + +const itIf = + features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && + !features.includes(TestEnvironmentFeatures.PERFORMANCE) && + !features.includes(TestEnvironmentFeatures.RUN_AUTOSCALING_TESTS_ONLY) && + instanceCount >= 2 + ? it + : it.skip; +const itIfNetworkOutages = features.includes(TestEnvironmentFeatures.NETWORK_OUTAGES_ENABLED) && instanceCount >= 2 ? itIf : it.skip; + +let env: TestEnvironment; +let driver: any; +let client: any; +let initClientFunc: (props: any) => any; + +let auroraTestUtility: AuroraTestUtility; + +async function initDefaultConfig(host: string, port: number, connectToProxy: boolean): Promise { + let config: any = { + user: env.databaseInfo.username, + host: host, + database: env.databaseInfo.defaultDbName, + password: env.databaseInfo.password, + port: port, + plugins: "gdbFailover", + failoverTimeoutMs: 250000, + activeHomeFailoverMode: "strict-writer", + inactiveHomeFailoverMode: "strict-writer", + enableTelemetry: true, + telemetryTracesBackend: "OTLP", + telemetryMetricsBackend: "OTLP" + }; + if (connectToProxy) { + config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; + } + config = DriverHelper.addDriverSpecificConfiguration(config, env.engine); + return config; +} + +describe("gdb failover", () => { + // Inherit shared failover tests with GDB-specific configuration + // This mirrors the Java pattern where GdbFailoverTest extends FailoverTest + describe( + "failover tests", + createFailoverTests({ + plugins: "gdbFailover", + getExtraConfig: () => ({ + // These settings mimic failover/failover2 plugin logic when connecting to non-GDB Aurora or RDS DB clusters. + activeHomeFailoverMode: "strict-writer", + inactiveHomeFailoverMode: "strict-writer" + }) + }) + ); + + // GDB-specific tests (overrides from Java GdbFailoverTest) + describe("gdb-specific tests", () => { + beforeEach(async () => { + logger.info(`Test started: ${expect.getState().currentTestName}`); + env = await TestEnvironment.getCurrent(); + + auroraTestUtility = new AuroraTestUtility(env.region); + driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + initClientFunc = DriverHelper.getClient(driver); + await ProxyHelper.enableAllConnectivity(); + await TestEnvironment.verifyClusterStatus(); + + client = null; + await PluginManager.releaseResources(); + }, 1320000); + + afterEach(async () => { + await ProxyHelper.enableAllConnectivity(); + + if (client !== null) { + try { + await client.end(); + } catch (error) { + // pass + } + } + await PluginManager.releaseResources(); + logger.info(`Test finished: ${expect.getState().currentTestName}`); + }, 1320000); + + itIfNetworkOutages( + "reader failover with home-reader-or-writer mode", + async () => { + const initialWriterId = env.proxyDatabaseInfo.writerInstanceId; + const initialWriterHost = env.proxyDatabaseInfo.writerInstanceEndpoint; + const initialWriterPort = env.proxyDatabaseInfo.instanceEndpointPort; + + const config = await initDefaultConfig(initialWriterHost, initialWriterPort, true); + config["activeHomeFailoverMode"] = "home-reader-or-writer"; + config["inactiveHomeFailoverMode"] = "home-reader-or-writer"; + + client = initClientFunc(config); + await client.connect(); + + await ProxyHelper.disableConnectivity(env.engine, initialWriterId!); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + }, + 1320000 + ); + + itIfNetworkOutages( + "reader failover with strict-home-reader mode", + async () => { + const initialWriterId = env.proxyDatabaseInfo.writerInstanceId; + const initialWriterHost = env.proxyDatabaseInfo.writerInstanceEndpoint; + const initialWriterPort = env.proxyDatabaseInfo.instanceEndpointPort; + + const config = await initDefaultConfig(initialWriterHost, initialWriterPort, true); + config["activeHomeFailoverMode"] = "strict-home-reader"; + config["inactiveHomeFailoverMode"] = "strict-home-reader"; + + client = initClientFunc(config); + await client.connect(); + + await ProxyHelper.disableConnectivity(env.engine, initialWriterId!); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(false); + }, + 1320000 + ); + + itIfNetworkOutages( + "writer reelected with home-reader-or-writer mode", + async () => { + const initialWriterId = env.proxyDatabaseInfo.writerInstanceId; + const initialWriterHost = env.proxyDatabaseInfo.writerInstanceEndpoint; + const initialWriterPort = env.proxyDatabaseInfo.instanceEndpointPort; + + const config = await initDefaultConfig(initialWriterHost, initialWriterPort, true); + config["activeHomeFailoverMode"] = "home-reader-or-writer"; + config["inactiveHomeFailoverMode"] = "home-reader-or-writer"; + config["wrapperQueryTimeout"] = 2000; + + client = initClientFunc(config); + await client.connect(); + + // Failover usually changes the writer instance, but we want to test re-election of the same writer, so we will + // simulate this by temporarily disabling connectivity to the writer. + await auroraTestUtility.simulateTemporaryFailure(initialWriterId!); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + }, + 1320000 + ); + }); +}); diff --git a/tests/integration/container/tests/fastest_response_strategy.test.ts b/tests/integration/container/tests/fastest_response_strategy.test.ts index ff44b9222..9e7cdb98c 100644 --- a/tests/integration/container/tests/fastest_response_strategy.test.ts +++ b/tests/integration/container/tests/fastest_response_strategy.test.ts @@ -89,8 +89,7 @@ describe("aurora fastest response strategy", () => { await TestEnvironment.verifyAllInstancesHasRightState("available"); await TestEnvironment.verifyAllInstancesUp(); - RdsHostListProvider.clearAll(); - PluginServiceImpl.clearHostAvailabilityCache(); + await PluginManager.releaseResources(); }, 1320000); afterEach(async () => { diff --git a/tests/integration/container/tests/iam_authentication.test.ts b/tests/integration/container/tests/iam_authentication.test.ts index 5b516c527..a38766630 100644 --- a/tests/integration/container/tests/iam_authentication.test.ts +++ b/tests/integration/container/tests/iam_authentication.test.ts @@ -23,7 +23,6 @@ import { readFileSync } from "fs"; import { logger } from "../../../../common/logutils"; import { TestEnvironmentFeatures } from "./utils/test_environment_features"; import { features } from "./config"; -import { jest } from "@jest/globals"; const itIf = !features.includes(TestEnvironmentFeatures.PERFORMANCE) && @@ -72,9 +71,6 @@ async function validateConnection() { describe("iam authentication", () => { beforeEach(async () => { logger.info(`Test started: ${expect.getState().currentTestName}`); - jest.useFakeTimers({ - doNotFake: ["nextTick"] - }); client = null; env = await TestEnvironment.getCurrent(); driver = DriverHelper.getDriverForDatabaseEngine(env.engine); @@ -93,11 +89,6 @@ describe("iam authentication", () => { logger.info(`Test finished: ${expect.getState().currentTestName}`); }, 1320000); - afterAll(async () => { - jest.runOnlyPendingTimers(); - jest.useRealTimers(); - }); - itIf( "iam wrong database username", async () => { diff --git a/tests/integration/container/tests/initial_connection_strategy.test.ts b/tests/integration/container/tests/initial_connection_strategy.test.ts index c540c72ab..98ef8551f 100644 --- a/tests/integration/container/tests/initial_connection_strategy.test.ts +++ b/tests/integration/container/tests/initial_connection_strategy.test.ts @@ -73,8 +73,7 @@ describe("aurora initial connection strategy", () => { await TestEnvironment.verifyAllInstancesHasRightState("available"); await TestEnvironment.verifyAllInstancesUp(); - RdsHostListProvider.clearAll(); - PluginServiceImpl.clearHostAvailabilityCache(); + await PluginManager.releaseResources(); numReaders = env.databaseInfo.instances.length - 1; }, 1320000); diff --git a/tests/integration/container/tests/parameterized_queries.test.ts b/tests/integration/container/tests/parameterized_queries.test.ts index a8ceac145..0be0f2a81 100644 --- a/tests/integration/container/tests/parameterized_queries.test.ts +++ b/tests/integration/container/tests/parameterized_queries.test.ts @@ -141,7 +141,7 @@ beforeEach(async () => { logger.info(`Test started: ${expect.getState().currentTestName}`); await TestEnvironment.verifyClusterStatus(); client = null; -}, 60000); +}, 1320000); afterEach(async () => { if (client !== null) { diff --git a/tests/integration/container/tests/performance.test.ts b/tests/integration/container/tests/performance.test.ts index f15d1268e..43d10ac1a 100644 --- a/tests/integration/container/tests/performance.test.ts +++ b/tests/integration/container/tests/performance.test.ts @@ -25,7 +25,7 @@ import { WrapperProperties, PluginManager } from "../../../../index"; import { features } from "./config"; import { PerfStat } from "./utils/perf_stat"; import { PerfTestUtility } from "./utils/perf_util"; -import { MonitorServiceImpl } from "../../../../common/lib/plugins/efm/monitor_service"; +import { HostMonitorServiceImpl as MonitorServiceImpl } from "../../../../common/lib/plugins/efm/base/host_monitor_service"; const itIf = features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && diff --git a/tests/integration/container/tests/pg_pool.test.ts b/tests/integration/container/tests/pg_pool.test.ts index 4b09f224f..2b4744d2d 100644 --- a/tests/integration/container/tests/pg_pool.test.ts +++ b/tests/integration/container/tests/pg_pool.test.ts @@ -156,6 +156,12 @@ describe("pg pool integration tests", () => { itIfPGTwoInstances("failover writer during multi-statement transaction", async () => { client = await factory(); + try { + await client.query("DROP TABLE IF EXISTS test_table"); + } catch { + // Ignore + } + const initialWriterId = await auroraTestUtility.queryInstanceId(client); expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); diff --git a/tests/integration/container/tests/read_write_splitting.test.ts b/tests/integration/container/tests/read_write_splitting.test.ts index f1355fe95..f3af23785 100644 --- a/tests/integration/container/tests/read_write_splitting.test.ts +++ b/tests/integration/container/tests/read_write_splitting.test.ts @@ -34,8 +34,6 @@ import { ProxyHelper } from "./utils/proxy_helper"; import { logger } from "../../../../common/logutils"; import { TestEnvironmentFeatures } from "./utils/test_environment_features"; import { features, instanceCount } from "./config"; -import { RdsHostListProvider } from "../../../../common/lib/host_list_provider/rds_host_list_provider"; -import { PluginServiceImpl } from "../../../../common/lib/plugin_service"; const itIf = !features.includes(TestEnvironmentFeatures.PERFORMANCE) && @@ -64,6 +62,8 @@ async function initConfig(host: string, port: number, connectToProxy: boolean, p port: port, plugins: plugins, enableTelemetry: true, + wrapperQueryTimeout: 10000, + wrapperConnectTimeout: 3000, telemetryTracesBackend: "OTLP", telemetryMetricsBackend: "OTLP" }; @@ -107,11 +107,11 @@ describe("aurora read write splitting", () => { await TestEnvironment.verifyAllInstancesHasRightState("available"); await TestEnvironment.verifyAllInstancesUp(); - RdsHostListProvider.clearAll(); - PluginServiceImpl.clearHostAvailabilityCache(); + await PluginManager.releaseResources(); }, 1320000); afterEach(async () => { + await ProxyHelper.enableAllConnectivity(); if (client !== null) { try { await client.end(); @@ -608,9 +608,9 @@ describe("aurora read write splitting", () => { connectionsSet.add(client); } } finally { - for (const connection of connectionsSet) { + connectionsSet.forEach(async (connection) => { await connection.end(); - } + }); } }, 1000000 @@ -662,9 +662,9 @@ describe("aurora read write splitting", () => { connectionsSet.add(client); } } finally { - for (const connection of connectionsSet) { + connectionsSet.forEach(async (connection) => { await connection.end(); - } + }); } }, 1000000 diff --git a/tests/integration/container/tests/session_state.test.ts b/tests/integration/container/tests/session_state.test.ts index 5f06d420a..015e8de9a 100644 --- a/tests/integration/container/tests/session_state.test.ts +++ b/tests/integration/container/tests/session_state.test.ts @@ -119,7 +119,7 @@ describe("session state", () => { expect(autoCommit[0][0].autocommit).toEqual(1); expect(transactionIsolation[0][0].level).toEqual("REPEATABLE-READ"); - await client.getPluginService().setCurrentClient(newClient.targetClient); + await client.pluginService.setCurrentClient(newClient.targetClient); expect(client.targetClient).not.toEqual(targetClient); expect(client.targetClient).toEqual(newTargetClient); @@ -154,7 +154,7 @@ describe("session state", () => { expect(schema.rows[0]["search_path"]).not.toEqual("testSessionState"); expect(transactionIsolation.rows[0]["transaction_isolation"]).toEqual("read committed"); - await client.getPluginService().setCurrentClient(newClient.targetClient); + await client.pluginService.setCurrentClient(newClient.targetClient); expect(client.targetClient).not.toEqual(targetClient); expect(client.targetClient).toEqual(newTargetClient); diff --git a/tests/integration/container/tests/utils/aurora_test_utility.ts b/tests/integration/container/tests/utils/aurora_test_utility.ts index b0e0bad85..bdfb9c71e 100644 --- a/tests/integration/container/tests/utils/aurora_test_utility.ts +++ b/tests/integration/container/tests/utils/aurora_test_utility.ts @@ -42,6 +42,7 @@ import { TestInstanceInfo } from "./test_instance_info"; import { TestEnvironmentInfo } from "./test_environment_info"; import { DatabaseEngine } from "./database_engine"; import { DatabaseEngineDeployment } from "./database_engine_deployment"; +import { ProxyHelper } from "./proxy_helper"; const instanceClass: string = "db.r5.large"; @@ -492,4 +493,38 @@ export class AuroraTestUtility { logger.debug("switchoverBlueGreenDeployment request is sent."); } } + + async simulateTemporaryFailure(instanceName: string, delayMs: number = 0, failureDurationMs: number = 5000): Promise { + const env = await TestEnvironment.getCurrent(); + const deployment = env.deployment; + const clusterEndpoint = env.proxyDatabaseInfo.clusterEndpoint; + const clusterReadOnlyEndpoint = env.proxyDatabaseInfo.clusterReadOnlyEndpoint; + + (async () => { + try { + if (delayMs > 0) { + await sleep(delayMs); + } + + await ProxyHelper.disableConnectivity(env.engine, instanceName); + + if (deployment === DatabaseEngineDeployment.RDS_MULTI_AZ_CLUSTER) { + await ProxyHelper.disableConnectivity(env.engine, clusterEndpoint); + await ProxyHelper.disableConnectivity(env.engine, clusterReadOnlyEndpoint); + } + + await sleep(failureDurationMs); + + await ProxyHelper.enableConnectivity(instanceName); + if (deployment === DatabaseEngineDeployment.RDS_MULTI_AZ_CLUSTER) { + await ProxyHelper.enableConnectivity(clusterEndpoint); + await ProxyHelper.enableConnectivity(clusterReadOnlyEndpoint); + } + } catch (e: any) { + logger.error(`Error during simulateTemporaryFailure: ${e.message}`); + } + })(); + + await sleep(500); + } } diff --git a/tests/integration/container/tests/utils/test_environment.ts b/tests/integration/container/tests/utils/test_environment.ts index cc2d52e96..a03caa1ca 100644 --- a/tests/integration/container/tests/utils/test_environment.ts +++ b/tests/integration/container/tests/utils/test_environment.ts @@ -37,12 +37,14 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"; import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-grpc"; import { logger } from "../../../../../common/logutils"; +import { RdsUtils } from "../../../../../common/lib/utils/rds_utils"; import pkgPg from "pg"; import { ConnectionOptions, createConnection } from "mysql2/promise"; import { readFileSync } from "fs"; export class TestEnvironment { private static env?: TestEnvironment; + private static sdk?: NodeSDK; private readonly _info: TestEnvironmentInfo; private proxies?: { [s: string]: ProxyInfo }; @@ -238,6 +240,14 @@ export class TestEnvironment { await TestEnvironment.initProxies(env); } + // Helps to eliminate problem with proxied endpoints. + RdsUtils.setPrepareHostFunc((host: string) => { + if (host.endsWith(".proxied")) { + return host.substring(0, host.length - ".proxied".length); + } + return host; + }); + const contextManager = new AsyncHooksContextManager(); contextManager.enable(); context.setGlobalContextManager(contextManager); @@ -272,6 +282,7 @@ export class TestEnvironment { // this enables the API to record telemetry sdk.start(); + TestEnvironment.sdk = sdk; // gracefully shut down the SDK on process exit process.on("SIGTERM", () => { sdk @@ -284,6 +295,17 @@ export class TestEnvironment { return env; } + static async shutdownTelemetry(): Promise { + if (TestEnvironment.sdk) { + try { + await TestEnvironment.sdk.shutdown(); + } catch (error) { + // ignore + } + TestEnvironment.sdk = undefined; + } + } + static async initProxies(environment: TestEnvironment) { if (environment.features.includes(TestEnvironmentFeatures.NETWORK_OUTAGES_ENABLED)) { environment.proxies = {}; diff --git a/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java b/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java index f4c8fb2e5..bfe94c656 100644 --- a/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java +++ b/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java @@ -13,6 +13,7 @@ import software.amazon.awssdk.services.rds.model.DBCluster; import software.amazon.awssdk.services.rds.model.DBInstance; +import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.net.UnknownHostException; @@ -1073,6 +1074,9 @@ private static void createTestContainer(TestEnvironmentConfig env) { .withEnv("AWS_SESSION_TOKEN", env.awsSessionToken); } + // Ensure the reports directory exists before mounting it into the container. + new File("../../../tests/integration/container/reports").mkdirs(); + env.testContainer.start(); } diff --git a/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java b/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java index 54f19846b..cb1bd3cd6 100644 --- a/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java +++ b/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java @@ -1117,6 +1117,7 @@ public void addAuroraAwsIamUser( } else { stmt.execute("GRANT ALL PRIVILEGES ON `%`.* TO '" + dbUser + "'@'%';"); } + stmt.execute("GRANT REPLICATION CLIENT ON *.* TO '" + dbUser + "'@'%';"); // BG switchover status needs it. stmt.execute("GRANT SELECT ON mysql.* TO '" + dbUser + "'@'%';"); diff --git a/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java b/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java index 88fc25c19..777b9a299 100644 --- a/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java +++ b/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java @@ -221,7 +221,6 @@ protected Long execInContainer( .execCreateCmd(containerId) .withAttachStdout(true) .withAttachStderr(true) - .withEnv(Arrays.asList("JEST_HTML_REPORTER_OUTPUT_PATH", "./tests/integration/container/reports/hello.html")) .withCmd(command); if (!StringUtils.isNullOrEmpty(workingDir)) { diff --git a/tests/plugin_benchmarks.ts b/tests/plugin_benchmarks.ts index d28f031a0..edcd56e76 100644 --- a/tests/plugin_benchmarks.ts +++ b/tests/plugin_benchmarks.ts @@ -25,8 +25,8 @@ import { SimpleHostAvailabilityStrategy } from "../common/lib/host_availability/ import { HostInfoBuilder } from "../common/lib"; import { PgClientWrapper } from "../common/lib/pg_client_wrapper"; import { NullTelemetryFactory } from "../common/lib/utils/telemetry/null_telemetry_factory"; -import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; import { ConnectionProviderManager } from "../common/lib/connection_provider_manager"; +import { FullServicesContainerImpl } from "../common/lib/utils/full_services_container"; const mockConnectionProvider = mock(); const mockPluginService = mock(PluginServiceImpl); @@ -47,8 +47,8 @@ when(mockPluginService.getCurrentClient()).thenReturn(mockClientWrapper.client); when(mockPluginService.getDriverDialect()).thenReturn(mockDialect); const connectionString = "my.domain.com"; -const pluginServiceManagerContainer = new PluginServiceManagerContainer(); -pluginServiceManagerContainer.pluginService = instance(mockPluginService); +const servicesContainer = mock(FullServicesContainerImpl); +when(servicesContainer.pluginService).thenReturn(instance(mockPluginService)); function getProps(plugins: string) { const props = new Map(); @@ -59,7 +59,7 @@ function getProps(plugins: string) { function getPluginManager(props: Map) { return new PluginManager( - pluginServiceManagerContainer, + servicesContainer, props, new ConnectionProviderManager(instance(mockConnectionProvider), null), new NullTelemetryFactory() diff --git a/tests/plugin_manager_benchmarks.ts b/tests/plugin_manager_benchmarks.ts index 1b84b5bcb..801a4f889 100644 --- a/tests/plugin_manager_benchmarks.ts +++ b/tests/plugin_manager_benchmarks.ts @@ -16,7 +16,6 @@ import { add, complete, configure, cycle, save, suite } from "benny"; import { ConnectionPlugin, ConnectionProvider, HostInfoBuilder, PluginManager } from "../common/lib"; -import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; import { instance, mock, when } from "ts-mockito"; import { SimpleHostAvailabilityStrategy } from "../common/lib/host_availability/simple_host_availability_strategy"; import { PluginServiceImpl } from "../common/lib/plugin_service"; @@ -32,6 +31,7 @@ import { ConnectionPluginFactory } from "../common/lib/plugin_factory"; import { DefaultPlugin } from "../common/lib/plugins/default_plugin"; import { AwsPGClient } from "../pg/lib"; import { ConfigurationProfileBuilder } from "../common/lib/profile/configuration_profile_builder"; +import { FullServicesContainerImpl } from "../common/lib/utils/full_services_container"; const mockConnectionProvider = mock(); const mockHostListProviderService = mock(); @@ -43,8 +43,8 @@ when(mockPluginService.getDialect()).thenReturn(new PgDatabaseDialect()); when(mockPluginService.getDriverDialect()).thenReturn(new NodePostgresDriverDialect()); when(mockPluginService.getCurrentClient()).thenReturn(mockClient); -const pluginServiceManagerContainer = new PluginServiceManagerContainer(); -pluginServiceManagerContainer.pluginService = instance(mockPluginService); +const servicesContainer = mock(FullServicesContainerImpl); +when(servicesContainer.pluginService).thenReturn(instance(mockPluginService)); const propsWithNoPlugins = new Map(); const propsWithPlugins = new Map(); @@ -53,7 +53,7 @@ WrapperProperties.PLUGINS.set(propsWithNoPlugins, ""); function getPluginManagerWithPlugins() { return new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsWithPlugins, new ConnectionProviderManager(instance(mockConnectionProvider), null), new NullTelemetryFactory() @@ -62,7 +62,7 @@ function getPluginManagerWithPlugins() { function getPluginManagerWithNoPlugins() { return new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsWithNoPlugins, new ConnectionProviderManager(instance(mockConnectionProvider), null), new NullTelemetryFactory() diff --git a/tests/plugin_manager_telemetry_benchmarks.ts b/tests/plugin_manager_telemetry_benchmarks.ts index 8420c8eab..3a0f19bba 100644 --- a/tests/plugin_manager_telemetry_benchmarks.ts +++ b/tests/plugin_manager_telemetry_benchmarks.ts @@ -16,7 +16,6 @@ import { add, complete, configure, cycle, save, suite } from "benny"; import { ConnectionPlugin, ConnectionProvider, HostInfoBuilder, PluginManager } from "../common/lib"; -import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; import { instance, mock, when } from "ts-mockito"; import { SimpleHostAvailabilityStrategy } from "../common/lib/host_availability/simple_host_availability_strategy"; import { PluginService, PluginServiceImpl } from "../common/lib/plugin_service"; @@ -44,6 +43,7 @@ import { ConnectionPluginFactory } from "../common/lib/plugin_factory"; import { ConfigurationProfileBuilder } from "../common/lib/profile/configuration_profile_builder"; import { AwsPGClient } from "../pg/lib"; import { resourceFromAttributes } from "@opentelemetry/resources"; +import { FullServicesContainerImpl } from "../common/lib/utils/full_services_container"; const mockConnectionProvider = mock(); const mockHostListProviderService = mock(); @@ -55,8 +55,8 @@ when(mockPluginService.getDialect()).thenReturn(new PgDatabaseDialect()); when(mockPluginService.getDriverDialect()).thenReturn(new NodePostgresDriverDialect()); when(mockPluginService.getCurrentClient()).thenReturn(mockClient); -const pluginServiceManagerContainer = new PluginServiceManagerContainer(); -pluginServiceManagerContainer.pluginService = instance(mockPluginService); +const servicesContainer = mock(FullServicesContainerImpl); +when(servicesContainer.pluginService).thenReturn(instance(mockPluginService)); const propsWithNoPlugins = new Map(); const propsWithPlugins = new Map(); @@ -80,7 +80,7 @@ async function createPlugins(numPlugins: number, pluginService: PluginService, c function getPluginManagerWithPlugins() { return new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsWithPlugins, new ConnectionProviderManager(instance(mockConnectionProvider), null), telemetryFactory @@ -89,7 +89,7 @@ function getPluginManagerWithPlugins() { function getPluginManagerWithNoPlugins() { return new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsWithNoPlugins, new ConnectionProviderManager(instance(mockConnectionProvider), null), telemetryFactory diff --git a/tests/plugin_telemetry_benchmarks.ts b/tests/plugin_telemetry_benchmarks.ts index 04d791e52..b6c391a58 100644 --- a/tests/plugin_telemetry_benchmarks.ts +++ b/tests/plugin_telemetry_benchmarks.ts @@ -17,7 +17,6 @@ import { anything, instance, mock, when } from "ts-mockito"; import { ConnectionProvider, HostInfoBuilder, PluginManager } from "../common/lib"; import { PluginServiceImpl } from "../common/lib/plugin_service"; -import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; import { WrapperProperties } from "../common/lib/wrapper_property"; import { add, complete, configure, cycle, save, suite } from "benny"; import { TestConnectionWrapper } from "./testplugin/test_connection_wrapper"; @@ -41,6 +40,7 @@ import { PgClientWrapper } from "../common/lib/pg_client_wrapper"; import { DriverDialect } from "../common/lib/driver_dialect/driver_dialect"; import { NodePostgresDriverDialect } from "../pg/lib/dialect/node_postgres_driver_dialect"; import { resourceFromAttributes } from "@opentelemetry/resources"; +import { FullServicesContainerImpl } from "../common/lib/utils/full_services_container"; const mockConnectionProvider = mock(); const mockPluginService = mock(PluginServiceImpl); @@ -60,8 +60,8 @@ when(mockPluginService.getCurrentClient()).thenReturn(mockClientWrapper.client); when(mockPluginService.getDriverDialect()).thenReturn(mockDialect); const connectionString = "my.domain.com"; -const pluginServiceManagerContainer = new PluginServiceManagerContainer(); -pluginServiceManagerContainer.pluginService = instance(mockPluginService); +const servicesContainer = mock(FullServicesContainerImpl); +when(servicesContainer.pluginService).thenReturn(instance(mockPluginService)); const propsExecute = new Map(); const propsReadWrite = new Map(); @@ -84,19 +84,19 @@ WrapperProperties.TELEMETRY_TRACES_BACKEND.set(propsReadWrite, "OTLP"); WrapperProperties.TELEMETRY_TRACES_BACKEND.set(props, "OTLP"); const pluginManagerExecute = new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsExecute, new ConnectionProviderManager(instance(mockConnectionProvider), null), telemetryFactory ); const pluginManagerReadWrite = new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsReadWrite, new ConnectionProviderManager(instance(mockConnectionProvider), null), telemetryFactory ); const pluginManager = new PluginManager( - pluginServiceManagerContainer, + servicesContainer, props, new ConnectionProviderManager(instance(mockConnectionProvider), null), new NullTelemetryFactory() diff --git a/tests/testplugin/benchmark_plugin_factory.ts b/tests/testplugin/benchmark_plugin_factory.ts index f1cd65e63..d4101cbaa 100644 --- a/tests/testplugin/benchmark_plugin_factory.ts +++ b/tests/testplugin/benchmark_plugin_factory.ts @@ -14,13 +14,14 @@ limitations under the License. */ -import { ConnectionPlugin, PluginService, AwsWrapperError } from "../../index"; +import { ConnectionPlugin, AwsWrapperError } from "../../index"; import { Messages } from "../../common/lib/utils/messages"; import { BenchmarkPlugin } from "./benchmark_plugin"; import { ConnectionPluginFactory } from "../../common/lib/plugin_factory"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; export class BenchmarkPluginFactory extends ConnectionPluginFactory { - async getInstance(pluginService: PluginService, properties: object): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: object): Promise { try { return new BenchmarkPlugin(); } catch (error: any) { diff --git a/tests/unit/aurora_initial_connection_strategy_plugin.test.ts b/tests/unit/aurora_initial_connection_strategy_plugin.test.ts index ea07276e5..920057ec5 100644 --- a/tests/unit/aurora_initial_connection_strategy_plugin.test.ts +++ b/tests/unit/aurora_initial_connection_strategy_plugin.test.ts @@ -109,7 +109,7 @@ describe("Aurora initial connection strategy plugin", () => { when(mockPluginService.connect(anything(), anything(), anything())).thenResolve(writerClient); expect(await plugin.connect(hostInfo, props, true, mockFunc)).toBe(writerClient); - verify(mockPluginService.forceRefreshHostList(writerClient)).never(); + verify(mockPluginService.forceRefreshHostList()).never(); }); it("test reader - not found", async () => { @@ -128,7 +128,7 @@ describe("Aurora initial connection strategy plugin", () => { when(mockPluginService.getHostInfoByStrategy(anything(), anything())).thenReturn(instance(mockReaderHostInfo)); expect(await plugin.connect(hostInfo, props, true, mockFunc)).toBe(readerClient); - verify(mockPluginService.forceRefreshHostList(readerClient)).never(); + verify(mockPluginService.forceRefreshHostList()).never(); }); it("test reader - resolves to writer", async () => { diff --git a/tests/unit/aws_secrets_manager_plugin.test.ts b/tests/unit/aws_secrets_manager_plugin.test.ts index e76278a14..4aa1271e1 100644 --- a/tests/unit/aws_secrets_manager_plugin.test.ts +++ b/tests/unit/aws_secrets_manager_plugin.test.ts @@ -192,6 +192,20 @@ describe("testSecretsManager", () => { expect(secretKey.region).toBe(expectedRegionParsedFromARN); }); + it.each([ + ["arn:aws:secretsmanager:us-east-1:123456789012:secret:mysecret", "us-east-1"], + ["arn:aws-us-gov:secretsmanager:us-gov-west-1:123456789012:secret:mysecret", "us-gov-west-1"], + ["arn:aws-cn:secretsmanager:cn-north-1:123456789012:secret:mysecret", "cn-north-1"], + ["arn:aws-iso:secretsmanager:us-iso-east-1:123456789012:secret:mysecret", "us-iso-east-1"], + ["arn:aws-iso-b:secretsmanager:us-isob-east-1:123456789012:secret:mysecret", "us-isob-east-1"] + ])("connect using partition-agnostic arn: %s", async (arn, expectedRegion) => { + const props = new Map(); + WrapperProperties.SECRET_ID.set(props, arn); + const testPlugin = new AwsSecretsManagerPlugin(instance(mockPluginService), props); + const secretKey = testPlugin.secretKey; + expect(secretKey.region).toBe(expectedRegion); + }); + it.each([ [TEST_ARN_1, "us-east-2"], [TEST_ARN_2, "us-west-1"], diff --git a/tests/unit/batching_event_publisher.test.ts b/tests/unit/batching_event_publisher.test.ts new file mode 100644 index 000000000..bb079c5c4 --- /dev/null +++ b/tests/unit/batching_event_publisher.test.ts @@ -0,0 +1,141 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { BatchingEventPublisher } from "../../common/lib/utils/events/batching_event_publisher"; +import { DataAccessEvent } from "../../common/lib/utils/events/data_access_event"; +import { Event, EventSubscriber } from "../../common/lib/utils/events/event"; + +class TestableEventPublisher extends BatchingEventPublisher { + constructor() { + super(0); // Pass 0 to avoid starting the interval + } + + protected initPublishingInterval(_messageIntervalMs: number): void { + // Do nothing. + } + + get subscriberCount(): number { + return this.subscribersMap.size; + } + + get pendingEventCount(): number { + return this.pendingEvents.size; + } + + async triggerSendMessages(): Promise { + await this.sendMessages(); + } +} + +// A simple class to use as the dataClass in DataAccessEvent +class TestDataClass {} + +describe("BatchingEventPublisher", () => { + let publisher: TestableEventPublisher; + let mockSubscriber: EventSubscriber; + let processEventCalls: Event[]; + + beforeEach(() => { + publisher = new TestableEventPublisher(); + processEventCalls = []; + mockSubscriber = { + processEvent: async (event: Event) => { + processEventCalls.push(event); + } + }; + }); + + afterEach(() => { + publisher.releaseResources(); + }); + + it("should publish events to subscribers and deduplicate", async () => { + const eventSubscriptions = new Set([DataAccessEvent]); + + publisher.subscribe(mockSubscriber, eventSubscriptions); + publisher.subscribe(mockSubscriber, eventSubscriptions); + expect(publisher.subscriberCount).toBe(1); + + const event = new DataAccessEvent(TestDataClass, "key"); + publisher.publish(event); + publisher.publish(event); + + await publisher.triggerSendMessages(); + + expect(publisher.pendingEventCount).toBe(0); + + expect(processEventCalls.length).toBe(1); + expect(processEventCalls[0]).toBe(event); + + publisher.unsubscribe(mockSubscriber, eventSubscriptions); + publisher.publish(event); + await publisher.triggerSendMessages(); + + expect(publisher.pendingEventCount).toBe(0); + + expect(processEventCalls.length).toBe(1); + }); + + it("should deliver immediate events synchronously", () => { + const immediateEvent: Event = { + isImmediateDelivery: true + }; + + const eventSubscriptions = new Set([immediateEvent.constructor as new (...args: any[]) => Event]); + publisher.subscribe(mockSubscriber, eventSubscriptions); + + publisher.publish(immediateEvent); + + expect(processEventCalls.length).toBe(1); + expect(processEventCalls[0]).toBe(immediateEvent); + + expect(publisher.pendingEventCount).toBe(0); + }); + + it("should not deliver events to unsubscribed subscribers", async () => { + const eventSubscriptions = new Set([DataAccessEvent]); + + publisher.subscribe(mockSubscriber, eventSubscriptions); + publisher.unsubscribe(mockSubscriber, eventSubscriptions); + + const event = new DataAccessEvent(TestDataClass, "key"); + publisher.publish(event); + await publisher.triggerSendMessages(); + + expect(processEventCalls.length).toBe(0); + }); + + it("should handle multiple subscribers", async () => { + const processEventCalls2: Event[] = []; + const mockSubscriber2: EventSubscriber = { + processEvent: async (event: Event) => { + processEventCalls2.push(event); + } + }; + + const eventSubscriptions = new Set([DataAccessEvent]); + + publisher.subscribe(mockSubscriber, eventSubscriptions); + publisher.subscribe(mockSubscriber2, eventSubscriptions); + + const event = new DataAccessEvent(TestDataClass, "key"); + publisher.publish(event); + await publisher.triggerSendMessages(); + + expect(processEventCalls.length).toBe(1); + expect(processEventCalls2.length).toBe(1); + }); +}); diff --git a/tests/unit/connection_plugin_chain_builder.test.ts b/tests/unit/connection_plugin_chain_builder.test.ts index ee152a198..b9cb9c780 100644 --- a/tests/unit/connection_plugin_chain_builder.test.ts +++ b/tests/unit/connection_plugin_chain_builder.test.ts @@ -30,23 +30,33 @@ import { ConnectionProviderManager } from "../../common/lib/connection_provider_ import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { AbstractConnectionPlugin } from "../../common/lib/abstract_connection_plugin"; import { ConnectionPluginFactory } from "../../common/lib/plugin_factory"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); const mockPluginServiceInstance: PluginService = instance(mockPluginService); const mockDefaultConnProvider: ConnectionProvider = mock(DriverConnectionProvider); const mockEffectiveConnProvider: ConnectionProvider = mock(DriverConnectionProvider); +const mockServicesContainer: FullServicesContainer = { + pluginService: mockPluginServiceInstance, + telemetryFactory: new NullTelemetryFactory() +} as unknown as FullServicesContainer; + describe("testConnectionPluginChainBuilder", () => { beforeAll(() => { when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); }); + afterEach(async () => { + await PluginManager.releaseResources(); + }); + it.each([["iam,staleDns,failover"], ["iam, staleDns, failover"]])("sort plugins", async (plugins) => { const props = new Map(); props.set(WrapperProperties.PLUGINS.name, plugins); const result = await ConnectionPluginChainBuilder.getPlugins( - mockPluginServiceInstance, + mockServicesContainer, props, new ConnectionProviderManager(mockDefaultConnProvider, mockEffectiveConnProvider), null @@ -65,7 +75,7 @@ describe("testConnectionPluginChainBuilder", () => { props.set(WrapperProperties.AUTO_SORT_PLUGIN_ORDER.name, false); const result = await ConnectionPluginChainBuilder.getPlugins( - mockPluginServiceInstance, + mockServicesContainer, props, new ConnectionProviderManager(mockDefaultConnProvider, mockEffectiveConnProvider), null @@ -84,7 +94,7 @@ describe("testConnectionPluginChainBuilder", () => { props.set(WrapperProperties.PLUGINS.name, "executeTime,connectTime,iam"); let result = await ConnectionPluginChainBuilder.getPlugins( - mockPluginServiceInstance, + mockServicesContainer, props, new ConnectionProviderManager(mockDefaultConnProvider, mockEffectiveConnProvider), null @@ -99,7 +109,7 @@ describe("testConnectionPluginChainBuilder", () => { props.set(WrapperProperties.PLUGINS.name, "iam,executeTime,connectTime,failover"); result = await ConnectionPluginChainBuilder.getPlugins( - mockPluginServiceInstance, + mockServicesContainer, props, new ConnectionProviderManager(mockDefaultConnProvider, mockEffectiveConnProvider), null @@ -120,7 +130,7 @@ describe("testConnectionPluginChainBuilder", () => { props.set(WrapperProperties.PLUGINS.name, "test"); const result = await ConnectionPluginChainBuilder.getPlugins( - mockPluginServiceInstance, + mockServicesContainer, props, new ConnectionProviderManager(mockDefaultConnProvider, mockEffectiveConnProvider), null @@ -133,8 +143,8 @@ describe("testConnectionPluginChainBuilder", () => { }); class TestPluginFactory extends ConnectionPluginFactory { - async getInstance(pluginService: PluginService, properties: Map): Promise { - return new TestPlugin(pluginService, properties); + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { + return new TestPlugin(servicesContainer.pluginService, properties); } } diff --git a/tests/unit/database_dialect.test.ts b/tests/unit/database_dialect.test.ts index 36b55a71f..b8ff34720 100644 --- a/tests/unit/database_dialect.test.ts +++ b/tests/unit/database_dialect.test.ts @@ -23,7 +23,6 @@ import { RdsPgDatabaseDialect } from "../../pg/lib/dialect/rds_pg_database_diale import { DatabaseDialect, DatabaseType } from "../../common/lib/database_dialect/database_dialect"; import { DatabaseDialectCodes } from "../../common/lib/database_dialect/database_dialect_codes"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; -import { PluginServiceManagerContainer } from "../../common/lib/plugin_service_manager_container"; import { AwsPGClient } from "../../pg/lib"; import { WrapperProperties } from "../../common/lib/wrapper_property"; import { HostInfoBuilder } from "../../common/lib/host_info_builder"; @@ -31,10 +30,18 @@ import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availabili import { ClientWrapper } from "../../common/lib/client_wrapper"; import { RdsMultiAZClusterMySQLDatabaseDialect } from "../../mysql/lib/dialect/rds_multi_az_mysql_database_dialect"; import { RdsMultiAZClusterPgDatabaseDialect } from "../../pg/lib/dialect/rds_multi_az_pg_database_dialect"; +import { GlobalAuroraMySQLDatabaseDialect } from "../../mysql/lib/dialect/global_aurora_mysql_database_dialect"; +import { GlobalAuroraPgDatabaseDialect } from "../../pg/lib/dialect/global_aurora_pg_database_dialect"; import { DatabaseDialectManager } from "../../common/lib/database_dialect/database_dialect_manager"; import { NodePostgresDriverDialect } from "../../pg/lib/dialect/node_postgres_driver_dialect"; import { mock } from "ts-mockito"; import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; +import { StorageService } from "../../common/lib/utils/storage/storage_service"; +import { ConnectionProvider } from "../../common/lib"; +import { TelemetryFactory } from "../../common/lib/utils/telemetry/telemetry_factory"; +import { MonitorService } from "../../common/lib/utils/monitoring/monitor_service"; +import { FullServicesContainerImpl } from "../../common/lib/utils/full_services_container"; +import { EventPublisher } from "../../common/lib/utils/events/event"; const LOCALHOST = "localhost"; const RDS_DATABASE = "database-1.xyz.us-east-2.rds.amazonaws.com"; @@ -44,14 +51,16 @@ const mysqlDialects: Map = new Map([ [DatabaseDialectCodes.MYSQL, new MySQLDatabaseDialect()], [DatabaseDialectCodes.RDS_MYSQL, new RdsMySQLDatabaseDialect()], [DatabaseDialectCodes.AURORA_MYSQL, new AuroraMySQLDatabaseDialect()], - [DatabaseDialectCodes.RDS_MULTI_AZ_MYSQL, new RdsMultiAZClusterMySQLDatabaseDialect()] + [DatabaseDialectCodes.RDS_MULTI_AZ_MYSQL, new RdsMultiAZClusterMySQLDatabaseDialect()], + [DatabaseDialectCodes.GLOBAL_AURORA_MYSQL, new GlobalAuroraMySQLDatabaseDialect()] ]); const pgDialects: Map = new Map([ [DatabaseDialectCodes.PG, new PgDatabaseDialect()], [DatabaseDialectCodes.RDS_PG, new RdsPgDatabaseDialect()], [DatabaseDialectCodes.AURORA_PG, new AuroraPgDatabaseDialect()], - [DatabaseDialectCodes.RDS_MULTI_AZ_PG, new RdsMultiAZClusterPgDatabaseDialect()] + [DatabaseDialectCodes.RDS_MULTI_AZ_PG, new RdsMultiAZClusterPgDatabaseDialect()], + [DatabaseDialectCodes.GLOBAL_AURORA_PG, new GlobalAuroraPgDatabaseDialect()] ]); const MYSQL_QUERY = "SHOW VARIABLES LIKE 'version_comment'"; @@ -180,7 +189,13 @@ const expectedDialectMapping: Map = ne ] ]); -const pluginServiceManagerContainer = new PluginServiceManagerContainer(); +const fullServicesContainer = new FullServicesContainerImpl( + mock(), + mock(), + mock(), + mock(), + mock() +); const mockClient = new AwsPGClient({}); const mockDriverDialect = mock(NodePostgresDriverDialect); @@ -275,14 +290,9 @@ describe("test database dialects", () => { }).build(); const mockClientWrapper: ClientWrapper = new PgClientWrapper(mockTargetClient, currentHostInfo, new Map()); - const pluginService = new PluginServiceImpl( - pluginServiceManagerContainer, - mockClient, - databaseType, - expectedDialect!.dialects, - props, - mockDriverDialect - ); + const pluginService = new PluginServiceImpl(fullServicesContainer, mockClient, databaseType, expectedDialect!.dialects, props, mockDriverDialect); + fullServicesContainer.hostListProviderService = pluginService; + fullServicesContainer.pluginService = pluginService; await pluginService.updateDialect(mockClientWrapper); expect(pluginService.getDialect()).toBe(expectedDialectClass); }); diff --git a/tests/unit/failover2_plugin.test.ts b/tests/unit/failover2_plugin.test.ts index b36a2baf3..8db14ad22 100644 --- a/tests/unit/failover2_plugin.test.ts +++ b/tests/unit/failover2_plugin.test.ts @@ -40,6 +40,7 @@ import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; import { DriverDialect } from "../../common/lib/driver_dialect/driver_dialect"; import { Failover2Plugin } from "../../common/lib/plugins/failover2/failover2_plugin"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; const builder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); @@ -57,7 +58,10 @@ const properties: Map = new Map(); let plugin: Failover2Plugin; function initializePlugin(mockPluginServiceInstance: PluginService): void { - plugin = new Failover2Plugin(mockPluginServiceInstance, properties, new RdsUtils()); + const mockContainer = { + pluginService: mockPluginServiceInstance + } as unknown as FullServicesContainer; + plugin = new Failover2Plugin(mockContainer, properties, new RdsUtils()); } describe("reader failover handler", () => { diff --git a/tests/unit/failover_plugin.test.ts b/tests/unit/failover_plugin.test.ts index 0cbbe34d1..42e510251 100644 --- a/tests/unit/failover_plugin.test.ts +++ b/tests/unit/failover_plugin.test.ts @@ -17,7 +17,6 @@ import { AwsClient } from "../../common/lib/aws_client"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { HostInfoBuilder } from "../../common/lib/host_info_builder"; -import { RdsHostListProvider } from "../../common/lib/host_list_provider/rds_host_list_provider"; import { PluginService, PluginServiceImpl } from "../../common/lib/plugin_service"; import { FailoverMode } from "../../common/lib/plugins/failover/failover_mode"; import { FailoverPlugin } from "../../common/lib/plugins/failover/failover_plugin"; @@ -44,6 +43,8 @@ import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { HostChangeOptions } from "../../common/lib/host_change_options"; import { Messages } from "../../common/lib/utils/messages"; +import { RdsHostListProvider } from "../../common/lib/host_list_provider/rds_host_list_provider"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; const builder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); @@ -61,6 +62,9 @@ const mockWriterResult: WriterFailoverResult = mock(WriterFailoverResult); const mockClientWrapper = new MySQLClientWrapper(undefined, mockHostInfo, new Map(), new MySQL2DriverDialect()); +const mockServicesContainer: FullServicesContainer = mock(); +const mockServicesContainerInstance = instance(mockServicesContainer); + const properties: Map = new Map(); let plugin: FailoverPlugin; @@ -76,10 +80,11 @@ function initializePlugin( readerFailoverHandler?: ClusterAwareReaderFailoverHandler, writerFailoverHandler?: ClusterAwareWriterFailoverHandler ): void { + when(mockServicesContainer.pluginService).thenReturn(mockPluginServiceInstance); plugin = readerFailoverHandler && writerFailoverHandler - ? new FailoverPlugin(mockPluginServiceInstance, properties, new RdsUtils(), readerFailoverHandler, writerFailoverHandler) - : new FailoverPlugin(mockPluginServiceInstance, properties, new RdsUtils()); + ? new FailoverPlugin(mockServicesContainerInstance, properties, new RdsUtils(), readerFailoverHandler, writerFailoverHandler) + : new FailoverPlugin(mockServicesContainerInstance, properties, new RdsUtils()); } describe("reader failover handler", () => { diff --git a/tests/unit/federated_auth_plugin.test.ts b/tests/unit/federated_auth_plugin.test.ts deleted file mode 100644 index d696e4511..000000000 --- a/tests/unit/federated_auth_plugin.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { HostInfo, HostRole, PluginManager } from "../../common/lib"; -import { FederatedAuthPlugin } from "../../common/lib/plugins/federated_auth/federated_auth_plugin"; -import { PluginServiceImpl } from "../../common/lib/plugin_service"; -import { IamAuthUtils, TokenInfo } from "../../common/lib/utils/iam_auth_utils"; -import { WrapperProperties } from "../../common/lib/wrapper_property"; -import { anything, instance, mock, spy, verify, when } from "ts-mockito"; -import { CredentialsProviderFactory } from "../../common/lib/plugins/federated_auth/credentials_provider_factory"; -import { DatabaseDialect } from "../../common/lib/database_dialect/database_dialect"; -import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; -import { jest } from "@jest/globals"; -import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; - -const testToken = "testToken"; -const defaultPort = 5432; -const pgCacheKey = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; -const dbUser = "iamUser"; -const expirationFiveMinutes = 5 * 60 * 1000; -const tokenCache = new Map(); - -const host = "pg.testdb.us-east-2.rds.amazonaws.com"; -const iamHost = "pg-123.testdb.us-east-2.rds.amazonaws.com"; -const hostInfo = new HostInfo(host, defaultPort, HostRole.WRITER); -const testTokenInfo = new TokenInfo(testToken, Date.now() + expirationFiveMinutes); - -const mockDialect = mock(); -const mockDialectInstance = instance(mockDialect); -const mockPluginService = mock(PluginServiceImpl); -const mockCredentialsProviderFactory = mock(); -const spyIamUtils = spy(IamAuthUtils); -const testCredentials = { - accessKeyId: "foo", - secretAccessKey: "bar", - sessionToken: "baz" -}; -const mockConnectFunc = jest.fn(() => { - return Promise.resolve(mock(PgClientWrapper)); -}); - -describe("federatedAuthTest", () => { - let spyPlugin: FederatedAuthPlugin; - let props: Map; - - beforeEach(() => { - when(mockPluginService.getDialect()).thenReturn(mockDialectInstance); - when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); - when(mockDialect.getDefaultPort()).thenReturn(defaultPort); - when(mockCredentialsProviderFactory.getAwsCredentialsProvider(anything(), anything(), anything())).thenResolve(instance(testCredentials)); - props = new Map(); - WrapperProperties.PLUGINS.set(props, "federatedAuth"); - WrapperProperties.DB_USER.set(props, dbUser); - spyPlugin = spy(new FederatedAuthPlugin(instance(mockPluginService), instance(mockCredentialsProviderFactory))); - }); - - afterEach(async () => { - await PluginManager.releaseResources(); - }); - - it("testCachedToken", async () => { - const spyPluginInstance = instance(spyPlugin); - FederatedAuthPlugin["tokenCache"].set(pgCacheKey, testTokenInfo); - - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - tokenCache.set(key, testTokenInfo); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testExpiredCachedToken", async () => { - const spyPluginInstance: FederatedAuthPlugin = instance(spyPlugin); - - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - const expiredToken = "expiredToken"; - const expiredTokenInfo = new TokenInfo(expiredToken, Date.now() - 300000); - - FederatedAuthPlugin["tokenCache"].set(key, expiredTokenInfo); - - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testNoCachedToken", async () => { - const spyPluginInstance = instance(spyPlugin); - - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testSpecifiedIamHostPortRegion", async () => { - const expectedHost = "pg.testdb.us-west-2.rds.amazonaws.com"; - const expectedPort = 9876; - const expectedRegion = "us-west-2"; - - WrapperProperties.IAM_HOST.set(props, expectedHost); - WrapperProperties.IAM_DEFAULT_PORT.set(props, expectedPort); - WrapperProperties.IAM_REGION.set(props, expectedRegion); - - const key = `us-west-2:pg.testdb.us-west-2.rds.amazonaws.com:${expectedPort}:iamUser`; - FederatedAuthPlugin["tokenCache"].set(key, testTokenInfo); - - const spyPluginInstance = instance(spyPlugin); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testIdpCredentialsFallback", async () => { - const expectedUser = "expectedUser"; - const expectedPassword = "expectedPassword"; - WrapperProperties.USER.set(props, expectedUser); - WrapperProperties.PASSWORD.set(props, expectedPassword); - - const spyPluginInstance = instance(spyPlugin); - - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - FederatedAuthPlugin["tokenCache"].set(key, testTokenInfo); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - expect(expectedUser).toBe(WrapperProperties.IDP_USERNAME.get(props)); - expect(expectedPassword).toBe(WrapperProperties.IDP_PASSWORD.get(props)); - }); - - it("testUsingIamHost", async () => { - WrapperProperties.IAM_HOST.set(props, iamHost); - const spyPluginInstance = instance(spyPlugin); - - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - verify(spyIamUtils.generateAuthenticationToken(iamHost, anything(), anything(), anything(), anything(), anything())).once(); - }); -}); diff --git a/tests/unit/gdb_region_utils.test.ts b/tests/unit/gdb_region_utils.test.ts new file mode 100644 index 000000000..243eae9e3 --- /dev/null +++ b/tests/unit/gdb_region_utils.test.ts @@ -0,0 +1,47 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { GDBRegionUtils } from "../../common/lib/utils/gdb_region_utils"; + +describe("GDBRegionUtils", () => { + let gdbRegionUtils: GDBRegionUtils; + + beforeEach(() => { + gdbRegionUtils = new GDBRegionUtils(); + }); + + describe("getRegionFromClusterArn", () => { + it.each([ + ["arn:aws:rds:us-east-1:123456789012:cluster:my-cluster", "us-east-1"], + ["arn:aws:rds:us-west-2:123456789012:cluster:my-cluster", "us-west-2"], + ["arn:aws:rds:eu-west-1:123456789012:cluster:my-cluster", "eu-west-1"], + ["arn:aws-us-gov:rds:us-gov-west-1:123456789012:cluster:my-cluster", "us-gov-west-1"], + ["arn:aws-us-gov:rds:us-gov-east-1:123456789012:cluster:my-cluster", "us-gov-east-1"], + ["arn:aws-cn:rds:cn-north-1:123456789012:cluster:my-cluster", "cn-north-1"], + ["arn:aws-cn:rds:cn-northwest-1:123456789012:cluster:my-cluster", "cn-northwest-1"], + ["arn:aws-iso:rds:us-iso-east-1:123456789012:cluster:my-cluster", "us-iso-east-1"], + ["arn:aws-iso-b:rds:us-isob-east-1:123456789012:cluster:my-cluster", "us-isob-east-1"] + ])("should extract region from partition-agnostic ARN: %s", (arn, expectedRegion) => { + const region = gdbRegionUtils.getRegionFromClusterArn(arn); + expect(region).toBe(expectedRegion); + }); + + it.each(["invalid-arn", "arn:aws:s3:::my-bucket", "arn:aws:rds", ""])("should return null for invalid ARN: %s", (invalidArn) => { + const region = gdbRegionUtils.getRegionFromClusterArn(invalidArn); + expect(region).toBeNull(); + }); + }); +}); diff --git a/tests/unit/host_monitoring_plugin.test.ts b/tests/unit/host_monitoring_plugin.test.ts index 02b70dd81..88c7c012c 100644 --- a/tests/unit/host_monitoring_plugin.test.ts +++ b/tests/unit/host_monitoring_plugin.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,15 +15,12 @@ */ import { WrapperProperties } from "../../common/lib/wrapper_property"; -import { HostMonitoringConnectionPlugin } from "../../common/lib/plugins/efm/host_monitoring_connection_plugin"; -import { DatabaseDialect } from "../../common/lib/database_dialect/database_dialect"; -import { PgDatabaseDialect } from "../../pg/lib/dialect/pg_database_dialect"; +import { HostMonitoringConnectionPlugin } from "../../common/lib/plugins/efm/v1/host_monitoring_connection_plugin"; import { anything, instance, mock, reset, verify, when } from "ts-mockito"; -import { RdsUtils } from "../../common/lib/utils/rds_utils"; -import { MonitorServiceImpl } from "../../common/lib/plugins/efm/monitor_service"; +import { HostMonitorServiceImpl } from "../../common/lib/plugins/efm/base/host_monitor_service"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; -import { MonitorConnectionContext } from "../../common/lib/plugins/efm/monitor_connection_context"; -import { AwsWrapperError, HostAvailability, HostInfo } from "../../common/lib"; +import { ConnectionContext, ConnectionContextImpl } from "../../common/lib/plugins/efm/base/connection_context"; +import { AwsWrapperError, HostInfo } from "../../common/lib"; import { RdsUrlType } from "../../common/lib/utils/rds_url_type"; import { HostChangeOptions } from "../../common/lib/host_change_options"; import { OldConnectionSuggestionAction } from "../../common/lib/old_connection_suggestion_action"; @@ -31,6 +28,8 @@ import { Messages } from "../../common/lib/utils/messages"; import { AwsPGClient } from "../../pg/lib"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; +import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; const FAILURE_DETECTION_TIME = 10; const FAILURE_DETECTION_INTERVAL = 100; @@ -38,12 +37,10 @@ const FAILURE_DETECTION_COUNT = 5; const MONITOR_METHOD_NAME = "query"; const NO_MONITOR_METHOD_NAME = "end"; -const mockDialect: DatabaseDialect = mock(PgDatabaseDialect); -const mockMonitorService: MonitorServiceImpl = mock(MonitorServiceImpl); -const mockMonitorConnectionContext: MonitorConnectionContext = mock(MonitorConnectionContext); +const mockMonitorService: HostMonitorServiceImpl = mock(HostMonitorServiceImpl); +const mockConnectionContext: ConnectionContextImpl = mock(ConnectionContextImpl); const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); const mockClient: AwsPGClient = mock(AwsPGClient); -const mockRdsUtils = mock(RdsUtils); const mockHostInfo1 = mock(HostInfo); const mockHostInfo2 = mock(HostInfo); @@ -59,18 +56,19 @@ function incrementQueryCounter() { const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); function initDefaultMockReturns() { - when(mockDialect.getHostAliasQuery()).thenReturn("any"); - when(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).thenResolve( - instance(mockMonitorConnectionContext) + when(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve( + instance(mockConnectionContext) ); + when(mockConnectionContext.isActiveContext()).thenReturn(true); + when(mockConnectionContext.isHostUnhealthy()).thenReturn(false); when(mockHostInfo1.host).thenReturn("host"); when(mockHostInfo1.port).thenReturn(1234); when(mockHostInfo2.host).thenReturn("host"); when(mockHostInfo2.port).thenReturn(1234); when(mockPluginService.getCurrentHostInfo()).thenReturn(instance(mockHostInfo1)); + when(mockPluginService.getRoutedHostInfo()).thenReturn(instance(mockHostInfo1)); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); when(mockClient.targetClient).thenReturn(mockClientWrapper); - when(mockRdsUtils.identifyRdsType(anything())).thenReturn(RdsUrlType.RDS_INSTANCE); properties.set(WrapperProperties.FAILURE_DETECTION_ENABLED.name, true); properties.set(WrapperProperties.FAILURE_DETECTION_TIME_MS.name, FAILURE_DETECTION_TIME); @@ -79,13 +77,18 @@ function initDefaultMockReturns() { } function initializePlugin() { - plugin = new HostMonitoringConnectionPlugin(instance(mockPluginService), properties, instance(mockRdsUtils), instance(mockMonitorService)); + const servicesContainer = { + pluginService: instance(mockPluginService), + telemetryFactory: new NullTelemetryFactory() + } as unknown as FullServicesContainer; + + plugin = new HostMonitoringConnectionPlugin(servicesContainer, properties, instance(mockMonitorService)); } describe("host monitoring plugin test", () => { beforeEach(() => { reset(mockMonitorService); - reset(mockMonitorConnectionContext); + reset(mockConnectionContext); reset(mockPluginService); reset(mockClient); @@ -99,7 +102,7 @@ describe("host monitoring plugin test", () => { initializePlugin(); await plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {}); - verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).never(); + verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).never(); verify(mockMonitorService.stopMonitoring(anything())).never(); expect(queryCounter).toBe(1); }); @@ -108,7 +111,7 @@ describe("host monitoring plugin test", () => { initializePlugin(); await plugin.execute(NO_MONITOR_METHOD_NAME, incrementQueryCounter, {}); - verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).never(); + verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).never(); verify(mockMonitorService.stopMonitoring(anything())).never(); expect(queryCounter).toBe(1); }); @@ -117,7 +120,7 @@ describe("host monitoring plugin test", () => { initializePlugin(); await plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {}); - verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).once(); + verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).once(); verify(mockMonitorService.stopMonitoring(anything())).once(); expect(queryCounter).toBe(1); }); @@ -126,55 +129,25 @@ describe("host monitoring plugin test", () => { initializePlugin(); const expectedError = new AwsWrapperError("Error thrown during isClientValid"); - when(mockMonitorConnectionContext.isHostUnhealthy).thenReturn(true); + when(mockConnectionContext.isHostUnhealthy()).thenReturn(true); when(mockPluginService.isClientValid(mockClientWrapper)).thenThrow(expectedError); await expect(plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {})).rejects.toThrow(expectedError); }); - it("execute cleanup when abort connection throws error", async () => { + it("execute cleanup when connection is invalid throws unavailable host error", async () => { initializePlugin(); when(mockPluginService.isClientValid(mockClientWrapper)).thenResolve(false); - when(mockMonitorConnectionContext.isHostUnhealthy).thenReturn(true); + when(mockConnectionContext.isHostUnhealthy()).thenReturn(true); const expectedError = new AwsWrapperError(Messages.get("HostMonitoringConnectionPlugin.unavailableHost", "host")); await expect(plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {})).rejects.toThrow(expectedError); - verify(mockPluginService.setAvailability(anything(), HostAvailability.NOT_AVAILABLE)).once(); }); - it("test connect", async () => { + it("notify connection changed resets monitoring host info", async () => { initializePlugin(); - when(mockRdsUtils.identifyRdsType(anything())).thenReturn(RdsUrlType.RDS_WRITER_CLUSTER); - await plugin.connect(instance(mockHostInfo1), properties, true, () => { - return Promise.resolve(mockClientWrapper); - }); - verify(mockPluginService.fillAliases(anything(), anything())).once(); + const result = await plugin.notifyConnectionChanged(new Set([HostChangeOptions.HOSTNAME])); + expect(result).toBe(OldConnectionSuggestionAction.NO_OPINION); }); - - it.each([[HostChangeOptions.WENT_DOWN], [HostChangeOptions.HOST_DELETED]])( - "notify connection changed when node went down", - async (options: HostChangeOptions) => { - initializePlugin(); - - await plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {}); - - const aliases1 = new Set(["alias1", "alias2"]); - const aliases2 = new Set(["alias3", "alias4"]); - - when(mockHostInfo1.allAliases).thenReturn(aliases1); - when(mockHostInfo2.allAliases).thenReturn(aliases2); - when(mockPluginService.getCurrentHostInfo()).thenReturn(instance(mockHostInfo1)); - - expect(await plugin.notifyConnectionChanged(new Set([options]))).toBe(OldConnectionSuggestionAction.NO_OPINION); - // NodeKeys should contain {"alias1", "alias2"}. - verify(mockMonitorService.stopMonitoringForAllConnections(aliases1)).once(); - - when(mockPluginService.getCurrentHostInfo()).thenReturn(instance(mockHostInfo2)); - expect(await plugin.notifyConnectionChanged(new Set([options]))).toBe(OldConnectionSuggestionAction.NO_OPINION); - // NotifyConnectionChanged should reset the monitoringHostSpec. - // NodeKeys should contain {"alias3", "alias4"} - verify(mockMonitorService.stopMonitoringForAllConnections(aliases2)).once(); - } - ); }); diff --git a/tests/unit/iam_authentication_plugin.test.ts b/tests/unit/iam_authentication_plugin.test.ts index c6d384d46..58d7ad7ba 100644 --- a/tests/unit/iam_authentication_plugin.test.ts +++ b/tests/unit/iam_authentication_plugin.test.ts @@ -21,7 +21,7 @@ import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availabili import { AwsClient } from "../../common/lib/aws_client"; import { WrapperProperties } from "../../common/lib/wrapper_property"; import fetch from "node-fetch"; -import { anything, instance, mock, spy, when } from "ts-mockito"; +import { anything, instance, mock, reset, spy, when } from "ts-mockito"; import { IamAuthUtils, TokenInfo } from "../../common/lib/utils/iam_auth_utils"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; @@ -58,22 +58,11 @@ const props = new Map(); const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); const mockClient: AwsClient = mock(AwsClient); -const spyIamAuthUtils = spy(IamAuthUtils); class IamAuthenticationPluginTestClass extends IamAuthenticationPlugin { put(key: string, token: TokenInfo) { IamAuthenticationPlugin.tokenCache.set(key, token); } - - public async generateAuthenticationToken( - hostInfo: HostInfo, - props: Map, - hostname: string, - port: number, - region: string - ): Promise { - return Promise.resolve(GENERATED_TOKEN); - } } async function testGenerateToken(info: HostInfo, plugin: IamAuthenticationPluginTestClass) { @@ -100,8 +89,11 @@ async function testToken(info: HostInfo, plugin: IamAuthenticationPlugin) { } describe("testIamAuth", () => { + let spyIamAuthUtils: IamAuthUtils; + beforeEach(() => { PluginManager.releaseResources(); + spyIamAuthUtils = spy(new IamAuthUtils()); props.clear(); props.set(WrapperProperties.USER.name, "postgresqlUser"); @@ -110,15 +102,21 @@ describe("testIamAuth", () => { when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); + when(spyIamAuthUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve( GENERATED_TOKEN ); }); + afterEach(() => { + reset(spyIamAuthUtils); + PluginManager.releaseResources(); + }); + it("testPostgresConnectValidTokenInCache", async () => { when(mockClient.defaultPort).thenReturn(DEFAULT_PG_PORT); - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(PG_CACHE_KEY, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); await testToken(PG_HOST_INFO, plugin); }); @@ -129,7 +127,7 @@ describe("testIamAuth", () => { when(mockClient.defaultPort).thenReturn(DEFAULT_MYSQL_PORT); - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(MYSQL_CACHE_KEY, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); await testToken(MYSQL_HOST_INFO, plugin); @@ -139,7 +137,7 @@ describe("testIamAuth", () => { props.set(WrapperProperties.IAM_DEFAULT_PORT.name, 0); const cacheKeyWithNewPort: string = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${PG_HOST_INFO_WITH_PORT.port}:postgresqlUser`; - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(cacheKeyWithNewPort, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); @@ -151,7 +149,7 @@ describe("testIamAuth", () => { when(mockClient.defaultPort).thenReturn(DEFAULT_PG_PORT); const cacheKeyWithNewPort: string = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${DEFAULT_PG_PORT}:postgresqlUser`; - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(cacheKeyWithNewPort, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); @@ -160,7 +158,7 @@ describe("testIamAuth", () => { it("testConnectExpiredTokenInCache", async () => { when(mockClient.defaultPort).thenReturn(DEFAULT_PG_PORT); - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(PG_CACHE_KEY, new TokenInfo(TEST_TOKEN, Date.now() - 300000)); await testGenerateToken(PG_HOST_INFO, plugin); @@ -168,13 +166,13 @@ describe("testIamAuth", () => { it("testConnectEmptyCache", async () => { when(mockClient.defaultPort).thenReturn(DEFAULT_PG_PORT); - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); await testGenerateToken(PG_HOST_INFO, plugin); }); it("testConnectWithSpecifiedPort", async () => { const cacheKeyWithSpecifiedPort: string = "us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:1234:postgresqlUser"; - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(cacheKeyWithSpecifiedPort, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); await testToken(PG_HOST_INFO_WITH_PORT, plugin); @@ -185,7 +183,7 @@ describe("testIamAuth", () => { props.set(WrapperProperties.IAM_DEFAULT_PORT.name, iamDefaultPort); const cacheKeyWithNewPort: string = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${iamDefaultPort}:postgresqlUser`; - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(cacheKeyWithNewPort, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); await testToken(PG_HOST_INFO_WITH_PORT, plugin); @@ -196,7 +194,7 @@ describe("testIamAuth", () => { when(mockClient.defaultPort).thenReturn(DEFAULT_PG_PORT); const cacheKeyWithNewRegion: string = `us-west-1:pg.testdb.us-west-1.rds.amazonaws.com:${DEFAULT_PG_PORT}:postgresqlUser`; - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(cacheKeyWithNewRegion, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); await testToken(PG_HOST_INFO_WITH_REGION, plugin); diff --git a/tests/unit/monitor_connection_context.test.ts b/tests/unit/monitor_connection_context.test.ts index 47e5eab01..3f11b3de3 100644 --- a/tests/unit/monitor_connection_context.test.ts +++ b/tests/unit/monitor_connection_context.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,113 +14,131 @@ limitations under the License. */ -import { instance, mock, spy, verify } from "ts-mockito"; -import { MonitorConnectionContext } from "../../common/lib/plugins/efm/monitor_connection_context"; -import { MonitorImpl } from "../../common/lib/plugins/efm/monitor"; -import { PluginServiceImpl } from "../../common/lib/plugin_service"; +import { mock } from "ts-mockito"; +import { ConnectionContextImpl } from "../../common/lib/plugins/efm/base/connection_context"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; +import { HostInfo } from "../../common/lib"; +import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { getCurrentTimeNano } from "../../common/lib/utils/utils"; -const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); -const mockMonitor = mock(MonitorImpl); -const mockTargetClient = { - end() { - throw new Error("close"); - } -}; -const mockClientWrapper = mock(MySQLClientWrapper); +const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); const FAILURE_DETECTION_TIME_MILLIS = 10; const FAILURE_DETECTION_INTERVAL_MILLIS = 100; const FAILURE_DETECTION_COUNT = 3; const VALIDATION_INTERVAL_MILLIS = 50; -let context: MonitorConnectionContext; +let context: ConnectionContextImpl; -describe("monitor connection context test", () => { +describe("connection context test", () => { beforeEach(() => { - context = new MonitorConnectionContext( - instance(mockMonitor), - null, + context = new ConnectionContextImpl( + mockClientWrapper, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT, - instance(mockPluginService), new NullTelemetryFactory().createCounter("counter") ); }); it("isHostUnhealthy with valid connection", async () => { - const currentTimeNano = Date.now(); - await context.setConnectionValid("test-node", true, currentTimeNano, currentTimeNano); - expect(context.isHostUnhealthy).toBe(false); - expect(context.failureCount).toBe(0); + const ctx = new ConnectionContextImpl( + mockClientWrapper, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); + // Timestamps must exceed the grace period relative to context's startMonitorTimeNano (hrtime-based) + const checkStart = getCurrentTimeNano() + (FAILURE_DETECTION_TIME_MILLIS + 100) * 1_000_000; + const checkEnd = checkStart + 1_000_000; + await ctx.updateConnectionStatus("test-node", checkStart, checkEnd, true); + expect(ctx.isHostUnhealthy()).toBe(false); }); - it("isHostUnhealthy with invalid connection", async () => { - const currentTimeNano = Date.now(); - await context.setConnectionValid("test-node", false, currentTimeNano, currentTimeNano); - expect(context.isHostUnhealthy).toBe(false); - expect(context.failureCount).toBe(1); + it("isHostUnhealthy with invalid connection - single failure", async () => { + const ctx = new ConnectionContextImpl( + mockClientWrapper, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); + const checkStart = getCurrentTimeNano() + (FAILURE_DETECTION_TIME_MILLIS + 100) * 1_000_000; + const checkEnd = checkStart + 1_000_000; + await ctx.updateConnectionStatus("test-node", checkStart, checkEnd, false); + // Single failure doesn't exceed threshold + expect(ctx.isHostUnhealthy()).toBe(false); }); - it("isHostUnhealthy exceeds failure detection count", async () => { - let currentTimeNano = Date.now(); - context.failureCount = 0; - context.resetInvalidHostStartTimeNano(); + it("isHostUnhealthy exceeds failure detection threshold", async () => { + const ctx = new ConnectionContextImpl( + mockClientWrapper, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); - for (let i = 0; i < 5; i++) { - const statusCheckStartTime = currentTimeNano; - const statusCheckEndTime = currentTimeNano + VALIDATION_INTERVAL_MILLIS * 1_000_000; + // Use timestamps far enough in the future to exceed the grace period (failureDetectionTimeMillis). + // The context's startMonitorTimeNano is hrtime-based (~now), so we simulate checks starting after the grace period. + const futureStartNano = getCurrentTimeNano() + (FAILURE_DETECTION_TIME_MILLIS + 100) * 1_000_000; - await context.setConnectionValid("test-node", false, statusCheckStartTime, statusCheckEndTime); - if (i >= FAILURE_DETECTION_COUNT - 1) { - expect(context.isHostUnhealthy).toBe(true); - } else { - expect(context.isHostUnhealthy).toBe(false); - } + // maxInvalidHostDuration = failureDetectionIntervalMillis * failureDetectionCount = 100 * 3 = 300ms + // Each check spans FAILURE_DETECTION_INTERVAL_MILLIS (100ms), so after 3 checks the duration exceeds threshold. + for (let i = 0; i < FAILURE_DETECTION_COUNT + 2; i++) { + const checkStart = futureStartNano + i * FAILURE_DETECTION_INTERVAL_MILLIS * 1_000_000; + const checkEnd = checkStart + FAILURE_DETECTION_INTERVAL_MILLIS * 1_000_000; + + await ctx.updateConnectionStatus("test-node", checkStart, checkEnd, false); - currentTimeNano += VALIDATION_INTERVAL_MILLIS * 1_000_000; + if (ctx.isHostUnhealthy()) { + break; + } } - const statusCheckStartTime = currentTimeNano; - const statusCheckEndTime = currentTimeNano + VALIDATION_INTERVAL_MILLIS * 1_000_000; - await context.setConnectionValid("test-node", false, statusCheckStartTime, statusCheckEndTime); - expect(context.isHostUnhealthy).toBe(true); + expect(ctx.isHostUnhealthy()).toBe(true); }); - it.each([[true], [false]])("updateConnectionStatus", async (isValid: boolean) => { - const currentTimeNano = Date.now(); - const statusCheckStartTime = Date.now() - FAILURE_DETECTION_TIME_MILLIS * 1_000_000; + it("updateConnectionStatus skips within grace period", async () => { + // Use a context with a long grace period + const ctx = new ConnectionContextImpl( + mockClientWrapper, + 30000, // 30s grace + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); - const contextSpy = spy(context); - await context.updateConnectionStatus("test-node", statusCheckStartTime, currentTimeNano, isValid); + // Call with a time that's within the grace period (elapsed < failureDetectionTime) + const now = getCurrentTimeNano(); + await ctx.updateConnectionStatus("test-node", now, now + 1_000_000, false); - verify(contextSpy.setConnectionValid("test-node", isValid, statusCheckStartTime, currentTimeNano)).once(); + // Should not have detected unhealthy since we're within grace period + expect(ctx.isHostUnhealthy()).toBe(false); }); - it("updateConnectionStatus - inactive context", async () => { - const currentTimeNano = Date.now(); - const statusCheckStartTime = Date.now() - 1000; - context.isActiveContext = false; + it("updateConnectionStatus skips for inactive context", async () => { + context.setInactive(); + expect(context.isActiveContext()).toBe(false); - const contextSpy = spy(context); - await context.updateConnectionStatus("test-node", statusCheckStartTime, currentTimeNano, true); + const now = getCurrentTimeNano(); + await context.updateConnectionStatus("test-node", now, now + 100_000_000, false); - verify(contextSpy.setConnectionValid("test-node", true, statusCheckStartTime, currentTimeNano)).never(); + // Should remain not-unhealthy since the context is inactive + expect(context.isHostUnhealthy()).toBe(false); }); - it("abort client ignores error", async () => { - context = new MonitorConnectionContext( - instance(mockMonitor), - mockClientWrapper, - FAILURE_DETECTION_TIME_MILLIS, - FAILURE_DETECTION_INTERVAL_MILLIS, - FAILURE_DETECTION_COUNT, - instance(mockPluginService), - new NullTelemetryFactory().createCounter("counter") - ); + it("setInactive marks context inactive", () => { + expect(context.isActiveContext()).toBe(true); + context.setInactive(); + expect(context.isActiveContext()).toBe(false); + }); + it("abortConnection does nothing for inactive context", async () => { + context.setInactive(); + // Should not throw await context.abortConnection(); }); }); diff --git a/tests/unit/monitor_impl.test.ts b/tests/unit/monitor_impl.test.ts index 8ba1116cc..2ba18316f 100644 --- a/tests/unit/monitor_impl.test.ts +++ b/tests/unit/monitor_impl.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,61 +14,56 @@ limitations under the License. */ -import { MonitorImpl } from "../../common/lib/plugins/efm/monitor"; -import { anything, instance, mock, reset, spy, verify, when } from "ts-mockito"; +import { HostMonitorImpl } from "../../common/lib/plugins/efm/base/host_monitor"; +import { ConnectionContextImpl } from "../../common/lib/plugins/efm/base/connection_context"; +import { anything, instance, mock, reset, when } from "ts-mockito"; import { PluginService, PluginServiceImpl } from "../../common/lib/plugin_service"; import { HostInfo } from "../../common/lib"; -import { AwsClient } from "../../common/lib/aws_client"; -import { MonitorConnectionContext } from "../../common/lib/plugins/efm/monitor_connection_context"; import { sleep } from "../../common/lib/utils/utils"; import { ClientWrapper } from "../../common/lib/client_wrapper"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; -class MonitorImplTest extends MonitorImpl { +class MonitorImplTest extends HostMonitorImpl { constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { super(pluginService, hostInfo, properties, monitorDisposalTimeMillis); } - override startRun() { - // do nothing + override checkConnectionStatus() { + // Exposes the protected checkConnectionStatus method for testing purposes. + return super.checkConnectionStatus(); } } const mockPluginService = mock(PluginServiceImpl); const mockHostInfo = mock(HostInfo); -const mockClient = mock(AwsClient); const mockClientWrapper: ClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); const SHORT_INTERVAL_MILLIS = 30; const properties = new Map(); -let monitor: MonitorImpl; -let monitorSpy: MonitorImpl; +let monitor: MonitorImplTest; describe("monitor impl test", () => { beforeEach(() => { reset(mockPluginService); - when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); when(mockPluginService.forceConnect(anything(), anything())).thenResolve(mockClientWrapper); when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); monitor = new MonitorImplTest(instance(mockPluginService), instance(mockHostInfo), properties, 0); - monitorSpy = spy(monitor); }); - afterEach(() => { - monitor.releaseResources(); + afterEach(async () => { + await monitor.releaseResources(); }); it("is client healthy with no existing client", async () => { - const status = await monitor.checkConnectionStatus(); + const [isValid, elapsedTimeNano] = await monitor.checkConnectionStatus(); - verify(mockPluginService.forceConnect(anything(), anything())).once(); - expect(status.isValid).toBe(true); - expect(status.elapsedTimeNano).toBeGreaterThanOrEqual(0); + expect(isValid).toBe(true); + expect(elapsedTimeNano).toBeGreaterThanOrEqual(0); }); it("is client healthy with existing client", async () => { @@ -77,13 +72,11 @@ describe("monitor impl test", () => { // Start up a monitoring client. await monitor.checkConnectionStatus(); - const status1 = await monitor.checkConnectionStatus(); - expect(status1.isValid).toBe(true); + const [isValid1] = await monitor.checkConnectionStatus(); + expect(isValid1).toBe(true); - const status2 = await monitor.checkConnectionStatus(); - expect(status2.isValid).toBe(true); - - verify(mockPluginService.isClientValid(mockClientWrapper)).twice(); + const [isValid2] = await monitor.checkConnectionStatus(); + expect(isValid2).toBe(true); }); it("is client healthy with error", async () => { @@ -92,31 +85,23 @@ describe("monitor impl test", () => { // Start up a monitoring client. await monitor.checkConnectionStatus(); - const status = await monitor.checkConnectionStatus(); - expect(status.isValid).toBe(false); - expect(status.elapsedTimeNano).toBeGreaterThanOrEqual(0); + const [isValid, elapsedTimeNano] = await monitor.checkConnectionStatus(); + expect(isValid).toBe(false); + expect(elapsedTimeNano).toBeGreaterThanOrEqual(0); }); it("run without context", async () => { - // Should end by itself. + // Should end by itself (monitorDisposalTimeMillis = 0). await monitor.run(); - verify(monitorSpy.checkConnectionStatus()).never(); }); it("run with context", async () => { - const monitorContextInstance = new MonitorConnectionContext( - monitor, - mockClientWrapper, - 30000, - 5000, - 3, - instance(mockPluginService), - new NullTelemetryFactory().createCounter("name") - ); - monitor.startMonitoring(monitorContextInstance); - // Should end by itself. + when(mockPluginService.isClientValid(anything())).thenResolve(true); + + const context = new ConnectionContextImpl(mockClientWrapper, 30000, 5000, 3, new NullTelemetryFactory().createCounter("counter")); + monitor.startMonitoring(context); monitor.run(); await sleep(SHORT_INTERVAL_MILLIS); - monitor.stopMonitoring(monitorContextInstance); + monitor.stopMonitoring(context); }); }); diff --git a/tests/unit/monitor_service_impl.test.ts b/tests/unit/monitor_service_impl.test.ts index 676d8c2bd..4093196c3 100644 --- a/tests/unit/monitor_service_impl.test.ts +++ b/tests/unit/monitor_service_impl.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,203 +14,168 @@ limitations under the License. */ -import { anything, capture, instance, mock, reset, verify, when } from "ts-mockito"; -import { MonitorImpl } from "../../common/lib/plugins/efm/monitor"; -import { MonitorServiceImpl } from "../../common/lib/plugins/efm/monitor_service"; -import { PluginService, PluginServiceImpl } from "../../common/lib/plugin_service"; +import { instance, mock, reset, when } from "ts-mockito"; +import { HostMonitorServiceImpl } from "../../common/lib/plugins/efm/base/host_monitor_service"; +import { HostMonitorImpl } from "../../common/lib/plugins/efm/base/host_monitor"; +import { PluginServiceImpl } from "../../common/lib/plugin_service"; import { HostInfo, HostInfoBuilder } from "../../common/lib"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; - -class MonitorImplTest extends MonitorImpl { - constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { - super(pluginService, hostInfo, properties, monitorDisposalTimeMillis); - } - - override startRun() { - // do nothing - } -} - -const mockPluginService = mock(PluginServiceImpl); -const mockMonitorA = mock(MonitorImpl); -const mockMonitorB = mock(MonitorImpl); -const mockHostInfo = mock(HostInfo); -const mockClientWrapper = mock(MySQLClientWrapper); +import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { MonitorServiceImpl } from "../../common/lib/utils/monitoring/monitor_service"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; +import { BatchingEventPublisher } from "../../common/lib/utils/events/batching_event_publisher"; +import { WrapperProperties } from "../../common/lib/wrapper_property"; const FAILURE_DETECTION_TIME_MILLIS = 10; const FAILURE_DETECTION_INTERVAL_MILLIS = 100; const FAILURE_DETECTION_COUNT = 3; -const NODE_KEYS = new Set(["any"]); -const properties = new Map(); -let monitorService: MonitorServiceImpl; +const mockPluginService = mock(PluginServiceImpl); + +const properties: Map = new Map(); +properties.set(WrapperProperties.MONITOR_DISPOSAL_TIME_MS.name, 600000); + +let monitorService: HostMonitorServiceImpl; +let coreMonitorService: MonitorServiceImpl; +let servicesContainer: FullServicesContainer; +let eventPublisher: BatchingEventPublisher; describe("monitor service impl test", () => { beforeEach(() => { reset(mockPluginService); - reset(mockMonitorA); - when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); - monitorService = new MonitorServiceImpl(instance(mockPluginService)); - monitorService.monitorSupplier = () => new MonitorImplTest(instance(mockPluginService), instance(mockHostInfo), properties, 0); + const telemetryFactory = new NullTelemetryFactory(); + when(mockPluginService.getTelemetryFactory()).thenReturn(telemetryFactory); + when(mockPluginService.isClientValid(undefined)).thenResolve(false); + + eventPublisher = new BatchingEventPublisher(60_000); + coreMonitorService = new MonitorServiceImpl(eventPublisher); + + servicesContainer = { + pluginService: instance(mockPluginService), + monitorService: coreMonitorService, + telemetryFactory: telemetryFactory + } as unknown as FullServicesContainer; + + monitorService = new HostMonitorServiceImpl(servicesContainer); }); afterEach(async () => { await monitorService.releaseResources(); + await coreMonitorService.releaseResources(); + eventPublisher.releaseResources(); }); - it("start monitoring", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); + it("start monitoring creates context", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - await monitorService.startMonitoring( + const context = await monitorService.startMonitoring( mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), + hostInfo, + properties, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT ); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg).toBeDefined(); - expect(arg).not.toBeNull(); - }); - it("start monitoring called multiple times", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); - - const runs = 5; - - for (let i = 0; i < runs; i++) { - await monitorService.startMonitoring( - mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), - FAILURE_DETECTION_TIME_MILLIS, - FAILURE_DETECTION_INTERVAL_MILLIS, - FAILURE_DETECTION_COUNT - ); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg).toBeDefined(); - expect(arg).not.toBeNull(); - } + expect(context).toBeDefined(); + expect(context).not.toBeNull(); + expect(context.isActiveContext()).toBe(true); + expect(context.isHostUnhealthy()).toBe(false); }); - it("start and stop monitoring", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); + it("stop monitoring sets context inactive", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); const context = await monitorService.startMonitoring( mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), + hostInfo, + properties, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT ); - await monitorService.stopMonitoring(context); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg[0]).toBe(context); - verify(mockMonitorA.stopMonitoring(anything())).once(); + monitorService.stopMonitoring(context); + expect(context.isActiveContext()).toBe(false); }); - it("stop monitoring called twice", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); + it("start monitoring reuses monitor for same host", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - const context = await monitorService.startMonitoring( + await monitorService.startMonitoring( mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), + hostInfo, + properties, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT ); - await monitorService.stopMonitoring(context); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg[0]).toBe(context); + const monitor1 = coreMonitorService.get(HostMonitorImpl, hostInfo.host); - await monitorService.stopMonitoring(context); - verify(mockMonitorA.stopMonitoring(anything())).twice(); - }); + await monitorService.startMonitoring( + mockClientWrapper, + hostInfo, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT + ); - it("stop monitoring for all connections with invalid node keys", async () => { - monitorService.stopMonitoringForAllConnections(new Set()); - monitorService.stopMonitoringForAllConnections(new Set(["foo"])); + const monitor2 = coreMonitorService.get(HostMonitorImpl, hostInfo.host); + expect(monitor2).toBe(monitor1); }); - it("stop monitoring for all connections", async () => { - const keysA = new Set(["monitorA"]); - const keysB = new Set(["monitorB"]); + it("start monitoring creates separate monitors for different hosts", async () => { + const hostInfoA = new HostInfoBuilder({ host: "host-a", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const hostInfoB = new HostInfoBuilder({ host: "host-b", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - monitorService.monitorSupplier = () => instance(mockMonitorA); - await monitorService.getMonitor( - keysA, - new HostInfoBuilder({ host: "test", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(), - properties + await monitorService.startMonitoring( + mockClientWrapper, + hostInfoA, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT ); - monitorService.monitorSupplier = () => instance(mockMonitorB); - await monitorService.getMonitor( - keysB, - new HostInfoBuilder({ host: "test", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(), - properties + await monitorService.startMonitoring( + mockClientWrapper, + hostInfoB, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT ); - monitorService.stopMonitoringForAllConnections(keysA); - verify(mockMonitorA.clearContexts()).once(); - - monitorService.stopMonitoringForAllConnections(keysB); - verify(mockMonitorB.clearContexts()).once(); + const monitorA = coreMonitorService.get(HostMonitorImpl, hostInfoA.host); + const monitorB = coreMonitorService.get(HostMonitorImpl, hostInfoB.host); + expect(monitorA).not.toBe(monitorB); }); - it("getMonitor called with multiple hosts in keys", async () => { - const keysA = new Set(["host1.domain", "host2.domain"]); - const keysB = new Set(["host2.domain"]); - - const monitorOne = await monitorService.getMonitor(keysA, instance(mockHostInfo), properties); - expect(monitorOne).not.toBeNull(); - - const monitorOneDupe = await monitorService.getMonitor(keysB, instance(mockHostInfo), properties); - expect(monitorOneDupe).toBe(monitorOne); - }); - - it("getMonitor called with different host keys", async () => { - const keys = new Set(["hostNEW.domain"]); - - const monitorOne = await monitorService.getMonitor(keys, instance(mockHostInfo), properties); - expect(monitorOne).not.toBeNull(); + it("release resources clears monitors", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - const monitorOneDupe = await monitorService.getMonitor(keys, instance(mockHostInfo), properties); - expect(monitorOneDupe).toBe(monitorOne); - - const monitorTwo = await monitorService.getMonitor(NODE_KEYS, instance(mockHostInfo), properties); - expect(monitorTwo).not.toBeNull(); - expect(monitorTwo).not.toBe(monitorOne); - }); - - it("getMonitor called with same keys in different sets", async () => { - const keysA = new Set(["hostA"]); - const keysB = new Set(["hostA", "hostB"]); - const keysC = new Set(["hostB"]); - - const monitorOne = await monitorService.getMonitor(keysA, instance(mockHostInfo), properties); - expect(monitorOne).not.toBeNull(); - - const monitorOneDupe = await monitorService.getMonitor(keysB, instance(mockHostInfo), properties); - expect(monitorOneDupe).toBe(monitorOne); + await monitorService.startMonitoring( + mockClientWrapper, + hostInfo, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT + ); - const monitorOneDupeTwo = await monitorService.getMonitor(keysC, instance(mockHostInfo), properties); - expect(monitorOneDupeTwo).toBe(monitorOne); - }); + await monitorService.releaseResources(); - it("startMonitoring with no host keys", async () => { - const keys: Set = new Set(); - expect(await monitorService.getMonitor(keys, instance(mockHostInfo), properties)).toBeNull(); + const monitor = coreMonitorService.get(HostMonitorImpl, hostInfo.host); + expect(monitor).toBeNull(); }); }); diff --git a/tests/unit/notification_pipeline.test.ts b/tests/unit/notification_pipeline.test.ts index cb1d315e1..0d4d95c5e 100644 --- a/tests/unit/notification_pipeline.test.ts +++ b/tests/unit/notification_pipeline.test.ts @@ -16,7 +16,6 @@ import { HostChangeOptions } from "../../common/lib/host_change_options"; import { OldConnectionSuggestionAction } from "../../common/lib/old_connection_suggestion_action"; -import { PluginServiceManagerContainer } from "../../common/lib/plugin_service_manager_container"; import { DefaultPlugin } from "../../common/lib/plugins/default_plugin"; import { instance, mock } from "ts-mockito"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; @@ -24,6 +23,7 @@ import { DriverConnectionProvider } from "../../common/lib/driver_connection_pro import { ConnectionProviderManager } from "../../common/lib/connection_provider_manager"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { PluginManager } from "../../common/lib"; +import { FullServicesContainer, FullServicesContainerImpl } from "../../common/lib/utils/full_services_container"; class TestPlugin extends DefaultPlugin { counter: number = 0; @@ -43,7 +43,7 @@ class TestPlugin extends DefaultPlugin { } } -const container: PluginServiceManagerContainer = new PluginServiceManagerContainer(); +const container: FullServicesContainer = mock(FullServicesContainerImpl); const props: Map = mock(Map); const hostListChanges: Map> = mock(Map>); const connectionChanges: Set = mock(Set); @@ -64,6 +64,10 @@ describe("notificationPipelineTest", () => { pluginManager["_plugins"] = [plugin]; }); + afterEach(async () => { + await PluginManager.releaseResources(); + }); + it("test_notifyConnectionChanged", async () => { const result: Set = await pluginManager.notifyConnectionChanged(connectionChanges, null); expect(plugin.counter).toBe(1); diff --git a/tests/unit/okta_auth_plugin.test.ts b/tests/unit/okta_auth_plugin.test.ts deleted file mode 100644 index 59a3c2811..000000000 --- a/tests/unit/okta_auth_plugin.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { anything, instance, mock, spy, verify, when } from "ts-mockito"; -import { PluginServiceImpl } from "../../common/lib/plugin_service"; -import { CredentialsProviderFactory } from "../../common/lib/plugins/federated_auth/credentials_provider_factory"; -import { IamAuthUtils, TokenInfo } from "../../common/lib/utils/iam_auth_utils"; -import { HostInfo, PluginManager } from "../../common/lib"; -import { WrapperProperties } from "../../common/lib/wrapper_property"; -import { DatabaseDialect } from "../../common/lib/database_dialect/database_dialect"; - -import { OktaAuthPlugin } from "../../common/lib/plugins/federated_auth/okta_auth_plugin"; -import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; -import { jest } from "@jest/globals"; -import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; - -const defaultPort = 1234; -const host = "pg.testdb.us-east-2.rds.amazonaws.com"; -const iamHost = "pg-123.testdb.us-east-2.rds.amazonaws.com"; -const hostInfo = new HostInfo(host, defaultPort); -const dbUser = "iamUser"; -const region = "us-east-2"; -const testToken = "someTestToken"; -const testTokenInfo = new TokenInfo(testToken, Date.now() + 300000); - -const mockPluginService = mock(PluginServiceImpl); -const mockDialect = mock(); -const mockDialectInstance = instance(mockDialect); -const testCredentials = { - accessKeyId: "foo", - secretAccessKey: "bar", - sessionToken: "baz" -}; -const spyIamUtils = spy(IamAuthUtils); -const mockCredentialsProviderFactory = mock(); -const mockConnectFunc = jest.fn(() => { - return Promise.resolve(mock(PgClientWrapper)); -}); - -describe("oktaAuthTest", () => { - let spyPlugin: OktaAuthPlugin; - let props: Map; - - beforeEach(() => { - when(mockPluginService.getDialect()).thenReturn(mockDialectInstance); - when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); - when(mockDialect.getDefaultPort()).thenReturn(defaultPort); - when(mockCredentialsProviderFactory.getAwsCredentialsProvider(anything(), anything(), anything())).thenResolve(testCredentials); - props = new Map(); - WrapperProperties.PLUGINS.set(props, "okta"); - WrapperProperties.DB_USER.set(props, dbUser); - spyPlugin = spy(new OktaAuthPlugin(instance(mockPluginService), instance(mockCredentialsProviderFactory))); - }); - - afterEach(async () => { - await PluginManager.releaseResources(); - }); - - it("testCachedToken", async () => { - const spyPluginInstance = instance(spyPlugin); - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - - OktaAuthPlugin["tokenCache"].set(key, testTokenInfo); - - await spyPluginInstance.connect(hostInfo, props, false, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testExpiredCachedToken", async () => { - const spyPluginInstance = instance(spyPlugin); - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - - const someExpiredToken = "someExpiredToken"; - const expiredTokenInfo = new TokenInfo(someExpiredToken, Date.now() - 300000); - - OktaAuthPlugin["tokenCache"].set(key, expiredTokenInfo); - - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, false, mockConnectFunc); - - verify( - spyIamUtils.generateAuthenticationToken(hostInfo.host, defaultPort, region, dbUser, testCredentials, instance(mockPluginService)) - ).called(); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testNoCachedToken", async () => { - const spyPluginInstance = instance(spyPlugin); - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - verify( - spyIamUtils.generateAuthenticationToken(hostInfo.host, defaultPort, region, dbUser, testCredentials, instance(mockPluginService)) - ).called(); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testSpecifiedIamHostPortRegion", async () => { - const spyPluginInstance = instance(spyPlugin); - const expectedHost = "pg.testdb.us-west-2.rds.amazonaws.com"; - const expectedPort = 9876; - const expectedRegion = "us-west-2"; - - WrapperProperties.IAM_HOST.set(props, expectedHost); - WrapperProperties.IAM_DEFAULT_PORT.set(props, expectedPort); - WrapperProperties.IAM_REGION.set(props, expectedRegion); - - const key = `us-west-2:pg.testdb.us-west-2.rds.amazonaws.com:${expectedPort}:iamUser`; - - OktaAuthPlugin["tokenCache"].set(key, testTokenInfo); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testIdpCredentialsFallback", async () => { - const spyPluginInstance = instance(spyPlugin); - const expectedUser = "expectedUser"; - const expectedPassword = "expectedPassword"; - - WrapperProperties.USER.set(props, expectedUser); - WrapperProperties.PASSWORD.set(props, expectedPassword); - - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - OktaAuthPlugin["tokenCache"].set(key, testTokenInfo); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - expect(expectedUser).toBe(WrapperProperties.IDP_USERNAME.get(props)); - expect(expectedPassword).toBe(WrapperProperties.IDP_PASSWORD.get(props)); - }); - - it("testUsingIamHost", async () => { - WrapperProperties.IAM_HOST.set(props, iamHost); - const spyPluginInstance = instance(spyPlugin); - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - verify(spyIamUtils.generateAuthenticationToken(iamHost, defaultPort, region, dbUser, testCredentials, instance(mockPluginService))).called(); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); -}); diff --git a/tests/unit/plugin_service.test.ts b/tests/unit/plugin_service.test.ts index 2d6b145bf..2b14f31cc 100644 --- a/tests/unit/plugin_service.test.ts +++ b/tests/unit/plugin_service.test.ts @@ -26,7 +26,7 @@ import { HostInfoBuilder } from "../../common/lib"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; import { DatabaseDialectCodes } from "../../common/lib/database_dialect/database_dialect_codes"; import { AwsClient } from "../../common/lib/aws_client"; -import { PluginServiceManagerContainer } from "../../common/lib/plugin_service_manager_container"; +import { FullServicesContainer, FullServicesContainerImpl } from "../../common/lib/utils/full_services_container"; import { AllowedAndBlockedHosts } from "../../common/lib/allowed_and_blocked_hosts"; import { DatabaseType } from "../../common/lib/database_dialect/database_dialect"; @@ -64,7 +64,7 @@ let pluginService: TestPluginService; describe("testCustomEndpoint", () => { beforeEach(() => { pluginService = new TestPluginService( - new PluginServiceManagerContainer(), + mock(FullServicesContainerImpl), mockAwsClient, DatabaseType.MYSQL, knownDialectsByCode, diff --git a/tests/unit/rds_host_list_provider.test.ts b/tests/unit/rds_host_list_provider.test.ts index df3b34c8b..6f675a8c8 100644 --- a/tests/unit/rds_host_list_provider.test.ts +++ b/tests/unit/rds_host_list_provider.test.ts @@ -14,24 +14,32 @@ limitations under the License. */ -import { RdsHostListProvider } from "../../common/lib/host_list_provider/rds_host_list_provider"; import { anything, instance, mock, reset, spy, verify, when } from "ts-mockito"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; import { AwsClient } from "../../common/lib/aws_client"; -import { AwsWrapperError, HostInfo, HostInfoBuilder, HostRole } from "../../common/lib"; +import { AwsWrapperError, HostInfo, HostInfoBuilder, PluginManager } from "../../common/lib"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { ConnectionUrlParser } from "../../common/lib/utils/connection_url_parser"; import { AwsPGClient } from "../../pg/lib"; -import { sleep } from "../../common/lib/utils/utils"; +import { convertMsToNanos, sleep } from "../../common/lib/utils/utils"; import { AuroraPgDatabaseDialect } from "../../pg/lib/dialect/aurora_pg_database_dialect"; import { ClientWrapper } from "../../common/lib/client_wrapper"; import { PgConnectionUrlParser } from "../../pg/lib/pg_connection_url_parser"; import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; +import { CoreServicesContainer } from "../../common/lib/utils/core_services_container"; +import { StorageService } from "../../common/lib/utils/storage/storage_service"; +import { Topology } from "../../common/lib/host_list_provider/topology"; +import { RdsHostListProvider } from "../../common/lib/host_list_provider/rds_host_list_provider"; +import { TopologyQueryResult, TopologyUtils } from "../../common/lib/host_list_provider/topology_utils"; +import { FullServicesContainerImpl } from "../../common/lib/utils/full_services_container"; const mockClient: AwsClient = mock(AwsPGClient); const mockDialect: AuroraPgDatabaseDialect = mock(AuroraPgDatabaseDialect); +const mockServiceContainer: FullServicesContainerImpl = mock(FullServicesContainerImpl); const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); const connectionUrlParser: ConnectionUrlParser = new PgConnectionUrlParser(); +const mockTopologyUtils: TopologyUtils = mock(TopologyUtils); +const storageService: StorageService = CoreServicesContainer.getInstance().storageService; const hosts: HostInfo[] = [ createHost({ @@ -51,18 +59,15 @@ const currentHostInfo = createHost({ }); const clientWrapper: ClientWrapper = new PgClientWrapper(undefined, currentHostInfo, new Map()); - const mockClientWrapper: ClientWrapper = mock(clientWrapper); -const defaultRefreshRateNano: number = 5 * 1_000_000_000; - function createHost(config: any): HostInfo { const info = new HostInfoBuilder(config); return info.build(); } function getRdsHostListProvider(originalHost: string): RdsHostListProvider { - const provider = new RdsHostListProvider(new Map(), originalHost, instance(mockPluginService)); + const provider = new RdsHostListProvider(new Map(), originalHost, instance(mockTopologyUtils), instance(mockServiceContainer)); provider.init(); return provider; } @@ -75,11 +80,15 @@ describe("testRdsHostListProvider", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(currentHostInfo); when(mockPluginService.getConnectionUrlParser()).thenReturn(connectionUrlParser); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); + when(mockClient.targetClient).thenReturn(mockClientWrapper); when(mockPluginService.getHostInfoBuilder()).thenReturn(new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() })); + when(mockServiceContainer.hostListProviderService).thenReturn(instance(mockPluginService)); + when(mockServiceContainer.pluginService).thenReturn(instance(mockPluginService)); + when(mockServiceContainer.storageService).thenReturn(storageService); }); - afterEach(() => { - RdsHostListProvider.clearAll(); + afterEach(async () => { + await PluginManager.releaseResources(); reset(mockDialect); reset(mockClientWrapper); @@ -92,13 +101,13 @@ describe("testRdsHostListProvider", () => { const spiedProvider = spy(rdsHostListProvider); const expected: HostInfo[] = hosts; - RdsHostListProvider.topologyCache.put(rdsHostListProvider.clusterId, expected, defaultRefreshRateNano); + storageService.set(rdsHostListProvider.clusterId, new Topology(expected)); - const result = await rdsHostListProvider.getTopology(mockClientWrapper, false); + const result = await rdsHostListProvider.getTopology(); expect(result.hosts.length).toEqual(2); expect(result.hosts).toEqual(expected); - verify(spiedProvider.queryForTopology(anything(), anything())).never(); + verify(spiedProvider.getCurrentTopology(anything(), anything())).never(); }); it("testGetTopology_withForceUpdate_returnsUpdatedTopology", async () => { @@ -108,7 +117,6 @@ describe("testRdsHostListProvider", () => { when(mockPluginService.isClientValid(anything())).thenResolve(true); - RdsHostListProvider.topologyCache.put(rdsHostListProvider.clusterId, hosts, defaultRefreshRateNano); const newHosts: HostInfo[] = [ createHost({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), @@ -117,13 +125,12 @@ describe("testRdsHostListProvider", () => { ]; when(mockClient.isValid()).thenResolve(true); - when(spiedProvider.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(newHosts)); + when(mockPluginService.isDialectConfirmed()).thenReturn(true); + when((spiedProvider as any).forceRefreshMonitor(anything(), anything())).thenReturn(Promise.resolve(newHosts)); - const result = await rdsHostListProvider.getTopology(mockClientWrapper, true); + const result = await rdsHostListProvider.getTopology(); expect(result.hosts.length).toEqual(1); expect(result.hosts).toEqual(newHosts); - - verify(spiedProvider.queryForTopology(anything(), anything())).atMost(1); }); it("testGetTopology_noForceUpdate_queryReturnsEmptyHostList", async () => { @@ -133,13 +140,13 @@ describe("testRdsHostListProvider", () => { spiedProvider.isInitialized = true; const expected: HostInfo[] = hosts; - RdsHostListProvider.topologyCache.put(rdsHostListProvider.clusterId, expected, defaultRefreshRateNano); - when(spiedProvider.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); + storageService.set(rdsHostListProvider.clusterId, new Topology(expected)); + when(spiedProvider.getCurrentTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); - const result = await rdsHostListProvider.getTopology(mockClientWrapper, false); + const result = await rdsHostListProvider.getTopology(); expect(result.hosts.length).toEqual(2); expect(result.hosts).toEqual(expected); - verify(spiedProvider.queryForTopology(anything(), anything())).atMost(1); + verify(spiedProvider.getCurrentTopology(anything(), anything())).atMost(1); }); it("testGetTopology_withForceUpdate_returnsInitialHostList", async () => { @@ -154,253 +161,58 @@ describe("testRdsHostListProvider", () => { const spiedProvider = spy(rdsHostListProvider); spiedProvider.clear(); - when(spiedProvider.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); + when(spiedProvider.getCurrentTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); - const result = await rdsHostListProvider.getTopology(mockClientWrapper, true); + const result = await rdsHostListProvider.getTopology(); expect(result.hosts).toBeTruthy(); for (let i = 0; i < result.hosts.length; i++) { expect(result.hosts[i].equals(initialHosts[i])).toBeTruthy(); } - verify(spiedProvider.queryForTopology(anything(), anything())).atMost(1); - }); - - it("testQueryForTopology_queryResultsInError", async () => { - const rdsHostListProvider = getRdsHostListProvider("someUrl"); - when(mockDialect.queryForTopology(anything(), anything())).thenThrow(new AwsWrapperError("bad things")); - - await expect(rdsHostListProvider.queryForTopology(instance(mockClientWrapper), instance(mockDialect))).rejects.toThrow(AwsWrapperError); + verify(spiedProvider.getCurrentTopology(anything(), anything())).atMost(1); }); it("testGetCachedTopology_returnCachedTopology", () => { const rdsHostListProvider = getRdsHostListProvider("foo"); const expected: HostInfo[] = hosts; - RdsHostListProvider.topologyCache.put(rdsHostListProvider.clusterId, expected, defaultRefreshRateNano); + storageService.set(rdsHostListProvider.clusterId, new Topology(expected)); - const result = rdsHostListProvider.getCachedTopology(); + const result = rdsHostListProvider.getStoredTopology(); expect(result).toEqual(expected); }); it("testGetCachedTopology_returnNull", async () => { let rdsHostListProvider = getRdsHostListProvider("foo"); - expect(rdsHostListProvider.getCachedTopology()).toBeNull(); + expect(rdsHostListProvider.getStoredTopology()).toBeNull(); rdsHostListProvider.clear(); rdsHostListProvider = getRdsHostListProvider("foo"); - RdsHostListProvider.topologyCache.put(rdsHostListProvider.clusterId, hosts, 1_000_000); - await sleep(2); - - expect(rdsHostListProvider.getCachedTopology()).toBeNull(); - }); - - it("testTopologyCache_noSuggestedClusterId", async () => { - RdsHostListProvider.clearAll(); - - when(mockPluginService.isClientValid(anything())).thenResolve(true); - - const provider1 = getRdsHostListProvider("cluster-a.xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider1 = spy(provider1); - - const topologyClusterA: HostInfo[] = [ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-1.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.WRITER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-2.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-3.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }) - ]; - - when(spiedProvider1.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(RdsHostListProvider.topologyCache.size()).toEqual(0); - - const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); - expect(topologyProvider1).toEqual(topologyClusterA); - - const provider2 = getRdsHostListProvider("cluster-b.xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider2 = spy(provider2); - - const topologyClusterB: HostInfo[] = [ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-b-1.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.WRITER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-b-2.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-b-3.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }) - ]; - when(spiedProvider2.queryForTopology(instance(mockClientWrapper), anything())).thenReturn(Promise.resolve(topologyClusterB)); - - expect(await provider2.refresh(instance(mockClientWrapper))).toEqual(topologyClusterB); - expect(RdsHostListProvider.topologyCache.size()).toEqual(2); - }); - - it("testTopologyCache_suggestedClusterIdForRds", async () => { - RdsHostListProvider.clearAll(); - - when(mockPluginService.isClientValid(anything())).thenResolve(true); - - const topologyClusterA: HostInfo[] = [ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-1.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.WRITER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-2.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-3.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }) - ]; - - const provider1 = getRdsHostListProvider("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider1 = spy(provider1); - - when(spiedProvider1.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(RdsHostListProvider.topologyCache.size()).toEqual(0); - - const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); - expect(topologyProvider1).toEqual(topologyClusterA); - - const provider2 = getRdsHostListProvider("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); - - expect(provider2.clusterId).toEqual(provider1.clusterId); - expect(provider1.isPrimaryClusterId).toBeTruthy(); - expect(provider2.isPrimaryClusterId).toBeTruthy(); - - expect(await provider2.refresh(mockClientWrapper)).toEqual(topologyClusterA); - expect(RdsHostListProvider.topologyCache.size()).toEqual(1); - }); - - it("testTopologyCache_suggestedClusterIdForInstance", async () => { - RdsHostListProvider.clearAll(); - when(mockPluginService.isClientValid(anything())).thenResolve(true); - - const topologyClusterA: HostInfo[] = [ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-1.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.WRITER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-2.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-3.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }) - ]; - - const provider1 = getRdsHostListProvider("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider1 = spy(provider1); - - when(spiedProvider1.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(RdsHostListProvider.topologyCache.size()).toEqual(0); - - const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); - expect(topologyProvider1).toEqual(topologyClusterA); - - const provider2 = getRdsHostListProvider("instance-a-3.xyz.us-east-2.rds.amazonaws.com"); - - expect(provider2.clusterId).toEqual(provider1.clusterId); - expect(provider1.isPrimaryClusterId).toBeTruthy(); - expect(provider2.isPrimaryClusterId).toBeTruthy(); - - expect(await provider2.refresh(mockClientWrapper)).toEqual(topologyClusterA); - expect(RdsHostListProvider.topologyCache.size()).toEqual(1); - }); - - it("testTopologyCache_acceptSuggestion", async () => { - RdsHostListProvider.clearAll(); - - when(mockPluginService.isClientValid(anything())).thenResolve(true); - - const topologyClusterA: HostInfo[] = [ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-1.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.WRITER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-2.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-3.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }) - ]; - - const provider1 = getRdsHostListProvider("instance-a-2.xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider1 = spy(provider1); - - when(spiedProvider1.queryForTopology(anything(), anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(RdsHostListProvider.topologyCache.size()).toEqual(0); - - const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); - expect(topologyProvider1).toEqual(topologyClusterA); - - const provider2 = getRdsHostListProvider("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider2 = spy(provider2); - - when(spiedProvider2.queryForTopology(anything(), anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(provider2.clusterId).not.toEqual(provider1.clusterId); - expect(provider1.isPrimaryClusterId).toBeFalsy(); - expect(provider2.isPrimaryClusterId).toBeTruthy(); - - expect(await provider2.refresh(instance(mockClientWrapper))).toEqual(topologyClusterA); - expect(RdsHostListProvider.topologyCache.size()).toEqual(2); - expect(RdsHostListProvider.suggestedPrimaryClusterIdCache.get(provider1.clusterId)).toEqual("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); + storageService.registerItemClassIfAbsent(Topology, true, convertMsToNanos(1)); + storageService.set(rdsHostListProvider.clusterId, new Topology(hosts)); + await sleep(2); - expect(await provider1.forceRefresh(instance(mockClientWrapper))).toEqual(topologyClusterA); - expect(provider2.clusterId).toEqual(provider1.clusterId); - expect(RdsHostListProvider.topologyCache.size()).toEqual(2); - expect(provider1.isPrimaryClusterId).toBeTruthy(); - expect(provider2.isPrimaryClusterId).toBeTruthy(); + expect(rdsHostListProvider.getStoredTopology()).toBeNull(); }); it("testIdentifyConnectionWithInvalidHostIdQuery", async () => { - when(mockDialect.identifyConnection(anything())).thenThrow(new AwsWrapperError("bad things")); + when(mockTopologyUtils.getInstanceId(anything())).thenThrow(new AwsWrapperError("bad things")); const rdsHostListProvider = getRdsHostListProvider("foo"); - await expect(rdsHostListProvider.identifyConnection(instance(mockClientWrapper), instance(mockDialect))).rejects.toThrow(AwsWrapperError); + await expect(rdsHostListProvider.identifyConnection(instance(mockClientWrapper))).rejects.toThrow(AwsWrapperError); }); it("testIdentifyConnectionHostInTopology", async () => { - when(mockDialect.queryForTopology(anything(), anything())).thenResolve([ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-1" + when(mockDialect.queryForTopology(anything())).thenResolve([ + new TopologyQueryResult({ + host: "instance-1", + isWriter: true, + weight: HostInfo.DEFAULT_WEIGHT, + lastUpdateTime: Date.now() }) ]); + when(await mockTopologyUtils.getInstanceId(anything())).thenReturn(["instance1", "instance1"]); const rdsHostListProvider = getRdsHostListProvider("foo"); const spiedProvider = spy(rdsHostListProvider); @@ -410,55 +222,7 @@ describe("testRdsHostListProvider", () => { }); when(spiedProvider.refresh()).thenReturn(Promise.resolve([])); - const res = await rdsHostListProvider.identifyConnection(instance(mockClientWrapper), instance(mockDialect)); + const res = await rdsHostListProvider.identifyConnection(instance(mockClientWrapper)); expect(res).toBeNull(); }); - - it("testGetTopology_staleRecord", async () => { - const hostName1: string = "hostName1"; - const hostName2: string = "hostName2"; - const cpuUtilization: number = 11.1; - const lag: number = 0.123; - const firstTimestamp: number = Date.now(); - const secondTimestamp: number = firstTimestamp + 100; - const weight = Math.round(lag) * 100 + Math.round(cpuUtilization); - - const expectedWriter: HostInfo = createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: hostName2, - weight: weight, - lastUpdateTime: secondTimestamp - }); - - when(mockDialect.queryForTopology(anything(), anything())).thenResolve([ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: hostName1, - role: HostRole.WRITER, - weight: Math.round(lag) * 100 + Math.round(cpuUtilization), - lastUpdateTime: firstTimestamp - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: hostName2, - role: HostRole.WRITER, - weight: Math.round(lag) * 100 + Math.round(cpuUtilization), - lastUpdateTime: secondTimestamp - }) - ]); - - when(mockPluginService.isClientValid(anything())).thenResolve(true); - when(mockPluginService.getDialect()).thenReturn(instance(mockDialect)); - - const rdsHostListProvider = getRdsHostListProvider("foo"); - const spiedProvider = spy(rdsHostListProvider); - rdsHostListProvider.isInitialized = false; - - when(spiedProvider.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); - - const result = await rdsHostListProvider.getTopology(instance(mockClientWrapper), true); - verify(spiedProvider.queryForTopology(anything(), anything())).atMost(1); - expect(result.hosts.length).toEqual(1); - expect(result.hosts[0].equals(expectedWriter)).toBeTruthy(); - }); }); diff --git a/tests/unit/read_write_splitting.test.ts b/tests/unit/read_write_splitting.test.ts index 2c4f1388b..775ca5836 100644 --- a/tests/unit/read_write_splitting.test.ts +++ b/tests/unit/read_write_splitting.test.ts @@ -29,7 +29,7 @@ import { PluginServiceImpl } from "../../common/lib/plugin_service"; import { AwsMySQLClient } from "../../mysql/lib"; import { anything, instance, mock, reset, spy, verify, when } from "ts-mockito"; import { HostListProviderService } from "../../common/lib/host_list_provider_service"; -import { ReadWriteSplittingPlugin } from "../../common/lib/plugins/read_write_splitting_plugin"; +import { ReadWriteSplittingPlugin } from "../../common/lib/plugins/read_write_splitting/read_write_splitting_plugin"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { MySQLDatabaseDialect } from "../../mysql/lib/dialect/mysql_database_dialect"; import { HostChangeOptions } from "../../common/lib/host_change_options"; @@ -82,6 +82,16 @@ const mockExecuteFuncThrowsFailoverSuccessError = jest.fn(() => { throw new FailoverSuccessError("test"); }); +class TestReadWriteSplitting extends ReadWriteSplittingPlugin { + getWriterTargetClient(): ClientWrapper | undefined { + return this.writerTargetClient; + } + + getReaderTargetClient(): ClientWrapper | undefined { + return this.readerCacheItem?.get(); + } +} + describe("reader write splitting test", () => { beforeEach(() => { when(mockPluginService.getHostListProvider()).thenReturn(instance(mockHostListProvider)); @@ -92,6 +102,8 @@ describe("reader write splitting test", () => { when(mockDriverDialect.connect(anything(), anything())).thenReturn(Promise.resolve(mockReaderWrapper)); when(mockDriverDialect.getQueryFromMethodArg("test")).thenReturn("test"); when(mockPluginService.getDriverDialect()).thenReturn(instance(mockDriverDialect)); + when(mockWriterClient.targetClient).thenReturn(instance(mockWriterWrapper)); + when(mockReaderClient.targetClient).thenReturn(instance(mockReaderWrapper)); properties.clear(); }); @@ -110,13 +122,14 @@ describe("reader write splitting test", () => { it("test set read only true", async () => { const mockPluginServiceInstance = instance(mockPluginService); when(mockPluginService.getAllHosts()).thenReturn(singleReaderTopology); - when(mockPluginService.getHostInfoByStrategy(anything(), anything())).thenReturn(readerHost1); + when(mockPluginService.getHostInfoByStrategy(anything(), anything(), anything())).thenReturn(readerHost1); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockWriterClient)); + when(await mockPluginService.isClientValid(anything())).thenReturn(true); when(await mockWriterClient.isValid()).thenReturn(true); when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(mockPluginService.connect(anything(), anything(), anything())).thenResolve(mockReaderWrapper); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -127,7 +140,7 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(true); verify(mockPluginService.refreshHostList()).once(); verify(mockPluginService.setCurrentClient(mockReaderWrapper, readerHost1)).once(); - expect(target.readerTargetClient).toBe(mockReaderWrapper); + expect(target.getReaderTargetClient()).toBe(mockReaderWrapper); }); it("test set read only false", async () => { @@ -140,7 +153,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(readerHost1); when(mockPluginService.connect(anything(), anything(), anything())).thenResolve(mockWriterWrapper); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -150,7 +163,7 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(false); verify(mockPluginService.setCurrentClient(mockWriterWrapper, writerHost)).once(); - expect(target.writerTargetClient).toEqual(mockWriterWrapper); + expect(target.getWriterTargetClient()).toEqual(mockWriterWrapper); }); it("test set read only true already on reader", async () => { @@ -164,7 +177,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(readerHost1); when(mockPluginService.connect(anything(), anything(), anything())).thenResolve(mockReaderWrapper); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderServiceInstance, @@ -174,8 +187,8 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(true); verify(mockPluginService.setCurrentClient(anything(), anything())).never(); - expect(target.readerTargetClient).toEqual(mockReaderWrapper); - expect(target.writerTargetClient).toEqual(undefined); + expect(target.getReaderTargetClient()).toEqual(mockReaderWrapper); + expect(target.getWriterTargetClient()).toEqual(undefined); }); it("test set read only false already on reader", async () => { @@ -188,7 +201,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(mockPluginService.connect(anything(), anything(), anything())).thenResolve(mockReaderWrapper); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderServiceInstance, @@ -198,8 +211,8 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(false); verify(mockPluginService.setCurrentClient(anything(), anything())).never(); - expect(target.writerTargetClient).toEqual(mockWriterWrapper); - expect(target.readerTargetClient).toEqual(undefined); + expect(target.getWriterTargetClient()).toEqual(mockWriterWrapper); + expect(target.getReaderTargetClient()).toEqual(undefined); }); it("test set read only true one host", async () => { @@ -214,7 +227,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(mockPluginService.connect(anything(), anything(), anything())).thenReturn(Promise.resolve(mockWriterWrapper)); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -225,8 +238,8 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(true); verify(mockPluginService.setCurrentClient(anything(), anything())).never(); - expect(target.readerTargetClient).toEqual(undefined); - expect(target.writerTargetClient).toEqual(mockWriterWrapper); + expect(target.getReaderTargetClient()).toEqual(undefined); + expect(target.getWriterTargetClient()).toEqual(mockWriterWrapper); }); it("test connect incorrect host role", async () => { @@ -239,10 +252,10 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(readerHost1); when(mockPluginService.acceptsStrategy(anything(), anything())).thenReturn(true); - when(mockHostListProviderService.isStaticHostListProvider()).thenReturn(false); + when(mockHostListProviderService.isDynamicHostListProvider()).thenReturn(true); when(mockHostListProviderService.getHostListProvider()).thenReturn(mockHostListProviderInstance); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderServiceInstance, @@ -264,7 +277,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(readerHost1); when(await mockPluginService.connect(writerHost, properties)).thenReject(); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -287,7 +300,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(mockPluginService.connect(readerHost1 || readerHost2, properties)).thenReject(); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderServiceInstance, @@ -297,7 +310,7 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(true); verify(mockPluginService.setCurrentClient(anything(), anything())).never(); - expect(target.readerTargetClient).toEqual(undefined); + expect(target.getReaderTargetClient()).toEqual(undefined); }); it("test set read only on closed connection", async () => { @@ -308,7 +321,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClosedWriterClient)); when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -318,7 +331,7 @@ describe("reader write splitting test", () => { await expect(async () => await target.switchClientIfRequired(true)).rejects.toThrow(AwsWrapperError); verify(mockPluginService.setCurrentClient(anything(), anything())).never(); - expect(target.readerTargetClient).toEqual(undefined); + expect(target.getReaderTargetClient()).toEqual(undefined); }); it("test execute failover to new writer", async () => { @@ -330,7 +343,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentClient()).thenReturn(mockNewWriterClient); when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(await mockPluginService.isClientValid(mockWriterWrapper)).thenReturn(true); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -352,7 +365,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentClient()).thenReturn(mockWriterClient); when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -372,7 +385,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(mockPluginService.acceptsStrategy(anything(), anything())).thenReturn(true); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -392,9 +405,10 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHostUnknownRole); when(mockPluginService.acceptsStrategy(anything(), anything())).thenReturn(true); - when(mockHostListProviderService.isStaticHostListProvider()).thenReturn(false); + when(mockPluginService.getHostRole(anything())).thenResolve(HostRole.UNKNOWN); + when(mockHostListProviderService.isDynamicHostListProvider()).thenReturn(true); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderServiceInstance, @@ -409,8 +423,9 @@ describe("reader write splitting test", () => { it("test pooled reader connection after set read only", async () => { const mockPluginServiceInstance = instance(mockPluginService); when(mockPluginService.getHosts()).thenReturn(singleReaderTopology); - when(mockPluginService.getHostInfoByStrategy(anything(), anything())).thenReturn(readerHost1); + when(mockPluginService.getHostInfoByStrategy(anything(), anything(), anything())).thenReturn(readerHost1); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockWriterClient)); + when(mockPluginService.isPooledClient()).thenReturn(true); when(await mockWriterClient.isValid()).thenReturn(true); when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost).thenReturn(writerHost).thenReturn(readerHost1); when(mockDriverDialect.connect(anything(), anything())).thenReturn(Promise.resolve(poolClientWrapper)); @@ -431,14 +446,15 @@ describe("reader write splitting test", () => { const spyTarget = instance(target); await spyTarget.switchClientIfRequired(true); await spyTarget.switchClientIfRequired(false); - verify(target.closeTargetClientIfIdle(poolClientWrapper)).once(); + verify(target.closeReaderClientIfIdle()).twice(); }); it("test pooled writer connection after set read only", async () => { const mockPluginServiceInstance = instance(mockPluginService); when(mockPluginService.getHosts()).thenReturn(singleReaderTopology); - when(mockPluginService.getHostInfoByStrategy(HostRole.READER, "random")).thenReturn(readerHost1); + when(mockPluginService.getHostInfoByStrategy(HostRole.READER, "random", singleReaderTopology)).thenReturn(readerHost1); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockWriterClient)); + when(mockPluginService.isPooledClient()).thenReturn(true); when(await mockWriterClient.isValid()).thenReturn(true); when(mockPluginService.getCurrentHostInfo()) .thenReturn(writerHost) @@ -480,6 +496,7 @@ describe("reader write splitting test", () => { await spyTarget.switchClientIfRequired(false); await spyTarget.switchClientIfRequired(true); - verify(target.closeTargetClientIfIdle(poolClientWrapper)).twice(); + verify(target.closeReaderClientIfIdle()).times(3); + verify(target.closeWriterClientIfIdle()).once(); }); }); diff --git a/tests/unit/region_utils.test.ts b/tests/unit/region_utils.test.ts new file mode 100644 index 000000000..7be55576f --- /dev/null +++ b/tests/unit/region_utils.test.ts @@ -0,0 +1,65 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { RegionUtils } from "../../common/lib/utils/region_utils"; + +describe("RegionUtils", () => { + let regionUtils: RegionUtils; + + beforeEach(() => { + regionUtils = new RegionUtils(); + }); + + describe("getRegion", () => { + it("should return null when props is invalid", async () => { + let result = await regionUtils.getRegion("region", undefined, undefined); + expect(result).toBeNull(); + + result = await regionUtils.getRegion("region", undefined, null); + expect(result).toBeNull(); + + const props = new Map(); + result = await regionUtils.getRegion("region", undefined, props); + expect(result).toBeNull(); + }); + + it.each([ + ["undefinedRegionKey", undefined], + ["nullRegionKey", null], + ["emptyRegionKey", ""] + ])("should return null when region key is invalid", async (regionKey: string, regionKeyVal: any) => { + const props = new Map(); + let result = await regionUtils.getRegion("region", undefined, props); + expect(result).toBeNull(); + + props.set(regionKey, regionKeyVal); + result = await regionUtils.getRegion(regionKey, undefined, props); + expect(result).toBeNull(); + }); + }); + + describe("getRegionFromRegionString", () => { + it.each([undefined, null, ""])("should return null for invalid regionString", (regionString: any) => { + const result = regionUtils.getRegionFromRegionString(regionString); + expect(result).toBeNull(); + }); + + it("should return region for valid region string", () => { + const result = regionUtils.getRegionFromRegionString("us-east-1"); + expect(result).toBe("us-east-1"); + }); + }); +}); diff --git a/tests/unit/saml_auth_plugin.test.ts b/tests/unit/saml_auth_plugin.test.ts new file mode 100644 index 000000000..76b55d246 --- /dev/null +++ b/tests/unit/saml_auth_plugin.test.ts @@ -0,0 +1,161 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { HostInfo, HostRole } from "../../common/lib"; +import { FederatedAuthPlugin } from "../../common/lib/plugins/federated_auth/federated_auth_plugin"; +import { OktaAuthPlugin } from "../../common/lib/plugins/federated_auth/okta_auth_plugin"; +import { BaseSamlAuthPlugin } from "../../common/lib/plugins/federated_auth/saml_auth_plugin"; +import { PluginServiceImpl } from "../../common/lib/plugin_service"; +import { IamAuthUtils, TokenInfo } from "../../common/lib/utils/iam_auth_utils"; +import { WrapperProperties } from "../../common/lib/wrapper_property"; +import { anything, instance, mock, reset, spy, verify, when } from "ts-mockito"; +import { CredentialsProviderFactory } from "../../common/lib/plugins/federated_auth/credentials_provider_factory"; +import { DatabaseDialect } from "../../common/lib/database_dialect/database_dialect"; +import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; +import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; + +const testToken = "testToken"; +const defaultPort = 5432; +const dbUser = "iamUser"; +const expirationFiveMinutes = 5 * 60 * 1000; + +const host = "pg.testdb.us-east-2.rds.amazonaws.com"; +const iamHost = "pg-123.testdb.us-east-2.rds.amazonaws.com"; +const testTokenInfo = new TokenInfo(testToken, Date.now() + expirationFiveMinutes); +const hostInfo = new HostInfo(host, defaultPort, HostRole.WRITER); + +const mockDialect = mock(); +const mockDialectInstance = instance(mockDialect); +const mockPluginService = mock(PluginServiceImpl); +const mockCredentialsProviderFactory = mock(); +const mockClientWrapper = mock(PgClientWrapper); +const testCredentials = { + accessKeyId: "foo", + secretAccessKey: "bar", + sessionToken: "baz" +}; +const mockConnectFunc = () => Promise.resolve(instance(mockClientWrapper)); + +describe.each([ + { pluginName: "federatedAuth", PluginClass: FederatedAuthPlugin }, + { pluginName: "okta", PluginClass: OktaAuthPlugin } +])("$pluginName plugin tests", ({ pluginName, PluginClass }) => { + let plugin: FederatedAuthPlugin | OktaAuthPlugin; + let spyIamAuthUtils: IamAuthUtils; + let props: Map; + + beforeEach(() => { + spyIamAuthUtils = spy(new IamAuthUtils()); + + when(mockPluginService.getDialect()).thenReturn(mockDialectInstance); + when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); + when(mockDialect.getDefaultPort()).thenReturn(defaultPort); + when(mockCredentialsProviderFactory.getAwsCredentialsProvider(anything(), anything(), anything())).thenResolve(testCredentials); + + props = new Map(); + WrapperProperties.PLUGINS.set(props, pluginName); + WrapperProperties.DB_USER.set(props, dbUser); + + plugin = new PluginClass(instance(mockPluginService), instance(mockCredentialsProviderFactory), instance(spyIamAuthUtils)); + }); + + afterEach(() => { + BaseSamlAuthPlugin.releaseResources(); + reset(spyIamAuthUtils); + }); + + it("testCachedToken", async () => { + const pgCacheKey = `us-east-2:${host}:${defaultPort}:${dbUser}`; + + BaseSamlAuthPlugin["tokenCache"].set(pgCacheKey, testTokenInfo); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + }); + + it("testExpiredCachedToken", async () => { + const key = `us-east-2:${host}:${defaultPort}:${dbUser}`; + const expiredToken = "expiredToken"; + const expiredTokenInfo = new TokenInfo(expiredToken, Date.now() - 300000); + + BaseSamlAuthPlugin["tokenCache"].set(key, expiredTokenInfo); + + when(spyIamAuthUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + }); + + it("testNoCachedToken", async () => { + when(spyIamAuthUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + }); + + it("testSpecifiedIamHostPortRegion", async () => { + const expectedHost = "pg.testdb.us-west-2.rds.amazonaws.com"; + const expectedPort = 9876; + const expectedRegion = "us-west-2"; + + WrapperProperties.IAM_HOST.set(props, expectedHost); + WrapperProperties.IAM_DEFAULT_PORT.set(props, expectedPort); + WrapperProperties.IAM_REGION.set(props, expectedRegion); + + const key = `${expectedRegion}:${expectedHost}:${expectedPort}:${dbUser}`; + BaseSamlAuthPlugin["tokenCache"].set(key, testTokenInfo); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + }); + + it("testIdpCredentialsFallback", async () => { + const expectedUser = "expectedUser"; + const expectedPassword = "expectedPassword"; + WrapperProperties.USER.set(props, expectedUser); + WrapperProperties.PASSWORD.set(props, expectedPassword); + + const key = `us-east-2:${host}:${defaultPort}:${dbUser}`; + BaseSamlAuthPlugin["tokenCache"].set(key, testTokenInfo); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + expect(WrapperProperties.IDP_USERNAME.get(props)).toBe(expectedUser); + expect(WrapperProperties.IDP_PASSWORD.get(props)).toBe(expectedPassword); + }); + + it("testUsingIamHost", async () => { + WrapperProperties.IAM_HOST.set(props, iamHost); + + when(spyIamAuthUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + verify(spyIamAuthUtils.generateAuthenticationToken(iamHost, anything(), anything(), anything(), anything(), anything())).once(); + }); +}); diff --git a/tests/unit/sliding_expiration_cache.test.ts b/tests/unit/sliding_expiration_cache.test.ts index 490e772ff..d9f2808b2 100644 --- a/tests/unit/sliding_expiration_cache.test.ts +++ b/tests/unit/sliding_expiration_cache.test.ts @@ -15,8 +15,8 @@ */ import { SlidingExpirationCache } from "../../common/lib/utils/sliding_expiration_cache"; -import { convertNanosToMs, sleep } from "../../common/lib/utils/utils"; import { SlidingExpirationCacheWithCleanupTask } from "../../common/lib/utils/sliding_expiration_cache_with_cleanup_task"; +import { convertMsToNanos, convertNanosToMs, sleep } from "../../common/lib/utils/utils"; class DisposableItem { shouldDispose: boolean; @@ -48,9 +48,9 @@ class AsyncDisposableItem { describe("test_sliding_expiration_cache", () => { it("test compute if absent", async () => { const target = new SlidingExpirationCache(BigInt(50_000_000)); - const result1 = target.computeIfAbsent(1, () => "a", BigInt(1)); + const result1 = target.computeIfAbsent(1, () => "a", convertMsToNanos(100)); const originalItemExpiration = target.map.get(1)!.expirationTimeNs; - const result2 = target.computeIfAbsent(1, () => "b", BigInt(5)); + const result2 = target.computeIfAbsent(1, () => "b", convertMsToNanos(500)); const updatedItemExpiration = target.map.get(1)?.expirationTimeNs; expect(updatedItemExpiration).toBeGreaterThan(originalItemExpiration); @@ -59,7 +59,7 @@ describe("test_sliding_expiration_cache", () => { expect(target.get(1)).toEqual("a"); await sleep(700); - const result3 = target.computeIfAbsent(1, () => "b", BigInt(5)); + const result3 = target.computeIfAbsent(1, () => "b", convertMsToNanos(500)); expect(result3).toEqual("b"); expect(target.get(1)).toEqual("b"); }); @@ -71,19 +71,19 @@ describe("test_sliding_expiration_cache", () => { (item) => item.dispose() ); const itemToRemove = new DisposableItem(true); - let result = target.computeIfAbsent("itemToRemove", () => itemToRemove, BigInt(15_000_000_000)); + let result = target.computeIfAbsent("itemToRemove", () => itemToRemove, convertMsToNanos(15_000)); expect(itemToRemove).toEqual(result); const itemToCleanup = new DisposableItem(true); - result = target.computeIfAbsent("itemToCleanup", () => itemToCleanup, BigInt(1)); + result = target.computeIfAbsent("itemToCleanup", () => itemToCleanup, convertMsToNanos(100)); expect(itemToCleanup).toEqual(result); const nonDisposableItem = new DisposableItem(false); - result = target.computeIfAbsent("nonDisposableItem", () => nonDisposableItem, BigInt(1)); + result = target.computeIfAbsent("nonDisposableItem", () => nonDisposableItem, convertMsToNanos(100)); expect(nonDisposableItem).toEqual(result); const nonExpiredItem = new DisposableItem(true); - result = target.computeIfAbsent("nonExpiredItem", () => nonExpiredItem, BigInt(15_000_000_000)); + result = target.computeIfAbsent("nonExpiredItem", () => nonExpiredItem, convertMsToNanos(15_000)); expect(nonExpiredItem).toEqual(result); await sleep(700); @@ -127,7 +127,7 @@ describe("test_sliding_expiration_cache", () => { expect(item2.disposed).toEqual(true); }); - it("test async cleanup thread", async () => { + it("test async cleanup task", async () => { const cleanupIntervalNanos = BigInt(300_000_000); // .3 seconds const disposeMs = 1000; const target = new SlidingExpirationCacheWithCleanupTask( diff --git a/tests/unit/stale_dns_helper.test.ts b/tests/unit/stale_dns_helper.test.ts index 086b75b8c..017b4c651 100644 --- a/tests/unit/stale_dns_helper.test.ts +++ b/tests/unit/stale_dns_helper.test.ts @@ -34,14 +34,6 @@ const mockHostListProviderService = mock(); const props: Map = new Map(); const writerInstance = new HostInfo("writer-host.XYZ.us-west-2.rds.amazonaws.com", 1234, HostRole.WRITER); -const writerCluster = new HostInfo("my-cluster.cluster-XYZ.us-west-2.rds.amazonaws.com", 1234, HostRole.WRITER); -const writerClusterInvalidClusterInetAddress = new HostInfo("my-cluster.cluster-invalid.us-west-2.rds.amazonaws.com", 1234, HostRole.WRITER); -const readerA = new HostInfo("reader-a-host.XYZ.us-west-2.rds.amazonaws.com", 1234, HostRole.READER, HostAvailability.AVAILABLE); -const readerB = new HostInfo("reader-b-host.XYZ.us-west-2.rds.amazonaws.com", 1234, HostRole.READER, HostAvailability.AVAILABLE); - -const clusterHostList = [writerCluster, readerA, readerB]; -const readerHostList = [readerA, readerB]; -const instanceHostList = [writerInstance, readerA, readerB]; const mockInitialConn = mock(AwsClient); const mockHostInfo = mock(HostInfo); @@ -87,208 +79,6 @@ describe("test_stale_dns_helper", () => { expect(returnConn).toBe(mockInitialClientWrapper); }); - it("test_get_verified_connection_cluster_inet_address_none", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - - when(target.lookupResult(anything())).thenReturn(); - - const returnConn = await targetInstance.getVerifiedConnection( - writerClusterInvalidClusterInetAddress.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockInitialClientWrapper).toBe(returnConn); - expect(mockConnectFunc).toHaveBeenCalled(); - }); - - it("test_get_verified_connection__no_writer_hostinfo", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - when(mockPluginService.getHosts()).thenReturn(readerHostList); - when(mockPluginService.getAllHosts()).thenReturn(readerHostList); - - when(mockPluginService.getCurrentHostInfo()).thenReturn(readerA); - - const lookupAddress = { address: "2.2.2.2", family: 0 }; - when(target.lookupResult(anything())).thenResolve(lookupAddress); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockConnectFunc).toHaveBeenCalled(); - expect(readerA.role).toBe(HostRole.READER); - verify(mockPluginService.forceRefreshHostList(anything())).once(); - expect(mockInitialClientWrapper).toBe(returnConn); - }); - - it("test_get_verified_connection__writer_rds_cluster_dns_true", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - - when(mockPluginService.getHosts()).thenReturn(clusterHostList); - when(mockPluginService.getAllHosts()).thenReturn(clusterHostList); - - const lookupAddress = { address: "5.5.5.5", family: 0 }; - when(target.lookupResult(anything())).thenResolve(lookupAddress); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockConnectFunc).toHaveBeenCalled(); - verify(mockPluginService.refreshHostList(anything())).once(); - expect(mockInitialClientWrapper).toBe(returnConn); - }); - - it("test_get_verified_connection__writer_host_address_none", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - when(mockPluginService.getHosts()).thenReturn(instanceHostList); - when(mockPluginService.getAllHosts()).thenReturn(instanceHostList); - - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - - const firstCall = { address: "5.5.5.5", family: 0 }; - const secondCall = { address: "", family: 0 }; - - when(target.lookupResult(anything())).thenResolve(firstCall, secondCall); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockConnectFunc).toHaveBeenCalled(); - expect(mockInitialClientWrapper).toBe(returnConn); - }); - - it("test_get_verified_connection__writer_host_info_none", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - when(mockPluginService.getHosts()).thenReturn(readerHostList); - when(mockPluginService.getAllHosts()).thenReturn(readerHostList); - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - - const firstCall = { address: "5.5.5.5", family: 0 }; - const secondCall = { address: "", family: 0 }; - - when(target.lookupResult(anything())).thenResolve(firstCall, secondCall); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockConnectFunc).toHaveBeenCalled(); - expect(mockInitialClientWrapper).toBe(returnConn); - verify(mockPluginService.connect(anything(), anything())).never(); - }); - - it("test_get_verified_connection__writer_host_address_equals_cluster_inet_address", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - when(mockPluginService.getHosts()).thenReturn(instanceHostList); - when(mockPluginService.getAllHosts()).thenReturn(instanceHostList); - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - - const firstCall = { address: "5.5.5.5", family: 0 }; - const secondCall = { address: "5.5.5.5", family: 0 }; - - when(target.lookupResult(anything())).thenResolve(firstCall, secondCall); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockConnectFunc).toHaveBeenCalled(); - expect(mockInitialClientWrapper).toBe(returnConn); - verify(mockPluginService.connect(anything(), anything())).never(); - }); - - it("test_get_verified_connection__writer_host_address_not_equals_cluster_inet_address", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - - when(mockPluginService.getHosts()).thenReturn(clusterHostList); - when(mockPluginService.getAllHosts()).thenReturn(clusterHostList); - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - targetInstance["writerHostInfo"] = writerCluster; - - const firstCall = { address: "5.5.5.5", family: 0 }; - const secondCall = { address: "8.8.8.8", family: 0 }; - - when(target.lookupResult(anything())).thenResolve(firstCall, secondCall); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - false, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockInitialConn.targetClient).not.toBe(returnConn); - expect(mockConnectFunc).toHaveBeenCalled(); - verify(mockPluginService.connect(anything(), anything())).once(); - }); - - it("test_get_verified_connection__initial_connection_writer_host_address_not_equals_cluster_inet_address", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - - when(mockPluginService.getHosts()).thenReturn(clusterHostList); - when(mockPluginService.getAllHosts()).thenReturn(clusterHostList); - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - targetInstance["writerHostInfo"] = writerCluster; - when(mockHostListProviderService.getInitialConnectionHostInfo()).thenReturn(writerCluster); - - const firstCall = { address: "5.5.5.5", family: 0 }; - const secondCall = { address: "8.8.8.8", family: 0 }; - - when(target.lookupResult(anything())).thenResolve(firstCall, secondCall); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - verify(mockPluginService.connect(anything(), anything())).once(); - expect(targetInstance["writerHostInfo"]).toBe(mockHostListProviderServiceInstance.getInitialConnectionHostInfo()); - expect(mockInitialConn.targetClient).not.toBe(returnConn); - }); - it("test_notify_host_list_changed", () => { const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); const targetInstance = instance(target); @@ -301,6 +91,5 @@ describe("test_stale_dns_helper", () => { targetInstance.notifyHostListChanged(changes); expect(targetInstance["writerHostInfo"]).toBeNull(); - expect(targetInstance["writerHostAddress"]).toBe(""); }); }); diff --git a/tests/unit/storage_service.test.ts b/tests/unit/storage_service.test.ts new file mode 100644 index 000000000..a52d739c9 --- /dev/null +++ b/tests/unit/storage_service.test.ts @@ -0,0 +1,57 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { StorageService } from "../../common/lib/utils/storage/storage_service"; +import { Topology } from "../../common/lib/host_list_provider/topology"; +import { CoreServicesContainer } from "../../common/lib/utils/core_services_container"; +import { HostInfoBuilder } from "../../common/lib"; +import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; + +describe("test_storage_service", () => { + let storageService: StorageService; + + beforeEach(() => { + storageService = CoreServicesContainer.getInstance().storageService; + }); + + afterEach(() => { + storageService.clearAll(); + }); + + it("should correctly identify Topology class from constructor", () => { + const topology = new Topology([]); + const secondTopology = new Topology([ + new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).withHost("foo").build() + ]); + const key = "test-key"; + + // Store the topology instance + storageService.set(key, topology); + + let retrieved = storageService.get(Topology, key); + expect(retrieved).toBeDefined(); + expect(retrieved).toBeInstanceOf(Topology); + expect(retrieved).toBe(topology); + + storageService.set(key, secondTopology); + + // Ensure the topology cache has been overwritten. + retrieved = storageService.get(Topology, key); + expect(retrieved).toBeDefined(); + expect(retrieved).toBeInstanceOf(Topology); + expect(retrieved).toBe(secondTopology); + }); +}); diff --git a/tests/unit/topology_utils.test.ts b/tests/unit/topology_utils.test.ts new file mode 100644 index 000000000..b3e927c0a --- /dev/null +++ b/tests/unit/topology_utils.test.ts @@ -0,0 +1,401 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TopologyQueryResult } from "../../common/lib/host_list_provider/topology_utils"; +import { AuroraTopologyUtils } from "../../common/lib/host_list_provider/aurora_topology_utils"; +import { anything, instance, mock, reset, when } from "ts-mockito"; +import { HostInfo, HostInfoBuilder, PluginManager } from "../../common/lib"; +import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; +import { AuroraPgDatabaseDialect } from "../../pg/lib/dialect/aurora_pg_database_dialect"; +import { ClientWrapper } from "../../common/lib/client_wrapper"; +import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; +import { HostRole } from "../../common/lib/host_role"; +import { HostAvailability } from "../../common/lib/host_availability/host_availability"; +import { PgDatabaseDialect } from "../../pg/lib/dialect/pg_database_dialect"; + +const mockDialect: AuroraPgDatabaseDialect = mock(AuroraPgDatabaseDialect); +const mockNonTopologyDialect: PgDatabaseDialect = mock(PgDatabaseDialect); + +const currentHostInfo = createHost({ + host: "foo", + port: 1234, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() +}); + +const clientWrapper: ClientWrapper = new PgClientWrapper(undefined, currentHostInfo, new Map()); +const mockClientWrapper: ClientWrapper = mock(clientWrapper); +const hostInfoBuilder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); + +function createHost(config: any): HostInfo { + const info = new HostInfoBuilder(config); + return info.build(); +} + +function getTopologyUtils(): AuroraTopologyUtils { + return new AuroraTopologyUtils(instance(mockDialect), hostInfoBuilder); +} + +describe("testTopologyUtils", () => { + beforeEach(() => { + reset(mockDialect); + reset(mockClientWrapper); + reset(mockNonTopologyDialect); + }); + + afterEach(async () => { + await PluginManager.releaseResources(); + }); + + it("testQueryForTopology_withNonTopologyAwareDialect_throwsError", async () => { + const hostInfoBuilder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); + const topologyUtils = new AuroraTopologyUtils(instance(mockNonTopologyDialect) as any, hostInfoBuilder); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + await expect( + topologyUtils.queryForTopology(mockClientWrapper, instance(mockNonTopologyDialect), initialHost, clusterInstanceTemplate) + ).rejects.toThrow(TypeError); + }); + + it("testQueryForTopology_returnsTopology", async () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const queryResults: TopologyQueryResult[] = [ + new TopologyQueryResult({ host: "instance-1", isWriter: true, weight: 100, lastUpdateTime: timestamp, port: 5432, id: "id-1" }), + new TopologyQueryResult({ host: "instance-2", isWriter: false, weight: 50, lastUpdateTime: timestamp, port: 5432, id: "id-2" }) + ]; + + when(mockDialect.queryForTopology(anything())).thenResolve(queryResults); + + const result = await topologyUtils.queryForTopology(mockClientWrapper, instance(mockDialect), initialHost, clusterInstanceTemplate); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(2); + expect(result[1].role).toEqual(HostRole.WRITER); // Writer should be at the bottom. + expect(result[0].role).toEqual(HostRole.READER); + }); + + it("testCreateHost_withAllParameters", () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const instanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 3306, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const result = topologyUtils.createHost("id-1", "instance-1", true, 100, timestamp, initialHost, instanceTemplate); + + expect(result).toBeTruthy(); + expect(result.host).toEqual("instance-1.cluster-endpoint"); + expect(result.port).toEqual(3306); + expect(result.role).toEqual(HostRole.WRITER); + expect(result.availability).toEqual(HostAvailability.AVAILABLE); + expect(result.weight).toEqual(100); + expect(result.lastUpdateTime).toEqual(timestamp); + expect(result.hostId).toEqual("id-1"); + }); + + it("testCreateHost_withMissingInstanceName", () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const instanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 3306, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const result = topologyUtils.createHost("id-1", "", false, 50, timestamp, initialHost, instanceTemplate); + + expect(result).toBeTruthy(); + expect(result.host).toEqual("?.cluster-endpoint"); + expect(result.role).toEqual(HostRole.READER); + }); + + it("testCreateHosts_withMultipleResults", () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 3306, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const queryResults: TopologyQueryResult[] = [ + new TopologyQueryResult({ host: "instance-1", isWriter: true, weight: 100, lastUpdateTime: timestamp, port: 3306, id: "id-1" }), + new TopologyQueryResult({ host: "instance-2", isWriter: false, weight: 50, lastUpdateTime: timestamp, port: 3306, id: "id-2" }), + new TopologyQueryResult({ host: "instance-3", isWriter: false, weight: 75, lastUpdateTime: timestamp, port: 3306, id: "id-3" }) + ]; + + const result = topologyUtils.createHosts(queryResults, initialHost, clusterInstanceTemplate); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(3); + expect(result[0].host).toEqual("instance-1.cluster-endpoint"); + expect(result[0].role).toEqual(HostRole.WRITER); + expect(result[1].host).toEqual("instance-2.cluster-endpoint"); + expect(result[1].role).toEqual(HostRole.READER); + expect(result[2].host).toEqual("instance-3.cluster-endpoint"); + expect(result[2].role).toEqual(HostRole.READER); + }); + + it("testCreateHosts_withEndpointInQueryResult", () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 3306, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const queryResults: TopologyQueryResult[] = [ + new TopologyQueryResult({ + host: "instance-1", + isWriter: true, + weight: 100, + lastUpdateTime: timestamp, + port: 3306, + id: "id-1", + endpoint: "custom-endpoint.com" + }) + ]; + + const result = topologyUtils.createHosts(queryResults, initialHost, clusterInstanceTemplate); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(1); + expect(result[0].host).toEqual("custom-endpoint.com"); + }); + + it("testCreateHosts_withMissingPort", () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const queryResults: TopologyQueryResult[] = [ + new TopologyQueryResult({ + host: "instance-1", + isWriter: true, + weight: 100, + lastUpdateTime: timestamp, + id: "id-1" + }) + ]; + + const result = topologyUtils.createHosts(queryResults, initialHost, clusterInstanceTemplate); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(1); + expect(result[0].port).toEqual(5432); + }); + + it("testVerifyWriter_withSingleWriter", async () => { + const topologyUtils = getTopologyUtils(); + + const writer = createHost({ + host: "writer-host", + port: 5432, + role: HostRole.WRITER, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const reader = createHost({ + host: "reader-host", + port: 5432, + role: HostRole.READER, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const allHosts = [writer, reader]; + const result = await topologyUtils["verifyWriter"](allHosts); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(2); + expect(result.filter((h) => h.role === HostRole.WRITER).length).toEqual(1); + }); + + it("testVerifyWriter_withMultipleWriters_selectsMostRecent", async () => { + const topologyUtils = getTopologyUtils(); + const oldTimestamp = Date.now() - 10000; + const newTimestamp = Date.now(); + + const oldWriter = createHost({ + host: "old-writer", + port: 5432, + role: HostRole.WRITER, + lastUpdateTime: oldTimestamp, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const newWriter = createHost({ + host: "new-writer", + port: 5432, + role: HostRole.WRITER, + lastUpdateTime: newTimestamp, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const reader = createHost({ + host: "reader-host", + port: 5432, + role: HostRole.READER, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const allHosts = [oldWriter, newWriter, reader]; + const result = await topologyUtils["verifyWriter"](allHosts); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(2); + expect(result.filter((h) => h.role === HostRole.WRITER).length).toEqual(1); + expect(result.find((h) => h.role === HostRole.WRITER)?.host).toEqual("new-writer"); + }); + + it("testVerifyWriter_withNoWriter_returnsNull", async () => { + const topologyUtils = getTopologyUtils(); + + const reader1 = createHost({ + host: "reader-1", + port: 5432, + role: HostRole.READER, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const reader2 = createHost({ + host: "reader-2", + port: 5432, + role: HostRole.READER, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const allHosts = [reader1, reader2]; + const result = await topologyUtils["verifyWriter"](allHosts); + + expect(result).toBeNull(); + }); + + it("testIsWriterInstance_returnsTrue", async () => { + const topologyUtils = getTopologyUtils(); + + when(mockDialect.getWriterId(anything())).thenResolve("writer-id"); + + const result = await topologyUtils.isWriterInstance(mockClientWrapper); + + expect(result).toBeTruthy(); + }); + + it("testIsWriterInstance_returnsFalse", async () => { + const topologyUtils = getTopologyUtils(); + + when(mockDialect.getWriterId(anything())).thenResolve(null); + + const result = await topologyUtils.isWriterInstance(mockClientWrapper); + + expect(result).toBeFalsy(); + }); + + it("testCreateHosts_withDuplicateHosts_keepsMostRecent", () => { + const topologyUtils = getTopologyUtils(); + const oldTimestamp = Date.now() - 10000; + const newTimestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 3306, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const queryResults: TopologyQueryResult[] = [ + new TopologyQueryResult({ host: "instance-1", isWriter: false, weight: 100, lastUpdateTime: oldTimestamp, port: 3306, id: "id-1" }), + new TopologyQueryResult({ host: "instance-1", isWriter: false, weight: 50, lastUpdateTime: newTimestamp, port: 3306, id: "id-1-updated" }) + ]; + + const result = topologyUtils.createHosts(queryResults, initialHost, clusterInstanceTemplate); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(1); + expect(result[0].host).toEqual("instance-1.cluster-endpoint"); + expect(result[0].lastUpdateTime).toEqual(newTimestamp); + expect(result[0].hostId).toEqual("id-1-updated"); + }); +}); diff --git a/tests/unit/writer_failover_handler.test.ts b/tests/unit/writer_failover_handler.test.ts index 2d4dcf468..6daf39869 100644 --- a/tests/unit/writer_failover_handler.test.ts +++ b/tests/unit/writer_failover_handler.test.ts @@ -28,6 +28,10 @@ import { PgDatabaseDialect } from "../../pg/lib/dialect/pg_database_dialect"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { DriverDialect } from "../../common/lib/driver_dialect/driver_dialect"; import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; +import { PluginManager } from "../../common/lib/plugin_manager"; +import { ServiceUtils } from "../../common/lib/utils/service_utils"; +import { HostListProviderService } from "../../common/lib/host_list_provider_service"; const builder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); @@ -45,6 +49,7 @@ const mockClientInstance = instance(mockClient); const mockPluginService = mock(PluginServiceImpl); const mockReaderFailover = mock(ClusterAwareReaderFailoverHandler); const mockDriverDialect: DriverDialect = mock(MySQL2DriverDialect); +const mockPluginManager = mock(PluginManager); const mockTargetClient = { client: 123 }; const mockClientWrapper: ClientWrapper = new MySQLClientWrapper( @@ -62,7 +67,23 @@ const mockClientWrapperB: ClientWrapper = new MySQLClientWrapper( mockDriverDialect ); +const mockServicesContainer = { + pluginService: null as any, + storageService: null as any, + monitorService: null as any, + eventPublisher: null as any, + defaultConnectionProvider: null as any, + telemetryFactory: null as any, + pluginManager: null as any, + hostListProviderService: null as any, + importantEventService: null as any +} as FullServicesContainer; + describe("writer failover handler", () => { + const originalServiceUtils = ServiceUtils.instance; + const mockServiceUtils = mock(ServiceUtils); + const mockHostListProviderService = mock(); + beforeEach(() => { writer.addAlias("writer-host"); newWriterHost.addAlias("new-writer-host"); @@ -70,11 +91,30 @@ describe("writer failover handler", () => { readerB.addAlias("reader-b-host"); when(mockPluginService.getDialect()).thenReturn(new PgDatabaseDialect()); + + // Mock ServiceUtils.createMinimalServiceContainerFrom to return a container + // that uses the same mock plugin service for both TaskA and TaskB. + const mockPluginManagerInstance = instance(mockPluginManager); + when(mockPluginManager.init()).thenResolve(); + when(mockServiceUtils.createMinimalServiceContainerFrom(anything(), anything())).thenReturn({ + pluginService: instance(mockPluginService), + pluginManager: mockPluginManagerInstance, + hostListProviderService: instance(mockHostListProviderService) + } as unknown as FullServicesContainer); + + // Replace the singleton instance with the mock. + Object.defineProperty(ServiceUtils, "instance", { get: () => instance(mockServiceUtils) }); }); afterEach(() => { reset(mockPluginService); reset(mockReaderFailover); + reset(mockPluginManager); + reset(mockServiceUtils); + reset(mockHostListProviderService); + + // Restore the original singleton instance. + Object.defineProperty(ServiceUtils, "instance", { get: () => originalServiceUtils }); }); it("test reconnect to writer - task B reader error", async () => { @@ -86,7 +126,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 5000, 2000, 2000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 5000, + 2000, + 2000 + ); const result = await target.failover(topology); expect(result.isConnected).toBe(true); @@ -111,7 +159,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 60000, 5000, 5000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 60000, + 5000, + 5000 + ); const result = await target.failover(topology); expect(result.isConnected).toBe(true); @@ -138,7 +194,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 60000, 2000, 2000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 60000, + 2000, + 2000 + ); const result: WriterFailoverResult = await target.failover(topology); expect(result.isConnected).toBe(true); @@ -168,6 +232,7 @@ describe("writer failover handler", () => { const target: ClusterAwareWriterFailoverHandler = new ClusterAwareWriterFailoverHandler( mockPluginServiceInstance, + mockServicesContainer, mockReaderFailoverInstance, properties, 60000, @@ -202,7 +267,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 60000, 5000, 2000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 60000, + 5000, + 2000 + ); const result: WriterFailoverResult = await target.failover(topology); expect(result.isConnected).toBe(true); @@ -211,7 +284,7 @@ describe("writer failover handler", () => { expect(result.topology.length).toBe(4); expect(result.topology[0].host).toBe("new-writer-host"); - verify(mockPluginService.forceRefreshHostList(anything())).atLeast(1); + verify(mockPluginService.forceRefreshHostList()).atLeast(1); verify(mockPluginService.setAvailability(newWriterHost.allAliases, HostAvailability.AVAILABLE)).once(); clearTimeout(timeoutId); }, 10000); @@ -238,7 +311,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 5000, 2000, 2000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 5000, + 2000, + 2000 + ); const startTime = Date.now(); const result = await target.failover(topology); @@ -264,7 +345,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 5000, 2000, 2000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 5000, + 2000, + 2000 + ); const result = await target.failover(topology); expect(result.isConnected).toBe(false);