fix: expand contact point hostnames to all DNS IPs at connection time (DRIVER-201) — Part 1/2

core/src/main/java/com/datastax/dse/driver/internal/core/insights/InsightsClient.java

@@ -313,7 +313,7 @@ private InsightsStartupData createStartupData() {
         .withDriverVersion(getDriverVersion(startupOptions))
         .withContactPoints(
             getResolvedContactPoints(
-                driverContext.getMetadataManager().getContactPoints().stream()
+                driverContext.getMetadataManager().getResolvedContactPoints().stream()
                     .map(n -> n.getEndPoint().resolve())
                     .filter(InetSocketAddress.class::isInstance)
                     .map(InetSocketAddress.class::cast)

core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java

@@ -701,8 +701,13 @@ public enum DefaultDriverOption implements DriverOption {
   CONTROL_CONNECTION_AGREEMENT_WARN("advanced.control-connection.schema-agreement.warn-on-failure"),
 
   /**
-   * Whether to forcibly add original contact points held by MetadataManager to the reconnection
-   * plan, in case there is no live nodes available according to LBP. Experimental.
+   * Whether to append the original contact points held by MetadataManager to the reconnection plan,
+   * after the live nodes reported by the load balancing policy. Defaults to {@code true}.
+   *
+   * <p>This is also the driver's DNS re-resolution path: contact points are expanded to their
+   * current DNS IPs at query-plan time, whereas metadata nodes hold an already-resolved endpoint
+   * that is never re-resolved. Keeping this enabled lets control-connection reconnects re-resolve
+   * the original hostnames and pick up new IPs once the live-node plan is exhausted.
    *
    * <p>Value-type: boolean
    */
@@ -837,7 +842,11 @@ public enum DefaultDriverOption implements DriverOption {
    * Whether to resolve the addresses passed to `basic.contact-points`.
    *
    * <p>Value-type: boolean
+   *
+   * @deprecated Contact points are now always kept as unresolved hostnames and expanded to all
+   *     their DNS-mapped IPs lazily at connection time. Setting this option has no effect.
    */
+  @Deprecated
   RESOLVE_CONTACT_POINTS("advanced.resolve-contact-points"),
 
   /**

core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java

@@ -369,7 +369,7 @@ protected static void fillWithDriverDefaults(OptionsMap map) {
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_INTERVAL, Duration.ofMillis(200));
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_TIMEOUT, Duration.ofSeconds(10));
     map.put(TypedDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, true);
-    map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, false);
+    map.put(TypedDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, true);
     map.put(TypedDriverOption.PREPARE_ON_ALL_NODES, true);
     map.put(TypedDriverOption.REPREPARE_ENABLED, true);
     map.put(TypedDriverOption.REPREPARE_CHECK_SYSTEM_TABLE, false);

core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java

@@ -600,7 +600,7 @@ public String toString() {
   public static final TypedDriverOption<Boolean> CONTROL_CONNECTION_AGREEMENT_WARN =
       new TypedDriverOption<>(
           DefaultDriverOption.CONTROL_CONNECTION_AGREEMENT_WARN, GenericType.BOOLEAN);
-  /** Whether to forcibly try original contacts if no live nodes are available */
+  /** Whether to append the original contact points to the reconnection plan (defaults to true) */
   public static final TypedDriverOption<Boolean> CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS =
       new TypedDriverOption<>(
           DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS, GenericType.BOOLEAN);
@@ -664,7 +664,13 @@ public String toString() {
   /** The coalescer reschedule interval. */
   public static final TypedDriverOption<Duration> COALESCER_INTERVAL =
       new TypedDriverOption<>(DefaultDriverOption.COALESCER_INTERVAL, GenericType.DURATION);
-  /** Whether to resolve the addresses passed to `basic.contact-points`. */
+  /**
+   * Whether to resolve the addresses passed to `basic.contact-points`.
+   *
+   * @deprecated Contact points are now always kept as unresolved hostnames and expanded to all
+   *     their DNS-mapped IPs lazily at connection time. Setting this option has no effect.
+   */
+  @Deprecated
   public static final TypedDriverOption<Boolean> RESOLVE_CONTACT_POINTS =
       new TypedDriverOption<>(DefaultDriverOption.RESOLVE_CONTACT_POINTS, GenericType.BOOLEAN);
   /**

core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java

@@ -166,11 +166,15 @@ protected DriverConfigLoader defaultConfigLoader(@Nullable ClassLoader classLoad
    * <p>Contact points can also be provided statically in the configuration. If both are specified,
    * they will be merged. If both are absent, the driver will default to 127.0.0.1:9042.
    *
-   * <p>Contrary to the configuration, DNS names with multiple A-records will not be handled here.
-   * If you need that, extract them manually with {@link java.net.InetAddress#getAllByName(String)}
-   * before calling this method. Similarly, if you need connect addresses to stay unresolved, make
-   * sure you pass unresolved instances here (see {@code advanced.resolve-contact-points} in the
-   * configuration for more explanations).
+   * <p>The driver automatically expands any contact point backed by an unresolved hostname to all
+   * its DNS-mapped IPs at query plan time (via {@code MetadataManager.getResolvedContactPoints()}),
+   * so passing a single hostname is sufficient to try all its IPs on initial connect. This applies
+   * equally to hostnames provided here programmatically (build an unresolved {@link
+   * InetSocketAddress} with {@link InetSocketAddress#createUnresolved(String, int)} to opt in) and
+   * to hostnames specified in the configuration. An already-resolved address passed here (the
+   * common case when constructing an {@code InetSocketAddress} directly from a hostname, which
+   * resolves eagerly) is used as provided, with no further expansion. The {@code
+   * advanced.resolve-contact-points} option is deprecated and has no effect.
    */
   @NonNull
   public SelfT addContactPoints(@NonNull Collection<InetSocketAddress> contactPoints) {
@@ -957,11 +961,11 @@ protected final CompletionStage<CqlSession> buildDefaultSessionAsync() {
         programmaticArguments = programmaticArgumentsBuilder.build();
       }
 
-      boolean resolveAddresses =
-          defaultConfig.getBoolean(DefaultDriverOption.RESOLVE_CONTACT_POINTS, false);
-
+      // RESOLVE_CONTACT_POINTS is deprecated: contact points are always kept as unresolved
+      // hostnames and expanded to all their DNS IPs at query plan time (before the first
+      // connection attempt) via MetadataManager.getResolvedContactPoints().
       Set<EndPoint> contactPoints =
-          ContactPoints.merge(programmaticContactPoints, configContactPoints, resolveAddresses);
+          ContactPoints.merge(programmaticContactPoints, configContactPoints, false);
 
       if (keyspace == null && defaultConfig.isDefined(DefaultDriverOption.SESSION_KEYSPACE)) {
         keyspace =

core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/helper/OptionalLocalDcHelper.java

@@ -26,7 +26,6 @@
 import edu.umd.cs.findbugs.annotations.NonNull;
 import java.util.ArrayList;
 import java.util.HashSet;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -68,73 +67,34 @@ public OptionalLocalDcHelper(
   @Override
   @NonNull
   public Optional<String> discoverLocalDc(@NonNull Map<UUID, Node> nodes) {
-    String localDcStr = context.getLocalDatacenter(profile.getName());
-    Optional<String> localDc;
-    if (localDcStr != null) {
-      LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDcStr);
-      localDc = Optional.of(localDcStr);
+    String localDc = context.getLocalDatacenter(profile.getName());
+    if (localDc != null) {
+      LOG.debug("[{}] Local DC set programmatically: {}", logPrefix, localDc);
     } else if (profile.isDefined(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER)) {
-      localDcStr = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER);
-      LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDcStr);
-      localDc = Optional.of(localDcStr);
-    } else {
-      localDc = Optional.empty();
-    }
-    if (localDc.isPresent()) {
-      checkLocalDatacenterCompatibility(
-          localDc.get(), context.getMetadataManager().getContactPoints());
-      // Also warn if the configured DC doesn't match any node in the cluster
-      if (!nodes.isEmpty()) {
-        boolean found = false;
-        for (Node node : nodes.values()) {
-          if (localDc.get().equals(node.getDatacenter())) {
-            found = true;
-            break;
-          }
-        }
-        if (!found) {
-          LOG.warn(
-              "[{}] Configured local DC '{}' does not match any node's datacenter"
-                  + " (available DCs: {}); please verify your configuration",
-              logPrefix,
-              localDc.get(),
-              formatDcs(nodes.values()));
-        }
-      }
+      localDc = profile.getString(DefaultDriverOption.LOAD_BALANCING_LOCAL_DATACENTER);
+      LOG.debug("[{}] Local DC set from configuration: {}", logPrefix, localDc);
     } else {
       LOG.debug("[{}] Local DC not set, DC awareness will be disabled", logPrefix);
+      return Optional.empty();
     }
-    return localDc;
-  }
-
-  /**
-   * Checks if the contact points are compatible with the local datacenter specified either through
-   * configuration, or programmatically.
-   *
-   * <p>The default implementation logs a warning when a contact point reports a datacenter
-   * different from the local one, and only for the default profile.
-   *
-   * @param localDc The local datacenter, as specified in the config, or programmatically.
-   * @param contactPoints The contact points provided when creating the session.
-   */
-  protected void checkLocalDatacenterCompatibility(
-      @NonNull String localDc, Set<? extends Node> contactPoints) {
-    if (profile.getName().equals(DriverExecutionProfile.DEFAULT_NAME)) {
-      Set<Node> badContactPoints = new LinkedHashSet<>();
-      for (Node node : contactPoints) {
-        if (!Objects.equals(localDc, node.getDatacenter())) {
-          badContactPoints.add(node);
+    if (!nodes.isEmpty()) {
+      boolean found = false;
+      for (Node node : nodes.values()) {
+        if (localDc.equals(node.getDatacenter())) {
+          found = true;
+          break;
         }
       }
-      if (!badContactPoints.isEmpty()) {
+      if (!found) {
         LOG.warn(
-            "[{}] You specified {} as the local DC, but some contact points are from a different DC: {}; "
-                + "please provide the correct local DC, or check your contact points",
+            "[{}] Configured local DC '{}' does not match any node's datacenter"
+                + " (available DCs: {}); please verify your configuration",
             logPrefix,
             localDc,
-            formatNodesAndDcs(badContactPoints));
+            formatDcs(nodes.values()));
       }
     }
+    return Optional.of(localDc);
   }
 
   /**

core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java

@@ -21,6 +21,7 @@
 import com.datastax.oss.driver.api.core.config.ClientRoutesConfig;
 import com.datastax.oss.driver.api.core.config.DefaultDriverOption;
 import com.datastax.oss.driver.api.core.metadata.EndPoint;
+import com.datastax.oss.driver.api.core.metadata.Node;
 import com.datastax.oss.driver.internal.core.adminrequest.AdminRequestHandler;
 import com.datastax.oss.driver.internal.core.adminrequest.AdminResult;
 import com.datastax.oss.driver.internal.core.adminrequest.AdminRow;
@@ -480,6 +481,22 @@ protected EndPoint buildNodeEndPoint(
     return new ClientRoutesEndPoint(this, hostId, broadcastInetAddress, fallback);
   }
 
+  @Override
+  public boolean reresolvesNodeAddresses() {
+    // ClientRoutesEndPoint re-resolves via the client route hostname on every connection attempt,
+    // but only when a route exists for that host_id (see ClientRoutesEndPoint#resolve()) -- for
+    // mixed/incomplete route sets it falls back to a static, non-re-resolving endpoint instead.
+    // Only report true when every currently-known node actually has a live route; otherwise the
+    // contact-point reconnection fallback must stay available for the nodes stuck on the fallback.
+    Map<UUID, ClientRouteRecord> routes = resolvedRoutesCache.get();
+    for (Node node : context.getMetadataManager().getMetadata().getNodes().values()) {
+      if (!routes.containsKey(node.getHostId())) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   /**
    * Builds the CQL query to fetch client routes.
    *

core/src/main/java/com/datastax/oss/driver/internal/core/metadata/CloudTopologyMonitor.java

@@ -44,4 +44,12 @@ protected EndPoint buildNodeEndPoint(
     UUID hostId = Objects.requireNonNull(row.getUuid("host_id"));
     return new SniEndPoint(cloudProxyAddress, hostId.toString());
   }
+
+  @Override
+  public boolean reresolvesNodeAddresses() {
+    // Nodes are reached through the cloud SNI proxy via SniEndPoint, which re-resolves the proxy
+    // hostname on every connection attempt. Appending the original contact points as a DNS
+    // re-resolution fallback is therefore unnecessary and could resurrect removed nodes.
+    return true;
+  }
 }

core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java

@@ -147,8 +147,7 @@ public Queue<Node> newQueryPlan(
     switch (stateRef.get()) {
       case BEFORE_INIT:
       case DURING_INIT:
-        // The contact points are not stored in the metadata yet:
-        List<Node> nodes = new ArrayList<>(context.getMetadataManager().getContactPoints());
+        List<Node> nodes = new ArrayList<>(context.getMetadataManager().getResolvedContactPoints());
         Collections.shuffle(nodes);
         return new ConcurrentLinkedQueue<>(nodes);
       case RUNNING:
@@ -164,15 +163,33 @@ public Queue<Node> newQueryPlan(
 
   @NonNull
   public Queue<Node> newControlReconnectionQueryPlan() {
+    // Read the state once, before building the regular plan. State transitions are monotonic
+    // (BEFORE_INIT -> DURING_INIT -> RUNNING -> ...), so this captured value is <= the value
+    // newQueryPlan() reads internally; that guarantees we never both build the plan from resolved
+    // contact points (pre-RUNNING branch of newQueryPlan) and append them again below.
+    State state = stateRef.get();
     Queue<Node> regularQueryPlan = newQueryPlan(null, DriverExecutionProfile.DEFAULT_NAME, null);
 
-    if (context
-        .getConfig()
-        .getDefaultProfile()
-        .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)) {
-      Set<DefaultNode> originalNodes = context.getMetadataManager().getContactPoints();
+    // Only append resolved contact points as an explicit fallback once the LBP is RUNNING: before
+    // that (BEFORE_INIT/DURING_INIT), newQueryPlan() above already built regularQueryPlan directly
+    // from getResolvedContactPoints(), so appending them again here would just duplicate every
+    // entry and re-trigger DNS resolution for no benefit.
+    if (state == State.RUNNING
+        && context
+            .getConfig()
+            .getDefaultProfile()
+            .getBoolean(DefaultDriverOption.CONTROL_CONNECTION_RECONNECT_CONTACT_POINTS)
+        && !context.getTopologyMonitor().reresolvesNodeAddresses()) {
+      // Use DNS-expanded contact points so every resolved IP is tried as a fallback, not just
+      // the first IP the JVM happens to return for the hostname.
+      //
+      // Skipped when the topology monitor re-resolves node addresses on its own (e.g. proxy-based
+      // monitors such as client routes or the cloud SNI proxy): those keep addresses fresh without
+      // this fallback, and appending raw contact points could resurrect nodes the monitor has
+      // authoritatively removed.
+      List<Node> resolvedNodes = context.getMetadataManager().getResolvedContactPoints();
       List<Node> contactNodes = new ArrayList<>();
-      for (DefaultNode node : originalNodes) {
+      for (Node node : resolvedNodes) {
         contactNodes.add(DefaultNode.newContactPoint(node.getEndPoint(), context));
       }
       Collections.shuffle(contactNodes);