resolveVariant(
mappedValue = mapValue(target, variant.value);
} catch (final NumberFormatException e) {
return error(
+ key,
defaultValue,
ErrorCode.PARSE_ERROR,
"Variant '"
diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DebugLoggingHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DebugLoggingHook.java
new file mode 100644
index 00000000000..7ee11175757
--- /dev/null
+++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DebugLoggingHook.java
@@ -0,0 +1,147 @@
+package datadog.trace.api.openfeature;
+
+import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration;
+import datadog.trace.bootstrap.config.provider.ConfigProvider;
+import dev.openfeature.sdk.EvaluationContext;
+import dev.openfeature.sdk.FlagEvaluationDetails;
+import dev.openfeature.sdk.Hook;
+import dev.openfeature.sdk.HookContext;
+import dev.openfeature.sdk.ImmutableStructure;
+import dev.openfeature.sdk.ProviderEvaluation;
+import dev.openfeature.sdk.Structure;
+import dev.openfeature.sdk.Value;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Logs raw evaluation diagnostics at the provider and OpenFeature hook boundaries.
+ *
+ * Raw values and context are logged only when both {@code DD_TRACE_DEBUG} and the application
+ * logger allow DEBUG. Logging and formatting failures never reach the evaluation.
+ */
+final class DebugLoggingHook implements Hook {
+ private static final Logger log = LoggerFactory.getLogger(DebugLoggingHook.class);
+
+ /** Cached {@code DD_TRACE_DEBUG}; it also gates the provider's other debug diagnostics. */
+ static final boolean TRACE_DEBUG = readTraceDebug();
+
+ private static final Structure VALUE_CONVERTER = new ImmutableStructure();
+
+ static DebugLoggingHook createIfEnabled() {
+ return isEnabled() ? new DebugLoggingHook() : null;
+ }
+
+ static void resolution(
+ String key,
+ Object defaultValue,
+ EvaluationContext context,
+ ProviderEvaluation result,
+ ServerConfiguration config) {
+ if (!isEnabled()) {
+ return;
+ }
+ try {
+ log.debug(
+ "Feature Flagging provider resolution: flag={}, targetingKey={}, context={}, defaultValue={}, resolvedValue={}, variant={}, reason={}, errorCode={}, errorMessage={}, metadata={}, configurationEnv={}, configurationCreatedAt={}",
+ key,
+ context == null ? null : context.getTargetingKey(),
+ raw(context),
+ raw(defaultValue),
+ raw(result.getValue()),
+ result.getVariant(),
+ result.getReason(),
+ result.getErrorCode(),
+ result.getErrorMessage(),
+ result.getFlagMetadata() == null
+ ? null
+ : raw(result.getFlagMetadata().asUnmodifiableMap()),
+ config == null || config.environment == null ? null : config.environment.name,
+ config == null ? null : config.createdAt);
+ } catch (Throwable failure) {
+ formattingFailed(failure);
+ }
+ }
+
+ static void resolutionError(String key, Object code, String message) {
+ if (!isEnabled()) {
+ return;
+ }
+ try {
+ log.debug(
+ "Feature Flagging provider resolution error: flag={}, errorCode={}, errorMessage={}",
+ key,
+ code,
+ message);
+ } catch (Throwable ignored) {
+ // Diagnostics must never interrupt evaluation.
+ }
+ }
+
+ @Override
+ public void finallyAfter(
+ final HookContext context,
+ final FlagEvaluationDetails details,
+ final Map hints) {
+ if (details == null || !isEnabled()) {
+ return;
+ }
+ try {
+ log.debug(
+ "Feature Flagging evaluation observed at finallyAfter: flag={}, targetingKey={}, context={}, defaultValue={}, observedValue={}, variant={}, reason={}, errorCode={}, errorMessage={}, metadata={}",
+ details.getFlagKey(),
+ context == null || context.getCtx() == null ? null : context.getCtx().getTargetingKey(),
+ context == null ? null : raw(context.getCtx()),
+ context == null ? null : raw(context.getDefaultValue()),
+ raw(details.getValue()),
+ details.getVariant(),
+ details.getReason(),
+ details.getErrorCode(),
+ details.getErrorMessage(),
+ details.getFlagMetadata() == null
+ ? null
+ : raw(details.getFlagMetadata().asUnmodifiableMap()));
+ } catch (Throwable failure) {
+ formattingFailed(failure);
+ }
+ }
+
+ private static boolean isEnabled() {
+ try {
+ return TRACE_DEBUG && log.isDebugEnabled();
+ } catch (Throwable ignored) {
+ return false;
+ }
+ }
+
+ /**
+ * Materialize inside the diagnostic boundary, including user supplied structure implementations.
+ */
+ private static String raw(Object value) {
+ if (value instanceof Value) {
+ value = VALUE_CONVERTER.convertValue((Value) value);
+ } else if (value instanceof Structure) {
+ value = ((Structure) value).asObjectMap();
+ }
+ return String.valueOf(value);
+ }
+
+ private static void formattingFailed(Throwable failure) {
+ try {
+ log.debug(
+ "Feature Flagging evaluation diagnostic formatting failed; raw detail unavailable",
+ failure);
+ } catch (Throwable ignored) {
+ // Diagnostics must never interrupt evaluation.
+ }
+ }
+
+ private static boolean readTraceDebug() {
+ try {
+ return ConfigProvider.getInstance().getBoolean("trace.debug", false);
+ } catch (Throwable ignored) {
+ // Configuration is unavailable without the agent or with an older agent.
+ return false;
+ }
+ }
+}
diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ExposureLoggingHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ExposureLoggingHook.java
index 67ab6b27cfd..f48ab528969 100644
--- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ExposureLoggingHook.java
+++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/ExposureLoggingHook.java
@@ -1,5 +1,7 @@
package datadog.trace.api.openfeature;
+import static datadog.trace.api.openfeature.Provider.debugQuietly;
+
import datadog.trace.api.featureflag.FeatureFlaggingGateway;
import dev.openfeature.sdk.EvaluationContext;
import dev.openfeature.sdk.FlagEvaluationDetails;
@@ -45,16 +47,22 @@ public void finallyAfter(
final Map hints) {
try {
if (details == null || details.getErrorCode() != null) {
+ debugQuietly(
+ log, "Feature Flagging exposure suppressed: evaluation failed or details unavailable");
return;
}
final ImmutableMetadata metadata = details.getFlagMetadata();
if (metadata == null) {
+ debugQuietly(log, "Feature Flagging exposure suppressed: metadata unavailable");
return;
}
// The ImmutableMetadata getters build a debug message for every absent key.
final Map values = metadata.asUnmodifiableMap();
if (!Boolean.TRUE.equals(values.get(DDEvaluator.METADATA_DO_LOG))
|| !Boolean.FALSE.equals(values.get(DDEvaluator.METADATA_EXPOSURE_CACHE_HIT))) {
+ debugQuietly(
+ log,
+ "Feature Flagging exposure suppressed: allocation logging disabled or exposure already recorded");
return;
}
final String flag = details.getFlagKey();
@@ -64,6 +72,9 @@ public void finallyAfter(
final String variantKey = details.getVariant();
final EvaluationContext context = ctx == null ? null : ctx.getCtx();
if (allocationKey == null || variantKey == null || context == null) {
+ debugQuietly(
+ log,
+ "Feature Flagging exposure suppressed: allocation, variant, or context unavailable");
return;
}
final Object serialValue = values.get(DDEvaluator.METADATA_SPLIT_SERIAL_ID);
@@ -81,6 +92,10 @@ public void finallyAfter(
serialId,
SplitFeatureMetadata.featuresWithPrefix(
metadata, DDEvaluator.METADATA_EXPOSURE_FEATURE_PREFIX)));
+ debugQuietly(log, "Feature Flagging exposure dispatched to agent listeners");
+ } else {
+ debugQuietly(
+ log, "Feature Flagging exposure suppressed: Datadog exposure logging disabled");
}
} catch (LinkageError e) {
// Never let exposure reporting break flag evaluation.
diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetrics.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetrics.java
index eba3b7ba4b5..2f28500f8c4 100644
--- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetrics.java
+++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalMetrics.java
@@ -1,5 +1,7 @@
package datadog.trace.api.openfeature;
+import static datadog.trace.api.openfeature.Provider.debugQuietly;
+
import dev.openfeature.sdk.ErrorCode;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.common.AttributeKey;
@@ -42,7 +44,7 @@ class FlagEvalMetrics implements Closeable {
.setDescription(METRIC_DESC)
.build();
- log.debug("Flag evaluation metrics initialized");
+ debugQuietly(log, "Flag evaluation metrics initialized");
} catch (NoClassDefFoundError e) {
log.error(
"OpenTelemetry API is not on the classpath — evaluation metrics disabled. Add"
diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java
index b5dc224392c..1c2a1ad6d2a 100644
--- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java
+++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java
@@ -89,7 +89,7 @@ public Provider(final Options options) {
// Precompute the immutable hook list once so getProviderHooks() (called on every evaluation)
// allocates nothing, including when the gate is off.
- final List hooks = new ArrayList<>(4);
+ final List hooks = new ArrayList<>(5);
if (flagEvalMetricsHook != null) {
hooks.add(flagEvalMetricsHook);
}
@@ -117,40 +117,81 @@ public Provider(final Options options) {
} catch (LinkageError e) {
log.warn("Feature flag exposure reporting is unavailable with the installed agent", e);
}
+ // Local debug diagnostics are not telemetry, so evaluation telemetry opt-out does not apply.
+ final DebugLoggingHook debugLoggingHook = DebugLoggingHook.createIfEnabled();
+ if (debugLoggingHook != null) {
+ hooks.add(debugLoggingHook);
+ } else if (DebugLoggingHook.TRACE_DEBUG) {
+ // DD_TRACE_DEBUG cannot raise application logger levels, so name the missing second gate.
+ try {
+ log.warn(
+ "DD_TRACE_DEBUG is enabled, but DEBUG is disabled for datadog.trace.api.openfeature in"
+ + " the application's logging configuration; provider debug diagnostics are off."
+ + " Enable DEBUG for that category before constructing the Provider.");
+ } catch (Throwable ignored) {
+ // A failing logging backend must not prevent provider construction.
+ }
+ }
this.providerHooks =
hooks.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(hooks);
// Announce the span-enrichment state at startup (matches the reference implementation).
// "enabled" only when the gate is on (the capture hook was constructed), otherwise "disabled".
- if (spanEnrichmentHook != null) {
- log.info("{} span enrichment enabled", METADATA);
- } else {
- log.info("{} span enrichment disabled", METADATA);
+ try {
+ if (spanEnrichmentHook != null) {
+ log.info("{} span enrichment enabled", METADATA);
+ } else {
+ log.info("{} span enrichment disabled", METADATA);
+ }
+ } catch (Throwable ignored) {
+ // A failing logging backend must not prevent provider construction.
}
}
@Override
public void initialize(final EvaluationContext context) throws Exception {
+ debugQuietly(
+ log,
+ "Feature Flagging provider initializing; timeout={} {}",
+ options.getTimeout(),
+ options.getUnit());
initializationState.set(InitializationState.INITIALIZING);
try {
evaluator = buildEvaluator();
if (!evaluator.initialize(options.getTimeout(), options.getUnit(), context)) {
if (markInitialConfigReceivedReady()) {
+ debugQuietly(log, "Feature Flagging provider initialized successfully");
return;
}
markInitializationError();
+ debugQuietly(
+ log,
+ "Feature Flagging provider initialization timed out waiting for usable configuration");
throw new ProviderNotReadyError(
"Provider timed-out while waiting for initial configuration");
}
if (!evaluator.hasConfiguration() || !markSuccessfulInitializationReady()) {
markInitializationError();
+ debugQuietly(
+ log, "Feature Flagging provider initialization failed: configuration unavailable");
throw new ProviderNotReadyError(
"Provider timed-out while waiting for initial configuration");
}
+ debugQuietly(log, "Feature Flagging provider initialized successfully");
} catch (final OpenFeatureError e) {
+ debugQuietly(
+ log,
+ "Feature Flagging provider initialization failed: errorCode={}",
+ e.getErrorCode(),
+ e);
markInitializationError();
throw e;
} catch (final Throwable e) {
+ debugQuietly(
+ log,
+ "Feature Flagging provider initialization failed: {}; verify the Java agent and feature flag configuration",
+ e.getClass().getSimpleName(),
+ e);
markInitializationError();
throw new FatalError("Failed to initialize provider, is the tracer configured?", e);
}
@@ -174,6 +215,7 @@ void onConfigurationChange() {
if (state == InitializationState.ERROR
&& initializationState.compareAndSet(
InitializationState.ERROR, InitializationState.READY)) {
+ debugQuietly(log, "Feature Flagging provider ready after configuration recovery");
emit(
ProviderEvent.PROVIDER_READY,
ProviderEventDetails.builder().message("Provider ready").build());
@@ -182,6 +224,7 @@ void onConfigurationChange() {
if (initializationState.get() != InitializationState.READY) {
return;
}
+ debugQuietly(log, "Feature Flagging provider configuration changed");
emit(
ProviderEvent.PROVIDER_CONFIGURATION_CHANGED,
ProviderEventDetails.builder().message("New configuration received").build());
@@ -195,6 +238,7 @@ private void onConfigurationUnavailable() {
if (!initializationState.compareAndSet(InitializationState.READY, InitializationState.ERROR)) {
return;
}
+ debugQuietly(log, "Feature Flagging provider configuration unavailable");
emit(
ProviderEvent.PROVIDER_ERROR,
ProviderEventDetails.builder()
@@ -249,6 +293,7 @@ Hook buildExposureLoggingHook() {
@Override
public void shutdown() {
+ debugQuietly(log, "Feature Flagging provider shutting down");
if (flagEvalMetrics != null) {
flagEvalMetrics.shutdown();
}
@@ -258,6 +303,35 @@ public void shutdown() {
if (evaluator != null) {
evaluator.shutdown();
}
+ debugQuietly(
+ log,
+ "Feature Flagging provider shutdown complete; configuration listener detached; agent owns exposure delivery and shutdown");
+ }
+
+ /**
+ * Logs a provider diagnostic only when {@code DD_TRACE_DEBUG} is enabled; a failing logging
+ * backend never changes provider behavior.
+ */
+ static void debugQuietly(final Logger log, final String message) {
+ if (!DebugLoggingHook.TRACE_DEBUG) {
+ return;
+ }
+ try {
+ log.debug(message);
+ } catch (Throwable ignored) {
+ // Diagnostics must never interrupt construction, initialization, evaluation, or shutdown.
+ }
+ }
+
+ static void debugQuietly(final Logger log, final String format, final Object... arguments) {
+ if (!DebugLoggingHook.TRACE_DEBUG) {
+ return;
+ }
+ try {
+ log.debug(format, arguments);
+ } catch (Throwable ignored) {
+ // Diagnostics must never interrupt construction, initialization, evaluation, or shutdown.
+ }
}
// Visible for tests: expose whether span enrichment is wired (gate-on) without leaking the impl.
diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DebugLoggingDisabledForkedTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DebugLoggingDisabledForkedTest.java
new file mode 100644
index 00000000000..dd5a1b6d33b
--- /dev/null
+++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DebugLoggingDisabledForkedTest.java
@@ -0,0 +1,78 @@
+package datadog.trace.api.openfeature;
+
+import static java.util.Collections.emptyMap;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyNoInteractions;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
+import dev.openfeature.sdk.EvaluationContext;
+import dev.openfeature.sdk.FlagEvaluationDetails;
+import dev.openfeature.sdk.FlagValueType;
+import dev.openfeature.sdk.HookContext;
+import dev.openfeature.sdk.ProviderEvaluation;
+import dev.openfeature.sdk.exceptions.ProviderNotReadyError;
+import org.junit.jupiter.api.Test;
+import org.slf4j.LoggerFactory;
+
+/** Exercises the cached debug gate in a fresh JVM with the application logger at DEBUG. */
+class DebugLoggingDisabledForkedTest {
+ static {
+ System.setProperty("dd.trace.debug", "false");
+ }
+
+ @Test
+ void skipsHookRegistrationAndRawInspection() {
+ Logger backend = (Logger) LoggerFactory.getLogger("datadog.trace.api.openfeature");
+ Level previousLevel = backend.getLevel();
+ ListAppender appender = new ListAppender<>();
+ appender.start();
+ backend.setLevel(Level.DEBUG);
+ backend.addAppender(appender);
+ try {
+ Provider provider =
+ new Provider(new Provider.Options().initTimeout(1, MILLISECONDS), mock(Evaluator.class));
+ try {
+ assertFalse(
+ provider.getProviderHooks().stream().anyMatch(DebugLoggingHook.class::isInstance));
+ assertNull(DebugLoggingHook.createIfEnabled());
+ EvaluationContext context = mock(EvaluationContext.class);
+ ProviderEvaluation result = mock(ProviderEvaluation.class);
+ FlagEvaluationDetails details = mock(FlagEvaluationDetails.class);
+ Object value = mock(Object.class);
+ HookContext hookContext =
+ HookContext.builder()
+ .flagKey("flag")
+ .type(FlagValueType.OBJECT)
+ .ctx(context)
+ .defaultValue(value)
+ .build();
+
+ DebugLoggingHook.resolution("flag", value, context, result, null);
+ DebugLoggingHook.resolutionError("flag", "ERROR", "raw error");
+ new DebugLoggingHook().finallyAfter(hookContext, details, emptyMap());
+
+ verifyNoInteractions(context, result, details, value);
+ assertThrows(ProviderNotReadyError.class, () -> provider.initialize(null));
+ ExposureLoggingHook.INSTANCE.finallyAfter(null, null, emptyMap());
+ } finally {
+ provider.shutdown();
+ }
+ assertTrue(appender.list.stream().noneMatch(event -> event.getLevel() == Level.DEBUG));
+ assertTrue(
+ appender.list.stream()
+ .noneMatch(event -> event.getFormattedMessage().startsWith("DD_TRACE_DEBUG")));
+ } finally {
+ backend.detachAppender(appender);
+ appender.stop();
+ backend.setLevel(previousLevel);
+ }
+ }
+}
diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DebugLoggingHookForkedTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DebugLoggingHookForkedTest.java
new file mode 100644
index 00000000000..4786e778a17
--- /dev/null
+++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/DebugLoggingHookForkedTest.java
@@ -0,0 +1,393 @@
+package datadog.trace.api.openfeature;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.turbo.TurboFilter;
+import ch.qos.logback.core.read.ListAppender;
+import ch.qos.logback.core.spi.FilterReply;
+import datadog.trace.api.featureflag.FeatureFlaggingGateway;
+import datadog.trace.api.featureflag.ufc.v1.Environment;
+import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration;
+import dev.openfeature.sdk.Client;
+import dev.openfeature.sdk.ErrorCode;
+import dev.openfeature.sdk.EvaluationContext;
+import dev.openfeature.sdk.EventDetails;
+import dev.openfeature.sdk.FlagEvaluationDetails;
+import dev.openfeature.sdk.FlagValueType;
+import dev.openfeature.sdk.Hook;
+import dev.openfeature.sdk.HookContext;
+import dev.openfeature.sdk.ImmutableMetadata;
+import dev.openfeature.sdk.ImmutableStructure;
+import dev.openfeature.sdk.MutableContext;
+import dev.openfeature.sdk.OpenFeatureAPI;
+import dev.openfeature.sdk.ProviderEvaluation;
+import dev.openfeature.sdk.ProviderEvent;
+import dev.openfeature.sdk.ProviderState;
+import dev.openfeature.sdk.Value;
+import dev.openfeature.sdk.exceptions.ProviderNotReadyError;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.slf4j.LoggerFactory;
+import org.slf4j.Marker;
+import org.tabletest.junit.TableTest;
+
+class DebugLoggingHookForkedTest {
+ static {
+ System.setProperty("dd.trace.debug", "true");
+ }
+
+ private final Logger logger = (Logger) LoggerFactory.getLogger(DebugLoggingHook.class);
+ private final ListAppender appender = new ListAppender<>();
+ private Level previousLevel;
+
+ @BeforeEach
+ void setup() {
+ previousLevel = logger.getLevel();
+ logger.setLevel(Level.DEBUG);
+ appender.start();
+ logger.addAppender(appender);
+ }
+
+ @AfterEach
+ void cleanup() {
+ logger.detachAppender(appender);
+ appender.stop();
+ logger.setLevel(previousLevel);
+ }
+
+ @TableTest({
+ "Scenario | Level | Registered | Warned",
+ "Debug enabled | DEBUG | true | false ",
+ "Debug filtered | INFO | false | true ",
+ "Errors only | ERROR | false | true "
+ })
+ void registersOnlyWhenDebugIsActiveAndWarnsOtherwise(
+ String scenario, String level, boolean registered, boolean warned) {
+ logger.setLevel(Level.toLevel(level));
+ Logger providerLogger = (Logger) LoggerFactory.getLogger(Provider.class);
+ ListAppender providerEvents = new ListAppender<>();
+ providerEvents.start();
+ providerLogger.addAppender(providerEvents);
+ Provider provider =
+ new Provider(new Provider.Options().initTimeout(1, MILLISECONDS), mock(Evaluator.class));
+ try {
+ assertEquals(
+ registered,
+ provider.getProviderHooks().stream().anyMatch(DebugLoggingHook.class::isInstance));
+ assertEquals(
+ warned,
+ providerEvents.list.stream()
+ .anyMatch(
+ event ->
+ event.getLevel() == Level.WARN
+ && event.getFormattedMessage().startsWith("DD_TRACE_DEBUG is enabled")));
+ } finally {
+ provider.shutdown();
+ providerLogger.detachAppender(providerEvents);
+ providerEvents.stop();
+ }
+ }
+
+ @Test
+ void logsRawOutcomeContextDefaultAndMetadata() {
+ EvaluationContext evaluationContext =
+ new MutableContext("user-123", singletonMap("country", new Value("FR")));
+ HookContext context =
+ HookContext.builder()
+ .flagKey("checkout")
+ .type(FlagValueType.STRING)
+ .defaultValue("secret-default")
+ .ctx(evaluationContext)
+ .build();
+ FlagEvaluationDetails details =
+ FlagEvaluationDetails.builder()
+ .flagKey("checkout")
+ .value("secret-value")
+ .variant("enabled")
+ .reason("TARGETING_MATCH")
+ .flagMetadata(
+ ImmutableMetadata.builder().addString("secret-key", "secret-metadata").build())
+ .build();
+
+ DebugLoggingHook.createIfEnabled().finallyAfter(context, details, emptyMap());
+
+ assertEquals(1, appender.list.size());
+ String message = appender.list.get(0).getFormattedMessage();
+ assertTrue(message.contains("flag=checkout"));
+ assertTrue(message.contains("reason=TARGETING_MATCH"));
+ assertTrue(message.contains("variant=enabled"));
+ assertTrue(message.contains("defaultValue=secret-default"));
+ assertTrue(message.contains("observedValue=secret-value"));
+ assertTrue(message.contains("secret-key=secret-metadata"));
+ assertTrue(message.contains("targetingKey=user-123"));
+ assertTrue(message.contains("country=FR"));
+ }
+
+ @Test
+ void logsRawFallbackAndErrorMessage() {
+ FlagEvaluationDetails details =
+ FlagEvaluationDetails.builder()
+ .flagKey("missing")
+ .value("secret-default")
+ .reason("ERROR")
+ .errorCode(ErrorCode.FLAG_NOT_FOUND)
+ .errorMessage("secret-error")
+ .build();
+
+ DebugLoggingHook.createIfEnabled().finallyAfter(null, details, emptyMap());
+
+ String message = appender.list.get(0).getFormattedMessage();
+ assertTrue(message.contains("reason=ERROR"));
+ assertTrue(message.contains("errorCode=FLAG_NOT_FOUND"));
+ assertTrue(message.contains("errorMessage=secret-error"));
+ assertTrue(message.contains("observedValue=secret-default"));
+ }
+
+ @Test
+ void logsFinalFallbackWhenAnApplicationAfterHookFails() throws Exception {
+ Evaluator evaluator = mock(Evaluator.class);
+ when(evaluator.initialize(anyLong(), any(), any())).thenReturn(true);
+ when(evaluator.hasConfiguration()).thenReturn(true);
+ when(evaluator.evaluate(any(), any(), any(), any()))
+ .thenReturn(
+ ProviderEvaluation.builder()
+ .value(true)
+ .variant("enabled")
+ .reason("TARGETING_MATCH")
+ .build());
+ OpenFeatureAPI api = OpenFeatureAPI.getInstance();
+ try {
+ api.setProviderAndWait(
+ new Provider(new Provider.Options().initTimeout(1, MILLISECONDS), evaluator));
+ Client client = api.getClient();
+ client.addHooks(
+ new Hook() {
+ @Override
+ public void after(
+ HookContext context,
+ FlagEvaluationDetails details,
+ Map hints) {
+ throw new IllegalStateException("secret-application-error");
+ }
+ });
+
+ assertFalse(client.getBooleanValue("checkout", false));
+
+ assertEquals(1, appender.list.size());
+ String message = appender.list.get(0).getFormattedMessage();
+ assertTrue(message.contains("reason=ERROR"));
+ assertTrue(message.contains("secret-application-error"));
+ assertTrue(message.contains("observedValue=false"));
+ } finally {
+ api.shutdown();
+ }
+ }
+
+ @Test
+ void doesNotInspectDetailsAfterDebugIsDisabled() {
+ DebugLoggingHook hook = DebugLoggingHook.createIfEnabled();
+ assertNotNull(hook);
+ logger.setLevel(Level.INFO);
+ FlagEvaluationDetails details = mock(FlagEvaluationDetails.class);
+
+ hook.finallyAfter(null, details, emptyMap());
+
+ verifyNoInteractions(details);
+ assertTrue(appender.list.isEmpty());
+ }
+
+ @ParameterizedTest
+ @MethodSource("rawValues")
+ void logsEveryValueTypeWithoutTruncation(Object value) {
+ FlagEvaluationDetails details =
+ FlagEvaluationDetails.builder().flagKey("raw").value(value).build();
+ DebugLoggingHook.createIfEnabled().finallyAfter(null, details, emptyMap());
+ String message = appender.list.get(0).getFormattedMessage();
+ String expected = value instanceof Value ? "tail-" + "x".repeat(4096) : value.toString();
+ assertTrue(message.contains(expected), message);
+ }
+
+ static Stream rawValues() {
+ return Stream.of(
+ true,
+ 42,
+ 1.25,
+ "untruncated-" + "x".repeat(4096),
+ new Value(
+ new ImmutableStructure(
+ singletonMap(
+ "nested", new Value(singletonList(new Value("tail-" + "x".repeat(4096))))))),
+ new Value(
+ singletonList(
+ new Value(
+ new ImmutableStructure(
+ singletonMap("nested", new Value("tail-" + "x".repeat(4096))))))));
+ }
+
+ @Test
+ void logsProviderFallbackWithConfigurationIdentity() {
+ ServerConfiguration config =
+ new ServerConfiguration(
+ "2026-10-21T00:00:00Z", "ufc/v1", false, new Environment("staging"), emptyMap());
+ DDEvaluator evaluator = new DDEvaluator(() -> {});
+ evaluator.accept(config);
+ ProviderEvaluation result =
+ evaluator.evaluate(String.class, "missing", "default", new MutableContext("user-123"));
+ assertEquals("default", result.getValue());
+ String message = appender.list.get(appender.list.size() - 1).getFormattedMessage();
+ assertTrue(message.contains("provider resolution:"));
+ assertTrue(message.contains("errorCode=FLAG_NOT_FOUND"));
+ assertTrue(message.contains("resolvedValue=default"));
+ assertTrue(message.contains("configurationEnv=staging"));
+ assertTrue(message.contains("configurationCreatedAt=" + config.createdAt));
+ }
+
+ @Test
+ void rawErrorLoggingDoesNotChangeReturnedMetadata() {
+ ServerConfiguration config = new ServerConfiguration(null, "ufc/v1", false, null, emptyMap());
+ config.invalidFlags = singletonMap("broken", "parse_error");
+ DDEvaluator evaluator = new DDEvaluator(() -> {});
+ evaluator.accept(config);
+ ProviderEvaluation result =
+ evaluator.evaluate(Boolean.class, "broken", false, new MutableContext());
+ assertEquals(ErrorCode.PARSE_ERROR, result.getErrorCode());
+ assertNull(result.getErrorMessage());
+ assertTrue(
+ appender.list.stream()
+ .anyMatch(
+ event ->
+ event
+ .getFormattedMessage()
+ .contains("errorMessage=invalid configuration for flag broken")));
+ }
+
+ @Test
+ void reportsFormattingFailureWithoutChangingEvaluation() {
+ Object broken =
+ new Object() {
+ @Override
+ public String toString() {
+ throw new AssertionError("broken formatting");
+ }
+ };
+ FlagEvaluationDetails details =
+ FlagEvaluationDetails.builder().flagKey("broken").value(broken).build();
+ assertDoesNotThrow(
+ () -> DebugLoggingHook.createIfEnabled().finallyAfter(null, details, emptyMap()));
+ assertTrue(
+ appender
+ .list
+ .get(0)
+ .getFormattedMessage()
+ .contains("formatting failed; raw detail unavailable"));
+ }
+
+ @Test
+ void disabledResolutionDoesNotInspectContextResultOrValues() {
+ logger.setLevel(Level.INFO);
+ EvaluationContext context = mock(EvaluationContext.class);
+ ProviderEvaluation result = mock(ProviderEvaluation.class);
+ Object value = mock(Object.class);
+ DebugLoggingHook.resolution("flag", value, context, result, null);
+ verifyNoInteractions(context, result, value);
+ assertTrue(appender.list.isEmpty());
+ }
+
+ @Test
+ void enabledTraceDebugLogsProviderLifecycle() {
+ Logger providerLogger = (Logger) LoggerFactory.getLogger(Provider.class);
+ Level previousLevel = providerLogger.getLevel();
+ ListAppender lifecycle = new ListAppender<>();
+ lifecycle.start();
+ providerLogger.setLevel(Level.DEBUG);
+ providerLogger.addAppender(lifecycle);
+ try {
+ new Provider(new Provider.Options(), mock(Evaluator.class)).shutdown();
+ assertTrue(
+ lifecycle.list.stream()
+ .anyMatch(
+ event ->
+ event.getLevel() == Level.DEBUG
+ && event
+ .getFormattedMessage()
+ .equals("Feature Flagging provider shutting down")));
+ } finally {
+ providerLogger.detachAppender(lifecycle);
+ lifecycle.stop();
+ providerLogger.setLevel(previousLevel);
+ }
+ }
+
+ @TableTest({
+ "Scenario | Fail level check",
+ "backend throws | false ",
+ "level check throws | true "
+ })
+ void loggerFailureDoesNotInterruptLifecycle(String scenario, boolean failLevelCheck)
+ throws Exception {
+ TurboFilter failure =
+ new TurboFilter() {
+ @Override
+ public FilterReply decide(
+ Marker marker,
+ Logger target,
+ Level level,
+ String format,
+ Object[] params,
+ Throwable throwable) {
+ if (target.getName().startsWith("datadog.trace.api.openfeature")
+ && (failLevelCheck || format != null)) {
+ throw new AssertionError("broken logger");
+ }
+ return FilterReply.NEUTRAL;
+ }
+ };
+ failure.start();
+ logger.getLoggerContext().addTurboFilter(failure);
+ OpenFeatureAPI api = OpenFeatureAPI.getInstance();
+ FeatureFlaggingGateway.dispatch((ServerConfiguration) null);
+ try {
+ Provider provider = new Provider(new Provider.Options().initTimeout(1, MILLISECONDS));
+ assertThrows(ProviderNotReadyError.class, () -> api.setProviderAndWait(provider));
+ Client client = api.getClient();
+ CompletableFuture recovered = new CompletableFuture<>();
+ client.on(ProviderEvent.PROVIDER_READY, recovered::complete);
+ ServerConfiguration config = new ServerConfiguration(null, "ufc/v1", false, null, emptyMap());
+ FeatureFlaggingGateway.dispatch(config);
+ recovered.get(5, TimeUnit.SECONDS);
+ assertEquals(ProviderState.READY, client.getProviderState());
+ assertEquals("fallback", client.getStringValue("missing", "fallback"));
+ assertDoesNotThrow(api::shutdown);
+ } finally {
+ logger.getLoggerContext().getTurboFilterList().remove(failure);
+ failure.stop();
+ api.shutdown();
+ FeatureFlaggingGateway.dispatch((ServerConfiguration) null);
+ }
+ }
+}
diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ExposureHookWithoutAgentTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ExposureHookWithoutAgentTest.java
index efd42915b6d..9a9b34a4cef 100644
--- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ExposureHookWithoutAgentTest.java
+++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ExposureHookWithoutAgentTest.java
@@ -2,6 +2,7 @@
import static java.util.Collections.singletonMap;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import dev.openfeature.sdk.EvaluationContext;
@@ -12,6 +13,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
import org.junit.jupiter.api.Test;
/**
@@ -54,6 +56,25 @@ void exposureHookWorksWithoutTheAgentClasses() throws Exception {
evaluationClass.getMethod("getFeatures").invoke(evaluation));
}
+ @Test
+ void rawDebugLoggingIsDisabledWithoutAgentConfiguration() throws Exception {
+ String previousDebug = System.getProperty("dd.trace.debug");
+ System.setProperty("dd.trace.debug", "true");
+ try {
+ ClassLoader loader = new AgentlessClassLoader(getClass().getClassLoader());
+ Class hookClass = Class.forName(DebugLoggingHook.class.getName(), true, loader);
+ Field traceDebug = hookClass.getDeclaredField("TRACE_DEBUG");
+ traceDebug.setAccessible(true);
+ assertFalse(traceDebug.getBoolean(null));
+ } finally {
+ if (previousDebug == null) {
+ System.clearProperty("dd.trace.debug");
+ } else {
+ System.setProperty("dd.trace.debug", previousDebug);
+ }
+ }
+ }
+
private static final class AgentlessClassLoader extends ClassLoader {
AgentlessClassLoader(final ClassLoader parent) {
super(parent);
@@ -63,12 +84,14 @@ private static final class AgentlessClassLoader extends ClassLoader {
protected Class loadClass(final String name, final boolean resolve)
throws ClassNotFoundException {
synchronized (getClassLoadingLock(name)) {
- if (name.startsWith("datadog.trace.api.featureflag.")
+ if (name.startsWith("datadog.trace.bootstrap.config.")
+ || name.startsWith("datadog.trace.api.featureflag.")
|| name.startsWith("datadog.trace.api.openfeature.DDEvaluator")) {
throw new ClassNotFoundException(name);
}
if (name.startsWith("datadog.trace.api.openfeature.ExposureHook")
- || name.equals("datadog.trace.api.openfeature.SplitFeatureMetadata")) {
+ || name.equals("datadog.trace.api.openfeature.SplitFeatureMetadata")
+ || name.equals("datadog.trace.api.openfeature.DebugLoggingHook")) {
Class loaded = findLoadedClass(name);
if (loaded == null) {
final byte[] bytes = classBytes(name);
diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java
index 2156eae7fdf..dab26a84e35 100644
--- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java
+++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java
@@ -324,8 +324,9 @@ public void testGetProviderHooksReturnsFlagEvalMetricsHook() {
Provider provider =
new Provider(new Options().initTimeout(10, MILLISECONDS), mock(Evaluator.class));
List hooks = provider.getProviderHooks();
- // OTel FlagEvalMetricsHook (index 0), FlagEvalLoggingHook (index 1), ExposureLoggingHook (last)
- assertThat(hooks.size(), equalTo(3));
+ // OTel FlagEvalMetricsHook (index 0), FlagEvalLoggingHook (index 1), ExposureLoggingHook (index
+ // 2), optional DebugLoggingHook (last)
+ assertThat(hooks.stream().filter(h -> !(h instanceof DebugLoggingHook)).count(), equalTo(3L));
assertThat(hooks.get(0) instanceof FlagEvalMetricsHook, equalTo(true));
assertThat(hooks.get(1) instanceof FlagEvalLoggingHook, equalTo(true));
assertThat(hooks.get(2) instanceof ExposureLoggingHook, equalTo(true));
@@ -343,7 +344,7 @@ Hook buildFlagEvalLoggingHook() {
List hooks = provider.getProviderHooks();
- assertThat(hooks.size(), equalTo(2));
+ assertThat(hooks.stream().filter(h -> !(h instanceof DebugLoggingHook)).count(), equalTo(2L));
assertThat(hooks.get(0) instanceof FlagEvalMetricsHook, equalTo(true));
assertThat(hooks.get(1) instanceof ExposureLoggingHook, equalTo(true));
}
@@ -412,7 +413,9 @@ public void testShutdownCleansUpEvaluator() throws Exception {
verify(evaluator).shutdown();
// After shutdown, getProviderHooks still returns the OTel, logging and exposure hooks
- assertThat(provider.getProviderHooks().size(), equalTo(3));
+ assertThat(
+ provider.getProviderHooks().stream().filter(h -> !(h instanceof DebugLoggingHook)).count(),
+ equalTo(3L));
}
private static void assertHasFlagEvalMetricsHook(final Provider provider) {
diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java
index f4fc35cce30..ba4b393a460 100644
--- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java
+++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java
@@ -123,6 +123,26 @@ public void init() {
started = true;
}
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Feature Flagging starting agentless configuration source at {}://{}:{}; pollIntervalMs={}, requestTimeoutSeconds={}, env={}, requestedEnv={}, endpointMode={}",
+ endpoint.scheme(),
+ endpoint.host(),
+ endpoint.port(),
+ pollIntervalMillis,
+ config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds(),
+ config.getEnv(),
+ endpoint.queryParameter("dd_env"),
+ config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() == null
+ ? "managed"
+ : "custom (DD_ENV is not appended)");
+ }
+ if (config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() == null
+ && isBlank(config.getApiKey())) {
+ LOGGER.warn(
+ "Feature Flagging agentless configuration requires DD_API_KEY for the Datadog endpoint");
+ }
+
// Complete the first poll cycle on the activation thread. This lets OpenFeature provider
// initialization observe a successful retry before it checks whether configuration is ready.
// No request occurs before application code activates the provider.
@@ -174,13 +194,17 @@ public void close() {
activePollingThread.interrupt();
}
executor.shutdownNow();
+ LOGGER.debug("Feature Flagging agentless configuration source stopped");
}
private void pollOnceSafely() {
try {
pollOnce();
} catch (final RuntimeException e) {
- LOGGER.debug("Unexpected error while polling Feature Flagging HTTP configuration source", e);
+ LOGGER.debug(
+ "Feature Flagging agentless configuration poll failed: {}",
+ e.getClass().getSimpleName(),
+ e);
}
}
@@ -192,7 +216,7 @@ private boolean fetchAndApply() {
}
if (isRetryableStatus(response.status)) {
ratelimitedLogger.warn(
- "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}",
+ "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}; retaining current configuration; retry on next poll",
MAX_ATTEMPTS,
response.status);
return false;
@@ -203,8 +227,10 @@ private boolean fetchAndApply() {
} catch (final IOException e) {
if (!closed) {
ratelimitedLogger.warn(
- "Feature Flagging agentless endpoint request failed after {} attempts",
+ "Feature Flagging agentless endpoint request failed after {} attempts: {}: {}; retaining current configuration; retry on next poll",
MAX_ATTEMPTS,
+ e.getClass().getSimpleName(),
+ e.getMessage(),
e);
}
return false;
@@ -213,6 +239,8 @@ private boolean fetchAndApply() {
private boolean apply(final UfcHttpResponse response) {
if (response.status == HttpURLConnection.HTTP_NOT_MODIFIED) {
+ LOGGER.debug(
+ "Feature Flagging agentless configuration unchanged (HTTP 304); retaining current configuration");
return true;
}
if (response.status == HttpURLConnection.HTTP_UNAUTHORIZED
@@ -222,21 +250,44 @@ private boolean apply(final UfcHttpResponse response) {
response.status);
return false;
}
- if (response.status != HttpURLConnection.HTTP_OK || response.body == null) {
+ if (response.status != HttpURLConnection.HTTP_OK) {
+ LOGGER.debug(
+ "Feature Flagging agentless configuration not applied: unexpected HTTP {}; retaining current configuration",
+ response.status);
+ return false;
+ }
+ if (response.body == null) {
+ LOGGER.debug(
+ "Feature Flagging agentless configuration rejected: missing response body; retaining current configuration");
return false;
}
final ServerConfiguration configuration;
try {
configuration = JsonApiUfcResponseParser.INSTANCE.parse(response.body);
} catch (final IOException | RuntimeException e) {
- LOGGER.debug("Feature Flagging HTTP configuration source returned malformed UFC payload", e);
+ LOGGER.debug(
+ "Feature Flagging agentless configuration rejected: malformed UFC payload ({}); retaining current configuration",
+ e.getClass().getSimpleName(),
+ e);
return false;
}
if (configuration == null) {
+ LOGGER.debug(
+ "Feature Flagging agentless configuration rejected: missing UFC data or flags; retaining current configuration");
return false;
}
FeatureFlaggingGateway.dispatch(configuration);
updateEtag(response.etag);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Feature Flagging agentless configuration applied: flags={}, env={}, configurationEnv={}, createdAt={}, invalidFlags={}, etag={}",
+ configuration.flags.size(),
+ config.getEnv(),
+ configuration.environment == null ? null : configuration.environment.name,
+ configuration.createdAt,
+ configuration.invalidFlags == null ? 0 : configuration.invalidFlags.size(),
+ etag);
+ }
return true;
}
@@ -261,7 +312,7 @@ private static HttpUrl endpointFromConfiguredBaseUrl(final String configuredBase
final HttpUrl parsed = HttpUrl.parse(configuredBaseUrl.trim());
if (parsed == null) {
throw new IllegalArgumentException(
- "Invalid Feature Flagging HTTP configuration source URL: " + configuredBaseUrl);
+ "Invalid Feature Flagging HTTP configuration source URL; verify DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL");
}
if ("/".equals(parsed.encodedPath()) || parsed.encodedPath().isEmpty()) {
return parsed
@@ -391,6 +442,7 @@ public HttpRetryPolicy create() {
if (cancelled.get()) {
call.cancel();
}
+ LOGGER.debug("Feature Flagging agentless configuration poll request starting");
return call;
};
return sendWithRetries(
@@ -399,6 +451,10 @@ public HttpRetryPolicy create() {
request,
response -> {
final int status = response.code();
+ if (response.isSuccessful()) {
+ LOGGER.debug(
+ "Feature Flagging agentless configuration poll returned HTTP {}", status);
+ }
final String responseEtag = response.header("ETag");
try (ResponseBody responseBody = response.body()) {
final byte[] body = responseBody != null ? responseBody.bytes() : null;
@@ -453,6 +509,14 @@ static final class AgentlessRetryPolicy extends HttpRetryPolicy {
@Override
public boolean shouldRetry(final Exception exception) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Feature Flagging agentless configuration poll attempt {}/{} failed: {}",
+ retryAttempt + 1,
+ MAX_ATTEMPTS,
+ exception.getClass().getSimpleName(),
+ exception);
+ }
return exception instanceof IOException
&& !Thread.currentThread().isInterrupted()
&& reserveRetry();
@@ -460,6 +524,13 @@ public boolean shouldRetry(final Exception exception) {
@Override
public boolean shouldRetry(@Nullable final Response response) {
+ if (response != null && LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Feature Flagging agentless configuration poll attempt {}/{} returned HTTP {}",
+ retryAttempt + 1,
+ MAX_ATTEMPTS,
+ response.code());
+ }
return response != null && isRetryableStatus(response.code()) && reserveRetry();
}
@@ -478,8 +549,10 @@ public void backoff() throws IOException {
throw new InterruptedIOException("Feature Flagging HTTP client is closed");
}
try {
- retrySleeper.sleep(
- retryDelayMillis(pollIntervalMillis, retryAttempt, jitter.getAsDouble()));
+ final long delayMillis =
+ retryDelayMillis(pollIntervalMillis, retryAttempt, jitter.getAsDouble());
+ LOGGER.debug("Feature Flagging agentless configuration retry in {} ms", delayMillis);
+ retrySleeper.sleep(delayMillis);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
throw new InterruptedIOException("Feature Flagging retry interrupted");
diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java
index 0584e5701ad..35b5eabd203 100644
--- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java
+++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java
@@ -9,6 +9,7 @@
import datadog.common.queue.MessagePassingBlockingQueue;
import datadog.common.queue.Queues;
import datadog.communication.BackendApi;
+import datadog.communication.HttpResponseException;
import datadog.communication.ddagent.SharedCommunicationObjects;
import datadog.trace.api.Config;
import datadog.trace.api.featureflag.FeatureFlaggingGateway;
@@ -144,6 +145,11 @@ public void requestShutdown() {
}
// Stop accepting new dispatches without waiting for producers already inside accept().
FeatureFlaggingGateway.removeExposureListener(this);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Feature Flagging exposure shutdown requested: queued={}; pending exposures get one final best-effort flush; processor completion is reported separately",
+ queue.size());
+ }
// Polling wakes within 100 ms. An interrupt can race with, or abort, the final POST.
serializer.requestShutdown();
}
@@ -152,6 +158,13 @@ public void requestShutdown() {
@Override
public void awaitTermination(final long deadlineNanos) {
FeatureFlagEventWriter.awaitThread(serializerThread, deadlineNanos);
+ // The final POST is not interrupted, so it can outlive the shutdown budget.
+ if (serializerThread.isAlive()
+ && serializerThread != Thread.currentThread()
+ && LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Feature Flagging exposure shutdown wait expired; final delivery continues in the background");
+ }
}
@Override
@@ -160,7 +173,12 @@ public void accept(final ExposureEvent event) {
// passes this check can race past the final drain and be dropped. Stale gateway dispatches
// entering after shutdown starts are rejected without extending the shutdown wait.
if (!closed.get()) {
- queue.offer(event);
+ final boolean queued = queue.offer(event);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Feature Flagging exposure {}", queued ? "queued" : "dropped: queue full");
+ }
+ } else if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Feature Flagging exposure dropped: writer shutting down");
}
}
@@ -233,7 +251,10 @@ public void run() {
}
}
}
- LOGGER.debug("exposure processor worker exited. submitting exposures stopped.");
+ LOGGER.debug(
+ "Feature Flagging exposure processor stopped after final flush; queued={}, buffered={}",
+ queue.size(),
+ buffer.size());
}
private void runDutyCycle() throws InterruptedException {
@@ -268,6 +289,7 @@ private boolean addToBuffer(final ExposureEvent event) {
buffer.add(event);
return true;
}
+ LOGGER.debug("Feature Flagging exposure suppressed: duplicate");
return false;
}
@@ -291,11 +313,23 @@ private void flush() {
return;
}
try {
+ LOGGER.debug("Feature Flagging exposure flush starting: count={}", this.buffer.size());
evpPublisher.post(EXPOSURES_ROUTE, payload);
+ LOGGER.debug("Feature Flagging exposure flush delivered: count={}", this.buffer.size());
} catch (AgentlessFeatureFlagBackendApi.UnavailableRouteException ignored) {
// The shared selector reports unavailable/recovered transitions once for both writers.
} catch (Exception e) {
- LOGGER.debug("Could not submit exposures", e);
+ if (e instanceof HttpResponseException) {
+ LOGGER.debug(
+ "Feature Flagging exposure flush failed: HTTP {}; dropping batch",
+ ((HttpResponseException) e).getStatusCode(),
+ e);
+ } else {
+ LOGGER.debug(
+ "Feature Flagging exposure flush failed: {}; dropping batch",
+ e.getClass().getSimpleName(),
+ e);
+ }
} finally {
// Best-effort delivery must not retry an ambiguously accepted batch. A later definitive
// proxy rejection could otherwise replay the same exposures through direct intake.
diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java
index e66a6231990..8e8106bad6d 100644
--- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java
+++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java
@@ -10,10 +10,14 @@
import datadog.trace.api.featureflag.FeatureFlaggingGateway;
import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration;
import javax.annotation.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class RemoteConfigServiceImpl
implements ConfigurationSourceService, ConfigurationChangesTypedListener {
+ private static final Logger LOGGER = LoggerFactory.getLogger(RemoteConfigServiceImpl.class);
+
private final ConfigurationPoller configurationPoller;
public RemoteConfigServiceImpl(final SharedCommunicationObjects sco, final Config config) {
@@ -22,6 +26,8 @@ public RemoteConfigServiceImpl(final SharedCommunicationObjects sco, final Confi
@Override
public void init() {
+ LOGGER.debug(
+ "Feature Flagging starting remote_config configuration source (Agent Remote Configuration)");
configurationPoller.addCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES);
configurationPoller.addListener(Product.FFE_FLAGS, UniversalFlagConfigParser.INSTANCE, this);
configurationPoller.start();
@@ -32,6 +38,7 @@ public void close() {
configurationPoller.removeCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES);
configurationPoller.removeListeners(Product.FFE_FLAGS);
configurationPoller.stop();
+ LOGGER.debug("Feature Flagging remote_config configuration source stopped");
}
@Override
@@ -40,5 +47,17 @@ public void accept(
@Nullable final ServerConfiguration configuration,
final PollingRateHinter pollingRateHinter) {
FeatureFlaggingGateway.dispatch(configuration);
+ if (configuration == null) {
+ LOGGER.debug("Feature Flagging remote_config configuration removed");
+ } else if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Feature Flagging remote_config configuration applied: key={}, flags={}, configurationEnv={}, createdAt={}, invalidFlags={}",
+ configKey,
+ // A payload without a flags member is not an empty configuration.
+ configuration.flags == null ? "missing" : configuration.flags.size(),
+ configuration.environment == null ? null : configuration.environment.name,
+ configuration.createdAt,
+ configuration.invalidFlags == null ? 0 : configuration.invalidFlags.size());
+ }
}
}
diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java
index 27fd3f70c70..39bfdf1bef3 100644
--- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java
+++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/UniversalFlagConfigParser.java
@@ -75,6 +75,12 @@ public ServerConfiguration deserialize(final byte[] content) throws IOException
final ServerConfiguration configuration = parse(reader);
requireEndOfDocument(reader);
return configuration;
+ } catch (IOException | RuntimeException e) {
+ LOGGER.debug(
+ "Feature Flagging remote_config configuration rejected: malformed UFC payload ({})",
+ e.getClass().getSimpleName(),
+ e);
+ throw e;
}
}
@@ -235,9 +241,7 @@ private static void validateAndCacheSemverComparands(final String flagKey, final
+ allocIdx
+ " rule has condition with operator \""
+ condition.operator
- + "\" and invalid semantic version \""
- + condition.value
- + "\"");
+ + "\" and invalid semantic version");
}
condition.semverComparand = parsed;
break;
@@ -365,28 +369,44 @@ public Map fromJson(@Nonnull final JsonReader reader) throws IOExc
return reader.nextNull();
}
final Map flags = new HashMap<>();
+ int receivedFlags = 0;
+ int rejectedFlags = 0;
reader.beginObject();
while (reader.hasNext()) {
final String flagKey = reader.nextName();
+ receivedFlags++;
final Object rawFlag = reader.readJsonValue();
try {
final Flag flag = flagAdapter.fromJsonValue(rawFlag);
if (flag != null) {
validateFlag(flagKey, flag);
flags.put(flagKey, flag);
+ } else {
+ rejectedFlags++;
+ LOGGER.debug(
+ "Feature Flagging rejected flag {} during configuration parsing: null flag",
+ flagKey);
}
} catch (JsonDataException | IllegalArgumentException error) {
+ rejectedFlags++;
INVALID_FLAGS_HOLDER.get().put(flagKey, INVALID_FLAG);
if (error instanceof InvalidSemverComparandException) {
INVALID_FLAGS_HOLDER.get().put(flagKey, INVALID_SEMVER_COMPARAND);
}
LOGGER.warn(
- "Dropping malformed FFE flag {} during remote config deserialization: {}",
+ "Feature Flagging rejected flag {} during configuration parsing: {}",
flagKey,
error.toString());
}
}
reader.endObject();
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Feature Flagging configuration flags parsed: receivedFlags={}, usableFlags={}, rejectedFlags={}",
+ receivedFlags,
+ flags.size(),
+ rejectedFlags);
+ }
return flags;
}
diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java
index b02bd05642b..3cb0fc7a8f4 100644
--- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java
+++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java
@@ -4,7 +4,9 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -15,6 +17,11 @@
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.spi.ThrowableProxy;
+import ch.qos.logback.core.read.ListAppender;
import datadog.communication.http.OkHttpUtils;
import datadog.logging.RatelimitedLogger;
import datadog.trace.agent.test.server.http.JavaTestHttpServer;
@@ -41,6 +48,7 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPOutputStream;
+import javax.net.ssl.SSLHandshakeException;
import okhttp3.Call;
import okhttp3.HttpUrl;
import okhttp3.MediaType;
@@ -49,12 +57,15 @@
import okhttp3.Response;
import okhttp3.ResponseBody;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import org.slf4j.LoggerFactory;
+import org.tabletest.junit.TableTest;
@ExtendWith(MockitoExtension.class)
class AgentlessConfigurationSourceTest {
@@ -755,7 +766,9 @@ void warnsRateLimitedAfterRetryableFailuresAreExhausted() throws Exception {
assertEquals(3, requests.size());
verify(ratelimitedLogger)
.warn(
- "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}", 3, 503);
+ "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}; retaining current configuration; retry on next poll",
+ 3,
+ 503);
verifyNoInteractions(listener);
} finally {
service.close();
@@ -793,8 +806,10 @@ void warnsRateLimitedAfterIoFailuresAreExhausted() throws Exception {
assertEquals(3, requests.size());
verify(ratelimitedLogger)
.warn(
- "Feature Flagging agentless endpoint request failed after {} attempts",
+ "Feature Flagging agentless endpoint request failed after {} attempts: {}: {}; retaining current configuration; retry on next poll",
3,
+ "SocketTimeoutException",
+ finalFailure.getMessage(),
finalFailure);
verifyNoInteractions(listener);
} finally {
@@ -1204,6 +1219,243 @@ private static Response okHttpResponse(
return builder.build();
}
+ @Nested
+ class Diagnostics {
+ private final Logger logger =
+ (Logger) LoggerFactory.getLogger(AgentlessConfigurationSource.class);
+ private final ListAppender appender = new ListAppender<>();
+ private Level previousLevel;
+
+ @BeforeEach
+ void captureLogs() {
+ previousLevel = logger.getLevel();
+ logger.setLevel(Level.DEBUG);
+ appender.start();
+ logger.addAppender(appender);
+ }
+
+ @AfterEach
+ void restoreLogging() {
+ logger.detachAppender(appender);
+ appender.stop();
+ logger.setLevel(previousLevel);
+ }
+
+ @TableTest({
+ "Scenario | Env | Custom URL | Requested ",
+ "managed environment | production | | production",
+ "custom no environment | production | 'https://flags.example.com/config' | ",
+ "custom scoped URL | production | 'https://flags.example.com/?dd_env=qa' | qa ",
+ "managed unset | | | "
+ })
+ void distinguishesSdkAndRequestedEnvironment(String env, String customUrl, String requested)
+ throws Exception {
+ Config config = config("datadoghq.com", env);
+ when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()).thenReturn(customUrl);
+ AgentlessConfigurationSource.UfcHttpClient client =
+ mock(AgentlessConfigurationSource.UfcHttpClient.class);
+ when(client.fetch(any(), any(), any())).thenReturn(response(304, null, null));
+ try (AgentlessConfigurationSource source =
+ new AgentlessConfigurationSource(
+ AgentlessConfigurationSource.endpoint(config),
+ config,
+ 30000,
+ client,
+ mock(ScheduledExecutorService.class))) {
+ source.init();
+ assertLogged(
+ "env="
+ + env
+ + ", requestedEnv="
+ + requested
+ + ", endpointMode="
+ + (customUrl == null ? "managed" : "custom (DD_ENV is not appended)"));
+ }
+ }
+
+ @Test
+ void reportsAppliedUnchangedAndRejectedConfigurationWithParseFailure() throws Exception {
+ final AgentlessConfigurationSource.UfcHttpClient client =
+ mock(AgentlessConfigurationSource.UfcHttpClient.class);
+ when(client.fetch(any(), any(), any()))
+ .thenReturn(
+ response(200, "configuration-v1", emptyConfig()),
+ response(304, null, null),
+ response(200, null, "{\"secret-payload\":"));
+ final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+ final AgentlessConfigurationSource service =
+ new AgentlessConfigurationSource(
+ HttpUrl.get("http://localhost" + CONFIG_PATH),
+ config("datadoghq.com", "production"),
+ 30_000,
+ client,
+ executor);
+ try {
+ assertTrue(service.pollOnce());
+ assertTrue(service.pollOnce());
+ assertFalse(service.pollOnce());
+ assertLogged("configuration applied: flags=0, env=production, configurationEnv=Staging");
+ assertLogged("etag=configuration-v1");
+ assertLogged("configuration unchanged (HTTP 304)");
+ assertNotNull(
+ assertLogged("configuration rejected: malformed UFC payload").getThrowableProxy());
+ assertEquals(
+ 1,
+ appender.list.stream()
+ .filter(e -> e.getFormattedMessage().contains("configuration applied:"))
+ .count());
+ assertNoSecretsInFormattedMessages();
+ } finally {
+ service.close();
+ }
+ }
+
+ @Test
+ void reportsAppliedConfigurationWithoutEnvironment() throws Exception {
+ final AgentlessConfigurationSource.UfcHttpClient client =
+ mock(AgentlessConfigurationSource.UfcHttpClient.class);
+ when(client.fetch(any(), any(), any()))
+ .thenReturn(
+ response(
+ 200, null, jsonApiResponse("universal-flag-configuration", "{\"flags\":{}}")));
+ try (AgentlessConfigurationSource service = service(client)) {
+ assertTrue(service.pollOnce());
+ assertLogged("configuration applied: flags=0, env=, configurationEnv=null, createdAt=null");
+ }
+ }
+
+ @Test
+ void omitsDebugDetailsAndApiKeyWarningForConfiguredEndpoint() throws Exception {
+ logger.setLevel(Level.INFO);
+ final Config config = config();
+ when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl())
+ .thenReturn("https://flags.example.com");
+ lenient().when(config.getApiKey()).thenReturn(null);
+ final AgentlessConfigurationSource.UfcHttpClient client =
+ mock(AgentlessConfigurationSource.UfcHttpClient.class);
+ when(client.fetch(any(), any(), any())).thenReturn(response(200, null, emptyConfig()));
+ FeatureFlaggingGateway.addConfigListener(listener);
+ final AgentlessConfigurationSource service =
+ new AgentlessConfigurationSource(
+ HttpUrl.get("https://flags.example.com" + CONFIG_PATH),
+ config,
+ 30_000,
+ client,
+ mock(ScheduledExecutorService.class));
+ try {
+ service.init();
+ verify(listener).accept(any(ServerConfiguration.class));
+ assertTrue(appender.list.isEmpty());
+ } finally {
+ service.close();
+ }
+ }
+
+ @Test
+ void logsMissingCredentialsAndOnlyEndpointOrigin() throws Exception {
+ final Config config = config();
+ when(config.getApiKey()).thenReturn(null);
+ final AgentlessConfigurationSource.UfcHttpClient client =
+ mock(AgentlessConfigurationSource.UfcHttpClient.class);
+ when(client.fetch(any(), any(), any())).thenReturn(response(401, null, null));
+ final ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
+ final AgentlessConfigurationSource service =
+ new AgentlessConfigurationSource(
+ HttpUrl.get(
+ "https://secret-user:secret-password@example.com/secret-path?key=secret-query"),
+ config,
+ 30_000,
+ client,
+ executor);
+ try {
+ service.init();
+ assertLogged("source at https://example.com:443");
+ assertLogged("requires DD_API_KEY");
+ assertLogged("HTTP 401; verify endpoint authentication");
+ assertNoSecretsInFormattedMessages();
+ } finally {
+ service.close();
+ }
+ }
+
+ @Test
+ void logsTransportFailureAndBackoffWithExceptionDetails() throws Exception {
+ final List requests = new ArrayList<>();
+ final SocketTimeoutException failure = new SocketTimeoutException("read timed out");
+ final AgentlessConfigurationSource.OkHttpUfcHttpClient client =
+ scriptedClient(
+ requests,
+ delay -> {},
+ () -> 1.0,
+ failure,
+ response(503, null, null),
+ response(200, null, emptyConfig()));
+ client.fetch(HttpUrl.get("https://example.com"), config(), null);
+ assertSame(
+ failure,
+ ((ThrowableProxy)
+ assertLogged("attempt 1/3 failed: SocketTimeoutException").getThrowableProxy())
+ .getThrowable());
+ assertLogged("attempt 2/3 returned HTTP 503");
+ assertLogged("retry in 5000 ms");
+ assertLogged("returned HTTP 200");
+ }
+
+ @Test
+ void retainsTransportRootCauseAtWarnWhenDebugIsDisabled() throws Exception {
+ logger.setLevel(Level.WARN);
+ final SSLHandshakeException failure = new SSLHandshakeException("PKIX path building failed");
+ final AgentlessConfigurationSource.UfcHttpClient client =
+ mock(AgentlessConfigurationSource.UfcHttpClient.class);
+ when(client.fetch(any(), any(), any())).thenThrow(failure);
+ try (AgentlessConfigurationSource service = service(client)) {
+ assertFalse(service.pollOnce());
+ final ILoggingEvent event =
+ assertLogged("SSLHandshakeException: PKIX path building failed");
+ assertEquals(Level.WARN, event.getLevel());
+ assertSame(failure, ((ThrowableProxy) event.getThrowableProxy()).getThrowable());
+ }
+ }
+
+ @Test
+ void retainsListenerFailureForErrorTelemetry() throws Exception {
+ final IllegalStateException failure =
+ new IllegalStateException("listener rejected configuration");
+ final FeatureFlaggingGateway.ConfigListener failingListener =
+ configuration -> {
+ throw failure;
+ };
+ FeatureFlaggingGateway.addConfigListener(failingListener);
+ final AgentlessConfigurationSource.UfcHttpClient client =
+ mock(AgentlessConfigurationSource.UfcHttpClient.class);
+ when(client.fetch(any(), any(), any())).thenReturn(response(200, null, emptyConfig()));
+ try (AgentlessConfigurationSource service = service(client)) {
+ service.init();
+ assertSame(
+ failure,
+ ((ThrowableProxy)
+ assertLogged("configuration poll failed: IllegalStateException")
+ .getThrowableProxy())
+ .getThrowable());
+ } finally {
+ FeatureFlaggingGateway.removeConfigListener(failingListener);
+ }
+ }
+
+ private ILoggingEvent assertLogged(final String text) {
+ return appender.list.stream()
+ .filter(e -> e.getFormattedMessage().contains(text))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("Missing log: " + text));
+ }
+
+ private void assertNoSecretsInFormattedMessages() {
+ for (final ILoggingEvent event : appender.list) {
+ assertFalse(event.getFormattedMessage().contains("secret"));
+ }
+ }
+ }
+
private static Config config() {
return config("datadoghq.com", "");
}
diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java
index ca12d057578..874e107c1c7 100644
--- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java
+++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java
@@ -15,6 +15,7 @@
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.AdditionalAnswers.delegatesTo;
import static org.mockito.ArgumentMatchers.any;
@@ -26,12 +27,18 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.spi.ThrowableProxy;
+import ch.qos.logback.core.read.ListAppender;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.Moshi;
import datadog.common.queue.MessagePassingBlockingQueue;
import datadog.common.queue.Queues;
import datadog.communication.BackendApi;
import datadog.communication.BackendApiFactory;
+import datadog.communication.HttpResponseException;
import datadog.communication.IntakeApi;
import datadog.communication.ddagent.DDAgentFeaturesDiscovery;
import datadog.communication.ddagent.SharedCommunicationObjects;
@@ -51,6 +58,7 @@
import datadog.trace.api.intake.Intake;
import datadog.trace.test.util.PollingConditions;
import java.io.ByteArrayInputStream;
+import java.io.IOException;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
@@ -65,6 +73,7 @@
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -80,6 +89,7 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
+import org.slf4j.LoggerFactory;
import org.tabletest.junit.TableTest;
class ExposureWriterTests {
@@ -168,6 +178,137 @@ void testExposureEventWrites(String service, String env, String version) throws
}
}
+ @TableTest({
+ "Scenario | Fails | Status | Outcome ",
+ "Successful delivery | false | | 'flush delivered' ",
+ "Transport failure | true | | 'flush failed: IOException; dropping batch'",
+ "HTTP rejection | true | 403 | 'flush failed: HTTP 403; dropping batch' "
+ })
+ void distinguishesQueuedFromDeliveredExposures(boolean fails, Integer status, String outcome)
+ throws Exception {
+ Logger logger = (Logger) LoggerFactory.getLogger(ExposureWriterImpl.class);
+ Level previousLevel = logger.getLevel();
+ ListAppender appender = new ListAppender<>();
+ appender.list = new CopyOnWriteArrayList<>();
+ appender.start();
+ logger.addAppender(appender);
+ logger.setLevel(Level.DEBUG);
+ BackendApi backend = mock(BackendApi.class);
+ IOException failure =
+ status == null
+ ? new IOException("connection reset")
+ : new HttpResponseException(status, "exposure batch rejected");
+ if (fails) {
+ when(backend.post(any(), any(), any(), any(), eq(false))).thenThrow(failure);
+ }
+ ExposureWriterImpl writer =
+ new ExposureWriterImpl(
+ 16,
+ 1,
+ MILLISECONDS,
+ () -> backend,
+ mockConfig("test-service"),
+ ExposureWriterImpl.SHUTDOWN_TIMEOUT_MILLIS);
+ try {
+ writer.init();
+ writer.accept(buildExposures(1).get(0));
+ poll.eventually(
+ () ->
+ assertTrue(
+ appender.list.stream().anyMatch(e -> e.getFormattedMessage().contains(outcome))));
+ assertTrue(
+ appender.list.stream()
+ .anyMatch(e -> e.getFormattedMessage().contains("exposure queued")));
+ assertEquals(
+ !fails,
+ appender.list.stream()
+ .anyMatch(e -> e.getFormattedMessage().contains("flush delivered")));
+ if (fails) {
+ ILoggingEvent event =
+ appender.list.stream()
+ .filter(e -> e.getFormattedMessage().contains(outcome))
+ .findFirst()
+ .get();
+ assertSame(failure, ((ThrowableProxy) event.getThrowableProxy()).getThrowable());
+ }
+ } finally {
+ writer.close();
+ logger.detachAppender(appender);
+ logger.setLevel(previousLevel);
+ appender.stop();
+ }
+ }
+
+ @TableTest({
+ "Scenario | Fails | Outcome ",
+ "in-flight delivered | false | 'flush delivered' ",
+ "in-flight timed out | true | 'flush failed: SocketTimeoutException; dropping batch'"
+ })
+ void reportsShutdownWithPendingAndInFlightExposures(boolean fails, String outcome)
+ throws Exception {
+ Logger logger = (Logger) LoggerFactory.getLogger(ExposureWriterImpl.class);
+ Level previous = logger.getLevel();
+ ListAppender appender = new ListAppender<>();
+ appender.list = new CopyOnWriteArrayList<>();
+ appender.start();
+ logger.addAppender(appender);
+ logger.setLevel(Level.DEBUG);
+ CountDownLatch inFlight = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicLong posts = new AtomicLong();
+ BackendApi backend = mock(BackendApi.class);
+ when(backend.post(any(), any(), any(), any(), eq(false)))
+ .thenAnswer(
+ invocation -> {
+ // Only the first delivery blocks; the final flush after it returns immediately.
+ if (posts.incrementAndGet() == 1) {
+ inFlight.countDown();
+ release.await();
+ if (fails) {
+ throw new SocketTimeoutException("destination timed out during shutdown");
+ }
+ }
+ return null;
+ });
+ List exposures = buildExposures(3);
+ ExposureWriterImpl writer =
+ new ExposureWriterImpl(16, 1, MILLISECONDS, () -> backend, mockConfig("shutdown"), 100);
+ try {
+ writer.init();
+ writer.accept(exposures.get(0));
+ assertTrue(inFlight.await(5000, MILLISECONDS));
+ writer.accept(exposures.get(1));
+ writer.close();
+ writer.accept(exposures.get(2));
+ assertTrue(writer.isSerializerThreadAlive());
+ release.countDown();
+ poll.eventually(() -> assertFalse(writer.isSerializerThreadAlive()));
+ assertTrue(
+ appender.list.stream()
+ .anyMatch(e -> e.getFormattedMessage().contains("shutdown requested: queued=1;")));
+ assertTrue(
+ appender.list.stream()
+ .anyMatch(e -> e.getFormattedMessage().contains("shutdown wait expired")));
+ assertTrue(
+ appender.list.stream()
+ .anyMatch(e -> e.getFormattedMessage().contains("dropped: writer shutting down")));
+ assertTrue(appender.list.stream().anyMatch(e -> e.getFormattedMessage().contains(outcome)));
+ assertTrue(
+ appender.list.stream()
+ .anyMatch(
+ e ->
+ e.getFormattedMessage()
+ .contains("stopped after final flush; queued=0, buffered=0")));
+ verify(backend, times(2)).post(any(), any(), any(), any(), eq(false));
+ } finally {
+ release.countDown();
+ writer.close();
+ logger.detachAppender(appender);
+ logger.setLevel(previous);
+ appender.stop();
+ }
+ }
+
@Test
void testExposureRequestsCarrySdkIdentity() throws Exception {
// Exercise the production writer and factory on the fixed Remote Config EVP v2 route.
@@ -194,6 +335,36 @@ void testExposureRequestsCarrySdkIdentity() throws Exception {
}
}
+ @Test
+ void logsExposuresDroppedWhenQueueIsFull() {
+ Logger logger = (Logger) LoggerFactory.getLogger(ExposureWriterImpl.class);
+ Level previousLevel = logger.getLevel();
+ ListAppender appender = new ListAppender<>();
+ appender.start();
+ logger.addAppender(appender);
+ logger.setLevel(Level.DEBUG);
+ Config config = mockConfig("test-service");
+
+ // The writer is not initialized, so no serializer thread drains the queue.
+ try (ExposureWriterImpl writer =
+ new ExposureWriterImpl(1 << 4, 100, MILLISECONDS, sharedCommunicationObjects, config)) {
+ for (ExposureEvent exposure : buildExposures(17)) {
+ writer.accept(exposure);
+ }
+
+ assertEquals(16, writer.queueSize());
+ assertEquals(
+ 1,
+ appender.list.stream()
+ .filter(e -> e.getFormattedMessage().contains("exposure dropped: queue full"))
+ .count());
+ } finally {
+ logger.detachAppender(appender);
+ appender.stop();
+ logger.setLevel(previousLevel);
+ }
+ }
+
@Test
void testAgentlessExposureEventWritesDirectlyWithApiKey() throws Exception {
Config config = mockConfig("test-service");
@@ -575,16 +746,36 @@ void testHttpFailureIsNotRetriedAtTransportLayer() throws Exception {
void testSerializationFailureDoesNotPoisonFollowingExposures() throws Exception {
Config config = mockConfig("test-service");
ExposureEvent validExposure = buildExposure();
+ Logger logger = (Logger) LoggerFactory.getLogger(ExposureWriterImpl.class);
+ Level previousLevel = logger.getLevel();
+ ListAppender appender = new ListAppender<>();
+ appender.list = new CopyOnWriteArrayList<>();
+ appender.start();
+ logger.addAppender(appender);
+ logger.setLevel(Level.ERROR);
try (ExposureWriterImpl writer =
new ExposureWriterImpl(1 << 4, 100, MILLISECONDS, sharedCommunicationObjects, config)) {
writer.init();
writer.accept(buildExposure(singletonMap("invalid", (Object) Double.NaN)));
- MILLISECONDS.sleep(300); // wait for the invalid batch to be dropped
+ poll.eventually(
+ () -> {
+ assertEquals(1, appender.list.size());
+ ILoggingEvent event = appender.list.get(0);
+ assertTrue(event.getFormattedMessage().contains("Could not serialize exposures"));
+ assertNotNull(event.getThrowableProxy());
+ assertEquals(
+ IllegalArgumentException.class.getName(), event.getThrowableProxy().getClassName());
+ assertTrue(event.getThrowableProxy().getMessage().contains("NaN"));
+ });
writer.accept(validExposure);
poll.eventually(() -> assertExposures(allExposures(), singletonList(validExposure)));
+ } finally {
+ logger.detachAppender(appender);
+ appender.stop();
+ logger.setLevel(previousLevel);
}
}
diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java
index fae2631ce2f..775d304b689 100644
--- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java
+++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/JsonApiUfcResponseParserTest.java
@@ -8,6 +8,10 @@
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
import com.squareup.moshi.JsonQualifier;
import com.squareup.moshi.Moshi;
import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration;
@@ -21,9 +25,78 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import org.slf4j.LoggerFactory;
class JsonApiUfcResponseParserTest {
+ @Test
+ void countsNullAndMalformedFlagsWithoutChangingTheirEvaluationSemantics() throws Exception {
+ Logger logger = (Logger) LoggerFactory.getLogger(UniversalFlagConfigParser.class);
+ Level previous = logger.getLevel();
+ ListAppender appender = new ListAppender<>();
+ appender.start();
+ logger.addAppender(appender);
+ logger.setLevel(Level.DEBUG);
+ try {
+ ServerConfiguration config = parse(wrap("{\"flags\":{\"null-flag\":null,\"bad\":42}}"));
+ assertTrue(config.flags.isEmpty());
+ assertEquals(1, config.invalidFlags.size());
+ assertFalse(config.invalidFlags.containsKey("null-flag"));
+ assertTrue(
+ appender.list.stream()
+ .anyMatch(
+ event ->
+ event
+ .getFormattedMessage()
+ .contains("receivedFlags=2, usableFlags=0, rejectedFlags=2")));
+ assertTrue(
+ appender.list.stream()
+ .anyMatch(event -> event.getFormattedMessage().contains("null flag")));
+ } finally {
+ logger.detachAppender(appender);
+ logger.setLevel(previous);
+ appender.stop();
+ }
+ }
+
+ @Test
+ void rejectedFlagWarningsRetainStructuralReasonsWithoutSemverComparands() throws Exception {
+ final Logger logger = (Logger) LoggerFactory.getLogger(UniversalFlagConfigParser.class);
+ final Level previousLevel = logger.getLevel();
+ final ListAppender appender = new ListAppender<>();
+ appender.start();
+ logger.addAppender(appender);
+ logger.setLevel(Level.WARN);
+ try {
+ final ServerConfiguration configuration =
+ parse(
+ wrap(
+ configWithFlags(
+ booleanFlag(
+ "missing-shards",
+ ",\"allocations\":[{\"key\":\"allocation\",\"splits\":[{}]}]"),
+ booleanFlag(
+ "invalid-semver",
+ allocation(
+ "allocation",
+ "[{\"conditions\":[{\"attribute\":\"version\",\"operator\":\"SEMVER_EQ\",\"value\":\"secret-comparand\"}]}]")))));
+
+ assertEquals("invalid_flag", configuration.invalidFlags.get("missing-shards"));
+ assertEquals("invalid_semver_comparand", configuration.invalidFlags.get("invalid-semver"));
+ assertEquals(2, appender.list.size());
+ assertTrue(appender.list.get(0).getFormattedMessage().contains("InvalidFlagException:"));
+ assertTrue(appender.list.get(0).getFormattedMessage().contains("split with missing shards"));
+ assertTrue(
+ appender.list.get(1).getFormattedMessage().contains("InvalidSemverComparandException:"));
+ assertTrue(appender.list.get(1).getFormattedMessage().contains("invalid semantic version"));
+ assertFalse(appender.list.get(1).getFormattedMessage().contains("secret-comparand"));
+ } finally {
+ logger.detachAppender(appender);
+ appender.stop();
+ logger.setLevel(previousLevel);
+ }
+ }
+
@Test
void parsesJsonApiMembersInAnyOrder() throws Exception {
final ServerConfiguration configuration =
diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java
index f11816a9bc0..ac334aa46e7 100644
--- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java
+++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/RemoteConfigServiceImplTest.java
@@ -17,6 +17,10 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
@@ -45,6 +49,8 @@
import java.util.Map;
import okio.Buffer;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
@@ -53,6 +59,7 @@
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import org.slf4j.LoggerFactory;
import org.tabletest.junit.TableTest;
@ExtendWith(MockitoExtension.class)
@@ -90,6 +97,81 @@ void testNewConfigReceived() throws Exception {
verify(poller).removeListeners(Product.FFE_FLAGS);
}
+ @Nested
+ class Diagnostics {
+ private final Logger logger = (Logger) LoggerFactory.getLogger(RemoteConfigServiceImpl.class);
+ private final ListAppender appender = new ListAppender<>();
+ private Level previousLevel;
+ private RemoteConfigServiceImpl service;
+
+ @BeforeEach
+ void captureLogs() {
+ previousLevel = logger.getLevel();
+ logger.setLevel(Level.DEBUG);
+ appender.start();
+ logger.addAppender(appender);
+ final SharedCommunicationObjects sco = mock(SharedCommunicationObjects.class);
+ when(sco.configurationPoller(any(Config.class))).thenReturn(mock(ConfigurationPoller.class));
+ service = new RemoteConfigServiceImpl(sco, Config.get());
+ FeatureFlaggingGateway.addConfigListener(listener);
+ }
+
+ @AfterEach
+ void restoreLogging() {
+ FeatureFlaggingGateway.dispatch((ServerConfiguration) null);
+ logger.detachAppender(appender);
+ appender.stop();
+ logger.setLevel(previousLevel);
+ }
+
+ @Test
+ void reportsAppliedConfiguration() throws Exception {
+ final ServerConfiguration config = deserialize(emptyConfig());
+
+ service.accept("datadog/2/FFE_FLAGS/config/config", config, mock(PollingRateHinter.class));
+
+ verify(listener).accept(config);
+ assertLogged(
+ "configuration applied: key=datadog/2/FFE_FLAGS/config/config, flags=0, configurationEnv=Test, createdAt=2024-04-17T19:40:53.716Z");
+ }
+
+ @Test
+ void reportsAppliedConfigurationWithoutFlagsOrEnvironment() {
+ final ServerConfiguration config = new ServerConfiguration(null, null, null, null, null);
+
+ service.accept("config", config, mock(PollingRateHinter.class));
+
+ verify(listener).accept(config);
+ assertLogged(
+ "configuration applied: key=config, flags=missing, configurationEnv=null, createdAt=null");
+ }
+
+ @Test
+ void reportsRemovedConfiguration() {
+ service.accept("config", null, mock(PollingRateHinter.class));
+
+ verify(listener).accept(null);
+ assertLogged("configuration removed");
+ }
+
+ @Test
+ void omitsAppliedConfigurationDetailsWhenDebugIsDisabled() throws Exception {
+ logger.setLevel(Level.INFO);
+ final ServerConfiguration config = deserialize(emptyConfig());
+
+ service.accept("config", config, mock(PollingRateHinter.class));
+
+ verify(listener).accept(config);
+ assertTrue(appender.list.isEmpty());
+ }
+
+ private void assertLogged(final String text) {
+ assertTrue(
+ appender.list.stream().anyMatch(e -> e.getFormattedMessage().contains(text)),
+ "Missing log: " + text);
+ }
+ }
+
@Test
void skipsMalformedFlagAllocationsAndKeepsValidFlag() throws Exception {
final ServerConfiguration config = deserialize(resource("malformed-allocations.json"));