Add debug logging for feature flag onboarding

products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java

@@ -52,6 +52,10 @@ static synchronized void start(
       return;
     }
 
+    LOGGER.debug(
+        "Feature Flagging enabled; configuration source={}, env={}",
+        config.getFeatureFlaggingConfigurationSource(),
+        config.getEnv());
     if (CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource())) {
       final FeatureFlaggingGateway.ActivationListener activationListener =
           () -> activateAgentless(sco, config, systemInitializer);
@@ -93,6 +97,8 @@ private static void initializeOrRollBack(
         SHUTDOWN_HOOK = hook;
       }
     } catch (final RuntimeException | Error e) {
+      LOGGER.debug(
+          "Feature Flagging system initialization failed: {}", e.getClass().getSimpleName());
       stop();
       throw e;
     }
@@ -164,6 +170,7 @@ static ConfigurationSourceService createConfigurationSourceService(
     final String configurationSource = config.getFeatureFlaggingConfigurationSource();
     if (CONFIGURATION_SOURCE_REMOTE_CONFIG.equals(configurationSource)) {
       if (!config.isRemoteConfigEnabled()) {
+        LOGGER.warn("Feature Flagging remote_config source requires DD_REMOTE_CONFIG_ENABLED=true");
         throw new IllegalStateException("Feature Flagging system started without RC");
       }
       return new RemoteConfigServiceImpl(sco, config);

products/feature-flagging/feature-flagging-api/README.md

@@ -253,3 +253,127 @@ This setting stops exposures only. Flag evaluation events have their own setting
   Configuration path. `offline` is reserved for startup-provided UFC bytes;
   until those bytes are implemented, no network source starts and evaluations
   use defaults.
+
+## Onboarding diagnostics
+
+Set `DD_TRACE_DEBUG=true` for the Java agent's configuration, writer, and transport logs.
+Also enable DEBUG for `datadog.trace.api.openfeature` in the application's SLF4J backend **before
+constructing `Provider`**. The provider and agent have separate logging backends; provider
+debug logs (lifecycle, exposure, and raw evaluation values, defaults, and targeting context)
+require both `DD_TRACE_DEBUG=true` (or `-Ddd.trace.debug=true`) and an application logger that
+permits DEBUG. The trace debug setting defaults to false and is read once, when the provider is
+first constructed; changing it requires restarting the application.
+`DD_TRACE_DEBUG` does not change application logger levels. If `DD_TRACE_DEBUG` is enabled but
+that category does not permit DEBUG when the provider is constructed, the provider logs a WARN
+that names the missing logger setting. No additional flag evaluation data
+setting is required for raw debug logs, including when `observe_full_evaluation_data` is false.
+That metadata setting continues to govern evaluation telemetry independently.
+
+For an application using Logback, save this as `logback-ffe-debug.xml`:
+
+```xml
+<configuration>
+  <appender name="STDERR" class="ch.qos.logback.core.ConsoleAppender">
+    <target>System.err</target>
+    <encoder>
+      <pattern>%date %-5level %logger - %msg%n%ex</pattern>
+    </encoder>
+  </appender>
+  <logger name="datadog.trace.api.openfeature" level="DEBUG"/>
+  <root level="INFO">
+    <appender-ref ref="STDERR"/>
+  </root>
+</configuration>
+```
+
+Launch the application with its existing SLF4J binding on the classpath:
+
+```shell
+DD_TRACE_DEBUG=true \
+DD_FEATURE_FLAGS_ENABLED=true \
+DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless \
+DD_ENV=production \
+java -javaagent:/path/to/dd-java-agent.jar \
+  -Dlogback.configurationFile=/path/to/logback-ffe-debug.xml \
+  -jar application.jar 2>ffe-debug.log
+```
+
+For the managed agentless endpoint, supply `DD_API_KEY` through the environment. For a custom
+endpoint, set `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL`; the SDK does not append
+`DD_ENV` to that URL. For Agent delivery, use `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=remote_config`
+and `DD_REMOTE_CONFIG_ENABLED=true`, with the Datadog Agent reachable from the application.
+Register the provider with `setProviderAndWait` after logging has been configured, then evaluate
+flags. With `slf4j-simple` instead of Logback, use
+`-Dorg.slf4j.simpleLogger.log.datadog.trace.api.openfeature=debug` before provider construction.
+With another binding, configure that category at DEBUG and allow it through appender filters.
+
+Both the agent's default output and the Logback example above go to stderr. Keep the full
+`ffe-debug.log`: filtering only for `Feature Flagging` loses shared Remote Config and HTTP
+transport logs. Relevant categories are `datadog.trace.api.openfeature` (provider),
+`com.datadog.featureflag` (configuration and writers), `datadog.remoteconfig`
+(Agent configuration polling), and `datadog.communication` (transport and delivery).
+Agent Remote Config logs describe the Java tracer's connection to the Agent; use the Datadog
+Agent's own logs to diagnose its upstream authentication and configuration fetches.
+
+Look for these diagnostic boundaries:
+
+- Setup names the source, SDK environment, timeouts, endpoint origin, and credential problems.
+  Agentless startup labels `requestedEnv` from the URL separately from `env`; application logs
+  label the delivered `configurationEnv`. `endpointMode=custom` means the SDK did not append
+  `DD_ENV`, even if the supplied URL already contains a `dd_env` query parameter.
+- Parsing reports usable and rejected flag counts. Application reports include the configuration's
+  creation time, agentless ETag or Remote Config key where available. `flags=0` on an applied
+  configuration means an available empty configuration. `flags=missing` means the Remote Config
+  payload had no `flags` member; evaluations against it return a `GENERAL` error. Removal or
+  absence of the whole configuration is reported separately. Failed refreshes report retention of the previous configuration. HTTP attempts,
+  retry delays, exhaustion, unchanged responses, and subsequent application are logged.
+- Provider initialization reports its timeout budget, initial availability, terminal outcome,
+  and unavailable/recovered transitions.
+- `provider resolution` records effective targeting inputs, raw default and resolved values,
+  variant, reason, errors, metadata, and the identity of the configuration used for that resolution.
+  Raw resolution errors are logged before any telemetry-specific error-message filtering.
+  `evaluation observed at finallyAfter` records the SDK details visible to that hook, including
+  fallback after an application `after` hook fails. It is an observation at that point: later
+  `finallyAfter` hooks can still mutate details, so it does not claim the application's final return.
+- Values and context are not truncated or redacted. OpenFeature's conversion methods expand nested
+  lists and structures. If conversion or formatting fails (including cycles or excessive nesting
+  that exhaust the JVM stack), a `diagnostic formatting failed; raw detail unavailable` message is
+  attempted. The evaluation proceeds unchanged. Backend filters, sinks, and log collectors can
+  impose their own limits, so check those when collecting large values.
+- Exposures report suppression reasons, queueing, overflow, flush counts, delivery, and failed
+  batches that are dropped. Dispatch to listeners and queueing do not confirm delivery.
+- Provider shutdown detaches its configuration listener; the Java agent owns the configuration
+  source and exposure writer. Exposure shutdown reports queued work, then the processor finishes
+  any in-flight delivery, drains the queue, and makes one final best-effort flush whose outcome is
+  logged like any other flush. Exposures dispatched after shutdown starts are dropped and logged.
+  If the final delivery outlives the shutdown wait, a message says it continues in the
+  background. The processor reports completion separately, with the counts left in its queue and
+  buffer.
+
+To disable these debug diagnostics, set `DD_TRACE_DEBUG=false` and restart the application
+(without a conflicting `-Ddd.trace.debug=true` override). Provider lifecycle, evaluation, and
+exposure debug diagnostics are then suppressed even if the application logger remains at DEBUG.
+A few debug messages for rare failures (an exposure callback, span enrichment, or metric recording
+failure, or an older agent missing a feature) follow only the application logger level. Lowering
+the application category to INFO suppresses all provider debug logs immediately. When either gate
+is off, raw values and context are not traversed or formatted. WARN, INFO, and ERROR messages
+retain their normal logging behavior. Logging and formatting failures in provider debug
+diagnostics are isolated from construction, initialization, readiness events, evaluation, and
+shutdown; WARN and ERROR messages are not isolated from a logging backend that throws.
+
+Run the focused suites with:
+
+```shell
+git submodule update --init -- dd-smoke-tests/openfeature/src/test/resources/ffe-system-test-data
+./gradlew :products:feature-flagging:feature-flagging-api:test \
+  :products:feature-flagging:feature-flagging-api:forkedTest \
+  :products:feature-flagging:feature-flagging-lib:test \
+  :products:feature-flagging:feature-flagging-agent:test -PtestJvm=11
+```
+
+For acceptance records, retain `git rev-parse HEAD` plus any working-tree patch, source mode,
+startup command, test/reproduction command, expected messages, observed transcript, and
+Pass/Fail/Not run/N/A with reason. Exercise agentless and Agent/Remote Config independently.
+Persistent-cache startup is N/A: this provider has no persistent configuration cache.
+
+Coverage follows the [SDK debug logging checklist](https://datadoghq.atlassian.net/wiki/spaces/PANA/pages/7276692209).

products/feature-flagging/feature-flagging-api/build.gradle.kts

@@ -56,6 +56,8 @@ dependencies {
   testImplementation(libs.bundles.junit5)
   testImplementation(libs.bundles.mockito)
   testImplementation(libs.moshi)
+  // OpenFeature uses SLF4J 2; the shared Java 8 test binding targets SLF4J 1.7.
+  testImplementation("ch.qos.logback:logback-classic:1.6.5")
 
   // The main source set gets the bootstrap/config types as compileOnly, so the JMH source set
   // needs them on its own compile and runtime classpath to drive the hook end to end.

products/feature-flagging/feature-flagging-api/gradle.lockfile

@@ -2,8 +2,8 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 # To regenerate this file, run: ./gradlew :products:feature-flagging:feature-flagging-api:dependencies --write-locks
-ch.qos.logback:logback-classic:1.2.13=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
-ch.qos.logback:logback-core:1.2.13=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+ch.qos.logback:logback-classic:1.6.5=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
+ch.qos.logback:logback-core:1.6.5=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.javaparser:javaparser-core:3.25.6=codenarc
 com.github.spotbugs:spotbugs-annotations:4.10.4=compileClasspath,jmhCompileClasspath,spotbugs
 com.github.spotbugs:spotbugs:4.10.4=spotbugs

products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java

@@ -1,5 +1,6 @@
 package datadog.trace.api.openfeature;
 
+import static datadog.trace.api.openfeature.Provider.debugQuietly;
 import static java.util.Arrays.asList;
 
 import datadog.trace.api.featureflag.FeatureFlaggingGateway;
@@ -225,6 +226,10 @@ public boolean initialize(
       final long timeout, final TimeUnit unit, final EvaluationContext context) throws Exception {
     FeatureFlaggingGateway.activate();
     FeatureFlaggingGateway.addConfigListener(this);
+    debugQuietly(
+        log,
+        "Feature Flagging provider configuration available at subscription: {}",
+        hasConfiguration());
     return initializationLatch.await(timeout, unit) || hasConfiguration();
   }
 
@@ -259,29 +264,42 @@ public <T> ProviderEvaluation<T> evaluate(
     // ProviderEvaluation returned, so the hook's consent decision is pinned to this evaluation's
     // config and cannot drift on a concurrent Remote Config swap.
     final ServerConfiguration config = configuration.get();
+    final ProviderEvaluation<T> result = resolve(target, key, defaultValue, context, config);
+    DebugLoggingHook.resolution(key, defaultValue, context, result, config);
+    return result;
+  }
+
+  private <T> ProviderEvaluation<T> resolve(
+      final Class<T> target,
+      final String key,
+      final T defaultValue,
+      final EvaluationContext context,
+      final ServerConfiguration config) {
     // Boolean.TRUE.equals covers both null (privacy-preserving default) and Boolean.FALSE without
     // an NPE — the field is boxed so a malformed UFC message doesn't abort the whole parse.
     final boolean observeFullEvaluationData =
         config != null && Boolean.TRUE.equals(config.observeFullEvaluationData);
     try {
       if (config == null) {
-        return error(defaultValue, ErrorCode.PROVIDER_NOT_READY, null, observeFullEvaluationData);
+        return error(
+            key, defaultValue, ErrorCode.PROVIDER_NOT_READY, null, observeFullEvaluationData);
       }
 
       if (context == null) {
-        return error(defaultValue, ErrorCode.INVALID_CONTEXT, null, observeFullEvaluationData);
+        return error(key, defaultValue, ErrorCode.INVALID_CONTEXT, null, observeFullEvaluationData);
       }
 
       final Flag flag = config.flags.get(key);
       if (flag == null) {
         if (config.invalidFlags != null && config.invalidFlags.containsKey(key)) {
           return error(
+              key,
               defaultValue,
               ErrorCode.PARSE_ERROR,
               "invalid configuration for flag " + key,
               observeFullEvaluationData);
         }
-        return error(defaultValue, ErrorCode.FLAG_NOT_FOUND, null, observeFullEvaluationData);
+        return error(key, defaultValue, ErrorCode.FLAG_NOT_FOUND, null, observeFullEvaluationData);
       }
 
       if (!flag.enabled) {
@@ -294,6 +312,7 @@ public <T> ProviderEvaluation<T> evaluate(
 
       if (flag.allocations == null) {
         return error(
+            key,
             defaultValue,
             ErrorCode.GENERAL,
             "Missing allocations for flag " + key,
@@ -332,7 +351,11 @@ public <T> ProviderEvaluation<T> evaluate(
             } else {
               if (targetingKey == null) {
                 return error(
-                    defaultValue, ErrorCode.TARGETING_KEY_MISSING, null, observeFullEvaluationData);
+                    key,
+                    defaultValue,
+                    ErrorCode.TARGETING_KEY_MISSING,
+                    null,
+                    observeFullEvaluationData);
               }
               // To match a split, subject must match ALL underlying shards
               boolean allShardsMatch = true;
@@ -366,12 +389,13 @@ public <T> ProviderEvaluation<T> evaluate(
           .flagMetadata(consentMetadata(observeFullEvaluationData))
           .build();
     } catch (final PatternSyntaxException e) {
-      return error(defaultValue, ErrorCode.PARSE_ERROR, e.getMessage(), observeFullEvaluationData);
+      return error(
+          key, defaultValue, ErrorCode.PARSE_ERROR, e.getMessage(), observeFullEvaluationData);
     } catch (final NumberFormatException e) {
       return error(
-          defaultValue, ErrorCode.TYPE_MISMATCH, e.getMessage(), observeFullEvaluationData);
+          key, defaultValue, ErrorCode.TYPE_MISMATCH, e.getMessage(), observeFullEvaluationData);
     } catch (final Exception e) {
-      return error(defaultValue, ErrorCode.GENERAL, e.getMessage(), observeFullEvaluationData);
+      return error(key, defaultValue, ErrorCode.GENERAL, e.getMessage(), observeFullEvaluationData);
     }
   }
 
@@ -382,10 +406,12 @@ private static ImmutableMetadata consentMetadata(final boolean observeFullEvalua
   }
 
   private static <T> ProviderEvaluation<T> error(
+      final String key,
       final T defaultValue,
       final ErrorCode code,
       final String errorMessage,
       final boolean observeFullEvaluationData) {
+    DebugLoggingHook.resolutionError(key, code, errorMessage);
     // Under consent-off the errorMessage is dropped: exception messages from the outer catch blocks
     // (NumberFormatException, generic Exception) can echo raw evaluation-context values, so they
     // must never reach any consumer of ProviderEvaluation.getErrorMessage() — not just our own
@@ -611,6 +637,7 @@ private static <T> ProviderEvaluation<T> resolveVariant(
     final Variant variant = flag.variations.get(variationKey);
     if (variant == null) {
       return error(
+          key,
           defaultValue,
           ErrorCode.GENERAL,
           "Variant not found for: " + variationKey,
@@ -619,6 +646,7 @@ private static <T> ProviderEvaluation<T> resolveVariant(
 
     if (!isTypeCompatible(target, flag.variationType)) {
       return error(
+          key,
           defaultValue,
           ErrorCode.TYPE_MISMATCH,
           "Requested type "
@@ -633,6 +661,7 @@ private static <T> ProviderEvaluation<T> resolveVariant(
       mappedValue = mapValue(target, variant.value);
     } catch (final NumberFormatException e) {
       return error(
+          key,
           defaultValue,
           ErrorCode.PARSE_ERROR,
           "Variant '"