Exports spans via OTLP gRPC to the ADOT collector extension (Lambda layer), which forwards to X-Ray. Used by + * {@code OtelXRayIntegrationTest} to verify spans appear correctly in X-Ray. + * + *
Expected trace structure in X-Ray: + * + *
+ * durable.invocation + * ├── durable.step:create-greeting + * │ └── durable.step:create-greeting [attempt 1] + * └── durable.step:transform + * └── durable.step:transform [attempt 1] + *+ */ +public class OtelXRayStepExample extends DurableHandler
This handler exercises the critical multi-invocation tracing scenario: + * + *
Exports spans via OTLP gRPC to the ADOT collector extension (Lambda layer), which forwards to X-Ray. + * + *
Used by {@code OtelXRayIntegrationTest} to verify that deterministic trace IDs correctly stitch spans from + * multiple invocations into a single X-Ray trace. + * + *
Expected trace structure in X-Ray: + * + *
+ * Trace (single trace ID across both invocations) + * ├── durable.invocation (invocation 1) + * │ ├── durable.step:before-wait + * │ │ └── durable.step:before-wait [attempt 1] + * │ └── durable.wait:pause (ended as PENDING) + * └── durable.invocation (invocation 2) + * ├── durable.wait:pause (completed) + * └── durable.step:after-wait + * └── durable.step:after-wait [attempt 1] + *+ * + *
All spans share the same deterministic trace ID derived from the execution ARN.
+ */
+public class OtelXRayWaitExample extends DurableHandler These tests deploy Lambda functions configured with:
+ *
+ * After invoking the function, the test queries the X-Ray API to verify:
+ *
+ * Enable with: {@code -Dtest.cloud.enabled=true}
+ */
+@EnabledIf("isEnabled")
+class OtelXRayIntegrationTest {
+
+ private static final Duration XRAY_INGESTION_DELAY = Duration.ofSeconds(15);
+ private static final int XRAY_QUERY_RETRIES = 3;
+ private static final Duration XRAY_RETRY_DELAY = Duration.ofSeconds(10);
+
+ private static String account;
+ private static String region;
+ private static String functionNameSuffix;
+ private static LambdaClient lambdaClient;
+ private static XRayClient xrayClient;
+
+ static boolean isEnabled() {
+ var enabled = "true".equals(System.getProperty("test.cloud.enabled"));
+ if (!enabled) {
+ System.out.println("⚠️ OTel X-Ray integration tests disabled. Enable with -Dtest.cloud.enabled=true");
+ }
+ return enabled;
+ }
+
+ @BeforeAll
+ static void setup() {
+ try {
+ DefaultCredentialsProvider.builder().build().resolveCredentials();
+ } catch (Exception e) {
+ throw new IllegalStateException("AWS credentials not available");
+ }
+
+ account = System.getProperty("test.aws.account");
+ region = System.getProperty("test.aws.region");
+ functionNameSuffix = System.getProperty("test.function.name.suffix", "-java17-runtime");
+
+ if (account == null || region == null) {
+ try (var sts = StsClient.create()) {
+ if (account == null) account = sts.getCallerIdentity().account();
+ if (region == null)
+ region = sts.serviceClientConfiguration().region().id();
+ }
+ }
+
+ lambdaClient = LambdaClient.builder()
+ .credentialsProvider(DefaultCredentialsProvider.builder().build())
+ .region(Region.of(region))
+ .build();
+
+ xrayClient = XRayClient.builder()
+ .credentialsProvider(DefaultCredentialsProvider.builder().build())
+ .region(Region.of(region))
+ .build();
+
+ System.out.println("☁️ Running OTel X-Ray integration tests against account " + account + " in " + region);
+ }
+
+ private static String arn(String functionName) {
+ return "arn:aws:lambda:" + region + ":" + account + ":function:" + functionName + functionNameSuffix
+ + ":$LATEST";
+ }
+
+ // ─── Test: Simple Steps (Single Invocation) ──────────────────────────
+
+ @Test
+ void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
+ var startTime = Instant.now();
+
+ // 1. Invoke the function (use unique input to avoid stale executions)
+ var runner = CloudDurableTestRunner.create(
+ arn("otel-xray-step-example"), GreetingRequest.class, String.class, lambdaClient);
+ var uniqueInput = "XRay-" + System.currentTimeMillis();
+ var result = runner.run(new GreetingRequest(uniqueInput));
+
+ assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result);
+ assertEquals("HELLO, " + uniqueInput.toUpperCase() + "!", result.getResult());
+
+ // 2. Wait for X-Ray ingestion
+ Thread.sleep(XRAY_INGESTION_DELAY.toMillis());
+
+ // 3. Query X-Ray for the trace
+ var traces = queryTracesWithRetry(startTime, Instant.now());
+
+ // 4. Find our trace (filter by annotation or just take the one matching our time window)
+ assertFalse(traces.isEmpty(), "Expected at least one trace in X-Ray after execution");
+
+ // Get full trace details
+ var traceIds = traces.stream().map(TraceSummary::id).toList();
+ var fullTraces = xrayClient.batchGetTraces(
+ BatchGetTracesRequest.builder().traceIds(traceIds).build());
+
+ // Find the trace that contains our durable spans
+ var durableTrace = fullTraces.traces().stream()
+ .filter(trace ->
+ trace.segments().stream().anyMatch(seg -> segmentContains(seg, "durable.step:create-greeting")))
+ .findFirst()
+ .orElse(null);
+
+ assertNotNull(
+ durableTrace,
+ "Expected to find a trace with durable.step:create-greeting segment. " + "Found " + traces.size()
+ + " traces in the time window.");
+
+ // 5. Verify span structure
+ var segmentDocuments =
+ durableTrace.segments().stream().map(Segment::document).toList();
+ var allSegmentText = String.join("\n", segmentDocuments);
+
+ // Verify expected span names appear in the trace
+ assertTrue(
+ allSegmentText.contains("durable.invocation"),
+ "Expected durable.invocation span in trace. Segments: " + summarizeSegments(segmentDocuments));
+ assertTrue(
+ allSegmentText.contains("durable.step:create-greeting"),
+ "Expected durable.step:create-greeting span in trace");
+ assertTrue(allSegmentText.contains("durable.step:transform"), "Expected durable.step:transform span in trace");
+
+ // Verify all segments share the same trace ID (single unified trace)
+ var uniqueTraceIds =
+ durableTrace.segments().stream().map(seg -> durableTrace.id()).collect(Collectors.toSet());
+ assertEquals(1, uniqueTraceIds.size(), "All segments should belong to a single trace");
+
+ System.out.println("✅ Simple steps test passed — "
+ + durableTrace.segments().size() + " segments in trace " + durableTrace.id());
+ }
+
+ // ─── Test: Wait + Resume (Multi-Invocation) ─────────────────────────
+
+ @Test
+ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
+ var startTime = Instant.now();
+
+ // 1. Invoke the function — will suspend on wait, then resume automatically
+ var runner = CloudDurableTestRunner.create(
+ arn("otel-xray-wait-example"), GreetingRequest.class, String.class, lambdaClient);
+ var uniqueInput = "Wait-" + System.currentTimeMillis();
+ var result = runner.run(new GreetingRequest(uniqueInput));
+
+ assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result);
+ assertTrue(result.getResult().contains("Resumed and completed"),
+ "Expected result to contain 'Resumed and completed', got: " + result.getResult());
+
+ // 2. Wait for X-Ray ingestion (extra time since multi-invocation takes longer)
+ Thread.sleep(XRAY_INGESTION_DELAY.plus(Duration.ofSeconds(5)).toMillis());
+
+ // 3. Query X-Ray for the trace
+ var traces = queryTracesWithRetry(startTime, Instant.now());
+
+ assertFalse(traces.isEmpty(), "Expected at least one trace in X-Ray after multi-invocation execution");
+
+ // Get full trace details
+ var traceIds = traces.stream().map(TraceSummary::id).toList();
+ var fullTraces = xrayClient.batchGetTraces(
+ BatchGetTracesRequest.builder().traceIds(traceIds).build());
+
+ // Find the trace containing our durable spans
+ var durableTrace = fullTraces.traces().stream()
+ .filter(trace ->
+ trace.segments().stream().anyMatch(seg -> segmentContains(seg, "durable.step:before-wait")))
+ .findFirst()
+ .orElse(null);
+
+ assertNotNull(
+ durableTrace,
+ "Expected to find a trace with durable.step:before-wait segment. " + "Found " + traces.size()
+ + " traces in the time window.");
+
+ // 4. Verify multi-invocation trace structure
+ var segmentDocuments =
+ durableTrace.segments().stream().map(Segment::document).toList();
+ var allSegmentText = String.join("\n", segmentDocuments);
+
+ // Verify spans from BOTH invocations appear in the same trace
+ assertTrue(
+ allSegmentText.contains("durable.step:before-wait"), "Expected before-wait span from first invocation");
+ assertTrue(
+ allSegmentText.contains("durable.step:after-wait"), "Expected after-wait span from second invocation");
+ assertTrue(allSegmentText.contains("durable.wait:pause"), "Expected wait:pause span in trace");
+
+ // Verify multiple invocation spans (one per Lambda invocation)
+ var invocationCount = countOccurrences(allSegmentText, "durable.invocation");
+ assertTrue(
+ invocationCount >= 2,
+ "Expected at least 2 invocation spans (multi-invocation), got " + invocationCount);
+
+ // Critical assertion: all segments under ONE trace (deterministic ID worked)
+ assertEquals(
+ 1,
+ Set.of(durableTrace.id()).size(),
+ "All segments should belong to a single trace — deterministic trace ID must work across invocations");
+
+ System.out.println(
+ "✅ Wait + resume test passed — " + durableTrace.segments().size() + " segments across "
+ + invocationCount + " invocations in trace " + durableTrace.id());
+ }
+
+ // ─── Helpers ─────────────────────────────────────────────────────────
+
+ /** Queries X-Ray for traces with retry logic to handle eventual consistency. */
+ private List
+ *
+ *
+ *
+ *
+ *
+ *