From 71fd5f0ad9b3073f0b9e40827322fdde87e169ca Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Wed, 6 Aug 2025 12:21:25 +0200 Subject: [PATCH 1/8] Expand timewindow for X-RAY trace summary search during E2E tests. --- .../testutils/tracing/TraceFetcher.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java index fc2d061cc..35ce0cb7e 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java @@ -149,21 +149,30 @@ private void getNestedSubSegments(List subsegments, Trace traceRes, * @return a list of trace ids */ private List getTraceIds() { + LOG.debug("Searching for traces from {} to {} with filter: {}", start, end, filterExpression); GetTraceSummariesResponse traceSummaries = xray.getTraceSummaries(GetTraceSummariesRequest.builder() .startTime(start) .endTime(end) - .timeRangeType(TimeRangeType.EVENT) + .timeRangeType(TimeRangeType.TRACE_ID) .sampling(false) .filterExpression(filterExpression) .build()); + + LOG.debug("Found {} trace summaries", + traceSummaries.hasTraceSummaries() ? traceSummaries.traceSummaries().size() : 0); + if (!traceSummaries.hasTraceSummaries()) { - throw new TraceNotFoundException("No trace id found"); + throw new TraceNotFoundException(String.format("No trace id found for filter '%s' between %s and %s", + filterExpression, start, end)); } List traceIds = traceSummaries.traceSummaries().stream().map(TraceSummary::id) .collect(Collectors.toList()); if (traceIds.isEmpty()) { - throw new TraceNotFoundException("No trace id found"); + throw new TraceNotFoundException( + String.format("Empty trace summary found for filter '%s' between %s and %s", + filterExpression, start, end)); } + LOG.debug("Found trace IDs: {}", traceIds); return traceIds; } @@ -183,9 +192,13 @@ public TraceFetcher build() { if (end == null) { end = start.plus(1, ChronoUnit.MINUTES); } - LOG.debug("Looking for traces from {} to {} with filter {} and excluded segments {}", start, end, - filterExpression, excludedSegments); - return new TraceFetcher(start, end, filterExpression, excludedSegments); + // Expand search window by 1 minute on each side to account for timing imprecisions + Instant expandedStart = start.minus(1, ChronoUnit.MINUTES); + Instant expandedEnd = end.plus(1, ChronoUnit.MINUTES); + LOG.debug( + "Looking for traces from {} to {} (expanded from {} to {}) with filter {} and excluded segments {}", + expandedStart, expandedEnd, start, end, filterExpression, excludedSegments); + return new TraceFetcher(expandedStart, expandedEnd, filterExpression, excludedSegments); } public Builder start(Instant start) { From 0d52313e5863dd451b3ab5bd59f56bac195127be Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Wed, 6 Aug 2025 16:56:05 +0200 Subject: [PATCH 2/8] Add retries for trace subsegments as well. --- powertools-e2e-tests/pom.xml | 10 ++++++++ .../testutils/tracing/TraceFetcher.java | 23 +++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/powertools-e2e-tests/pom.xml b/powertools-e2e-tests/pom.xml index ccd87e95e..d12395676 100644 --- a/powertools-e2e-tests/pom.xml +++ b/powertools-e2e-tests/pom.xml @@ -150,6 +150,16 @@ 2.4 test + + com.fasterxml.jackson.core + jackson-databind + test + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + test + org.aspectj aspectjrt diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java index 35ce0cb7e..89d6b91f0 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java @@ -27,7 +27,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; @@ -46,9 +48,10 @@ * Class in charge of retrieving the actual traces of a Lambda execution on X-Ray */ public class TraceFetcher { - - private static final ObjectMapper MAPPER = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final ObjectMapper MAPPER = JsonMapper.builder() + .disable(MapperFeature.REQUIRE_HANDLERS_FOR_JAVA8_TIMES) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .build(); private static final Logger LOG = LoggerFactory.getLogger(TraceFetcher.class); private static final SdkHttpClient httpClient = UrlConnectionHttpClient.builder().build(); private static final Region region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1")); @@ -104,7 +107,7 @@ private Trace getTrace(List traceIds) { .traceIds(traceIds) .build()); if (!tracesResponse.hasTraces()) { - throw new TraceNotFoundException("No trace found"); + throw new TraceNotFoundException(String.format("No trace found for traceIds %s", traceIds)); } Trace traceRes = new Trace(); tracesResponse.traces().forEach(trace -> { @@ -113,15 +116,21 @@ private Trace getTrace(List traceIds) { try { SegmentDocument document = MAPPER.readValue(segment.document(), SegmentDocument.class); if ("AWS::Lambda::Function".equals(document.getOrigin()) && document.hasSubsegments()) { - getNestedSubSegments(document.getSubsegments(), traceRes, - Collections.emptyList()); + getNestedSubSegments(document.getSubsegments(), traceRes, Collections.emptyList()); + } else if ("AWS::Lambda::Function".equals(document.getOrigin())) { + LOG.debug( + "Found AWS::Lambda::Function SegmentDocument with no subsegments. Retrying {}", + MAPPER.writeValueAsString(document)); + throw new TraceNotFoundException( + "Found AWS::Lambda::Function SegmentDocument with no subsegments."); } - } catch (JsonProcessingException e) { LOG.error("Failed to parse segment document: " + e.getMessage()); throw new RuntimeException(e); } }); + } else { + throw new TraceNotFoundException(String.format("No segments found in trace %s", trace.id())); } }); return traceRes; From 1c4dccc5e5593556c8beb661de41a50b90bf4df7 Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Wed, 6 Aug 2025 17:33:57 +0200 Subject: [PATCH 3/8] Add log.debug for subsegement population. --- .../lambda/powertools/testutils/tracing/TraceFetcher.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java index 89d6b91f0..6a0c5ec39 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java @@ -109,6 +109,7 @@ private Trace getTrace(List traceIds) { if (!tracesResponse.hasTraces()) { throw new TraceNotFoundException(String.format("No trace found for traceIds %s", traceIds)); } + Trace traceRes = new Trace(); tracesResponse.traces().forEach(trace -> { if (trace.hasSegments()) { @@ -116,6 +117,7 @@ private Trace getTrace(List traceIds) { try { SegmentDocument document = MAPPER.readValue(segment.document(), SegmentDocument.class); if ("AWS::Lambda::Function".equals(document.getOrigin()) && document.hasSubsegments()) { + LOG.debug("Populating subsegments for document {}", MAPPER.writeValueAsString(document)); getNestedSubSegments(document.getSubsegments(), traceRes, Collections.emptyList()); } else if ("AWS::Lambda::Function".equals(document.getOrigin())) { LOG.debug( @@ -133,6 +135,7 @@ private Trace getTrace(List traceIds) { throw new TraceNotFoundException(String.format("No segments found in trace %s", trace.id())); } }); + return traceRes; } From 90218d74a31d72f106546ad8a2ca43b843716999 Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Wed, 6 Aug 2025 17:42:13 +0200 Subject: [PATCH 4/8] Add retries for custom subsegment population. --- .../lambda/powertools/testutils/tracing/TraceFetcher.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java index 6a0c5ec39..9a63a619c 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java @@ -109,7 +109,7 @@ private Trace getTrace(List traceIds) { if (!tracesResponse.hasTraces()) { throw new TraceNotFoundException(String.format("No trace found for traceIds %s", traceIds)); } - + Trace traceRes = new Trace(); tracesResponse.traces().forEach(trace -> { if (trace.hasSegments()) { @@ -119,6 +119,12 @@ private Trace getTrace(List traceIds) { if ("AWS::Lambda::Function".equals(document.getOrigin()) && document.hasSubsegments()) { LOG.debug("Populating subsegments for document {}", MAPPER.writeValueAsString(document)); getNestedSubSegments(document.getSubsegments(), traceRes, Collections.emptyList()); + // If only the default (excluded) subsegments were populated we need to keep retrying for + // our custom subsegments. They might appear later. + if (traceRes.getSubsegments().isEmpty()) { + throw new TraceNotFoundException( + "Found AWS::Lambda::Function SegmentDocument with no non-excluded subsegments."); + } } else if ("AWS::Lambda::Function".equals(document.getOrigin())) { LOG.debug( "Found AWS::Lambda::Function SegmentDocument with no subsegments. Retrying {}", From 120d062211432a63629ea8452915f328bd8695b5 Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Thu, 7 Aug 2025 10:15:32 +0200 Subject: [PATCH 5/8] Add longer retries for TracingE2ET and capability to pass custom retry config to RetryUtils. --- .../powertools/testutils/RetryUtils.java | 32 ++++++++++++++++++- .../testutils/tracing/TraceFetcher.java | 9 +++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/RetryUtils.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/RetryUtils.java index 054e9aa8e..ce64f04ea 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/RetryUtils.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/RetryUtils.java @@ -47,7 +47,21 @@ private RetryUtils() { */ @SafeVarargs public static Retry createRetry(String name, Class... retryOnThrowables) { - RetryConfig config = RetryConfig.from(DEFAULT_RETRY_CONFIG) + return createRetry(name, DEFAULT_RETRY_CONFIG, retryOnThrowables); + } + + /** + * Creates a retry instance with custom configuration for the specified throwable types. + * + * @param name the name for the retry instance + * @param customConfig the custom retry configuration + * @param retryOnThrowables the throwable classes to retry on + * @return configured Retry instance + */ + @SafeVarargs + public static Retry createRetry(String name, RetryConfig customConfig, + Class... retryOnThrowables) { + RetryConfig config = RetryConfig.from(customConfig) .retryExceptions(retryOnThrowables) .build(); @@ -72,4 +86,20 @@ public static Supplier withRetry(Supplier supplier, String name, Retry retry = createRetry(name, retryOnThrowables); return Retry.decorateSupplier(retry, supplier); } + + /** + * Decorates a supplier with custom retry logic for the specified throwable types. + * + * @param supplier the supplier to decorate + * @param name the name for the retry instance + * @param customConfig the custom retry configuration + * @param retryOnThrowables the throwable classes to retry on + * @return decorated supplier with retry logic + */ + @SafeVarargs + public static Supplier withRetry(Supplier supplier, String name, RetryConfig customConfig, + Class... retryOnThrowables) { + Retry retry = createRetry(name, customConfig, retryOnThrowables); + return Retry.decorateSupplier(retry, supplier); + } } diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java index 9a63a619c..0aed9e811 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/tracing/TraceFetcher.java @@ -14,6 +14,7 @@ package software.amazon.lambda.powertools.testutils.tracing; +import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Arrays; @@ -31,6 +32,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.json.JsonMapper; +import io.github.resilience4j.retry.RetryConfig; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; @@ -93,7 +95,12 @@ public Trace fetchTrace() { return getTrace(traceIds); }; - return RetryUtils.withRetry(supplier, "trace-fetcher", TraceNotFoundException.class).get(); + RetryConfig customConfig = RetryConfig.custom() + .maxAttempts(120) // 120 attempts over 10 minutes + .waitDuration(Duration.ofSeconds(5)) // 5 seconds between attempts + .build(); + + return RetryUtils.withRetry(supplier, "trace-fetcher", customConfig, TraceNotFoundException.class).get(); } /** From 4a534cdfcf2473c9774c0619d60cc9c9e8611f3d Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Thu, 7 Aug 2025 12:10:14 +0200 Subject: [PATCH 6/8] Add 1 minute padding around MetricsFetcher. --- .../lambda/powertools/testutils/metrics/MetricsFetcher.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java index f856d8f2f..7f6ad3bd8 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java @@ -14,6 +14,7 @@ package software.amazon.lambda.powertools.testutils.metrics; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -74,8 +75,9 @@ public List fetchMetrics(Instant start, Instant end, int period, String LOG.debug("Get Metrics for namespace {}, start {}, end {}, metric {}, dimensions {}", namespace, start, end, metricName, dimensionsList); GetMetricDataResponse metricData = cloudwatch.getMetricData(GetMetricDataRequest.builder() - .startTime(start) - .endTime(end) + // Add 1 minute padding around start and end time to account time imprecisions + .startTime(start.minus(Duration.ofMinutes(1))) + .endTime(end.plus(Duration.ofMinutes(1))) .metricDataQueries(MetricDataQuery.builder() .id(metricName.toLowerCase(Locale.ROOT)) .metricStat(MetricStat.builder() From 939365940180254056c5701303b0c849d96b8022 Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Thu, 7 Aug 2025 13:55:11 +0200 Subject: [PATCH 7/8] Revert 1 minute time padding for MetricsFetcher. --- .../lambda/powertools/testutils/metrics/MetricsFetcher.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java index 7f6ad3bd8..f856d8f2f 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/metrics/MetricsFetcher.java @@ -14,7 +14,6 @@ package software.amazon.lambda.powertools.testutils.metrics; -import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -75,9 +74,8 @@ public List fetchMetrics(Instant start, Instant end, int period, String LOG.debug("Get Metrics for namespace {}, start {}, end {}, metric {}, dimensions {}", namespace, start, end, metricName, dimensionsList); GetMetricDataResponse metricData = cloudwatch.getMetricData(GetMetricDataRequest.builder() - // Add 1 minute padding around start and end time to account time imprecisions - .startTime(start.minus(Duration.ofMinutes(1))) - .endTime(end.plus(Duration.ofMinutes(1))) + .startTime(start) + .endTime(end) .metricDataQueries(MetricDataQuery.builder() .id(metricName.toLowerCase(Locale.ROOT)) .metricStat(MetricStat.builder() From e046ac375062d51a095a8009a4aa6f615b30a8f5 Mon Sep 17 00:00:00 2001 From: Philipp Page Date: Thu, 7 Aug 2025 13:55:44 +0200 Subject: [PATCH 8/8] Add retry loop around datapoint fetching for first metrics datapoints. --- .../amazon/lambda/powertools/MetricsE2ET.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java index feb9537d5..35f8b5ba3 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/MetricsE2ET.java @@ -34,7 +34,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import software.amazon.lambda.powertools.testutils.DataNotReadyException; import software.amazon.lambda.powertools.testutils.Infrastructure; +import software.amazon.lambda.powertools.testutils.RetryUtils; import software.amazon.lambda.powertools.testutils.lambda.InvocationResult; import software.amazon.lambda.powertools.testutils.metrics.MetricsFetcher; @@ -89,16 +91,20 @@ void test_recordMetrics() { { "FunctionName", functionName }, { "Service", SERVICE } }).collect(Collectors.toMap(data -> data[0], data -> data[1]))); assertThat(coldStart.get(0)).isEqualTo(1); - List orderMetrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), - 60, NAMESPACE, - "orders", Collections.singletonMap("Environment", "test")); + List orderMetrics = RetryUtils.withRetry(() -> { + List metrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), + 60, NAMESPACE, "orders", Collections.singletonMap("Environment", "test")); + if (metrics.get(0) != 2.0) { + throw new DataNotReadyException("Expected 2.0 orders but got " + metrics.get(0)); + } + return metrics; + }, "orderMetricsRetry", DataNotReadyException.class).get(); assertThat(orderMetrics.get(0)).isEqualTo(2); List productMetrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), 60, NAMESPACE, "products", Collections.singletonMap("Environment", "test")); // When searching across a 1 minute time period with a period of 60 we find both metrics and the sum is 12 - assertThat(productMetrics.get(0)).isEqualTo(12); orderMetrics = metricsFetcher.fetchMetrics(invocationResult.getStart(), invocationResult.getEnd(), 60,