Truncate step_info and error reason in ILM execution state and history (#125054)

This commit adds a limit to the `step_info` contained in
`LifecycleExcutionState` so that large step info messages will not be
stored in the cluster state. Additionally, when generating an ILM
history failure, the full exception that is "stringified" is truncated
to the same limit, ensuring that we do not accidentally index gigantic
documents in the history store.

The default limit is 1024 characters.

Resolves #124181

Author: dakrone

docs/changelog/125054.yaml

@@ -0,0 +1,6 @@
+pr: 125054
+summary: Truncate `step_info` and error reason in ILM execution state and history
+area: ILM+SLM
+type: enhancement
+issues:
+ - 124181

server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Strings;
 
 import java.util.Collections;
 import java.util.HashMap;
@@ -42,6 +43,7 @@ public record LifecycleExecutionState(
 ) {
 
     public static final String ILM_CUSTOM_METADATA_KEY = "ilm";
+    public static final int MAXIMUM_STEP_INFO_STRING_LENGTH = 1024;
 
     private static final String PHASE = "phase";
     private static final String ACTION = "action";
@@ -267,6 +269,17 @@ public Map<String, String> asMap() {
         return Collections.unmodifiableMap(result);
     }
 
+    public static String truncateWithExplanation(String input) {
+        if (input != null && input.length() > MAXIMUM_STEP_INFO_STRING_LENGTH) {
+            return Strings.cleanTruncate(input, MAXIMUM_STEP_INFO_STRING_LENGTH)
+                + "... ("
+                + (input.length() - MAXIMUM_STEP_INFO_STRING_LENGTH)
+                + " chars truncated)";
+        } else {
+            return input;
+        }
+    }
+
     public static class Builder {
         private String phase;
         private String action;
@@ -308,7 +321,7 @@ public Builder setFailedStep(String failedStep) {
         }
 
         public Builder setStepInfo(String stepInfo) {
-            this.stepInfo = stepInfo;
+            this.stepInfo = truncateWithExplanation(stepInfo);
             return this;
         }
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleExecutionStateTests.java

@@ -14,6 +14,8 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class LifecycleExecutionStateTests extends ESTestCase {
 
     public void testConversion() {
@@ -22,6 +24,20 @@ public void testConversion() {
         assertEquals(customMetadata, parsed.asMap());
     }
 
+    public void testTruncatingStepInfo() {
+        Map<String, String> custom = createCustomMetadata();
+        LifecycleExecutionState state = LifecycleExecutionState.fromCustomMetadata(custom);
+        assertThat(custom.get("step_info"), equalTo(state.stepInfo()));
+        String longStepInfo = randomAlphanumericOfLength(LifecycleExecutionState.MAXIMUM_STEP_INFO_STRING_LENGTH + 100);
+        LifecycleExecutionState newState = LifecycleExecutionState.builder(state).setStepInfo(longStepInfo).build();
+        // Length includes the post suffix
+        assertThat(newState.stepInfo().length(), equalTo(LifecycleExecutionState.MAXIMUM_STEP_INFO_STRING_LENGTH + 25));
+        assertThat(
+            newState.stepInfo().substring(LifecycleExecutionState.MAXIMUM_STEP_INFO_STRING_LENGTH, 1049),
+            equalTo("... (100 chars truncated)")
+        );
+    }
+
     public void testEmptyValuesAreNotSerialized() {
         LifecycleExecutionState empty = LifecycleExecutionState.builder().build();
         assertEquals(new HashMap<String, String>().entrySet(), empty.asMap().entrySet());

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryItem.java

@@ -86,7 +86,15 @@ public static ILMHistoryItem failure(
         Exception error
     ) {
         Objects.requireNonNull(error, "ILM failures require an attached exception");
-        return new ILMHistoryItem(index, policyId, timestamp, indexAge, false, executionState, exceptionToString(error));
+        String fullErrorString = exceptionToString(error);
+        String truncatedErrorString = LifecycleExecutionState.truncateWithExplanation(fullErrorString);
+        if (truncatedErrorString.equals(fullErrorString) == false) {
+            // Append a closing quote and closing brace to attempt to make it valid JSON.
+            // There is no requirement that it actually be valid JSON, so this is
+            // best-effort, but does not cause problems if it is still invalid.
+            truncatedErrorString += "\"}";
+        }
+        return new ILMHistoryItem(index, policyId, timestamp, indexAge, false, executionState, truncatedErrorString);
     }
 
     @Override

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryItemTests.java

@@ -13,8 +13,13 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Map;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.equalTo;
@@ -111,4 +116,35 @@ public void testToXContent() throws IOException {
                 \\"stack_trace\\":\\"java.lang.IllegalArgumentException: failure""".replaceAll("\\s", "")));
         }
     }
+
+    public void testTruncateLongError() throws IOException {
+        String longError = randomAlphaOfLength(LifecycleExecutionState.MAXIMUM_STEP_INFO_STRING_LENGTH + 20);
+
+        ILMHistoryItem failure = ILMHistoryItem.failure(
+            "index",
+            "policy",
+            1234L,
+            100L,
+            LifecycleExecutionState.EMPTY_STATE,
+            new IllegalArgumentException(longError)
+        );
+
+        try (XContentBuilder builder = jsonBuilder()) {
+            failure.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            String json = Strings.toString(builder);
+            try (XContentParser p = XContentFactory.xContent(XContentType.JSON).createParser(XContentParserConfiguration.EMPTY, json)) {
+                Map<String, Object> item = p.map();
+                assertThat(
+                    item.get("error_details"),
+                    equalTo(
+                        "{\"type\":\"illegal_argument_exception\",\"reason\":\""
+                            // We subtract a number of characters here due to the truncation being based
+                            // on the length of the whole string, not just the "reason" part.
+                            + longError.substring(0, LifecycleExecutionState.MAXIMUM_STEP_INFO_STRING_LENGTH - 47)
+                            + "... (5126 chars truncated)\"}"
+                    )
+                );
+            }
+        }
+    }
 }