Require System Index Descriptors to allow a specific suffix (#78355)

Previously, the leading `.` at the start of system and associated index
descriptor patterns was not properly replaced before being compiled as a
regular expression, leading to any single-character prefix matching.

This commit fixes the issue and adds tests to catch it.

Author: gwbrown

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java

@@ -64,6 +64,7 @@ public class GeoIpDownloader extends AllocatedPersistentTask {
 
     public static final String GEOIP_DOWNLOADER = "geoip-downloader";
     static final String DATABASES_INDEX = ".geoip_databases";
+    static final String DATABASES_INDEX_PATTERN = DATABASES_INDEX + "*";
     static final int MAX_CHUNK_SIZE = 1024 * 1024;
 
     private final Client client;

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java

@@ -66,6 +66,7 @@
 import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
 import static org.elasticsearch.ingest.IngestService.INGEST_ORIGIN;
 import static org.elasticsearch.ingest.geoip.GeoIpDownloader.DATABASES_INDEX;
+import static org.elasticsearch.ingest.geoip.GeoIpDownloader.DATABASES_INDEX_PATTERN;
 import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER;
 
 public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, SystemIndexPlugin, Closeable, PersistentTaskPlugin, ActionPlugin {
@@ -160,7 +161,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
         SystemIndexDescriptor geoipDatabasesIndex = SystemIndexDescriptor.builder()
-            .setIndexPattern(DATABASES_INDEX)
+            .setIndexPattern(DATABASES_INDEX_PATTERN)
             .setDescription("GeoIP databases")
             .setMappings(mappings())
             .setSettings(Settings.builder()

modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java

@@ -37,14 +37,14 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
         .build();
 
     public static final SystemIndexDescriptor APM_AGENT_CONFIG_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
-        .setIndexPattern(".apm-agent-configuration")
+        .setIndexPattern(".apm-agent-configuration*")
         .setDescription("system index for APM agent configuration")
         .setType(Type.EXTERNAL_UNMANAGED)
         .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN)
         .build();
 
     public static final SystemIndexDescriptor APM_CUSTOM_LINK_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
-        .setIndexPattern(".apm-custom-link")
+        .setIndexPattern(".apm-custom-link*")
         .setDescription("system index for APM custom links")
         .setType(Type.EXTERNAL_UNMANAGED)
         .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN)

modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java

@@ -24,7 +24,7 @@ public void testKibanaIndexNames() {
                 .stream()
                 .map(SystemIndexDescriptor::getIndexPattern)
                 .collect(Collectors.toUnmodifiableList()),
-            contains(".kibana_*", ".reporting-*", ".apm-agent-configuration", ".apm-custom-link")
+            contains(".kibana_*", ".reporting-*", ".apm-agent-configuration*", ".apm-custom-link*")
         );
     }
 }

server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java

@@ -84,7 +84,7 @@ public List<ActionFilter> getActionFilters() {
 
         @Override
         public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-            return List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, "Test system index"));
+            return List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME + "*", "Test system index"));
         }
 
         @Override

server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java

@@ -960,7 +960,7 @@ public static class AnotherSystemIndexTestPlugin extends Plugin implements Syste
 
         @Override
         public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-            return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System indices for tests"));
+            return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME + "*", "System indices for tests"));
         }
 
         @Override
@@ -981,7 +981,7 @@ public static class AssociatedIndicesTestPlugin extends Plugin implements System
 
         @Override
         public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-            return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System & associated indices for tests"));
+            return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME + "*", "System & associated indices for tests"));
         }
 
         @Override

server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java

@@ -40,7 +40,12 @@
  * creating the system index, upgrading its mappings, and creating an alias.
  */
 public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<SystemIndexDescriptor> {
-    /** A pattern, either with a wildcard or simple regex. Indices that match one of these patterns are considered system indices. */
+    /**
+     * A pattern, either with a wildcard or simple regex. Indices that match one of these patterns are considered system indices.
+     * Note that this pattern must not overlap with any other {@link SystemIndexDescriptor}s and must allow an alphanumeric suffix
+     * (see {@link SystemIndices#UPGRADED_INDEX_SUFFIX} for the specific suffix that's checked) to ensure that there's a name within the
+     * pattern we can use to create a new index when upgrading.
+     * */
     private final String indexPattern;
 
     /**
@@ -107,7 +112,9 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
     /**
      * Creates a descriptor for system indices matching the supplied pattern. These indices will not be managed
      * by Elasticsearch internally.
-     * @param indexPattern The pattern of index names that this descriptor will be used for. Must start with a '.' character.
+     * @param indexPattern The pattern of index names that this descriptor will be used for. Must start with a '.' character, must not
+     *                     overlap with any other descriptor patterns, and must allow a suffix (see note on
+     *                     {@link SystemIndexDescriptor#indexPattern} for details).
      * @param description The name of the plugin responsible for this system index.
      */
     public SystemIndexDescriptor(String indexPattern, String description) {
@@ -118,7 +125,9 @@ public SystemIndexDescriptor(String indexPattern, String description) {
     /**
      * Creates a descriptor for system indices matching the supplied pattern. These indices will not be managed
      * by Elasticsearch internally.
-     * @param indexPattern The pattern of index names that this descriptor will be used for. Must start with a '.' character.
+     * @param indexPattern The pattern of index names that this descriptor will be used for. Must start with a '.' character, must not
+     *                            overlap with any other descriptor patterns, and must allow a suffix (see note on
+     *                            {@link SystemIndexDescriptor#indexPattern} for details).
      * @param description The name of the plugin responsible for this system index.
      * @param type The {@link Type} of system index
      * @param allowedElasticProductOrigins A list of allowed origin values that should be allowed access in the case of external system
@@ -133,7 +142,9 @@ public SystemIndexDescriptor(String indexPattern, String description, Type type,
      * Creates a descriptor for system indices matching the supplied pattern. These indices will be managed
      * by Elasticsearch internally if mappings or settings are provided.
      *
-     * @param indexPattern The pattern of index names that this descriptor will be used for. Must start with a '.' character.
+     * @param indexPattern The pattern of index names that this descriptor will be used for. Must start with a '.' character, must not
+     *                            overlap with any other descriptor patterns, and must allow a suffix (see note on
+     *                            {@link SystemIndexDescriptor#indexPattern} for details).
      * @param primaryIndex The primary index name of this descriptor. Used when creating the system index for the first time.
      * @param description The name of the plugin responsible for this system index.
      * @param mappings The mappings to apply to this index when auto-creating, if appropriate
@@ -314,7 +325,9 @@ public SystemIndexDescriptor(String indexPattern, String description, Type type,
 
 
     /**
-     * @return The pattern of index names that this descriptor will be used for.
+     * @return The pattern of index names that this descriptor will be used for. Must start with a '.' character, must not
+     *         overlap with any other descriptor patterns, and must allow a suffix (see note on
+     *         {@link SystemIndexDescriptor#indexPattern} for details).
      */
     @Override
     public String getIndexPattern() {

server/src/main/java/org/elasticsearch/indices/SystemIndices.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.indices;
 
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
@@ -58,6 +59,7 @@
 public class SystemIndices {
     public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed";
     public static final String EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_external_system_index_access_origin";
+    public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-8";
 
     private static final Automaton EMPTY = Automata.makeEmpty();
 
@@ -83,6 +85,7 @@ TASKS_FEATURE_NAME, new Feature(TASKS_FEATURE_NAME, "Manages task results", List
     public SystemIndices(Map<String, Feature> pluginAndModulesDescriptors) {
         featureDescriptors = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors);
         checkForOverlappingPatterns(featureDescriptors);
+        ensurePatternsAllowSuffix(featureDescriptors);
         checkForDuplicateAliases(this.getSystemIndexDescriptors());
         Automaton systemIndexAutomata = buildIndexAutomaton(featureDescriptors);
         this.systemIndexRunAutomaton = new CharacterRunAutomaton(systemIndexAutomata);
@@ -99,6 +102,35 @@ public SystemIndices(Map<String, Feature> pluginAndModulesDescriptors) {
         this.systemNameRunAutomaton = new CharacterRunAutomaton(systemNameAutomaton);
     }
 
+    static void ensurePatternsAllowSuffix(Map<String, Feature> features) {
+        String suffixPattern = "*" + UPGRADED_INDEX_SUFFIX;
+        final List<String> descriptorsWithNoRoomForSuffix = features.entrySet()
+            .stream()
+            .flatMap(
+                feature -> feature.getValue().getIndexDescriptors()
+                    .stream()
+                    // The below filter & map are inside the enclosing flapMap so we have access to both the feature and the descriptor
+                    .filter(descriptor -> overlaps(descriptor.getIndexPattern(), suffixPattern) == false)
+                    .map(
+                        descriptor -> new ParameterizedMessage(
+                            "pattern [{}] from feature [{}]",
+                            descriptor.getIndexPattern(),
+                            feature.getKey()
+                        ).getFormattedMessage()
+                    )
+            )
+            .collect(Collectors.toList());
+        if (descriptorsWithNoRoomForSuffix.isEmpty() == false) {
+            throw new IllegalStateException(
+                new ParameterizedMessage(
+                    "the following system index patterns do not allow suffix [{}] required to allow upgrades: [{}]",
+                    UPGRADED_INDEX_SUFFIX,
+                    descriptorsWithNoRoomForSuffix
+                ).getFormattedMessage()
+            );
+        }
+    }
+
     private static void checkForDuplicateAliases(Collection<SystemIndexDescriptor> descriptors) {
         final Map<String, Integer> aliasCounts = new HashMap<>();
 

server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java

@@ -176,7 +176,7 @@ public void testDeprecationWarningEmittedWhenRequestingNonExistingAliasInSystemP
         SystemIndices systemIndices = new SystemIndices(Collections.singletonMap(
             this.getTestName(),
             new SystemIndices.Feature(this.getTestName(), "test feature",
-                Collections.singletonList(new SystemIndexDescriptor(".y", "an index that doesn't exist")))));
+                Collections.singletonList(new SystemIndexDescriptor(".y*", "an index that doesn't exist")))));
 
         GetAliasesRequest request = new GetAliasesRequest(".y");
         ImmutableOpenMap<String, List<AliasMetadata>> aliases = ImmutableOpenMap.<String, List<AliasMetadata>>builder()
@@ -229,7 +229,7 @@ public void testNetNewSystemIndicesDontErrorWhenNotRequested() {
         String[] concreteIndices;
 
         SystemIndexDescriptor netNewDescriptor = SystemIndexDescriptor.builder()
-            .setIndexPattern(".b")
+            .setIndexPattern(".b*")
             .setAliasName(".y")
             .setPrimaryIndex(".b")
             .setDescription(this.getTestName())

server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java

@@ -58,7 +58,7 @@ public class TransportUpdateSettingsActionTests extends ESTestCase {
         Map.of("test-feature", new SystemIndices.Feature(
             "test-feature",
             "a test feature",
-            List.of(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "test"))
+            List.of(new SystemIndexDescriptor(SYSTEM_INDEX_NAME + "*", "test"))
         ))
     );