diff --git a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml index 98ded638773ce..cb4a58ee453a7 100644 --- a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml +++ b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml @@ -13,6 +13,7 @@ + diff --git a/dev-tools/protoc_exe_sha256.sh b/dev-tools/protoc_exe_sha256.sh new file mode 100644 index 0000000000000..58e31f4cdc50d --- /dev/null +++ b/dev-tools/protoc_exe_sha256.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". +# + +# Script to download all .exe files from protobuf protoc repository and generate SHA256 checksums +# URL to download from +VERSION="4.32.0" +URL="https://repo1.maven.org/maven2/com/google/protobuf/protoc/${VERSION}/" +DOWNLOAD_DIR="protoc-${VERSION}-executables" + +# Create download directory if it doesn't exist +mkdir -p "${DOWNLOAD_DIR}" +cd "${DOWNLOAD_DIR}" || { echo "Failed to create/enter download directory"; exit 1; } + +# Get the HTML content, extract links to .exe files (but not .exe.md5 etc.) +# Using grep with lookahead assertion to ensure we don't match .exe followed by something else +curl -s "${URL}" | grep -o 'href="[^"]*\.exe"' | grep -v -E 'jsonl' | grep -v -E '\.exe\.[^"]+' | sed 's/href="//g' | sed 's/"//g' > exe_files.txt + +if [ ! -s exe_files.txt ]; then + echo "No .exe files found at ${URL}" + exit 1 +fi + +echo "Found $(wc -l < exe_files.txt | tr -d ' ') .exe files. Downloading..." + +# Download each file +while IFS= read -r file; do + curl -s -O "${URL}${file}" +done < exe_files.txt + +echo "Generating SHA256 checksums..." + +# Generate SHA256 checksums for all downloaded .exe files +if command -v shasum &> /dev/null; then + # macOS/some Linux + shasum -a 256 *.exe > SHA256SUMS.txt +elif command -v sha256sum &> /dev/null; then + # Most Linux distributions + sha256sum *.exe > SHA256SUMS.txt +else + echo "Neither shasum nor sha256sum command found. Cannot generate checksums." + exit 1 +fi + +# Print the checksums +cat SHA256SUMS.txt + +cd .. +rm -rf "${DOWNLOAD_DIR}" diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index ff3a06c604a06..7fc0bdb824855 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -272,6 +272,11 @@ + + + + + @@ -702,6 +707,11 @@ + + + + + @@ -862,6 +872,11 @@ + + + + + @@ -872,11 +887,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1709,6 +1766,16 @@ + + + + + + + + + + @@ -1724,11 +1791,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1874,6 +1991,11 @@ + + + + + diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index 51bb04185cfde..32b3e44866a6d 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -52,6 +52,10 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("data_stream/30_auto_create_data_stream/Don't initialize failure store during data stream auto-creation on successful index", "Configuring the failure store via data stream templates is not supported anymore.") task.skipTest("data_stream/150_tsdb/TSDB failures go to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + // TODO remove these after removing exact _tsid assertions in 8.x + task.skipTest("data_stream/150_tsdb/dynamic templates", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.") + task.skipTest("data_stream/150_tsdb/dynamic templates - conflicting aliases", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.") + task.skipTest("data_stream/150_tsdb/dynamic templates with nesting", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.") task.skipTest("data_stream/170_modify_data_stream/Modify a data stream's failure store", "Configuring the failure store via data stream templates is not supported anymore.") diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index eaef99d86a86e..1d19dfdc5670f 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -14,6 +14,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; @@ -24,6 +26,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -34,6 +37,7 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; @@ -58,12 +62,14 @@ import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -320,16 +326,12 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { ComposableIndexTemplate.builder() .indexPatterns(List.of("k8s*")) .template( - new Template( - Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset").build(), - new CompressedXContent(mappingTemplate), - null - ) + new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(mappingTemplate), null) ) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); - client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request)); } public void testInvalidTsdbTemplatesMissingSettings() throws Exception { @@ -621,6 +623,191 @@ public void testReindexing() throws Exception { ); } + public void testAddDimensionToMapping() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template( + new Template( + Settings.builder().put("index.mode", "time_series").build(), + new CompressedXContent(MAPPING_TEMPLATE), + null + ) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + // create data stream + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset"))); + + // put mapping with k8s.pod.uid as another time series dimension + var putMappingRequest = new PutMappingRequest(dataStreamName).source(""" + { + "properties": { + "k8s.pod.name": { + "type": "keyword", + "time_series_dimension": true + } + } + } + """, XContentType.JSON); + assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet()); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name")); + + indexWithPodNames(dataStreamName, Instant.now(), Map.of(), "dog", "cat"); + } + + public void testDynamicStringDimensions() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template(new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(""" + { + "_doc": { + "dynamic_templates": [ + { + "labels": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""), null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + // doesn't populate index.dimensions because the "labels" dynamic template doesn't have a path_math + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + // index doc + BulkResponse bulkResponse = client().prepareBulk() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .add( + client().prepareIndex(dataStreamName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(DOC.replace("$time", formatInstant(Instant.now())), XContentType.JSON) + ) + .get(); + assertThat(bulkResponse.hasFailures(), is(false)); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + } + + public void testDynamicDimensions() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template(new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(""" + + { + "_doc": { + "dynamic_templates": [ + { + "label": { + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""), null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + // doesn't populate index.dimensions because the "label" dynamic template doesn't have a path_math + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + // index doc + indexWithPodNames(dataStreamName, Instant.now(), Map.of("k8s.pod.name", "label"), "dog", "cat"); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + } + + private void indexWithPodNames(String dataStreamName, Instant timestamp, Map dynamicTemplates, String... podNames) { + // index doc + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (String podName : podNames) { + bulkRequestBuilder.add( + client().prepareIndex(dataStreamName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(DOC.replace("$time", formatInstant(timestamp)).replace("dog", podName), XContentType.JSON) + .request() + .setDynamicTemplates(dynamicTemplates) + ); + } + + BulkResponse bulkResponse = bulkRequestBuilder.get(); + assertThat(bulkResponse.hasFailures(), is(false)); + } + + private T getSetting(String dataStreamName, Setting setting) { + GetIndexResponse getIndexResponse = safeGet( + indicesAdmin().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(dataStreamName)) + ); + assertThat(getIndexResponse.getIndices().length, equalTo(1)); + Settings settings = getIndexResponse.getSettings().get(getIndexResponse.getIndices()[0]); + return setting.get(settings); + } + static String formatInstant(Instant instant) { return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index a76dac5db4540..c32eb05ddda65 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -184,7 +184,7 @@ public void testIndexingGettingAndSearching() throws Exception { // validate index: var getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(index)).actionGet(); - assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), equalTo("[attributes.*]")); + assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), equalTo("[attributes.*]")); // validate mapping var mapping = getIndexResponse.mappings().get(index).getSourceAsMap(); assertMap( diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index 9be0c18d18213..a7fc4b102866f 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -433,9 +433,10 @@ public void testSimulateTsdbDataStreamTemplate() throws Exception { assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.time_series.start_time"), notNullValue()); assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.time_series.end_time"), notNullValue()); assertThat( - ObjectPath.evaluate(responseBody, "template.settings.index.routing_path"), + ObjectPath.evaluate(responseBody, "template.settings.index.dimensions"), containsInAnyOrder("metricset", "k8s.pod.uid", "pod.labels.*") ); + assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.routing_path"), nullValue()); assertThat(ObjectPath.evaluate(responseBody, "overlapping"), empty()); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index ac3987dfa2771..2b4b87b368bc1 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.CheckedFunction; @@ -23,21 +24,20 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.PassThroughObjectMapper; import java.io.IOException; import java.io.UncheckedIOException; import java.time.Instant; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Locale; +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DIMENSIONS; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; /** @@ -119,13 +119,15 @@ public Settings getAdditionalIndexSettings( if (indexTemplateAndCreateRequestSettings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey()) == false && combinedTemplateMappings.isEmpty() == false) { - List routingPaths = findRoutingPaths( + List dimensions = new ArrayList<>(); + boolean matchesAllDimensions = findDimensionFields( indexName, indexTemplateAndCreateRequestSettings, - combinedTemplateMappings + combinedTemplateMappings, + dimensions ); - if (routingPaths.isEmpty() == false) { - builder.putList(INDEX_ROUTING_PATH.getKey(), routingPaths); + if (dimensions.isEmpty() == false) { + builder.putList(matchesAllDimensions ? INDEX_DIMENSIONS.getKey() : INDEX_ROUTING_PATH.getKey(), dimensions); } } return builder.build(); @@ -137,15 +139,53 @@ public Settings getAdditionalIndexSettings( } /** - * Find fields in mapping that are of type keyword and time_series_dimension enabled. + * This is called when mappings are updated, so that the {@link IndexMetadata#INDEX_DIMENSIONS} + * and {@link IndexMetadata#INDEX_ROUTING_PATH} settings are updated to match the new mappings. + * Updates {@link IndexMetadata#INDEX_DIMENSIONS} if a new dimension field is added to the mappings, + * or sets {@link IndexMetadata#INDEX_ROUTING_PATH} if a new dimension field is added that doesn't allow for matching all + * dimension fields via a wildcard pattern. + */ + @Override + public Settings onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper documentMapper) { + List indexDimensions = INDEX_DIMENSIONS.get(indexMetadata.getSettings()); + if (indexDimensions.isEmpty()) { + return Settings.EMPTY; + } + assert indexMetadata.getIndexMode() == IndexMode.TIME_SERIES; + List newIndexDimensions = new ArrayList<>(indexDimensions.size()); + boolean matchesAllDimensions = findDimensionFields(newIndexDimensions, documentMapper); + if (indexDimensions.equals(newIndexDimensions)) { + return Settings.EMPTY; + } + if (matchesAllDimensions) { + return Settings.builder().putList(INDEX_DIMENSIONS.getKey(), newIndexDimensions).build(); + } else { + return Settings.builder().putList(INDEX_ROUTING_PATH.getKey(), newIndexDimensions).build(); + } + } + + /** + * Find fields in mapping that are time_series_dimension enabled. * Using MapperService here has an overhead, but allows the mappings from template to * be merged correctly and fetching the fields without manually parsing the mappings. - * + *

* Alternatively this method can instead parse mappings into map of maps and merge that and * iterate over all values to find the field that can serve as routing value. But this requires * mapping specific logic to exist here. + * + * @param indexName the name of the index for which the dimension fields are being found + * @param allSettings the settings of the index + * @param combinedTemplateMappings the combined mappings from index templates + * (if any) that are applied to the index + * @param dimensions a list to which the found dimension fields will be added + * @return true if all potential dimension fields can be matched via the dimensions in the list, false otherwise */ - private List findRoutingPaths(String indexName, Settings allSettings, List combinedTemplateMappings) { + private boolean findDimensionFields( + String indexName, + Settings allSettings, + List combinedTemplateMappings, + List dimensions + ) { var tmpIndexMetadata = IndexMetadata.builder(indexName); int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(allSettings); @@ -169,57 +209,61 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li // Create MapperService just to extract keyword dimension fields: try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata.build())) { mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); - List routingPaths = new ArrayList<>(); - for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) { - extractPath(routingPaths, fieldMapper); - } - for (var objectMapper : mapperService.documentMapper().mappers().objectMappers().values()) { - if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { - if (passThroughObjectMapper.containsDimensions()) { - routingPaths.add(passThroughObjectMapper.fullPath() + ".*"); - } - } - } - for (var template : mapperService.getAllDynamicTemplates()) { - if (template.pathMatch().isEmpty()) { - continue; - } - - var templateName = "__dynamic__" + template.name(); - var mappingSnippet = template.mappingForName(templateName, KeywordFieldMapper.CONTENT_TYPE); - String mappingSnippetType = (String) mappingSnippet.get("type"); - if (mappingSnippetType == null) { - continue; - } + DocumentMapper documentMapper = mapperService.documentMapper(); + return findDimensionFields(dimensions, documentMapper); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - MappingParserContext parserContext = mapperService.parserContext(); - for (Iterator iterator = template.pathMatch().iterator(); iterator.hasNext();) { - var mapper = parserContext.typeParser(mappingSnippetType) - .parse(iterator.next(), mappingSnippet, parserContext) - .build(MapperBuilderContext.root(false, false)); - extractPath(routingPaths, mapper); - if (iterator.hasNext()) { - // Since FieldMapper.parse modifies the Map passed in (removing entries for "type"), that means - // that only the first pathMatch passed in gets recognized as a time_series_dimension. - // To avoid this, each parsing call uses a new mapping snippet. - // Note that a shallow copy of the mappingSnippet map is not enough if there are multi-fields. - mappingSnippet = template.mappingForName(templateName, KeywordFieldMapper.CONTENT_TYPE); - } + /** + * Finds the dimension fields in the provided document mapper and adds them to the provided list. + * + * @param dimensions the list to which the found dimension fields will be added + * @param documentMapper the document mapper from which to extract the dimension fields + * @return true if all potential dimension fields can be matched via the dimensions in the list, false otherwise + */ + private static boolean findDimensionFields(List dimensions, DocumentMapper documentMapper) { + for (var objectMapper : documentMapper.mappers().objectMappers().values()) { + if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { + if (passThroughObjectMapper.containsDimensions()) { + dimensions.add(passThroughObjectMapper.fullPath() + ".*"); } } - return routingPaths; - } catch (IOException e) { - throw new UncheckedIOException(e); } + boolean matchesAllDimensions = true; + for (var template : documentMapper.mapping().getRoot().dynamicTemplates()) { + if (template.isTimeSeriesDimension() == false) { + continue; + } + if (template.isSimplePathMatch() == false) { + // If the template is not using a simple path match, the dimensions list can't match all potential dimensions. + // For example, if the dynamic template matches by mapping type (all strings are mapped as dimensions), + // the coordinating node can't rely on the dimensions list to match all dimensions. + // In this case, the index.routing_path setting will be used instead. + matchesAllDimensions = false; + } + if (template.pathMatch().isEmpty() == false) { + dimensions.addAll(template.pathMatch()); + } + } + + for (var fieldMapper : documentMapper.mappers().fieldMappers()) { + extractPath(dimensions, fieldMapper); + } + return matchesAllDimensions; } /** - * Helper method that adds the name of the mapper to the provided list if it is a keyword dimension field. + * Helper method that adds the name of the mapper to the provided list. */ - private static void extractPath(List routingPaths, Mapper mapper) { - if (mapper instanceof KeywordFieldMapper keywordFieldMapper) { - if (keywordFieldMapper.fieldType().isDimension()) { - routingPaths.add(mapper.fullPath()); + private static void extractPath(List dimensions, Mapper mapper) { + if (mapper instanceof FieldMapper fieldMapper && fieldMapper.fieldType().isDimension()) { + String path = mapper.fullPath(); + // don't add if the path already matches via a wildcard pattern in the list + // e.g. if "path.*" is already added, "path.foo" should not be added + if (Regex.simpleMatch(dimensions, path) == false) { + dimensions.add(path); } } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index b85b3f6e7ae39..bdd94de1f92c6 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -33,7 +33,6 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.FORMATTER; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -70,6 +69,18 @@ public void testGetAdditionalIndexSettings() throws Exception { "field3": { "type": "keyword", "time_series_dimension": true + }, + "field4": { + "type": "long", + "time_series_dimension": true + }, + "field5": { + "type": "ip", + "time_series_dimension": true + }, + "field6": { + "type": "boolean", + "time_series_dimension": true } } } @@ -91,7 +102,7 @@ public void testGetAdditionalIndexSettings() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("field3")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); } public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throws Exception { @@ -105,7 +116,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw "_doc": { "properties": { "field1": { - "type": "keyword" + "type": "keyword", "time_series_dimension": true }, "field2": { @@ -206,7 +217,7 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field3")); } public void testGetAdditionalIndexSettingsNoMappings() { @@ -461,7 +472,7 @@ public void testGenerateRoutingPathFromDynamicTemplate() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); } public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntries() throws Exception { @@ -502,10 +513,10 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat( - IndexMetadata.INDEX_ROUTING_PATH.get(result), + IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") ); - List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); + List routingPathList = IndexMetadata.INDEX_DIMENSIONS.get(result); assertEquals(3, routingPathList.size()); } @@ -552,10 +563,10 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat( - IndexMetadata.INDEX_ROUTING_PATH.get(result), + IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") ); - List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); + List routingPathList = IndexMetadata.INDEX_DIMENSIONS.get(result); assertEquals(3, routingPathList.size()); } @@ -605,7 +616,7 @@ public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch() assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); } public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() throws Exception { @@ -652,8 +663,8 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro Settings result = generateTsdbSettings(mapping, now); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); - assertEquals(2, IndexMetadata.INDEX_ROUTING_PATH.get(result).size()); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + assertEquals(2, IndexMetadata.INDEX_DIMENSIONS.get(result).size()); } public void testGenerateRoutingPathFromPassThroughObject() throws Exception { @@ -665,7 +676,12 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception { "labels": { "type": "passthrough", "time_series_dimension": true, - "priority": 2 + "priority": 2, + "properties": { + "label1": { + "type": "keyword" + } + } }, "metrics": { "type": "passthrough", @@ -683,7 +699,7 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception { assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("labels.*")); } private Settings generateTsdbSettings(String mapping, Instant now) throws IOException { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 598bc90217574..f393fafb5067e 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -148,13 +148,12 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" ]} --- "aggregate the tsid": - requires: cluster_features: ["gte_v8.13.0"] - reason: _tsid hahing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: @@ -169,9 +168,8 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} + - length: {aggregations.tsids.buckets: 2} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -368,7 +366,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -387,7 +384,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -406,7 +402,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -425,7 +420,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: search: @@ -443,7 +437,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -520,7 +513,6 @@ dynamic templates - conflicting aliases: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -539,7 +531,6 @@ dynamic templates - conflicting aliases: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -699,7 +690,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -718,7 +708,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -737,7 +726,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -756,7 +744,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -775,7 +762,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -794,7 +780,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java index 6580a5af3d005..11bfa05abeccf 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.core.Nullable; import org.elasticsearch.ingest.AbstractProcessor; @@ -17,11 +18,9 @@ import org.elasticsearch.ingest.Processor; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.function.Function; -import java.util.regex.Pattern; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; @@ -221,11 +220,6 @@ public RerouteProcessor create( */ static final class DataStreamValueSource { - private static final int MAX_LENGTH = 100; - private static final String REPLACEMENT = "_"; - private static final Pattern DISALLOWED_IN_TYPE = Pattern.compile("[\\\\/*?\"<>| ,#:-]"); - private static final Pattern DISALLOWED_IN_DATASET = Pattern.compile("[\\\\/*?\"<>| ,#:-]"); - private static final Pattern DISALLOWED_IN_NAMESPACE = Pattern.compile("[\\\\/*?\"<>| ,#:]"); static final DataStreamValueSource TYPE_VALUE_SOURCE = type("{{" + DATA_STREAM_TYPE + "}}"); static final DataStreamValueSource DATASET_VALUE_SOURCE = dataset("{{" + DATA_STREAM_DATASET + "}}"); static final DataStreamValueSource NAMESPACE_VALUE_SOURCE = namespace("{{" + DATA_STREAM_NAMESPACE + "}}"); @@ -235,24 +229,15 @@ static final class DataStreamValueSource { private final Function sanitizer; public static DataStreamValueSource type(String type) { - return new DataStreamValueSource(type, ds -> sanitizeDataStreamField(ds, DISALLOWED_IN_TYPE)); + return new DataStreamValueSource(type, DataStream::sanitizeType); } public static DataStreamValueSource dataset(String dataset) { - return new DataStreamValueSource(dataset, ds -> sanitizeDataStreamField(ds, DISALLOWED_IN_DATASET)); + return new DataStreamValueSource(dataset, DataStream::sanitizeDataset); } public static DataStreamValueSource namespace(String namespace) { - return new DataStreamValueSource(namespace, nsp -> sanitizeDataStreamField(nsp, DISALLOWED_IN_NAMESPACE)); - } - - private static String sanitizeDataStreamField(String s, Pattern disallowedInDataset) { - if (s == null) { - return null; - } - s = s.toLowerCase(Locale.ROOT); - s = s.substring(0, Math.min(s.length(), MAX_LENGTH)); - return disallowedInDataset.matcher(s).replaceAll(REPLACEMENT); + return new DataStreamValueSource(namespace, DataStream::sanitizeNamespace); } private DataStreamValueSource(String value, Function sanitizer) { diff --git a/server/build.gradle b/server/build.gradle index 20557aefba731..50ccb3eaf210a 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -59,6 +59,7 @@ dependencies { // utilities api project(":libs:cli") implementation 'com.carrotsearch:hppc:0.8.1' + api 'com.dynatrace.hash4j:hash4j:0.25.0' // precentil ranks aggregation api 'org.hdrhistogram:HdrHistogram:2.1.9' diff --git a/server/licenses/hash4j-LICENSE.txt b/server/licenses/hash4j-LICENSE.txt new file mode 100644 index 0000000000000..261eeb9e9f8b2 --- /dev/null +++ b/server/licenses/hash4j-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/server/licenses/hash4j-NOTICE.txt b/server/licenses/hash4j-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 549c603b13980..0c60fd9347006 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -54,6 +54,7 @@ requires org.apache.lucene.queryparser; requires org.apache.lucene.sandbox; requires org.apache.lucene.suggest; + requires hash4j; exports org.elasticsearch; exports org.elasticsearch.action; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 5a026b6e1660b..af2d9edb9474c 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -365,6 +365,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_LOOKUP_JOIN_ON_MANY_FIELDS = def(9_139_0_00); public static final TransportVersion SIMULATE_INGEST_EFFECTIVE_MAPPING = def(9_140_0_00); public static final TransportVersion RESOLVE_INDEX_MODE_ADDED = def(9_141_0_00); + public static final TransportVersion INGEST_REQUEST_INCLUDE_TSID = def(9_142_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index e8bd38b01414f..2bc51b252386c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -52,6 +52,8 @@ public class CreateIndexClusterStateUpdateRequest { private ComposableIndexTemplate matchingTemplate; + private boolean settingsSystemProvided = false; + /** * @deprecated project id ought always be specified */ @@ -223,6 +225,19 @@ public CreateIndexClusterStateUpdateRequest setMatchingTemplate(ComposableIndexT return this; } + /** + * Indicates whether the {@link #settings} of this request are system provided. + * If this is true, private settings will be allowed to be set in the request. + */ + public CreateIndexClusterStateUpdateRequest settingsSystemProvided(boolean settingsSystemProvided) { + this.settingsSystemProvided = settingsSystemProvided; + return this; + } + + public boolean settingsSystemProvided() { + return settingsSystemProvided; + } + @Override public String toString() { return "CreateIndexClusterStateUpdateRequest{" diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 01968596db932..357504fe87d1b 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -415,7 +415,8 @@ static boolean executeBulkItemRequest( request.routing(), request.getDynamicTemplates(), request.getIncludeSourceOnError(), - meteringParserDecorator + meteringParserDecorator, + request.tsid() ); result = primary.applyIndexOperationOnPrimary( version, @@ -743,7 +744,11 @@ private static Engine.Result performOpOnReplica( indexRequest.id(), indexRequest.source(), indexRequest.getContentType(), - indexRequest.routing() + indexRequest.routing(), + Map.of(), + true, + XContentMeteringParserDecorator.NOOP, + indexRequest.tsid() ); result = replica.applyIndexOperationOnReplica( primaryResponse.getSeqNo(), diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index b52f5447b9311..c1772dcd2e1dd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -206,7 +206,8 @@ private ValidationResult validateMappings( request.routing(), request.getDynamicTemplates(), request.getIncludeSourceOnError(), - XContentMeteringParserDecorator.NOOP + XContentMeteringParserDecorator.NOOP, + request.tsid() ); ProjectMetadata project = projectResolver.getProjectMetadata(clusterService.state()); diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 707ea0919a91f..a3210bcbb4983 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.index; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchGenerationException; @@ -152,6 +153,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement * rawTimestamp field is used on the coordinate node, it doesn't need to be serialised. */ private Object rawTimestamp; + private BytesRef tsid; public IndexRequest(StreamInput in) throws IOException { this(null, in); @@ -216,6 +218,13 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { includeSourceOnError = in.readBoolean(); } // else default value is true + + if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_TSID)) { + tsid = in.readBytesRef(); + if (tsid.length == 0) { + tsid = null; // no tsid set + } + } } public IndexRequest() { @@ -353,6 +362,22 @@ public String routing() { return this.routing; } + /** + * When {@link org.elasticsearch.cluster.metadata.IndexMetadata#INDEX_DIMENSIONS} is populated, + * the coordinating node will calculate _tsid during routing and set it on the request. + * For time series indices where the setting is not populated, the _tsid will be created in the data node during document parsing. + *

+ * The _tsid can not be directly set by a user, it is set by the coordinating node. + */ + public IndexRequest tsid(BytesRef tsid) { + this.tsid = tsid; + return this; + } + + public BytesRef tsid() { + return this.tsid; + } + /** * Sets the ingest pipeline to be executed before indexing the document */ @@ -815,6 +840,9 @@ private void writeBody(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { out.writeBoolean(includeSourceOnError); } + if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_TSID)) { + out.writeBytesRef(tsid); + } } @Override @@ -917,7 +945,7 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, ProjectMetadata project) @Override public int route(IndexRouting indexRouting) { - return indexRouting.indexShard(id, routing, contentType, source); + return indexRouting.indexShard(id, routing, tsid, contentType, source); } public IndexRequest setRequireAlias(boolean requireAlias) { diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index 3041e6bf5e274..620cd90cbebfc 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.index; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; @@ -50,6 +51,7 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder, ToXContentO public static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd"); public static final String TIMESTAMP_FIELD_NAME = "@timestamp"; + private static final int MAX_LENGTH = 100; + private static final String REPLACEMENT = "_"; + private static final Pattern DISALLOWED_IN_TYPE = Pattern.compile("[\\\\/*?\"<>| ,#:-]"); + private static final Pattern DISALLOWED_IN_DATASET = Pattern.compile("[\\\\/*?\"<>| ,#:-]"); + private static final Pattern DISALLOWED_IN_NAMESPACE = Pattern.compile("[\\\\/*?\"<>| ,#:]"); + + public static String sanitizeType(String type) { + return sanitizeDataStreamField(type, DISALLOWED_IN_TYPE); + } + + public static String sanitizeDataset(String dataset) { + return sanitizeDataStreamField(dataset, DISALLOWED_IN_DATASET); + } + + public static String sanitizeNamespace(String namespace) { + return sanitizeDataStreamField(namespace, DISALLOWED_IN_NAMESPACE); + } + + private static String sanitizeDataStreamField(String s, Pattern disallowedInDataset) { + if (s == null) { + return null; + } + s = s.toLowerCase(Locale.ROOT); + s = s.substring(0, Math.min(s.length(), MAX_LENGTH)); + return disallowedInDataset.matcher(s).replaceAll(REPLACEMENT); + } + // Timeseries indices' leaf readers should be sorted by desc order of their timestamp field, as it allows search time optimizations public static final Comparator TIMESERIES_LEAF_READERS_SORTER = Comparator.comparingLong((LeafReader r) -> { try { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index ef29a74fd47a5..dfac3a21cd4c3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -514,6 +514,27 @@ public Iterator> settings() { Property.ServerlessPublic ); + /** + * Populated when an index that belongs to a time_series data stream is created or its mappings are updated. + * This setting is used so that the coordinating node knows which fields are time series dimensions + * as it doesn't have access to mappings. + * When this setting is populated, an optimization kicks in that allows the coordinating node to create the tsid and the routing hash + * in one go. + * Otherwise, the coordinating node only creates the routing hash based on {@link #INDEX_ROUTING_PATH} and the tsid is created + * during document parsing, effectively requiring two passes over the document. + *

+ * The condition for this optimization to kick in is that all possible dimension fields can be identified + * via a list of wildcard patterns. + * If that's not the case (for example when certain types of dynamic templates are used), + * the {@link #INDEX_ROUTING_PATH} is populated instead. + */ + public static final Setting> INDEX_DIMENSIONS = Setting.stringListSetting( + "index.dimensions", + Setting.Property.IndexScope, + Property.Dynamic, + Property.PrivateIndex + ); + /** * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use. * TODO: Remove in 9.0 @@ -576,6 +597,7 @@ public Iterator> settings() { private final int routingFactor; private final int routingPartitionSize; private final List routingPaths; + private final List dimensions; private final int numberOfShards; private final int numberOfReplicas; @@ -689,6 +711,7 @@ private IndexMetadata( final int routingNumShards, final int routingPartitionSize, final List routingPaths, + final List dimensions, final ActiveShardCount waitForActiveShards, final ImmutableOpenMap rolloverInfos, final boolean isSystem, @@ -744,6 +767,7 @@ private IndexMetadata( this.routingFactor = routingNumShards / numberOfShards; this.routingPartitionSize = routingPartitionSize; this.routingPaths = routingPaths; + this.dimensions = dimensions; this.waitForActiveShards = waitForActiveShards; this.rolloverInfos = rolloverInfos; this.isSystem = isSystem; @@ -803,6 +827,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -865,6 +890,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -935,6 +961,7 @@ public IndexMetadata withSetPrimaryTerm(int shardId, long primaryTerm) { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -996,6 +1023,7 @@ public IndexMetadata withTimestampRanges(IndexLongFieldRange timestampRange, Ind this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -1052,6 +1080,7 @@ public IndexMetadata withIncrementedVersion() { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.dimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -1166,6 +1195,10 @@ public List getRoutingPaths() { return routingPaths; } + public List getDimensions() { + return dimensions; + } + public int getTotalNumberOfShards() { return totalNumberOfShards; } @@ -2378,6 +2411,7 @@ IndexMetadata build(boolean repair) { } final List routingPaths = INDEX_ROUTING_PATH.get(settings); + final List dimensions = INDEX_DIMENSIONS.get(settings); final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE); @@ -2457,6 +2491,7 @@ IndexMetadata build(boolean repair) { getRoutingNumShards(), routingPartitionSize, routingPaths, + dimensions, waitForActiveShards, rolloverInfos.build(), isSystem, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index ca612eb20747a..4f4263dcaff04 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1561,12 +1561,12 @@ private static void validateActiveShardCount(ActiveShardCount waitForActiveShard private void validate(CreateIndexClusterStateUpdateRequest request, ProjectMetadata projectMetadata, RoutingTable routingTable) { validateIndexName(request.index(), projectMetadata, routingTable); - validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings); + validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings && request.settingsSystemProvided() == false); } public void validateIndexSettings(String indexName, final Settings settings, final boolean forbidPrivateIndexSettings) throws IndexCreationException { - List validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings); + List validationErrors = getIndexSettingsValidationErrors(settings, null, forbidPrivateIndexSettings); if (validationErrors.isEmpty() == false) { ValidationException validationException = new ValidationException(); @@ -1575,21 +1575,30 @@ public void validateIndexSettings(String indexName, final Settings settings, fin } } - List getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) { + List getIndexSettingsValidationErrors( + final Settings settings, + @Nullable Settings systemProvided, + final boolean forbidPrivateIndexSettings + ) { List validationErrors = validateIndexCustomPath(settings, env.sharedDataDir()); if (forbidPrivateIndexSettings) { - validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings)); + validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, systemProvided, indexScopedSettings)); } return validationErrors; } - private static List validatePrivateSettingsNotExplicitlySet(Settings settings, IndexScopedSettings indexScopedSettings) { + private static List validatePrivateSettingsNotExplicitlySet( + Settings settings, + @Nullable Settings systemProvided, + IndexScopedSettings indexScopedSettings + ) { List validationErrors = new ArrayList<>(); for (final String key : settings.keySet()) { final Setting setting = indexScopedSettings.get(key); if (setting == null) { assert indexScopedSettings.isPrivateSetting(key) : "expected [" + key + "] to be private but it was not"; - } else if (setting.isPrivateIndex()) { + } else if (setting.isPrivateIndex() && (systemProvided == null || settings.get(key).equals(systemProvided.get(key)) == false)) { + // if the setting is system provided, they're not set by the user validationErrors.add("private index setting [" + key + "] can not be set explicitly"); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 14c480b521c30..3496faa4b9fc1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -789,6 +789,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo final var combinedSettings = resolveSettings(indexTemplate, projectMetadata.componentTemplates()); // First apply settings sourced from index setting providers: var finalSettings = Settings.builder(); + var additionalSettingsBuilder = Settings.builder(); for (var provider : indexSettingProviders) { var newAdditionalSettings = provider.getAdditionalIndexSettings( VALIDATE_INDEX_NAME, @@ -799,9 +800,11 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo combinedSettings, combinedMappings ); - MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, finalSettings); - finalSettings.put(newAdditionalSettings); + MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, additionalSettingsBuilder); + additionalSettingsBuilder.put(newAdditionalSettings); } + Settings additionalSettings = additionalSettingsBuilder.build(); + finalSettings.put(additionalSettings); // Then apply setting from component templates: finalSettings.put(combinedSettings); // Then finally apply settings resolved from index template: @@ -811,7 +814,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo var templateToValidate = indexTemplate.toBuilder().template(Template.builder(finalTemplate).settings(finalSettings)).build(); - validate(name, templateToValidate); + validate(name, templateToValidate, additionalSettings); validateDataStreamsStillReferenced(projectMetadata, name, templateToValidate); validateLifecycle(projectMetadata, name, templateToValidate, globalRetentionSettings.get(false)); validateDataStreamOptions(projectMetadata, name, templateToValidate, globalRetentionSettings.get(true)); @@ -2041,18 +2044,19 @@ public static void validateTemplate(Settings validateSettings, CompressedXConten } public void validate(String name, ComponentTemplate template) { - validate(name, template.template(), Collections.emptyList()); + validate(name, template.template(), Collections.emptyList(), null); } - private void validate(String name, ComposableIndexTemplate template) { - validate(name, template.template(), template.indexPatterns()); + private void validate(String name, ComposableIndexTemplate template, Settings systemProvided) { + validate(name, template.template(), template.indexPatterns(), systemProvided); } - private void validate(String name, Template template, List indexPatterns) { + private void validate(String name, Template template, List indexPatterns, @Nullable Settings systemProvided) { Optional