From 659aa26dd287ad9657a8f8849e1e501372049934 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Tue, 11 Nov 2025 15:14:38 +0530 Subject: [PATCH 01/10] Add preserve_dots paramater to ObjectMapper Signed-off-by: Mohit Kumar --- .../opensearch/index/mapper/ObjectMapper.java | 33 ++++++++++++++++++- .../index/mapper/RootObjectMapper.java | 5 ++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index fb21c8d2071d9..582cc66b447b0 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -81,6 +81,7 @@ public static class Defaults { public static final boolean ENABLED = true; public static final Nested NESTED = Nested.NO; public static final Dynamic DYNAMIC = null; // not set, inherited from root + public static final Explicit PRESERVE_DOTS = new Explicit<>(false, false); } /** @@ -202,6 +203,8 @@ public static class Builder extends Mapper.Builder { protected Dynamic dynamic = Defaults.DYNAMIC; + protected Explicit preserveDots = Defaults.PRESERVE_DOTS; + protected final List mappersBuilders = new ArrayList<>(); public Builder(String name) { @@ -224,6 +227,11 @@ public T nested(Nested nested) { return builder; } + public T preserveDots(boolean preserveDots) { + this.preserveDots = new Explicit<>(preserveDots, true); + return builder; + } + public T add(Mapper.Builder builder) { mappersBuilders.add(builder); return this.builder; @@ -250,6 +258,7 @@ public ObjectMapper build(BuilderContext context) { enabled, nested, dynamic, + preserveDots, mappers, context.indexSettings() ); @@ -263,10 +272,11 @@ protected ObjectMapper createMapper( Explicit enabled, Nested nested, Dynamic dynamic, + Explicit preserveDots, Map mappers, @Nullable Settings settings ) { - return new ObjectMapper(name, fullPath, enabled, nested, dynamic, mappers, settings); + return new ObjectMapper(name, fullPath, enabled, nested, dynamic, preserveDots, mappers, settings); } } @@ -324,6 +334,9 @@ protected static boolean parseObjectOrDocumentTypeProperties( } else if (fieldName.equals("enabled")) { builder.enabled(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".enabled")); return true; + } else if (fieldName.equals("preserve_dots")) { + builder.preserveDots(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".preserve_dots")); + return true; } else if (fieldName.equals("derived")) { if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) { // nothing to do here, empty (to support "derived: []" case) @@ -623,6 +636,8 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map preserveDots; + private volatile CopyOnWriteHashMap mappers; ObjectMapper( @@ -631,6 +646,7 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map enabled, Nested nested, Dynamic dynamic, + Explicit preserveDots, Map mappers, Settings settings ) { @@ -643,6 +659,7 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map(); } else { @@ -726,6 +743,14 @@ public final Dynamic dynamic() { return dynamic; } + public boolean preserveDots() { + return preserveDots.value(); + } + + public Explicit preserveDotsExplicit() { + return preserveDots; + } + /** * Returns the parent {@link ObjectMapper} instance of the specified object mapper or null if there * isn't any. @@ -787,6 +812,9 @@ protected void doMerge(final ObjectMapper mergeWith, MergeReason reason) { if (mergeWith.enabled.explicit()) { this.enabled = mergeWith.enabled; } + if (mergeWith.preserveDotsExplicit().explicit()) { + this.preserveDots = mergeWith.preserveDotsExplicit(); + } } else if (isEnabled() != mergeWith.isEnabled()) { throw new MapperException("the [enabled] parameter can't be updated for the object mapping [" + name() + "]"); } @@ -845,6 +873,9 @@ public void toXContent(XContentBuilder builder, Params params, ToXContent custom if (isEnabled() != Defaults.ENABLED) { builder.field("enabled", enabled.value()); } + if (preserveDotsExplicit().explicit()) { + builder.field("preserve_dots", preserveDots.value()); + } if (custom != null) { custom.toXContent(builder, params); diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java index 11f8d38249c56..70cb64dbc6641 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java @@ -123,6 +123,7 @@ protected ObjectMapper createMapper( Explicit enabled, Nested nested, Dynamic dynamic, + Explicit preserveDots, Map mappers, @Nullable Settings settings ) { @@ -131,6 +132,7 @@ protected ObjectMapper createMapper( name, enabled, dynamic, + preserveDots, mappers, dynamicDateTimeFormatters, dynamicTemplates, @@ -295,6 +297,7 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam String name, Explicit enabled, Dynamic dynamic, + Explicit preserveDots, Map mappers, Explicit dynamicDateTimeFormatters, Explicit dynamicTemplates, @@ -302,7 +305,7 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam Explicit numericDetection, Settings settings ) { - super(name, name, enabled, Nested.NO, dynamic, mappers, settings); + super(name, name, enabled, Nested.NO, dynamic, preserveDots, mappers, settings); this.dynamicTemplates = dynamicTemplates; this.dynamicDateTimeFormatters = dynamicDateTimeFormatters; this.dateDetection = dateDetection; From 5405ee6bae10623fdb5371d9cfbc123d213b1ca3 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Tue, 11 Nov 2025 16:32:41 +0530 Subject: [PATCH 02/10] Extend RootObjectMapper to support preserve_dots Signed-off-by: Mohit Kumar --- .../index/mapper/RootObjectMapper.java | 9 +++++- .../index/mapper/RootObjectMapperTests.java | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java index 70cb64dbc6641..a3303e0b4568d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java @@ -230,7 +230,10 @@ protected static void parseContextAwareGroupingField( } protected boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, ParserContext parserContext) { - if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) { + if (fieldName.equals("preserve_dots")) { + builder.preserveDots(nodeBooleanValue(fieldNode, "preserve_dots")); + return true; + } else if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) { if (fieldNode instanceof List) { List formatters = new ArrayList<>(); for (Object formatter : (List) fieldNode) { @@ -426,6 +429,10 @@ protected void doMerge(ObjectMapper mergeWith, MergeReason reason) { protected void doXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { final boolean includeDefaults = params.paramAsBoolean("include_defaults", false); + if (preserveDotsExplicit().explicit() || includeDefaults) { + builder.field("preserve_dots", preserveDots()); + } + if (dynamicDateTimeFormatters.explicit() || includeDefaults) { builder.startArray("dynamic_date_formats"); for (DateFormatter dateTimeFormatter : dynamicDateTimeFormatters.value()) { diff --git a/server/src/test/java/org/opensearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/RootObjectMapperTests.java index 054ff2ff8bbc6..2cbd6b9542efa 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RootObjectMapperTests.java @@ -45,6 +45,36 @@ public class RootObjectMapperTests extends OpenSearchSingleNodeTestCase { + public void testPreserveDots() throws Exception { + MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .field("preserve_dots", false) + .endObject() + .endObject() + .toString(); + MapperService mapperService = createIndex("test").mapperService(); + DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); + assertEquals(mapping, mapper.mappingSource().toString()); + + // update with a different explicit value + String mapping2 = XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .field("preserve_dots", true) + .endObject() + .endObject() + .toString(); + mapper = mapperService.merge("type", new CompressedXContent(mapping2), reason); + assertEquals(mapping2, mapper.mappingSource().toString()); + + // update with an implicit value: no change + String mapping3 = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().toString(); + mapper = mapperService.merge("type", new CompressedXContent(mapping3), reason); + assertEquals(mapping2, mapper.mappingSource().toString()); + } + public void testNumericDetection() throws Exception { MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); String mapping = XContentFactory.jsonBuilder() From f9fa2ba040f7fc9a8d16c4161f52230db259be64 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Tue, 11 Nov 2025 19:16:07 +0530 Subject: [PATCH 03/10] Implement flat field parsing in DocumentParser Signed-off-by: Mohit Kumar --- .../index/mapper/DocumentParser.java | 187 +++++++++++++++++- .../index/mapper/DocumentParserTests.java | 98 +++++++++ 2 files changed, 284 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index 8267627573c16..4634d62a5f36e 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -548,7 +548,14 @@ private static ParseContext nestedContext(ParseContext context, ObjectMapper map private static void parseObjectOrField(ParseContext context, Mapper mapper) throws IOException { if (mapper instanceof ObjectMapper objectMapper) { - parseObjectOrNested(context, objectMapper); + // Check if preserve_dots is enabled for this object mapper + if (objectMapper.preserveDots()) { + // Parse as flat field - do not expand dots into nested objects + parseFieldWithDots(context, objectMapper); + } else { + // Existing behavior: parse as nested object + parseObjectOrNested(context, objectMapper); + } } else if (mapper instanceof FieldMapper fieldMapper) { fieldMapper.parse(context); parseCopyFields(context, fieldMapper.copyTo().copyToFields()); @@ -561,6 +568,184 @@ private static void parseObjectOrField(ParseContext context, Mapper mapper) thro } } + /** + * Parses fields with dots as literal field names when preserve_dots is enabled. + * This method treats dotted field names (e.g., "user.name", "metrics.cpu.usage") as flat fields + * rather than expanding them into nested object hierarchies. + * + * @param context the parse context + * @param mapper the object mapper with preserve_dots enabled + * @throws IOException if parsing fails + */ + private static void parseFieldWithDots(ParseContext context, ObjectMapper mapper) throws IOException { + if (mapper.isEnabled() == false) { + context.parser().skipChildren(); + return; + } + + XContentParser parser = context.parser(); + XContentParser.Token token = parser.currentToken(); + + if (token == XContentParser.Token.VALUE_NULL) { + // the object is null, simply bail + return; + } + + String currentFieldName = parser.currentName(); + if (token.isValue()) { + throw new MapperParsingException( + "object mapping for [" + + mapper.name() + + "] tried to parse field [" + + currentFieldName + + "] as object, but found a concrete value" + ); + } + + // if we are at the end of the previous object, advance + if (token == XContentParser.Token.END_OBJECT) { + token = parser.nextToken(); + } + if (token == XContentParser.Token.START_OBJECT) { + // if we are just starting an OBJECT, advance, this is the object we are parsing, we need the name first + token = parser.nextToken(); + } + + try { + assert token == XContentParser.Token.FIELD_NAME || token == XContentParser.Token.END_OBJECT; + context.incrementFieldCurrentDepth(); + context.checkFieldDepthLimit(); + + while (token != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + // For preserve_dots, treat the entire field name (including dots) as a literal field name + // Do not split on dots - this is the key difference from normal object parsing + + // Look up the mapper using the complete dotted field name + Mapper fieldMapper = mapper.getMapper(currentFieldName); + + if (fieldMapper != null) { + // Found an existing mapper for this dotted field name + token = parser.nextToken(); + if (fieldMapper instanceof FieldMapper fm) { + fm.parse(context); + parseCopyFields(context, fm.copyTo().copyToFields()); + } else if (fieldMapper instanceof ObjectMapper om) { + // Even with preserve_dots, we can have nested ObjectMappers + // for explicitly defined sub-objects + parseObjectOrNested(context, om); + } else { + throw new MapperParsingException( + "Cannot parse field [" + currentFieldName + "] with mapper type [" + + fieldMapper.getClass().getSimpleName() + "]" + ); + } + } else { + // No existing mapper - handle dynamic mapping + token = parser.nextToken(); + ObjectMapper.Dynamic dynamic = dynamicOrDefault(mapper, context); + + switch (dynamic) { + case STRICT: + throw new StrictDynamicMappingException( + dynamic.name().toLowerCase(Locale.ROOT), + mapper.fullPath(), + currentFieldName + ); + case TRUE: + case STRICT_ALLOW_TEMPLATES: + case FALSE_ALLOW_TEMPLATES: + // Determine the field type based on the token + XContentFieldType fieldType = getFieldType(token); + + Mapper.Builder builder = findTemplateBuilder( + context, + currentFieldName, + fieldType, + dynamic, + mapper.fullPath() + ); + + if (builder == null) { + if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { + if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) { + parser.skipChildren(); + } + break; + } + // Create a default field mapper based on the value type + builder = createBuilderFromDynamicValue( + context, + token, + currentFieldName, + dynamic, + mapper.fullPath() + ); + } + + if (builder != null) { + Mapper.BuilderContext builderContext = new Mapper.BuilderContext( + context.indexSettings().getSettings(), + context.path() + ); + Mapper dynamicMapper = builder.build(builderContext); + context.addDynamicMapper(dynamicMapper); + + if (dynamicMapper instanceof FieldMapper fm) { + fm.parse(context); + parseCopyFields(context, fm.copyTo().copyToFields()); + } else if (dynamicMapper instanceof ObjectMapper om) { + parseObjectOrNested(context, om); + } + } + break; + case FALSE: + // Dynamic mapping is disabled, skip the field + if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) { + parser.skipChildren(); + } + break; + } + } + } else if (token == null) { + throw new MapperParsingException( + "object mapping for [" + + mapper.name() + + "] tried to parse field [" + + currentFieldName + + "] as object, but got EOF, has a concrete value been provided to it?" + ); + } + token = parser.nextToken(); + } + } finally { + context.decrementFieldCurrentDepth(); + } + } + + /** + * Determines the XContentFieldType based on the parser token. + * Used for dynamic field mapping when preserve_dots is enabled. + */ + private static XContentFieldType getFieldType(XContentParser.Token token) { + if (token == XContentParser.Token.START_OBJECT) { + return XContentFieldType.OBJECT; + } else if (token == XContentParser.Token.START_ARRAY) { + return XContentFieldType.OBJECT; // Arrays are treated as objects for template matching + } else if (token == XContentParser.Token.VALUE_STRING) { + return XContentFieldType.STRING; + } else if (token == XContentParser.Token.VALUE_NUMBER) { + return XContentFieldType.LONG; // Default to LONG, will be refined by createBuilderFromDynamicValue + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + return XContentFieldType.BOOLEAN; + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + return XContentFieldType.BINARY; + } else { + return XContentFieldType.STRING; // Default fallback + } + } + private static void parseObject(final ParseContext context, ObjectMapper mapper, String currentFieldName, String[] paths) throws IOException { assert currentFieldName != null; diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java index 5460ee8f2b326..318e2de0076d4 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java @@ -2600,4 +2600,102 @@ public void testGenerateGroupingCriteriaFromScript() throws Exception { assertEquals("300", doc.docs().getFirst().getGroupingCriteria()); } + public void testPreserveDotsWithFlatFields() throws Exception { + // Test that when preserve_dots=true, dotted field names are treated as literal field names + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("preserve_dots", true); + b.startObject("properties"); + { + b.startObject("metrics.cpu.usage"); + { + b.field("type", "float"); + } + b.endObject(); + b.startObject("metrics.memory.used"); + { + b.field("type", "long"); + } + b.endObject(); + } + b.endObject(); + })); + + ParsedDocument doc = mapper.parse(source(b -> { + b.field("metrics.cpu.usage", 75.5); + b.field("metrics.memory.used", 8589934592L); + })); + + // Verify that fields are stored with their complete dotted names + IndexableField cpuField = doc.rootDoc().getField("metrics.cpu.usage"); + assertNotNull("Field 'metrics.cpu.usage' should exist", cpuField); + assertEquals(75.5f, cpuField.numericValue().floatValue(), 0.001f); + + IndexableField memoryField = doc.rootDoc().getField("metrics.memory.used"); + assertNotNull("Field 'metrics.memory.used' should exist", memoryField); + assertEquals(8589934592L, memoryField.numericValue().longValue()); + + // Verify that no nested object structure was created + assertNull("No 'metrics' object should exist", doc.rootDoc().getField("metrics")); + } + + public void testPreserveDotsWithDynamicFields() throws Exception { + // Test that dynamic fields work correctly with preserve_dots=true + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("preserve_dots", true); + b.field("dynamic", "true"); + })); + + ParsedDocument doc = mapper.parse(source(b -> { + b.field("user.name", "John Doe"); + b.field("user.age", 30); + b.field("metrics.response.time", 150); + })); + + // Verify that dynamic fields are created with dotted names + IndexableField nameField = doc.rootDoc().getField("user.name"); + assertNotNull("Dynamic field 'user.name' should exist", nameField); + + IndexableField ageField = doc.rootDoc().getField("user.age"); + assertNotNull("Dynamic field 'user.age' should exist", ageField); + + IndexableField responseTimeField = doc.rootDoc().getField("metrics.response.time"); + assertNotNull("Dynamic field 'metrics.response.time' should exist", responseTimeField); + } + + public void testPreserveDotsDefaultBehavior() throws Exception { + // Test that when preserve_dots is not set (default=false), normal nested object behavior is preserved + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.startObject("properties"); + { + b.startObject("user"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("name"); + { + b.field("type", "text"); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + + ParsedDocument doc = mapper.parse(source(b -> { + b.startObject("user"); + { + b.field("name", "Jane Doe"); + } + b.endObject(); + })); + + // Verify that nested object structure is created (default behavior) + IndexableField nameField = doc.rootDoc().getField("user.name"); + assertNotNull("Field 'user.name' should exist in nested structure", nameField); + } + } From bd8e0c63608b28e44b50ca337a9cf34c9ce072f5 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Wed, 12 Nov 2025 23:14:16 +0530 Subject: [PATCH 04/10] Implement validation in MapperService Signed-off-by: Mohit Kumar --- .../opensearch/index/mapper/ObjectMapper.java | 80 ++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index 582cc66b447b0..79167e76f9dbe 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -787,6 +787,9 @@ public ObjectMapper merge(Mapper mergeWith) { @Override public void validate(MappingLookup mappers) { + // Validate flat field compatibility when preserve_dots is enabled + validateFlatFieldCompatibility(); + for (Mapper mapper : this.mappers.values()) { mapper.validate(mappers); } @@ -815,8 +818,14 @@ protected void doMerge(final ObjectMapper mergeWith, MergeReason reason) { if (mergeWith.preserveDotsExplicit().explicit()) { this.preserveDots = mergeWith.preserveDotsExplicit(); } - } else if (isEnabled() != mergeWith.isEnabled()) { - throw new MapperException("the [enabled] parameter can't be updated for the object mapping [" + name() + "]"); + } else { + if (isEnabled() != mergeWith.isEnabled()) { + throw new MapperException("the [enabled] parameter can't be updated for the object mapping [" + name() + "]"); + } + // Validate preserve_dots immutability (except for MAPPING_RECOVERY) + if (reason != MergeReason.MAPPING_RECOVERY) { + validatePreserveDotsImmutability(mergeWith, reason); + } } for (Mapper mergeWithMapper : mergeWith) { @@ -824,6 +833,10 @@ protected void doMerge(final ObjectMapper mergeWith, MergeReason reason) { Mapper merged; if (mergeIntoMapper == null) { + // Validate that we're not adding nested objects when preserve_dots is enabled + if (reason != MergeReason.INDEX_TEMPLATE && reason != MergeReason.MAPPING_RECOVERY) { + validateNestedObjectRejection(mergeWithMapper); + } merged = mergeWithMapper; } else if (mergeIntoMapper instanceof ObjectMapper objectMapper) { merged = objectMapper.merge(mergeWithMapper, reason); @@ -847,6 +860,69 @@ protected void doMerge(final ObjectMapper mergeWith, MergeReason reason) { } } + /** + * Validates that preserve_dots parameter is not being modified after index creation. + * + * @param mergeWith The ObjectMapper being merged + * @param reason The reason for the merge + * @throws MapperException if preserve_dots is being changed + */ + private void validatePreserveDotsImmutability(ObjectMapper mergeWith, MergeReason reason) { + if (this.preserveDotsExplicit().explicit() && mergeWith.preserveDotsExplicit().explicit()) { + if (this.preserveDots() != mergeWith.preserveDots()) { + throw new MapperException( + "Cannot update parameter [preserve_dots] from [" + + this.preserveDots() + + "] to [" + + mergeWith.preserveDots() + + "] for object mapping [" + + name() + + "]" + ); + } + } + } + + /** + * Validates that nested objects are not being added when preserve_dots is enabled. + * + * @param newMapper The new mapper being added + * @throws MapperException if attempting to add nested object when preserve_dots=true + */ + private void validateNestedObjectRejection(Mapper newMapper) { + if (this.preserveDots() && newMapper instanceof ObjectMapper) { + throw new MapperException( + "Cannot add nested object field [" + + newMapper.name() + + "] when preserve_dots is enabled for [" + + name() + + "]. Use flat field notation instead." + ); + } + } + + /** + * Validates that no nested object definitions exist when preserve_dots is enabled. + * This is called during index creation to ensure compatibility. + * + * @throws MapperParsingException if nested objects are found when preserve_dots=true + */ + private void validateFlatFieldCompatibility() { + if (this.preserveDots()) { + for (Mapper childMapper : this.mappers.values()) { + if (childMapper instanceof ObjectMapper) { + throw new MapperParsingException( + "Cannot define nested object [" + + childMapper.name() + + "] when preserve_dots is enabled for parent [" + + name() + + "]. All fields must be flat when preserve_dots=true." + ); + } + } + } + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { toXContent(builder, params, null); From ecf3e8bb308d32edfbf8e190994672e977a1ae7f Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Thu, 13 Nov 2025 18:15:27 +0530 Subject: [PATCH 05/10] Add index template support for preserve_dots tests Signed-off-by: Mohit Kumar --- .../template/SimpleIndexTemplateIT.java | 177 ++++++++++++++++++ .../MetadataCreateIndexServiceTests.java | 158 ++++++++++++++++ 2 files changed, 335 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java index 14be51e977745..0b65b4fcf5964 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java @@ -1078,4 +1078,181 @@ public void testAwarenessReplicaBalance() throws IOException { } } + public void testIndexTemplateWithPreserveDots() throws Exception { + // Create an index template with preserve_dots enabled + client().admin() + .indices() + .preparePutTemplate("metrics_template") + .setPatterns(Collections.singletonList("metrics-*")) + .setMapping( + XContentFactory.jsonBuilder() + .startObject() + .field("preserve_dots", true) + .startObject("properties") + .startObject("cpu.usage") + .field("type", "float") + .endObject() + .startObject("memory.used") + .field("type", "long") + .endObject() + .endObject() + .endObject() + ) + .get(); + + // Create an index that matches the template pattern + assertAcked(prepareCreate("metrics-001")); + + // Verify that preserve_dots setting was applied from the template + ClusterState state = client().admin().cluster().prepareState().get().getState(); + IndexMetadata indexMetadata = state.metadata().index("metrics-001"); + assertNotNull(indexMetadata); + + // Get the mapping and verify preserve_dots is set + String mapping = indexMetadata.mapping().source().toString(); + assertTrue("Mapping should contain preserve_dots", mapping.contains("preserve_dots")); + assertTrue("preserve_dots should be set to true", mapping.contains("\"preserve_dots\":true")); + + // Index a document with dotted field names + client().prepareIndex("metrics-001") + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .field("cpu.usage", 75.5) + .field("memory.used", 8589934592L) + .endObject() + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + + // Query using the complete dotted field name + SearchResponse searchResponse = client().prepareSearch("metrics-001") + .setQuery(QueryBuilders.rangeQuery("cpu.usage").gte(70.0)) + .get(); + + assertHitCount(searchResponse, 1); + } + + public void testIndexTemplateWithPreserveDotsAtNestedLevel() throws Exception { + // Create an index template with preserve_dots enabled on a nested object + client().admin() + .indices() + .preparePutTemplate("mixed_template") + .setPatterns(Collections.singletonList("mixed-*")) + .setMapping( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("user") + .field("type", "object") + .startObject("properties") + .startObject("name") + .field("type", "text") + .endObject() + .endObject() + .endObject() + .startObject("metrics") + .field("type", "object") + .field("preserve_dots", true) + .startObject("properties") + .startObject("cpu.usage") + .field("type", "float") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ) + .get(); + + // Create an index that matches the template pattern + assertAcked(prepareCreate("mixed-001")); + + // Verify that preserve_dots setting was applied from the template + ClusterState state = client().admin().cluster().prepareState().get().getState(); + IndexMetadata indexMetadata = state.metadata().index("mixed-001"); + assertNotNull(indexMetadata); + + // Get the mapping and verify preserve_dots is set on the metrics object + String mapping = indexMetadata.mapping().source().toString(); + assertTrue("Mapping should contain preserve_dots", mapping.contains("preserve_dots")); + + // Index a document with both nested and flat dotted fields + client().prepareIndex("mixed-001") + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .startObject("user") + .field("name", "John Doe") + .endObject() + .startObject("metrics") + .field("cpu.usage", 85.0) + .endObject() + .endObject() + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + + // Query using both field types + SearchResponse searchResponse = client().prepareSearch("mixed-001") + .setQuery(QueryBuilders.boolQuery() + .must(QueryBuilders.matchQuery("user.name", "John")) + .must(QueryBuilders.rangeQuery("metrics.cpu.usage").gte(80.0))) + .get(); + + assertHitCount(searchResponse, 1); + } + + public void testMultipleTemplatesWithPreserveDots() throws Exception { + // Create first template with lower order + client().admin() + .indices() + .preparePutTemplate("template_1") + .setPatterns(Collections.singletonList("test-*")) + .setOrder(0) + .setMapping( + XContentFactory.jsonBuilder() + .startObject() + .field("preserve_dots", false) + .startObject("properties") + .startObject("field1") + .field("type", "text") + .endObject() + .endObject() + .endObject() + ) + .get(); + + // Create second template with higher order and preserve_dots enabled + client().admin() + .indices() + .preparePutTemplate("template_2") + .setPatterns(Collections.singletonList("test-*")) + .setOrder(1) + .setMapping( + XContentFactory.jsonBuilder() + .startObject() + .field("preserve_dots", true) + .startObject("properties") + .startObject("field.dotted") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + ) + .get(); + + // Create an index that matches both templates + assertAcked(prepareCreate("test-001")); + + // Verify that preserve_dots from the higher order template was applied + ClusterState state = client().admin().cluster().prepareState().get().getState(); + IndexMetadata indexMetadata = state.metadata().index("test-001"); + assertNotNull(indexMetadata); + + String mapping = indexMetadata.mapping().source().toString(); + assertTrue("Mapping should contain preserve_dots", mapping.contains("preserve_dots")); + assertTrue("preserve_dots should be set to true from higher order template", mapping.contains("\"preserve_dots\":true")); + } + } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 9526491a1b6dd..9ce4578752d77 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -3010,6 +3010,164 @@ private void verifyRemoteStoreIndexSettings( assertEquals(translogBufferInterval, indexSettings.get(INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey())); } + public void testTemplateWithPreserveDots() throws Exception { + // Create a template with preserve_dots enabled + IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> { + try { + builder.putMapping( + "type", + XContentFactory.jsonBuilder() + .startObject() + .field("preserve_dots", true) + .startObject("properties") + .startObject("cpu.usage") + .field("type", "float") + .endObject() + .startObject("memory.used") + .field("type", "long") + .endObject() + .endObject() + .endObject() + .toString() + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + // Parse mappings with the template + Map parsedMappings = MetadataCreateIndexService.parseV1Mappings( + "", + Collections.singletonList(templateMetadata.getMappings()), + NamedXContentRegistry.EMPTY + ); + + // Verify preserve_dots is present in the parsed mappings + assertThat(parsedMappings, hasKey(MapperService.SINGLE_MAPPING_NAME)); + Map doc = (Map) parsedMappings.get(MapperService.SINGLE_MAPPING_NAME); + assertThat(doc, hasKey("preserve_dots")); + assertEquals(true, doc.get("preserve_dots")); + + // Verify the dotted field properties are present + assertThat(doc, hasKey("properties")); + Map properties = (Map) doc.get("properties"); + assertThat(properties, hasKey("cpu.usage")); + assertThat(properties, hasKey("memory.used")); + } + + public void testTemplateWithPreserveDotsAtNestedLevel() throws Exception { + // Create a template with preserve_dots enabled on a nested object + IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> { + try { + builder.putMapping( + "type", + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("user") + .field("type", "object") + .startObject("properties") + .startObject("name") + .field("type", "text") + .endObject() + .endObject() + .endObject() + .startObject("metrics") + .field("type", "object") + .field("preserve_dots", true) + .startObject("properties") + .startObject("cpu.usage") + .field("type", "float") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString() + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + // Parse mappings with the template + Map parsedMappings = MetadataCreateIndexService.parseV1Mappings( + "", + Collections.singletonList(templateMetadata.getMappings()), + NamedXContentRegistry.EMPTY + ); + + // Verify the structure + assertThat(parsedMappings, hasKey(MapperService.SINGLE_MAPPING_NAME)); + Map doc = (Map) parsedMappings.get(MapperService.SINGLE_MAPPING_NAME); + assertThat(doc, hasKey("properties")); + Map properties = (Map) doc.get("properties"); + + // Verify user object (without preserve_dots) + assertThat(properties, hasKey("user")); + Map userObject = (Map) properties.get("user"); + assertThat(userObject, hasKey("properties")); + + // Verify metrics object (with preserve_dots) + assertThat(properties, hasKey("metrics")); + Map metricsObject = (Map) properties.get("metrics"); + assertThat(metricsObject, hasKey("preserve_dots")); + assertEquals(true, metricsObject.get("preserve_dots")); + assertThat(metricsObject, hasKey("properties")); + Map metricsProperties = (Map) metricsObject.get("properties"); + assertThat(metricsProperties, hasKey("cpu.usage")); + } + + public void testMultipleTemplatesWithPreserveDots() throws Exception { + // Create first template with preserve_dots=false + CompressedXContent template1Mapping = new CompressedXContent( + XContentFactory.jsonBuilder() + .startObject() + .field("preserve_dots", false) + .startObject("properties") + .startObject("field1") + .field("type", "text") + .endObject() + .endObject() + .endObject() + .toString() + ); + + // Create second template with preserve_dots=true + CompressedXContent template2Mapping = new CompressedXContent( + XContentFactory.jsonBuilder() + .startObject() + .field("preserve_dots", true) + .startObject("properties") + .startObject("field.dotted") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .toString() + ); + + // Parse mappings with both templates (template2 should override template1) + List templateMappings = Arrays.asList(template1Mapping, template2Mapping); + Map parsedMappings = MetadataCreateIndexService.parseV1Mappings( + "", + templateMappings, + NamedXContentRegistry.EMPTY + ); + + // Verify preserve_dots from the last template is applied + assertThat(parsedMappings, hasKey(MapperService.SINGLE_MAPPING_NAME)); + Map doc = (Map) parsedMappings.get(MapperService.SINGLE_MAPPING_NAME); + assertThat(doc, hasKey("preserve_dots")); + assertEquals(true, doc.get("preserve_dots")); + + // Verify both field1 and field.dotted are present + assertThat(doc, hasKey("properties")); + Map properties = (Map) doc.get("properties"); + assertThat(properties, hasKey("field1")); + assertThat(properties, hasKey("field.dotted")); + } + private DiscoveryNode getRemoteNode() { Map attributes = new HashMap<>(); attributes.put(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-cluster-rep-1"); From e505b3e359cfd5905a78833474cf2cef0f8806a7 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Fri, 14 Nov 2025 12:17:11 +0530 Subject: [PATCH 06/10] error handling with descriptive messages Signed-off-by: Mohit Kumar --- .../index/mapper/DocumentParser.java | 24 ++++++++------ .../opensearch/index/mapper/ObjectMapper.java | 33 ++++++++++++++----- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index 4634d62a5f36e..91a0a33a277eb 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -594,11 +594,12 @@ private static void parseFieldWithDots(ParseContext context, ObjectMapper mapper String currentFieldName = parser.currentName(); if (token.isValue()) { throw new MapperParsingException( - "object mapping for [" - + mapper.name() - + "] tried to parse field [" + "Document structure incompatibility: Field [" + currentFieldName - + "] as object, but found a concrete value" + + "] in object [" + + mapper.name() + + "] is defined as an object with preserve_dots=true, but received a concrete value. " + + "Expected format: an object containing flat dotted field names (e.g., {\"field.name\": \"value\"})." ); } @@ -637,8 +638,11 @@ private static void parseFieldWithDots(ParseContext context, ObjectMapper mapper parseObjectOrNested(context, om); } else { throw new MapperParsingException( - "Cannot parse field [" + currentFieldName + "] with mapper type [" - + fieldMapper.getClass().getSimpleName() + "]" + "Document structure incompatibility: Cannot parse field [" + + currentFieldName + + "] with mapper type [" + + fieldMapper.getClass().getSimpleName() + + "]. Expected a field mapper or object mapper." ); } } else { @@ -710,11 +714,11 @@ private static void parseFieldWithDots(ParseContext context, ObjectMapper mapper } } else if (token == null) { throw new MapperParsingException( - "object mapping for [" - + mapper.name() - + "] tried to parse field [" + "Document structure incompatibility: Unexpected end of document while parsing field [" + currentFieldName - + "] as object, but got EOF, has a concrete value been provided to it?" + + "] in object [" + + mapper.name() + + "]. Expected format: an object containing flat dotted field names with their values." ); } token = parser.nextToken(); diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index 79167e76f9dbe..01012a232fafd 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -335,7 +335,17 @@ protected static boolean parseObjectOrDocumentTypeProperties( builder.enabled(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".enabled")); return true; } else if (fieldName.equals("preserve_dots")) { - builder.preserveDots(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".preserve_dots")); + try { + builder.preserveDots(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".preserve_dots")); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Configuration error: Parameter [preserve_dots] must be a boolean value (true or false), but got [" + + fieldNode + + "]. " + + e.getMessage(), + e + ); + } return true; } else if (fieldName.equals("derived")) { if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) { @@ -862,10 +872,11 @@ protected void doMerge(final ObjectMapper mergeWith, MergeReason reason) { /** * Validates that preserve_dots parameter is not being modified after index creation. + * Provides descriptive error message including the field name and attempted change. * * @param mergeWith The ObjectMapper being merged * @param reason The reason for the merge - * @throws MapperException if preserve_dots is being changed + * @throws MapperException if preserve_dots is being changed (Requirement 3.4) */ private void validatePreserveDotsImmutability(ObjectMapper mergeWith, MergeReason reason) { if (this.preserveDotsExplicit().explicit() && mergeWith.preserveDotsExplicit().explicit()) { @@ -877,7 +888,7 @@ private void validatePreserveDotsImmutability(ObjectMapper mergeWith, MergeReaso + mergeWith.preserveDots() + "] for object mapping [" + name() - + "]" + + "]. The preserve_dots setting is immutable after index creation." ); } } @@ -885,18 +896,20 @@ private void validatePreserveDotsImmutability(ObjectMapper mergeWith, MergeReaso /** * Validates that nested objects are not being added when preserve_dots is enabled. + * Provides clear indication of flat vs nested conflict with field name. * * @param newMapper The new mapper being added - * @throws MapperException if attempting to add nested object when preserve_dots=true + * @throws MapperException if attempting to add nested object when preserve_dots=true (Requirement 3.1) */ private void validateNestedObjectRejection(Mapper newMapper) { if (this.preserveDots() && newMapper instanceof ObjectMapper) { throw new MapperException( - "Cannot add nested object field [" + "Field mapping conflict: Cannot add nested object field [" + newMapper.name() + "] when preserve_dots is enabled for [" + name() - + "]. Use flat field notation instead." + + "]. With preserve_dots=true, all fields must use flat field notation (e.g., 'field.name' as a single field) " + + "rather than nested object structures." ); } } @@ -904,19 +917,21 @@ private void validateNestedObjectRejection(Mapper newMapper) { /** * Validates that no nested object definitions exist when preserve_dots is enabled. * This is called during index creation to ensure compatibility. + * Provides clear indication of flat vs nested conflict with field name. * - * @throws MapperParsingException if nested objects are found when preserve_dots=true + * @throws MapperParsingException if nested objects are found when preserve_dots=true (Requirement 3.1) */ private void validateFlatFieldCompatibility() { if (this.preserveDots()) { for (Mapper childMapper : this.mappers.values()) { if (childMapper instanceof ObjectMapper) { throw new MapperParsingException( - "Cannot define nested object [" + "Field mapping conflict: Cannot define nested object [" + childMapper.name() + "] when preserve_dots is enabled for parent [" + name() - + "]. All fields must be flat when preserve_dots=true." + + "]. With preserve_dots=true, all fields must use flat field notation (e.g., 'field.name' as a single field) " + + "rather than nested object structures." ); } } From 3c56a978d95a8edaf1d13952704a85235e3390fc Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Fri, 14 Nov 2025 15:35:23 +0530 Subject: [PATCH 07/10] Removing redundancy Signed-off-by: Mohit Kumar --- .../main/java/org/opensearch/index/mapper/ObjectMapper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index 01012a232fafd..7c409a30b73bd 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -876,7 +876,7 @@ protected void doMerge(final ObjectMapper mergeWith, MergeReason reason) { * * @param mergeWith The ObjectMapper being merged * @param reason The reason for the merge - * @throws MapperException if preserve_dots is being changed (Requirement 3.4) + * @throws MapperException if preserve_dots is being changed */ private void validatePreserveDotsImmutability(ObjectMapper mergeWith, MergeReason reason) { if (this.preserveDotsExplicit().explicit() && mergeWith.preserveDotsExplicit().explicit()) { @@ -899,7 +899,7 @@ private void validatePreserveDotsImmutability(ObjectMapper mergeWith, MergeReaso * Provides clear indication of flat vs nested conflict with field name. * * @param newMapper The new mapper being added - * @throws MapperException if attempting to add nested object when preserve_dots=true (Requirement 3.1) + * @throws MapperException if attempting to add nested object when preserve_dots=true */ private void validateNestedObjectRejection(Mapper newMapper) { if (this.preserveDots() && newMapper instanceof ObjectMapper) { @@ -919,7 +919,7 @@ private void validateNestedObjectRejection(Mapper newMapper) { * This is called during index creation to ensure compatibility. * Provides clear indication of flat vs nested conflict with field name. * - * @throws MapperParsingException if nested objects are found when preserve_dots=true (Requirement 3.1) + * @throws MapperParsingException if nested objects are found when preserve_dots=true */ private void validateFlatFieldCompatibility() { if (this.preserveDots()) { From 96af23bf20c5824cee854207ff9ae4d92c8e07d3 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Sat, 29 Nov 2025 22:34:46 +0530 Subject: [PATCH 08/10] Adding more edge cases, autoflatten and test cases Signed-off-by: Mohit Kumar --- .../metadata/MetadataCreateIndexService.java | 16 ++ .../index/mapper/DocumentParser.java | 152 ++++++++++- .../opensearch/index/mapper/ObjectMapper.java | 140 ++++++----- .../index/mapper/RootObjectMapper.java | 17 +- .../MetadataCreateIndexServiceTests.java | 64 ++--- .../index/mapper/DocumentParserTests.java | 237 +++++++++++++++--- .../FieldAliasMapperValidationTests.java | 7 +- .../index/mapper/RootObjectMapperTests.java | 30 --- 8 files changed, 492 insertions(+), 171 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 02659c7e0e706..ae20efa6a3b26 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -957,6 +957,22 @@ static Map parseV1Mappings( mappings = templateMapping; } else { XContentHelper.mergeDefaults(mappings, templateMapping); + // Special handling for disable_objects: later templates should override earlier ones. + // mergeDefaults only adds missing keys, so we need to explicitly override disable_objects + // to ensure the last template's value wins (consistent with template priority ordering). + Object templateDocObj = templateMapping.get(MapperService.SINGLE_MAPPING_NAME); + if (templateDocObj instanceof Map) { + @SuppressWarnings("unchecked") + Map templateDoc = (Map) templateDocObj; + if (templateDoc.containsKey("disable_objects")) { + Object mappingsDocObj = mappings.get(MapperService.SINGLE_MAPPING_NAME); + if (mappingsDocObj instanceof Map) { + @SuppressWarnings("unchecked") + Map mappingsDoc = (Map) mappingsDocObj; + mappingsDoc.put("disable_objects", templateDoc.get("disable_objects")); + } + } + } } } } diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index 91a0a33a277eb..f785f65144ce3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -232,6 +232,50 @@ private static String[] splitAndValidatePath(String fullFieldPath) { } } + /** + * Split a field path, respecting disable_objects settings on parent mappers. + * If a parent mapper has disable_objects=true, the path is not split beyond that parent. + */ + private static String[] splitPathRespectingDisableObjects(String fieldPath, DocumentMapper docMapper) { + // First check if the root mapper has disable_objects=true + // If so, treat the entire field path as a single flat field + if (docMapper.mapping().root().disableObjects()) { + return new String[] { fieldPath }; + } + + String[] allParts = splitAndValidatePath(fieldPath); + + // Check each level to see if we encounter a mapper with disable_objects=true + StringBuilder currentPath = new StringBuilder(); + for (int i = 0; i < allParts.length - 1; i++) { + if (i > 0) { + currentPath.append("."); + } + currentPath.append(allParts[i]); + + ObjectMapper parentMapper = docMapper.objectMappers().get(currentPath.toString()); + if (parentMapper != null && parentMapper.disableObjects()) { + // This parent has disable_objects=true, so don't split beyond this point + // Return: [parts up to parent, remaining path as single element] + String[] result = new String[i + 2]; + System.arraycopy(allParts, 0, result, 0, i + 1); + // Join the remaining parts back together + StringBuilder remaining = new StringBuilder(); + for (int j = i + 1; j < allParts.length; j++) { + if (j > i + 1) { + remaining.append("."); + } + remaining.append(allParts[j]); + } + result[i + 1] = remaining.toString(); + return result; + } + } + + // No parent with disable_objects=true found, return the fully split path + return allParts; + } + /** Creates a Mapping containing any dynamically added fields, or returns null if there were no dynamic mappings. */ static Mapping createDynamicUpdate(Mapping mapping, DocumentMapper docMapper, List dynamicMappers) { if (dynamicMappers.isEmpty()) { @@ -244,7 +288,7 @@ static Mapping createDynamicUpdate(Mapping mapping, DocumentMapper docMapper, Li Iterator dynamicMapperItr = dynamicMappers.iterator(); List parentMappers = new ArrayList<>(); Mapper firstUpdate = dynamicMapperItr.next(); - parentMappers.add(createUpdate(mapping.root(), splitAndValidatePath(firstUpdate.name()), 0, firstUpdate)); + parentMappers.add(createUpdate(mapping.root(), splitPathRespectingDisableObjects(firstUpdate.name(), docMapper), 0, firstUpdate)); Mapper previousMapper = null; while (dynamicMapperItr.hasNext()) { Mapper newMapper = dynamicMapperItr.next(); @@ -256,7 +300,7 @@ static Mapping createDynamicUpdate(Mapping mapping, DocumentMapper docMapper, Li continue; } previousMapper = newMapper; - String[] nameParts = splitAndValidatePath(newMapper.name()); + String[] nameParts = splitPathRespectingDisableObjects(newMapper.name(), docMapper); // We first need the stack to only contain mappers in common with the previously processed mapper // For example, if the first mapper processed was a.b.c, and we now have a.d, the stack will contain @@ -442,13 +486,23 @@ private static void innerParseObject( while (token != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); - paths = splitAndValidatePath(currentFieldName); + // When disable_objects is enabled, treat dotted field names as literal field names + if (mapper.disableObjects()) { + paths = new String[] { currentFieldName }; + } else { + paths = splitAndValidatePath(currentFieldName); + } if (containsDisabledObjectMapper(mapper, paths)) { parser.nextToken(); parser.skipChildren(); } } else if (token == XContentParser.Token.START_OBJECT) { - parseObject(context, mapper, currentFieldName, paths); + // When disable_objects is enabled, flatten nested objects to dotted notation + if (mapper.disableObjects()) { + flattenObject(context, mapper, currentFieldName); + } else { + parseObject(context, mapper, currentFieldName, paths); + } } else if (token == XContentParser.Token.START_ARRAY) { parseArray(context, mapper, currentFieldName, paths); } else if (token == XContentParser.Token.VALUE_NULL) { @@ -494,6 +548,80 @@ private static void generateGroupingCriteria(ParseContext context) { } } + /** + * Flattens a nested object structure to dotted field notation when disable_objects is enabled. + * For example, {"foo": {"bar": 6}} becomes "foo.bar": 6 + * + * @param context the parse context + * @param mapper the object mapper with disable_objects enabled + * @param prefix the current field name prefix + * @throws IOException if parsing fails + */ + private static void flattenObject(ParseContext context, ObjectMapper mapper, String prefix) throws IOException { + XContentParser parser = context.parser(); + XContentParser.Token token = parser.nextToken(); // Move to first field or END_OBJECT + + while (token != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + String fieldName = parser.currentName(); + String flattenedFieldName = prefix + "." + fieldName; + token = parser.nextToken(); // Move to the value + + if (token == XContentParser.Token.START_OBJECT) { + // Recursively flatten nested objects + flattenObject(context, mapper, flattenedFieldName); + } else if (token == XContentParser.Token.START_ARRAY) { + // Handle arrays by flattening each element + flattenArray(context, mapper, flattenedFieldName); + } else if (token == XContentParser.Token.VALUE_NULL) { + // Handle null values + String[] paths = new String[] { flattenedFieldName }; + parseNullValue(context, mapper, flattenedFieldName, paths); + } else if (token.isValue()) { + // Handle primitive values + String[] paths = new String[] { flattenedFieldName }; + parseValue(context, mapper, flattenedFieldName, token, paths); + } + } else { + throw new MapperParsingException( + "Unexpected token [" + token + "] while flattening object for field [" + prefix + "]" + ); + } + token = parser.nextToken(); + } + } + + /** + * Flattens an array when disable_objects is enabled. + * Each array element is processed and flattened if it's an object. + * + * @param context the parse context + * @param mapper the object mapper with disable_objects enabled + * @param fieldName the flattened field name + * @throws IOException if parsing fails + */ + private static void flattenArray(ParseContext context, ObjectMapper mapper, String fieldName) throws IOException { + XContentParser parser = context.parser(); + XContentParser.Token token = parser.nextToken(); // Move to first array element or END_ARRAY + + String[] paths = new String[] { fieldName }; + + while (token != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.START_OBJECT) { + // Flatten nested objects within the array + flattenObject(context, mapper, fieldName); + } else if (token == XContentParser.Token.START_ARRAY) { + // Nested arrays - recursively flatten + flattenArray(context, mapper, fieldName); + } else if (token == XContentParser.Token.VALUE_NULL) { + parseNullValue(context, mapper, fieldName, paths); + } else if (token.isValue()) { + parseValue(context, mapper, fieldName, token, paths); + } + token = parser.nextToken(); + } + } + private static void nested(ParseContext context, ObjectMapper.Nested nested) { ParseContext.Document nestedDoc = context.doc(); ParseContext.Document parentDoc = nestedDoc.getParent(); @@ -548,8 +676,8 @@ private static ParseContext nestedContext(ParseContext context, ObjectMapper map private static void parseObjectOrField(ParseContext context, Mapper mapper) throws IOException { if (mapper instanceof ObjectMapper objectMapper) { - // Check if preserve_dots is enabled for this object mapper - if (objectMapper.preserveDots()) { + // Check if disable_objects is enabled for this object mapper + if (objectMapper.disableObjects()) { // Parse as flat field - do not expand dots into nested objects parseFieldWithDots(context, objectMapper); } else { @@ -569,12 +697,12 @@ private static void parseObjectOrField(ParseContext context, Mapper mapper) thro } /** - * Parses fields with dots as literal field names when preserve_dots is enabled. + * Parses fields with dots as literal field names when disable_objects is enabled. * This method treats dotted field names (e.g., "user.name", "metrics.cpu.usage") as flat fields * rather than expanding them into nested object hierarchies. * * @param context the parse context - * @param mapper the object mapper with preserve_dots enabled + * @param mapper the object mapper with disable_objects enabled * @throws IOException if parsing fails */ private static void parseFieldWithDots(ParseContext context, ObjectMapper mapper) throws IOException { @@ -598,7 +726,7 @@ private static void parseFieldWithDots(ParseContext context, ObjectMapper mapper + currentFieldName + "] in object [" + mapper.name() - + "] is defined as an object with preserve_dots=true, but received a concrete value. " + + "] is defined as an object with disable_objects=true, but received a concrete value. " + "Expected format: an object containing flat dotted field names (e.g., {\"field.name\": \"value\"})." ); } @@ -620,7 +748,7 @@ private static void parseFieldWithDots(ParseContext context, ObjectMapper mapper while (token != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); - // For preserve_dots, treat the entire field name (including dots) as a literal field name + // For disable_objects, treat the entire field name (including dots) as a literal field name // Do not split on dots - this is the key difference from normal object parsing // Look up the mapper using the complete dotted field name @@ -633,7 +761,7 @@ private static void parseFieldWithDots(ParseContext context, ObjectMapper mapper fm.parse(context); parseCopyFields(context, fm.copyTo().copyToFields()); } else if (fieldMapper instanceof ObjectMapper om) { - // Even with preserve_dots, we can have nested ObjectMappers + // Even with disable_objects, we can have nested ObjectMappers // for explicitly defined sub-objects parseObjectOrNested(context, om); } else { @@ -730,7 +858,7 @@ private static void parseFieldWithDots(ParseContext context, ObjectMapper mapper /** * Determines the XContentFieldType based on the parser token. - * Used for dynamic field mapping when preserve_dots is enabled. + * Used for dynamic field mapping when disable_objects is enabled. */ private static XContentFieldType getFieldType(XContentParser.Token token) { if (token == XContentParser.Token.START_OBJECT) { diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index 7c409a30b73bd..1bcfec1e64c6f 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -81,7 +81,7 @@ public static class Defaults { public static final boolean ENABLED = true; public static final Nested NESTED = Nested.NO; public static final Dynamic DYNAMIC = null; // not set, inherited from root - public static final Explicit PRESERVE_DOTS = new Explicit<>(false, false); + public static final Explicit DISABLE_OBJECTS = new Explicit<>(false, false); } /** @@ -203,7 +203,7 @@ public static class Builder extends Mapper.Builder { protected Dynamic dynamic = Defaults.DYNAMIC; - protected Explicit preserveDots = Defaults.PRESERVE_DOTS; + protected Explicit disableObjects = Defaults.DISABLE_OBJECTS; protected final List mappersBuilders = new ArrayList<>(); @@ -227,8 +227,8 @@ public T nested(Nested nested) { return builder; } - public T preserveDots(boolean preserveDots) { - this.preserveDots = new Explicit<>(preserveDots, true); + public T disableObjects(boolean disableObjects) { + this.disableObjects = new Explicit<>(disableObjects, true); return builder; } @@ -258,11 +258,27 @@ public ObjectMapper build(BuilderContext context) { enabled, nested, dynamic, - preserveDots, + disableObjects, mappers, context.indexSettings() ); + // Validate flat field compatibility during build + if (Boolean.TRUE.equals(disableObjects.value())) { + for (Mapper childMapper : mappers.values()) { + if (childMapper instanceof ObjectMapper) { + throw new MapperParsingException( + "Field mapping conflict: Cannot add nested object field [" + + childMapper.name() + + "] when disable_objects is enabled for [" + + name + + "]. With disable_objects=true, all fields must use flat field notation (e.g., 'field.name' as a single field) " + + "rather than nested object structures." + ); + } + } + } + return objectMapper; } @@ -272,11 +288,11 @@ protected ObjectMapper createMapper( Explicit enabled, Nested nested, Dynamic dynamic, - Explicit preserveDots, + Explicit disableObjects, Map mappers, @Nullable Settings settings ) { - return new ObjectMapper(name, fullPath, enabled, nested, dynamic, preserveDots, mappers, settings); + return new ObjectMapper(name, fullPath, enabled, nested, dynamic, disableObjects, mappers, settings); } } @@ -334,12 +350,12 @@ protected static boolean parseObjectOrDocumentTypeProperties( } else if (fieldName.equals("enabled")) { builder.enabled(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".enabled")); return true; - } else if (fieldName.equals("preserve_dots")) { + } else if (fieldName.equals("disable_objects")) { try { - builder.preserveDots(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".preserve_dots")); + builder.disableObjects(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".disable_objects")); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( - "Configuration error: Parameter [preserve_dots] must be a boolean value (true or false), but got [" + "Configuration error: Parameter [disable_objects] must be a boolean value (true or false), but got [" + fieldNode + "]. " + e.getMessage(), @@ -599,19 +615,29 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map fieldBuilder = typeParser.parse(realFieldName, propNode, parserContext); - for (int i = fieldNameParts.length - 2; i >= 0; --i) { - ObjectMapper.Builder intermediate = new ObjectMapper.Builder<>(fieldNameParts[i]); - intermediate.add(fieldBuilder); - fieldBuilder = intermediate; + + // When disable_objects is enabled, treat dotted field names as literal field names + // and don't create intermediate object mappers + if (Boolean.TRUE.equals(objBuilder.disableObjects.value())) { + // Use the full field name as-is without splitting + Mapper.Builder fieldBuilder = typeParser.parse(fieldName, propNode, parserContext); + objBuilder.add(fieldBuilder); + } else { + // Standard behavior: split dotted names and create intermediate object mappers + String[] fieldNameParts = fieldName.split("\\."); + // field name is just ".", which is invalid + if (fieldNameParts.length < 1) { + throw new MapperParsingException("Invalid field name " + fieldName); + } + String realFieldName = fieldNameParts[fieldNameParts.length - 1]; + Mapper.Builder fieldBuilder = typeParser.parse(realFieldName, propNode, parserContext); + for (int i = fieldNameParts.length - 2; i >= 0; --i) { + ObjectMapper.Builder intermediate = new ObjectMapper.Builder<>(fieldNameParts[i]); + intermediate.add(fieldBuilder); + fieldBuilder = intermediate; + } + objBuilder.add(fieldBuilder); } - objBuilder.add(fieldBuilder); propNode.remove("type"); DocumentMapperParser.checkNoRemainingFields(fieldName, propNode, parserContext.indexVersionCreated()); iterator.remove(); @@ -646,7 +672,7 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map preserveDots; + private Explicit disableObjects; private volatile CopyOnWriteHashMap mappers; @@ -656,7 +682,7 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map enabled, Nested nested, Dynamic dynamic, - Explicit preserveDots, + Explicit disableObjects, Map mappers, Settings settings ) { @@ -669,7 +695,7 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map(); } else { @@ -753,12 +779,12 @@ public final Dynamic dynamic() { return dynamic; } - public boolean preserveDots() { - return preserveDots.value(); + public boolean disableObjects() { + return disableObjects.value(); } - public Explicit preserveDotsExplicit() { - return preserveDots; + public Explicit disableObjectsExplicit() { + return disableObjects; } /** @@ -797,7 +823,7 @@ public ObjectMapper merge(Mapper mergeWith) { @Override public void validate(MappingLookup mappers) { - // Validate flat field compatibility when preserve_dots is enabled + // Validate flat field compatibility when disable_objects is enabled validateFlatFieldCompatibility(); for (Mapper mapper : this.mappers.values()) { @@ -825,16 +851,16 @@ protected void doMerge(final ObjectMapper mergeWith, MergeReason reason) { if (mergeWith.enabled.explicit()) { this.enabled = mergeWith.enabled; } - if (mergeWith.preserveDotsExplicit().explicit()) { - this.preserveDots = mergeWith.preserveDotsExplicit(); + if (mergeWith.disableObjectsExplicit().explicit()) { + this.disableObjects = mergeWith.disableObjectsExplicit(); } } else { if (isEnabled() != mergeWith.isEnabled()) { throw new MapperException("the [enabled] parameter can't be updated for the object mapping [" + name() + "]"); } - // Validate preserve_dots immutability (except for MAPPING_RECOVERY) + // Validate disable_objects immutability (except for MAPPING_RECOVERY) if (reason != MergeReason.MAPPING_RECOVERY) { - validatePreserveDotsImmutability(mergeWith, reason); + validateDisableObjectsImmutability(mergeWith, reason); } } @@ -843,7 +869,7 @@ protected void doMerge(final ObjectMapper mergeWith, MergeReason reason) { Mapper merged; if (mergeIntoMapper == null) { - // Validate that we're not adding nested objects when preserve_dots is enabled + // Validate that we're not adding nested objects when disable_objects is enabled if (reason != MergeReason.INDEX_TEMPLATE && reason != MergeReason.MAPPING_RECOVERY) { validateNestedObjectRejection(mergeWithMapper); } @@ -871,66 +897,66 @@ protected void doMerge(final ObjectMapper mergeWith, MergeReason reason) { } /** - * Validates that preserve_dots parameter is not being modified after index creation. + * Validates that disable_objects parameter is not being modified after index creation. * Provides descriptive error message including the field name and attempted change. * * @param mergeWith The ObjectMapper being merged * @param reason The reason for the merge - * @throws MapperException if preserve_dots is being changed + * @throws MapperException if disable_objects is being changed */ - private void validatePreserveDotsImmutability(ObjectMapper mergeWith, MergeReason reason) { - if (this.preserveDotsExplicit().explicit() && mergeWith.preserveDotsExplicit().explicit()) { - if (this.preserveDots() != mergeWith.preserveDots()) { + private void validateDisableObjectsImmutability(ObjectMapper mergeWith, MergeReason reason) { + if (this.disableObjectsExplicit().explicit() && mergeWith.disableObjectsExplicit().explicit()) { + if (this.disableObjects() != mergeWith.disableObjects()) { throw new MapperException( - "Cannot update parameter [preserve_dots] from [" - + this.preserveDots() + "Cannot update parameter [disable_objects] from [" + + this.disableObjects() + "] to [" - + mergeWith.preserveDots() + + mergeWith.disableObjects() + "] for object mapping [" + name() - + "]. The preserve_dots setting is immutable after index creation." + + "]. The disable_objects setting is immutable after index creation." ); } } } /** - * Validates that nested objects are not being added when preserve_dots is enabled. + * Validates that nested objects are not being added when disable_objects is enabled. * Provides clear indication of flat vs nested conflict with field name. * * @param newMapper The new mapper being added - * @throws MapperException if attempting to add nested object when preserve_dots=true + * @throws MapperException if attempting to add nested object when disable_objects=true */ private void validateNestedObjectRejection(Mapper newMapper) { - if (this.preserveDots() && newMapper instanceof ObjectMapper) { + if (this.disableObjects() && newMapper instanceof ObjectMapper) { throw new MapperException( "Field mapping conflict: Cannot add nested object field [" + newMapper.name() - + "] when preserve_dots is enabled for [" + + "] when disable_objects is enabled for [" + name() - + "]. With preserve_dots=true, all fields must use flat field notation (e.g., 'field.name' as a single field) " + + "]. With disable_objects=true, all fields must use flat field notation (e.g., 'field.name' as a single field) " + "rather than nested object structures." ); } } /** - * Validates that no nested object definitions exist when preserve_dots is enabled. + * Validates that no nested object definitions exist when disable_objects is enabled. * This is called during index creation to ensure compatibility. * Provides clear indication of flat vs nested conflict with field name. * - * @throws MapperParsingException if nested objects are found when preserve_dots=true + * @throws MapperParsingException if nested objects are found when disable_objects=true */ private void validateFlatFieldCompatibility() { - if (this.preserveDots()) { + if (this.disableObjects()) { for (Mapper childMapper : this.mappers.values()) { if (childMapper instanceof ObjectMapper) { throw new MapperParsingException( - "Field mapping conflict: Cannot define nested object [" + "Field mapping conflict: Cannot add nested object field [" + childMapper.name() - + "] when preserve_dots is enabled for parent [" + + "] when disable_objects is enabled for [" + name() - + "]. With preserve_dots=true, all fields must use flat field notation (e.g., 'field.name' as a single field) " + + "]. With disable_objects=true, all fields must use flat field notation (e.g., 'field.name' as a single field) " + "rather than nested object structures." ); } @@ -964,8 +990,8 @@ public void toXContent(XContentBuilder builder, Params params, ToXContent custom if (isEnabled() != Defaults.ENABLED) { builder.field("enabled", enabled.value()); } - if (preserveDotsExplicit().explicit()) { - builder.field("preserve_dots", preserveDots.value()); + if (disableObjectsExplicit().explicit()) { + builder.field("disable_objects", disableObjects.value()); } if (custom != null) { diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java index a3303e0b4568d..fb8662af5663d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java @@ -123,7 +123,7 @@ protected ObjectMapper createMapper( Explicit enabled, Nested nested, Dynamic dynamic, - Explicit preserveDots, + Explicit disableObjects, Map mappers, @Nullable Settings settings ) { @@ -132,7 +132,7 @@ protected ObjectMapper createMapper( name, enabled, dynamic, - preserveDots, + disableObjects, mappers, dynamicDateTimeFormatters, dynamicTemplates, @@ -230,10 +230,7 @@ protected static void parseContextAwareGroupingField( } protected boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, ParserContext parserContext) { - if (fieldName.equals("preserve_dots")) { - builder.preserveDots(nodeBooleanValue(fieldNode, "preserve_dots")); - return true; - } else if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) { + if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) { if (fieldNode instanceof List) { List formatters = new ArrayList<>(); for (Object formatter : (List) fieldNode) { @@ -300,7 +297,7 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam String name, Explicit enabled, Dynamic dynamic, - Explicit preserveDots, + Explicit disableObjects, Map mappers, Explicit dynamicDateTimeFormatters, Explicit dynamicTemplates, @@ -308,7 +305,7 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam Explicit numericDetection, Settings settings ) { - super(name, name, enabled, Nested.NO, dynamic, preserveDots, mappers, settings); + super(name, name, enabled, Nested.NO, dynamic, disableObjects, mappers, settings); this.dynamicTemplates = dynamicTemplates; this.dynamicDateTimeFormatters = dynamicDateTimeFormatters; this.dateDetection = dateDetection; @@ -429,10 +426,6 @@ protected void doMerge(ObjectMapper mergeWith, MergeReason reason) { protected void doXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { final boolean includeDefaults = params.paramAsBoolean("include_defaults", false); - if (preserveDotsExplicit().explicit() || includeDefaults) { - builder.field("preserve_dots", preserveDots()); - } - if (dynamicDateTimeFormatters.explicit() || includeDefaults) { builder.startArray("dynamic_date_formats"); for (DateFormatter dateTimeFormatter : dynamicDateTimeFormatters.value()) { diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 9ce4578752d77..ce4d27cca387e 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -3010,15 +3010,15 @@ private void verifyRemoteStoreIndexSettings( assertEquals(translogBufferInterval, indexSettings.get(INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey())); } - public void testTemplateWithPreserveDots() throws Exception { - // Create a template with preserve_dots enabled + public void testTemplateWithDisableObjects() throws Exception { IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> { try { builder.putMapping( "type", XContentFactory.jsonBuilder() .startObject() - .field("preserve_dots", true) + .startObject(MapperService.SINGLE_MAPPING_NAME) + .field("disable_objects", true) .startObject("properties") .startObject("cpu.usage") .field("type", "float") @@ -3028,6 +3028,7 @@ public void testTemplateWithPreserveDots() throws Exception { .endObject() .endObject() .endObject() + .endObject() .toString() ); } catch (IOException e) { @@ -3035,28 +3036,26 @@ public void testTemplateWithPreserveDots() throws Exception { } }); - // Parse mappings with the template Map parsedMappings = MetadataCreateIndexService.parseV1Mappings( "", Collections.singletonList(templateMetadata.getMappings()), NamedXContentRegistry.EMPTY ); - // Verify preserve_dots is present in the parsed mappings assertThat(parsedMappings, hasKey(MapperService.SINGLE_MAPPING_NAME)); Map doc = (Map) parsedMappings.get(MapperService.SINGLE_MAPPING_NAME); - assertThat(doc, hasKey("preserve_dots")); - assertEquals(true, doc.get("preserve_dots")); + assertThat(doc, hasKey("disable_objects")); + assertEquals(true, doc.get("disable_objects")); - // Verify the dotted field properties are present assertThat(doc, hasKey("properties")); Map properties = (Map) doc.get("properties"); + assertThat(properties, hasKey("cpu.usage")); assertThat(properties, hasKey("memory.used")); } - public void testTemplateWithPreserveDotsAtNestedLevel() throws Exception { - // Create a template with preserve_dots enabled on a nested object + public void testTemplateWithDisableObjectsAtNestedLevel() throws Exception { + // Create a template with disable_objects enabled on a nested object IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> { try { builder.putMapping( @@ -3074,7 +3073,7 @@ public void testTemplateWithPreserveDotsAtNestedLevel() throws Exception { .endObject() .startObject("metrics") .field("type", "object") - .field("preserve_dots", true) + .field("disable_objects", true) .startObject("properties") .startObject("cpu.usage") .field("type", "float") @@ -3100,74 +3099,83 @@ public void testTemplateWithPreserveDotsAtNestedLevel() throws Exception { // Verify the structure assertThat(parsedMappings, hasKey(MapperService.SINGLE_MAPPING_NAME)); Map doc = (Map) parsedMappings.get(MapperService.SINGLE_MAPPING_NAME); - assertThat(doc, hasKey("properties")); - Map properties = (Map) doc.get("properties"); + // assertThat(doc, hasKey("properties")); + Map properties = doc; + // Map properties = (Map) doc.get("properties"); - // Verify user object (without preserve_dots) + // Verify user object (without disable_objects) assertThat(properties, hasKey("user")); Map userObject = (Map) properties.get("user"); assertThat(userObject, hasKey("properties")); - // Verify metrics object (with preserve_dots) + // Verify metrics object (with disable_objects) assertThat(properties, hasKey("metrics")); Map metricsObject = (Map) properties.get("metrics"); - assertThat(metricsObject, hasKey("preserve_dots")); - assertEquals(true, metricsObject.get("preserve_dots")); + assertThat(metricsObject, hasKey("disable_objects")); + assertEquals(true, metricsObject.get("disable_objects")); assertThat(metricsObject, hasKey("properties")); Map metricsProperties = (Map) metricsObject.get("properties"); assertThat(metricsProperties, hasKey("cpu.usage")); } - public void testMultipleTemplatesWithPreserveDots() throws Exception { - // Create first template with preserve_dots=false + public void testMultipleTemplatesWithDisableObjects() throws Exception { + // Template 1: no disable_objects (defaults to false) CompressedXContent template1Mapping = new CompressedXContent( XContentFactory.jsonBuilder() .startObject() - .field("preserve_dots", false) + .startObject(MapperService.SINGLE_MAPPING_NAME) .startObject("properties") .startObject("field1") .field("type", "text") .endObject() .endObject() .endObject() + .endObject() .toString() ); - // Create second template with preserve_dots=true + // Template 2: disable_objects=true CompressedXContent template2Mapping = new CompressedXContent( XContentFactory.jsonBuilder() .startObject() - .field("preserve_dots", true) + .startObject(MapperService.SINGLE_MAPPING_NAME) + .field("disable_objects", true) .startObject("properties") .startObject("field.dotted") .field("type", "keyword") .endObject() .endObject() .endObject() + .endObject() .toString() ); - // Parse mappings with both templates (template2 should override template1) List templateMappings = Arrays.asList(template1Mapping, template2Mapping); + Map parsedMappings = MetadataCreateIndexService.parseV1Mappings( "", templateMappings, NamedXContentRegistry.EMPTY ); - // Verify preserve_dots from the last template is applied assertThat(parsedMappings, hasKey(MapperService.SINGLE_MAPPING_NAME)); - Map doc = (Map) parsedMappings.get(MapperService.SINGLE_MAPPING_NAME); - assertThat(doc, hasKey("preserve_dots")); - assertEquals(true, doc.get("preserve_dots")); - // Verify both field1 and field.dotted are present + Map doc = + (Map) parsedMappings.get(MapperService.SINGLE_MAPPING_NAME); + + // disable_objects from last template wins + assertThat(doc, hasKey("disable_objects")); + assertEquals(true, doc.get("disable_objects")); + + // properties are at root of doc assertThat(doc, hasKey("properties")); Map properties = (Map) doc.get("properties"); + assertThat(properties, hasKey("field1")); assertThat(properties, hasKey("field.dotted")); } + private DiscoveryNode getRemoteNode() { Map attributes = new HashMap<>(); attributes.put(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-cluster-rep-1"); diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java index 318e2de0076d4..878933f6e25bd 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java @@ -2600,20 +2600,29 @@ public void testGenerateGroupingCriteriaFromScript() throws Exception { assertEquals("300", doc.docs().getFirst().getGroupingCriteria()); } - public void testPreserveDotsWithFlatFields() throws Exception { - // Test that when preserve_dots=true, dotted field names are treated as literal field names + public void testDisableObjectsWithFlatFields() throws Exception { + // Test that when disable_objects=true on an object, dotted field names are treated as literal field names DocumentMapper mapper = createDocumentMapper(topMapping(b -> { - b.field("preserve_dots", true); b.startObject("properties"); { - b.startObject("metrics.cpu.usage"); - { - b.field("type", "float"); - } - b.endObject(); - b.startObject("metrics.memory.used"); + b.startObject("metrics"); { - b.field("type", "long"); + b.field("type", "object"); + b.field("disable_objects", true); + b.startObject("properties"); + { + b.startObject("cpu.usage"); + { + b.field("type", "float"); + } + b.endObject(); + b.startObject("memory.used"); + { + b.field("type", "long"); + } + b.endObject(); + } + b.endObject(); } b.endObject(); } @@ -2621,8 +2630,12 @@ public void testPreserveDotsWithFlatFields() throws Exception { })); ParsedDocument doc = mapper.parse(source(b -> { - b.field("metrics.cpu.usage", 75.5); - b.field("metrics.memory.used", 8589934592L); + b.startObject("metrics"); + { + b.field("cpu.usage", 75.5); + b.field("memory.used", 8589934592L); + } + b.endObject(); })); // Verify that fields are stored with their complete dotted names @@ -2634,36 +2647,46 @@ public void testPreserveDotsWithFlatFields() throws Exception { assertNotNull("Field 'metrics.memory.used' should exist", memoryField); assertEquals(8589934592L, memoryField.numericValue().longValue()); - // Verify that no nested object structure was created - assertNull("No 'metrics' object should exist", doc.rootDoc().getField("metrics")); + // Verify that no intermediate nested object structure was created for cpu or memory + assertNull("No 'metrics.cpu' object should exist", doc.rootDoc().getField("metrics.cpu")); + assertNull("No 'metrics.memory' object should exist", doc.rootDoc().getField("metrics.memory")); } - public void testPreserveDotsWithDynamicFields() throws Exception { - // Test that dynamic fields work correctly with preserve_dots=true + public void testDisableObjectsWithDynamicFields() throws Exception { + // Test that dynamic fields work correctly with disable_objects=true DocumentMapper mapper = createDocumentMapper(topMapping(b -> { - b.field("preserve_dots", true); b.field("dynamic", "true"); + b.startObject("properties"); + { + b.startObject("metrics"); + { + b.field("type", "object"); + b.field("disable_objects", true); + } + b.endObject(); + } + b.endObject(); })); ParsedDocument doc = mapper.parse(source(b -> { - b.field("user.name", "John Doe"); - b.field("user.age", 30); - b.field("metrics.response.time", 150); + b.startObject("metrics"); + { + b.field("cpu.usage", 75.5); + b.field("memory.used", 8589934592L); + } + b.endObject(); })); - // Verify that dynamic fields are created with dotted names - IndexableField nameField = doc.rootDoc().getField("user.name"); - assertNotNull("Dynamic field 'user.name' should exist", nameField); - - IndexableField ageField = doc.rootDoc().getField("user.age"); - assertNotNull("Dynamic field 'user.age' should exist", ageField); + // Verify that dynamic fields are created with dotted names within the metrics object + IndexableField cpuField = doc.rootDoc().getField("metrics.cpu.usage"); + assertNotNull("Dynamic field 'metrics.cpu.usage' should exist", cpuField); - IndexableField responseTimeField = doc.rootDoc().getField("metrics.response.time"); - assertNotNull("Dynamic field 'metrics.response.time' should exist", responseTimeField); + IndexableField memoryField = doc.rootDoc().getField("metrics.memory.used"); + assertNotNull("Dynamic field 'metrics.memory.used' should exist", memoryField); } - public void testPreserveDotsDefaultBehavior() throws Exception { - // Test that when preserve_dots is not set (default=false), normal nested object behavior is preserved + public void testDisableObjectsDefaultBehavior() throws Exception { + // Test that when disable_objects is not set (default=false), normal nested object behavior is preserved DocumentMapper mapper = createDocumentMapper(topMapping(b -> { b.startObject("properties"); { @@ -2698,4 +2721,158 @@ public void testPreserveDotsDefaultBehavior() throws Exception { assertNotNull("Field 'user.name' should exist in nested structure", nameField); } + public void testRootLevelDisableObjectsWithDynamicFields() throws Exception { + // Test that when disable_objects=true at root level, all dynamic fields are treated as flat fields + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("disable_objects", true); + b.field("dynamic", "true"); + b.startObject("properties"); + { + b.startObject("metrics.cpu.usage"); + { + b.field("type", "float"); + } + b.endObject(); + } + b.endObject(); + })); + + // First document with a predefined field + ParsedDocument doc1 = mapper.parse(source(b -> { + b.field("metrics.cpu.usage", 55.5); + })); + + IndexableField cpuUsageField = doc1.rootDoc().getField("metrics.cpu.usage"); + assertNotNull("Field 'metrics.cpu.usage' should exist", cpuUsageField); + + // Second document with a new dynamic field that shares a prefix + ParsedDocument doc2 = mapper.parse(source(b -> { + b.field("metrics.cpu", 99.1); + })); + + IndexableField cpuField = doc2.rootDoc().getField("metrics.cpu"); + assertNotNull("Dynamic field 'metrics.cpu' should exist as a flat field", cpuField); + + // Verify the mapping update contains the new flat field + Mapping mappingUpdate = doc2.dynamicMappingsUpdate(); + assertNotNull("Dynamic mapping update should be created", mappingUpdate); + + // The new field should be added as a flat field at root level + Mapper cpuMapper = mappingUpdate.root().getMapper("metrics.cpu"); + assertNotNull("Mapper for 'metrics.cpu' should exist in dynamic update", cpuMapper); + assertThat("Mapper should be a FieldMapper", cpuMapper, instanceOf(FieldMapper.class)); + } + + public void testDisableObjectsAutomaticFlattening() throws Exception { + // Test that when disable_objects=true, nested object notation is automatically flattened + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("disable_objects", true); + b.field("dynamic", "true"); + b.startObject("properties"); + { + b.startObject("foo.bar"); + { + b.field("type", "long"); + } + b.endObject(); + } + b.endObject(); + })); + + // Test case 1: Flat notation (should work as before) + ParsedDocument doc1 = mapper.parse(source(b -> { + b.field("foo.bar", 5); + })); + + IndexableField fooBarField1 = doc1.rootDoc().getField("foo.bar"); + assertNotNull("Field 'foo.bar' should exist", fooBarField1); + assertEquals(5L, fooBarField1.numericValue().longValue()); + + // Test case 2: Nested object notation (should be automatically flattened) + ParsedDocument doc2 = mapper.parse(source(b -> { + b.startObject("foo"); + { + b.field("bar", 6); + } + b.endObject(); + })); + + IndexableField fooBarField2 = doc2.rootDoc().getField("foo.bar"); + assertNotNull("Field 'foo.bar' should exist after flattening", fooBarField2); + assertEquals(6L, fooBarField2.numericValue().longValue()); + + // Verify no intermediate 'foo' object was created + assertNull("No 'foo' object should exist", doc2.rootDoc().getField("foo")); + } + + public void testDisableObjectsAutomaticFlatteningDeepNesting() throws Exception { + // Test automatic flattening with deeply nested objects + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("disable_objects", true); + b.field("dynamic", "true"); + })); + + ParsedDocument doc = mapper.parse(source(b -> { + b.startObject("a"); + { + b.startObject("b"); + { + b.startObject("c"); + { + b.field("d", "value"); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + + // Verify the deeply nested structure was flattened to "a.b.c.d" + IndexableField field = doc.rootDoc().getField("a.b.c.d"); + assertNotNull("Field 'a.b.c.d' should exist after flattening", field); + assertEquals("value", field.stringValue()); + + // Verify no intermediate objects were created + assertNull("No 'a' object should exist", doc.rootDoc().getField("a")); + assertNull("No 'a.b' object should exist", doc.rootDoc().getField("a.b")); + assertNull("No 'a.b.c' object should exist", doc.rootDoc().getField("a.b.c")); + } + + public void testDisableObjectsAutomaticFlatteningWithMultipleFields() throws Exception { + // Test automatic flattening with multiple fields in nested objects + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("disable_objects", true); + b.field("dynamic", "true"); + })); + + ParsedDocument doc = mapper.parse(source(b -> { + b.startObject("user"); + { + b.field("name", "John"); + b.field("age", 30); + b.startObject("address"); + { + b.field("city", "NYC"); + b.field("zip", "10001"); + } + b.endObject(); + } + b.endObject(); + })); + + // Verify all fields were flattened correctly + assertNotNull("Field 'user.name' should exist", doc.rootDoc().getField("user.name")); + assertEquals("John", doc.rootDoc().getField("user.name").stringValue()); + + assertNotNull("Field 'user.age' should exist", doc.rootDoc().getField("user.age")); + assertEquals(30L, doc.rootDoc().getField("user.age").numericValue().longValue()); + + assertNotNull("Field 'user.address.city' should exist", doc.rootDoc().getField("user.address.city")); + assertEquals("NYC", doc.rootDoc().getField("user.address.city").stringValue()); + + assertNotNull("Field 'user.address.zip' should exist", doc.rootDoc().getField("user.address.zip")); + assertEquals("10001", doc.rootDoc().getField("user.address.zip").stringValue()); + } + } diff --git a/server/src/test/java/org/opensearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/opensearch/index/mapper/FieldAliasMapperValidationTests.java index 92de2707078f3..0428b71a38843 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FieldAliasMapperValidationTests.java @@ -40,6 +40,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.Map; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -219,7 +220,8 @@ private static ObjectMapper createObjectMapper(String name) { new Explicit<>(true, false), ObjectMapper.Nested.NO, ObjectMapper.Dynamic.FALSE, - emptyMap(), + new Explicit<>(false, false), + (Map) (Map) emptyMap(), SETTINGS ); } @@ -231,7 +233,8 @@ private static ObjectMapper createNestedObjectMapper(String name) { new Explicit<>(true, false), ObjectMapper.Nested.newNested(), ObjectMapper.Dynamic.FALSE, - emptyMap(), + new Explicit<>(false, false), + (Map) (Map) emptyMap(), SETTINGS ); } diff --git a/server/src/test/java/org/opensearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/RootObjectMapperTests.java index 2cbd6b9542efa..054ff2ff8bbc6 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RootObjectMapperTests.java @@ -45,36 +45,6 @@ public class RootObjectMapperTests extends OpenSearchSingleNodeTestCase { - public void testPreserveDots() throws Exception { - MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); - String mapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("type") - .field("preserve_dots", false) - .endObject() - .endObject() - .toString(); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); - assertEquals(mapping, mapper.mappingSource().toString()); - - // update with a different explicit value - String mapping2 = XContentFactory.jsonBuilder() - .startObject() - .startObject("type") - .field("preserve_dots", true) - .endObject() - .endObject() - .toString(); - mapper = mapperService.merge("type", new CompressedXContent(mapping2), reason); - assertEquals(mapping2, mapper.mappingSource().toString()); - - // update with an implicit value: no change - String mapping3 = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().toString(); - mapper = mapperService.merge("type", new CompressedXContent(mapping3), reason); - assertEquals(mapping2, mapper.mappingSource().toString()); - } - public void testNumericDetection() throws Exception { MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); String mapping = XContentFactory.jsonBuilder() From bd83c0bc8b9309a706e23690fb4c26d990d31175 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Sat, 29 Nov 2025 22:34:46 +0530 Subject: [PATCH 09/10] Adding more edge cases, autoflatten and test cases Signed-off-by: Mohit Kumar --- .../template/SimpleIndexTemplateIT.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java index 0b65b4fcf5964..ebcb0547eec2c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java @@ -1078,8 +1078,8 @@ public void testAwarenessReplicaBalance() throws IOException { } } - public void testIndexTemplateWithPreserveDots() throws Exception { - // Create an index template with preserve_dots enabled + public void testIndexTemplateWithdisableObjects() throws Exception { + // Create an index template with disable_objects enabled client().admin() .indices() .preparePutTemplate("metrics_template") @@ -1087,7 +1087,7 @@ public void testIndexTemplateWithPreserveDots() throws Exception { .setMapping( XContentFactory.jsonBuilder() .startObject() - .field("preserve_dots", true) + .field("disable_objects", true) .startObject("properties") .startObject("cpu.usage") .field("type", "float") @@ -1103,15 +1103,15 @@ public void testIndexTemplateWithPreserveDots() throws Exception { // Create an index that matches the template pattern assertAcked(prepareCreate("metrics-001")); - // Verify that preserve_dots setting was applied from the template + // Verify that disable_objects setting was applied from the template ClusterState state = client().admin().cluster().prepareState().get().getState(); IndexMetadata indexMetadata = state.metadata().index("metrics-001"); assertNotNull(indexMetadata); - // Get the mapping and verify preserve_dots is set + // Get the mapping and verify disable_objects is set String mapping = indexMetadata.mapping().source().toString(); - assertTrue("Mapping should contain preserve_dots", mapping.contains("preserve_dots")); - assertTrue("preserve_dots should be set to true", mapping.contains("\"preserve_dots\":true")); + assertTrue("Mapping should contain disable_objects", mapping.contains("disable_objects")); + assertTrue("disable_objects should be set to true", mapping.contains("\"disable_objects\":true")); // Index a document with dotted field names client().prepareIndex("metrics-001") @@ -1133,8 +1133,8 @@ public void testIndexTemplateWithPreserveDots() throws Exception { assertHitCount(searchResponse, 1); } - public void testIndexTemplateWithPreserveDotsAtNestedLevel() throws Exception { - // Create an index template with preserve_dots enabled on a nested object + public void testIndexTemplateWithdisableObjectsAtNestedLevel() throws Exception { + // Create an index template with disable_objects enabled on a nested object client().admin() .indices() .preparePutTemplate("mixed_template") @@ -1153,7 +1153,7 @@ public void testIndexTemplateWithPreserveDotsAtNestedLevel() throws Exception { .endObject() .startObject("metrics") .field("type", "object") - .field("preserve_dots", true) + .field("disable_objects", true) .startObject("properties") .startObject("cpu.usage") .field("type", "float") @@ -1168,14 +1168,14 @@ public void testIndexTemplateWithPreserveDotsAtNestedLevel() throws Exception { // Create an index that matches the template pattern assertAcked(prepareCreate("mixed-001")); - // Verify that preserve_dots setting was applied from the template + // Verify that disable_objects setting was applied from the template ClusterState state = client().admin().cluster().prepareState().get().getState(); IndexMetadata indexMetadata = state.metadata().index("mixed-001"); assertNotNull(indexMetadata); - // Get the mapping and verify preserve_dots is set on the metrics object + // Get the mapping and verify disable_objects is set on the metrics object String mapping = indexMetadata.mapping().source().toString(); - assertTrue("Mapping should contain preserve_dots", mapping.contains("preserve_dots")); + assertTrue("Mapping should contain disable_objects", mapping.contains("disable_objects")); // Index a document with both nested and flat dotted fields client().prepareIndex("mixed-001") @@ -1203,7 +1203,7 @@ public void testIndexTemplateWithPreserveDotsAtNestedLevel() throws Exception { assertHitCount(searchResponse, 1); } - public void testMultipleTemplatesWithPreserveDots() throws Exception { + public void testMultipleTemplatesWithDisableObjects() throws Exception { // Create first template with lower order client().admin() .indices() @@ -1213,7 +1213,7 @@ public void testMultipleTemplatesWithPreserveDots() throws Exception { .setMapping( XContentFactory.jsonBuilder() .startObject() - .field("preserve_dots", false) + .field("disable_objects", false) .startObject("properties") .startObject("field1") .field("type", "text") @@ -1223,7 +1223,7 @@ public void testMultipleTemplatesWithPreserveDots() throws Exception { ) .get(); - // Create second template with higher order and preserve_dots enabled + // Create second template with higher order and disable_objects enabled client().admin() .indices() .preparePutTemplate("template_2") @@ -1232,7 +1232,7 @@ public void testMultipleTemplatesWithPreserveDots() throws Exception { .setMapping( XContentFactory.jsonBuilder() .startObject() - .field("preserve_dots", true) + .field("disable_objects", true) .startObject("properties") .startObject("field.dotted") .field("type", "keyword") @@ -1245,14 +1245,14 @@ public void testMultipleTemplatesWithPreserveDots() throws Exception { // Create an index that matches both templates assertAcked(prepareCreate("test-001")); - // Verify that preserve_dots from the higher order template was applied + // Verify that disable_objects from the higher order template was applied ClusterState state = client().admin().cluster().prepareState().get().getState(); IndexMetadata indexMetadata = state.metadata().index("test-001"); assertNotNull(indexMetadata); String mapping = indexMetadata.mapping().source().toString(); - assertTrue("Mapping should contain preserve_dots", mapping.contains("preserve_dots")); - assertTrue("preserve_dots should be set to true from higher order template", mapping.contains("\"preserve_dots\":true")); + assertTrue("Mapping should contain disable_objects", mapping.contains("disable_objects")); + assertTrue("disable_objects should be set to true from higher order template", mapping.contains("\"disable_objects\":true")); } } From c711dcb77bae3c0f3a0d446cf440dcab06f0552b Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Wed, 3 Dec 2025 14:13:36 +0530 Subject: [PATCH 10/10] Adding autoflattening at nested level as well Signed-off-by: Mohit Kumar --- .../index/mapper/DocumentParser.java | 114 +++++++++--------- 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index f785f65144ce3..82bfd011fa87a 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -490,7 +490,8 @@ private static void innerParseObject( if (mapper.disableObjects()) { paths = new String[] { currentFieldName }; } else { - paths = splitAndValidatePath(currentFieldName); + // Use splitPathRespectingDisableObjects to handle cases where child mappers have disable_objects=true + paths = splitPathRespectingDisableObjects(currentFieldName, context.docMapper()); } if (containsDisabledObjectMapper(mapper, paths)) { parser.nextToken(); @@ -776,68 +777,71 @@ private static void parseFieldWithDots(ParseContext context, ObjectMapper mapper } else { // No existing mapper - handle dynamic mapping token = parser.nextToken(); - ObjectMapper.Dynamic dynamic = dynamicOrDefault(mapper, context); - switch (dynamic) { - case STRICT: - throw new StrictDynamicMappingException( - dynamic.name().toLowerCase(Locale.ROOT), - mapper.fullPath(), - currentFieldName - ); - case TRUE: - case STRICT_ALLOW_TEMPLATES: - case FALSE_ALLOW_TEMPLATES: - // Determine the field type based on the token - XContentFieldType fieldType = getFieldType(token); - - Mapper.Builder builder = findTemplateBuilder( - context, - currentFieldName, - fieldType, - dynamic, - mapper.fullPath() - ); - - if (builder == null) { - if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { - if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) { - parser.skipChildren(); - } - break; - } - // Create a default field mapper based on the value type - builder = createBuilderFromDynamicValue( + // When disable_objects is enabled and we encounter a nested object, flatten it + if (token == XContentParser.Token.START_OBJECT) { + flattenObject(context, mapper, currentFieldName); + } else if (token == XContentParser.Token.START_ARRAY) { + flattenArray(context, mapper, currentFieldName); + } else { + // Handle primitive values with dynamic mapping + ObjectMapper.Dynamic dynamic = dynamicOrDefault(mapper, context); + + switch (dynamic) { + case STRICT: + throw new StrictDynamicMappingException( + dynamic.name().toLowerCase(Locale.ROOT), + mapper.fullPath(), + currentFieldName + ); + case TRUE: + case STRICT_ALLOW_TEMPLATES: + case FALSE_ALLOW_TEMPLATES: + // Determine the field type based on the token + XContentFieldType fieldType = getFieldType(token); + + Mapper.Builder builder = findTemplateBuilder( context, - token, currentFieldName, + fieldType, dynamic, mapper.fullPath() ); - } - - if (builder != null) { - Mapper.BuilderContext builderContext = new Mapper.BuilderContext( - context.indexSettings().getSettings(), - context.path() - ); - Mapper dynamicMapper = builder.build(builderContext); - context.addDynamicMapper(dynamicMapper); - if (dynamicMapper instanceof FieldMapper fm) { - fm.parse(context); - parseCopyFields(context, fm.copyTo().copyToFields()); - } else if (dynamicMapper instanceof ObjectMapper om) { - parseObjectOrNested(context, om); + if (builder == null) { + if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) { + break; + } + // Create a default field mapper based on the value type + builder = createBuilderFromDynamicValue( + context, + token, + currentFieldName, + dynamic, + mapper.fullPath() + ); } - } - break; - case FALSE: - // Dynamic mapping is disabled, skip the field - if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) { - parser.skipChildren(); - } - break; + + if (builder != null) { + Mapper.BuilderContext builderContext = new Mapper.BuilderContext( + context.indexSettings().getSettings(), + context.path() + ); + Mapper dynamicMapper = builder.build(builderContext); + context.addDynamicMapper(dynamicMapper); + + if (dynamicMapper instanceof FieldMapper fm) { + fm.parse(context); + parseCopyFields(context, fm.copyTo().copyToFields()); + } else if (dynamicMapper instanceof ObjectMapper om) { + parseObjectOrNested(context, om); + } + } + break; + case FALSE: + // Dynamic mapping is disabled, skip the field + break; + } } } } else if (token == null) {