Merge branch '9.2' into mergify/bp/9.2/pr-49348

Author: andrzej-stencel

CHANGELOG.asciidoc

@@ -17,6 +17,7 @@ https://github.com/elastic/beats/compare/v9.0.0-rc1\...v9.0.0[View commits]
 
 *Metricbeat*
 
+- Added `mode` and `codec` fields to the `elasticsearch.index` metricset.
 - Add a warning log to `metricbeat.vsphere` in case vSphere connection has been configured as insecure. {pull}43104[43104]
 
 

changelog/fragments/1772635870-es-index-mode-&-codec-in-metricbeat.yaml

@@ -0,0 +1,45 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user's deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: Addition of Elasticsearch index mode and codec settings in Metricbeat index stats module.
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: metricbeat
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+# pr: https://github.com/owner/repo/1234
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

docs/reference/filebeat/filebeat-input-aws-cloudwatch.md

@@ -10,12 +10,16 @@ applies_to:
 # AWS CloudWatch input [filebeat-input-aws-cloudwatch]
 
 
-`aws-cloudwatch` input can be used to retrieve all logs from all log streams in a specific log group. `filterLogEvents` AWS API is used to list log events from the specified log group. Amazon CloudWatch Logs can be used to store log files from Amazon Elastic Compute Cloud(EC2), AWS CloudTrail, Route53, and other sources.
+The `aws-cloudwatch` input can be used to retrieve all logs from all log streams in a specific log group. The `FilterLogEvents` AWS API is used to list log events from the specified log group. Amazon CloudWatch Logs can be used to store log files from Amazon Elastic Compute Cloud(EC2), AWS CloudTrail, Route53, and other sources.
 
 A log group is a group of log streams that share the same retention, monitoring, and access control settings. You can define log groups and specify which streams to put into each group. There is no limit on the number of log streams that can belong to one log group.
 
 A log stream is a sequence of log events that share the same source. Each separate source of logs in CloudWatch Logs makes up a separate log stream.
 
+:::{important}
+This input uses the AWS `FilterLogEvents` API, which is only supported for log groups that use the **Standard** log class. Log groups that use the **Infrequent Access** log class are not supported. For more information about CloudWatch Logs log classes, refer to the [AWS documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatch_Logs_Log_Classes.html).
+:::
+
 ```yaml
 filebeat.inputs:
 - type: aws-cloudwatch

docs/reference/metricbeat/exported-fields-elasticsearch.md

@@ -1370,6 +1370,14 @@ index
 :   type: keyword
 
 
+**`elasticsearch.index.mode`**
+:   type: keyword
+
+
+**`elasticsearch.index.codec`**
+:   type: keyword
+
+
 **`elasticsearch.index.name`**
 :   Index name.
 

docs/reference/metricbeat/metricbeat-metricset-elasticsearch-index.md

@@ -170,6 +170,8 @@ Here is an example document generated by this metricset:
         "tier_preference": "data_content",
         "creation_date": 1731657995821,
         "version": "8505000",
+        "mode": "time_series",
+        "codec": "default",
         "hidden": true,
         "shards": {
           "total": 1,

metricbeat/module/elasticsearch/fields.go

@@ -147,6 +147,8 @@
         "tier_preference": "data_content",
         "creation_date": 1731657995821,
         "version": "8505000",
+        "mode": "time_series",
+        "codec": "default",
         "hidden": true,
         "shards": {
           "total": 1,

metricbeat/module/elasticsearch/index/_meta/data.json

@@ -23,6 +23,10 @@
       type: date
     - name: version
       type: keyword
+    - name: mode
+      type: keyword
+    - name: codec
+      type: keyword
     - name: name
       type: keyword
       description: >

metricbeat/module/elasticsearch/index/_meta/fields.yml

@@ -45,6 +45,8 @@ type Index struct {
 	TierPreference string     `json:"tier_preference,omitempty"`
 	CreationDate   string     `json:"creation_date,omitempty"`
 	Version        string     `json:"version,omitempty"`
+	Mode           string     `json:"mode,omitempty"`
+	Codec          string     `json:"codec,omitempty"`
 	Shards         shardStats `json:"shards"`
 }
 
@@ -196,7 +198,7 @@ func eventsMapping(
 	}
 
 	indicesSettingsPattern := "*,-.*"
-	indicesSettingsFilterPaths := []string{"*.settings.index.creation_date", "*.settings.index.**._tier_preference", "*.settings.index.version.created"}
+	indicesSettingsFilterPaths := []string{"*.settings.index.creation_date", "*.settings.index.**._tier_preference", "*.settings.index.version.created", "*.settings.index.mode", "*.settings.index.codec"}
 	indicesSettings, err := elasticsearch.GetIndexSettings(httpClient, httpClient.GetURI(), indicesSettingsPattern, indicesSettingsFilterPaths)
 	if err != nil {
 		return fmt.Errorf("failure retrieving index settings from Elasticsearch: %w", err)
@@ -336,6 +338,13 @@ func addIndexSettings(idx *Index, indicesSettings mapstr.M) error {
 
 	idx.Version = indexVersion
 
+	if indexMode, err := getIndexSettingForIndex(indexSettings, idx.Index, "index.mode"); err == nil {
+		idx.Mode = indexMode
+	}
+	if indexCodec, err := getIndexSettingForIndex(indexSettings, idx.Index, "index.codec"); err == nil {
+		idx.Codec = indexCodec
+	}
+
 	return nil
 }
 

metricbeat/module/elasticsearch/index/data.go

@@ -97,8 +97,6 @@ processors:
         logs_dynamic_pipeline:
             enabled: true
         logs_index: logs-integration-{{ .Namespace }}
-        mapping:
-            mode: bodymap
         max_conns_per_host: 1
         password: {{ .Password }}
         retry: