Support policy tests in input packages with input type otelcol

internal/stack/_static/kibana.yml.tmpl

@@ -41,9 +41,18 @@ xpack.fleet.registryUrl: "https://package-registry:8080"
 xpack.fleet.agents.enabled: true
 xpack.fleet.agents.fleet_server.hosts: ["{{ fact "fleet_url" }}"]
 
-{{ if and (not (semverLessThan $version "8.7.0")) (semverLessThan $version "8.10.0-SNAPSHOT") }}
-xpack.fleet.enableExperimental: ["experimentalDataStreamSettings"] # Enable experimental toggles in Fleet UI
-{{ end }}
+{{/* Feature flags introduced in 8.3.0 https://github.com/elastic/kibana/pull/130253 */}}
+{{- if (not (semverLessThan $version "8.3.0")) }}
+xpack.fleet.enableExperimental: [
+{{- if and (not (semverLessThan $version "8.7.0")) (semverLessThan $version "8.10.0-SNAPSHOT") }}
+  # Enable experimental toggles in Fleet UI
+  "experimentalDataStreamSettings"
+{{- else if (not (semverLessThan $version "9.2.0-SNAPSHOT")) }}
+  # Support for otelcol input
+  "enableOtelIntegrations"
+{{- end }}
+]
+{{- end }}
 
 xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012"
 

internal/testrunner/runners/policy/policy.go

@@ -5,6 +5,7 @@
 package policy
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"errors"
@@ -13,13 +14,15 @@ import (
 	"path/filepath"
 	"regexp"
 	"slices"
+	"strconv"
 	"strings"
 
 	"github.com/pmezard/go-difflib/difflib"
 	"gopkg.in/yaml.v3"
 
 	"github.com/elastic/elastic-package/internal/common"
 	"github.com/elastic/elastic-package/internal/kibana"
+	"github.com/elastic/elastic-package/internal/logger"
 )
 
 func dumpExpectedAgentPolicy(ctx context.Context, kibanaClient *kibana.Client, testPath string, policyID string) error {
@@ -71,6 +74,9 @@ func comparePolicies(expected, found []byte) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("failed to prepare found policy: %w", err)
 	}
+	logger.Tracef("expected policy after cleaning:\n%s", want)
+	logger.Tracef("found policy after cleaning:\n%s", got)
+
 	if bytes.Equal(want, got) {
 		return "", nil
 	}
@@ -166,10 +172,35 @@ var policyEntryFilters = []policyEntryFilter{
 	}},
 }
 
+var uniqueOtelComponentIDReplace = policyEntryReplace{
+	regexp:  regexp.MustCompile(`^(\s{2,})([^/]+)/([^:]+):(\s\{\}|\s*)$`),
+	replace: "$1$2/componentid-%s:$4",
+}
+
+// otelComponentIDsRegexp is the regex to find otel components sections and their IDs to replace them with controlled values.
+// It matches sections like:
+//
+//	 extensions:
+//		  health_check/4391d954-1ffe-4014-a256-5eda78a71828: {}
+//
+//	 receivers:
+//	     httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7:
+//	        collection_interval: 1m
+//	        targets:
+//	            - endpoints:
+//	                - https://epr.elastic.co
+//	              method: GET
+var otelComponentIDsRegexp = regexp.MustCompile(`(?m)^(?:extensions|receivers|processors|connectors|exporters):(?:\s\{\}\n|\n(?:\s{2,}.+\n)+)`)
+
 // cleanPolicy prepares a policy YAML as returned by the download API to be compared with other
 // policies. This preparation is based on removing contents that are generated, or replace them
 // by controlled values.
 func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, error) {
+	// Replacement of the OTEL component IDs needs to be done before unmarshalling the YAML.
+	// The OTEL IDs are keys in maps, and using the policyEntryFilter with memberReplace does
+	// not ensure to keep the same ordering.
+	policy = replaceOtelComponentIDs(policy)
+
 	var policyMap common.MapStr
 	err := yaml.Unmarshal(policy, &policyMap)
 	if err != nil {
@@ -184,6 +215,47 @@ func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, err
 	return yaml.Marshal(policyMap)
 }
 
+// replaceOtelComponentIDs finds OTel Collector component IDs in the policy and replaces them with controlled values.
+// It also replaces references to those IDs in service.extensions and service.pipelines.
+func replaceOtelComponentIDs(policy []byte) []byte {
+	replacementsDone := map[string]string{}
+
+	policy = otelComponentIDsRegexp.ReplaceAllFunc(policy, func(match []byte) []byte {
+		count := 0
+		scanner := bufio.NewScanner(bytes.NewReader(match))
+		var section strings.Builder
+		for scanner.Scan() {
+			line := scanner.Text()
+			if uniqueOtelComponentIDReplace.regexp.MatchString(line) {
+				originalOtelID, _, _ := strings.Cut(strings.TrimSpace(line), ":")
+
+				replacement := fmt.Sprintf(uniqueOtelComponentIDReplace.replace, strconv.Itoa(count))
+				count++
+				line = uniqueOtelComponentIDReplace.regexp.ReplaceAllString(line, replacement)
+
+				// store the otel ID replaced without the space indentation and the colon to be replaced later
+				// (e.g. http_check/4391d954-1ffe-4014-a256-5eda78a71828 replaced by http_check/componentid-0)
+				replacementsDone[originalOtelID], _, _ = strings.Cut(strings.TrimSpace(string(line)), ":")
+			}
+			section.WriteString(line + "\n")
+		}
+
+		return []byte(section.String())
+	})
+
+	// Replace references in arrays to the otel component IDs replaced before.
+	// These references can be in:
+	// service.extensions
+	// service.pipelines.<signal>.(receivers|processors|exporters)
+	for original, replacement := range replacementsDone {
+		originalArrayItem := fmt.Sprintf("- %s", original)
+		replacedArrayItem := fmt.Sprintf("- %s", replacement)
+
+		policy = bytes.ReplaceAll(policy, []byte(originalArrayItem), []byte(replacedArrayItem))
+	}
+	return policy
+}
+
 func cleanPolicyMap(policyMap common.MapStr, entries []policyEntryFilter) (common.MapStr, error) {
 	for _, entry := range entries {
 		v, err := policyMap.GetValue(entry.name)
@@ -218,10 +290,14 @@ func cleanPolicyMap(policyMap common.MapStr, entries []policyEntryFilter) (commo
 			if !ok {
 				return nil, fmt.Errorf("expected map, found %T", v)
 			}
+			regexp := entry.memberReplace.regexp
+			replacement := entry.memberReplace.replace
 			for k, e := range m {
-				if entry.memberReplace.regexp.MatchString(k) {
+				key := k
+				if regexp.MatchString(k) {
 					delete(m, k)
-					m[entry.memberReplace.replace] = e
+					key = regexp.ReplaceAllString(k, replacement)
+					m[key] = e
 				}
 			}
 		default:

internal/testrunner/runners/policy/policy_test.go

@@ -121,7 +121,7 @@ output_permissions:
                 - monitor
         _elastic_agent_monitoring:
             indices: []
-        uuid-for-permissions-on-related-indices:
+        8d024b11-4e82-4192-8e7f-be71d1b13aac:
             indices:
                 - names:
                     - metrics-*-*
@@ -221,7 +221,7 @@ output_permissions:
                 - monitor
         _elastic_agent_monitoring:
             indices: []
-        uuid-for-permissions-on-related-indices:
+        bfe4f402-df02-4673-8a71-fd5b29f1e2f3:
             indices:
                 - names:
                     - metrics-*-*
@@ -285,6 +285,194 @@ secret_references:
 `,
 			equal: false,
 		},
+		{
+			title: "otel ids",
+			expected: `
+inputs: []
+output_permissions:
+    default:
+        _elastic_agent_checks:
+            cluster:
+                - monitor
+        _elastic_agent_monitoring:
+            indices: []
+        05c98f91-203c-44a9-bee7-dd621c9bd37e:
+            indices:
+                - names:
+                    - logs-*-*
+                  privileges:
+                    - auto_configure
+                    - create_doc
+extensions:
+    health_check/31c94f44-214a-4778-8a36-acc2634096f7: {}
+processors:
+    batch/11c35ad0-4351-49d4-9c78-fa679ce9d950:
+        send_batch_size: 10
+        timeout: 1s
+    batch/e6e379c5-6446-4090-af10-a9e5f8fc4640:
+        send_batch_size: 10000
+        timeout: 10s
+receivers:
+    httpcheck/4bae34b3-8f66-49c1-b04f-d58af1b5f743:
+        collection_interval: 1m
+        targets:
+            - endpoints:
+                - https://epr.elastic.co
+              method: GET
+secret_references: []
+service:
+    extensions:
+        - health_check/31c94f44-214a-4778-8a36-acc2634096f7
+    pipelines:
+        logs:
+            receivers:
+                - httpcheck/4bae34b3-8f66-49c1-b04f-d58af1b5f743
+            processors:
+                - batch/11c35ad0-4351-49d4-9c78-fa679ce9d950
+                - batch/e6e379c5-6446-4090-af10-a9e5f8fc4640
+
+`,
+			found: `
+inputs: []
+output_permissions:
+    default:
+        _elastic_agent_checks:
+            cluster:
+                - monitor
+        aeb4d606-2d90-4b41-b231-27bfad6dea09:
+            indices:
+                - names:
+                    - logs-*-*
+                  privileges:
+                    - auto_configure
+                    - create_doc
+        _elastic_agent_monitoring:
+            indices: []
+extensions:
+    health_check/4391d954-1ffe-4014-a256-5eda78a71829: {}
+processors:
+    batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b:
+        send_batch_size: 10
+        timeout: 1s
+    batch/8ec6ee99-2176-4231-9668-908069c77784:
+        send_batch_size: 10000
+        timeout: 10s
+receivers:
+    httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7:
+        collection_interval: 1m
+        targets:
+            - endpoints:
+                - https://epr.elastic.co
+              method: GET
+secret_references: []
+service:
+    extensions:
+        - health_check/4391d954-1ffe-4014-a256-5eda78a71829
+    pipelines:
+        logs:
+            receivers:
+                - httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7
+            processors:
+                - batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b
+                - batch/8ec6ee99-2176-4231-9668-908069c77784
+
+`,
+			equal: true,
+		},
+		{
+			title: "otel hardcode expected ids",
+			expected: `
+inputs: []
+output_permissions:
+    default:
+        _elastic_agent_checks:
+            cluster:
+                - monitor
+        _elastic_agent_monitoring:
+            indices: []
+        05c98f91-203c-44a9-bee7-dd621c9bd37e:
+            indices:
+                - names:
+                    - logs-*-*
+                  privileges:
+                    - auto_configure
+                    - create_doc
+extensions:
+    health_check/componentid-0: {}
+processors:
+    batch/componentid-0:
+        send_batch_size: 10
+        timeout: 1s
+    batch/componentid-1:
+        send_batch_size: 10000
+        timeout: 10s
+receivers:
+    httpcheck/componentid-0:
+        collection_interval: 1m
+        targets:
+            - endpoints:
+                - https://epr.elastic.co
+              method: GET
+secret_references: []
+service:
+    extensions:
+        - health_check/componentid-0
+    pipelines:
+        logs:
+            receivers:
+                - httpcheck/componentid-0
+            processors:
+                - batch/componentid-0
+                - batch/componentid-1
+
+`,
+			found: `
+inputs: []
+output_permissions:
+    default:
+        _elastic_agent_checks:
+            cluster:
+                - monitor
+        aeb4d606-2d90-4b41-b231-27bfad6dea09:
+            indices:
+                - names:
+                    - logs-*-*
+                  privileges:
+                    - auto_configure
+                    - create_doc
+        _elastic_agent_monitoring:
+            indices: []
+extensions:
+    health_check/4391d954-1ffe-4014-a256-5eda78a71828: {}
+processors:
+    batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b:
+        send_batch_size: 10
+        timeout: 1s
+    batch/8ec6ee99-2176-4231-9668-908069c77784:
+        send_batch_size: 10000
+        timeout: 10s
+receivers:
+    httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7:
+        collection_interval: 1m
+        targets:
+            - endpoints:
+                - https://epr.elastic.co
+              method: GET
+secret_references: []
+service:
+    extensions:
+        - health_check/4391d954-1ffe-4014-a256-5eda78a71828
+    pipelines:
+        logs:
+            receivers:
+                - httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7
+            processors:
+                - batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b
+                - batch/8ec6ee99-2176-4231-9668-908069c77784
+
+`,
+			equal: true,
+		},
 	}
 
 	for _, c := range cases {

test/packages/parallel/httpcheck.stack_version

@@ -0,0 +1 @@
+9.2.0-SNAPSHOT