add support for kibana policy overrides (#3094)

Author: leehinman

docs/howto/system_testing.md

@@ -977,6 +977,23 @@ In this case, `elastic-package test system` will fail with an error and print a
 skip_ignored_fields:
   - field.to.ignore
 ```
+### Kibana policy overrides
+
+If you need to test a system test with a Kibana policy override you can do that by setting the environment variable `ELASTIC_PACKAGE_KIBANA_POLICY_OVERRIDES` to be a path to a yaml file that contains the policy override.  For example:
+
+```shell
+ELASTIC_PACKAGE_KIBANA_POLICY_OVERRIDES=/tmp/overrides.yml elastic-package test system
+```
+
+and the `/tmp/overrides.yml` file has the following contents:
+
+```yaml
+agent:
+  monitoring:
+    _runtime_experimental: otel
+```
+
+Will result in the system test running with the agent monitoring using the `otel` runtime.
 
 ## Continuous Integration
 

internal/kibana/policies.go

@@ -16,15 +16,16 @@ import (
 
 // Policy represents an Agent Policy in Fleet.
 type Policy struct {
-	ID                   string   `json:"id,omitempty"`
-	Name                 string   `json:"name"`
-	Description          string   `json:"description"`
-	Namespace            string   `json:"namespace"`
-	Revision             int      `json:"revision,omitempty"`
-	MonitoringEnabled    []string `json:"monitoring_enabled,omitempty"`
-	MonitoringOutputID   string   `json:"monitoring_output_id,omitempty"`
-	DataOutputID         string   `json:"data_output_id,omitempty"`
-	IsDefaultFleetServer bool     `json:"is_default_fleet_server,omitempty"`
+	ID                   string         `json:"id,omitempty"`
+	Name                 string         `json:"name"`
+	Description          string         `json:"description"`
+	Namespace            string         `json:"namespace"`
+	Revision             int            `json:"revision,omitempty"`
+	MonitoringEnabled    []string       `json:"monitoring_enabled,omitempty"`
+	MonitoringOutputID   string         `json:"monitoring_output_id,omitempty"`
+	DataOutputID         string         `json:"data_output_id,omitempty"`
+	IsDefaultFleetServer bool           `json:"is_default_fleet_server,omitempty"`
+	Overrides            map[string]any `json:"overrides,omitempty"`
 }
 
 // DownloadedPolicy represents a policy as returned by the download policy API.

internal/testrunner/runners/system/tester.go

@@ -178,6 +178,7 @@ var (
 	dumpScenarioDocsEnv          = environment.WithElasticPackagePrefix("TEST_DUMP_SCENARIO_DOCS")
 	fieldValidationTestMethodEnv = environment.WithElasticPackagePrefix("FIELD_VALIDATION_TEST_METHOD")
 	prefixServiceTestRunIDEnv    = environment.WithElasticPackagePrefix("PREFIX_SERVICE_TEST_RUN_ID")
+	kibanaPolicyOverridesEnv     = environment.WithElasticPackagePrefix("KIBANA_POLICY_OVERRIDES")
 )
 
 type fieldValidationMethod int
@@ -230,6 +231,8 @@ type tester struct {
 
 	globalTestConfig testrunner.GlobalRunnerTestConfig
 
+	kibanaPolicyOverrides map[string]any
+
 	// Execution order of following handlers is defined in runner.TearDown() method.
 	removeAgentHandler        func(context.Context) error
 	deleteTestPolicyHandler   func(context.Context) error
@@ -348,6 +351,20 @@ func NewSystemTester(options SystemTesterOptions) (*tester, error) {
 		r.fieldValidationMethod = method
 	}
 
+	// kibana policy overrides
+	v, ok = os.LookupEnv(kibanaPolicyOverridesEnv)
+	if ok {
+		data, err := os.ReadFile(v)
+		if err != nil {
+			return nil, fmt.Errorf("error reading file %q specified in %s: %w", v, kibanaPolicyOverridesEnv, err)
+		}
+		overrides := make(map[string]any)
+		if err = yaml.Unmarshal(data, &overrides); err != nil {
+			return nil, fmt.Errorf("error reading yaml from %s: %w", v, err)
+		}
+		r.kibanaPolicyOverrides = overrides
+	}
+
 	return &r, nil
 }
 
@@ -1374,6 +1391,7 @@ func (r *tester) createOrGetKibanaPolicies(ctx context.Context, serviceStateData
 			Name:        fmt.Sprintf("ep-test-system-%s-%s-%s-%s-%s", r.testFolder.Package, r.testFolder.DataStream, r.serviceVariant, r.configFileName, testTime),
 			Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.testFolder.Package, r.testFolder.DataStream),
 			Namespace:   common.CreateTestRunID(),
+			Overrides:   r.kibanaPolicyOverrides,
 		}
 		// Assign the data_output_id to the agent policy to configure the output to logstash. The value is inferred from stack/_static/kibana.yml.tmpl
 		// TODO: Migrate from stack.logstash_enabled to the stack config.