Merge branch 'main' into 970-new-agent-version-condition

Author: teresaromero

.buildkite/pipeline.test-with-integrations-repo.yml

@@ -1,5 +1,5 @@
 env:
-  SETUP_GVM_VERSION: 'v0.5.1' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
+  SETUP_GVM_VERSION: 'v0.6.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   GO_LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   GH_CLI_VERSION: "2.29.0"
   JQ_VERSION: "1.7"

.buildkite/pipeline.yml

@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  SETUP_GVM_VERSION: 'v0.5.1' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
+  SETUP_GVM_VERSION: 'v0.6.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   SETUP_MAGE_VERSION: "latest"
   DOCKER_COMPOSE_VERSION: "v2.17.2"
   GO_LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"

code/go/internal/validator/semantic/validate_input_policy_template_template_path.go

@@ -0,0 +1,98 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
+)
+
+const (
+	inputPackageType string = "input"
+)
+
+var (
+	errRequiredTemplatePath  = errors.New("template_path is required for input type packages")
+	errFailedToReadManifest  = errors.New("failed to read manifest")
+	errFailedToParseManifest = errors.New("failed to parse manifest")
+	errTemplateNotFound      = errors.New("template file not found")
+	errInvalidPackageType    = errors.New("invalid package type")
+)
+
+type inputPolicyTemplate struct {
+	Name         string `yaml:"name"`
+	TemplatePath string `yaml:"template_path"` // input type packages require this field
+}
+
+type inputPackageManifest struct { // package manifest
+	Type            string                `yaml:"type"`
+	PolicyTemplates []inputPolicyTemplate `yaml:"policy_templates"`
+}
+
+// ValidateInputPackagesPolicyTemplates validates the policy template entries of an input package
+func ValidateInputPackagesPolicyTemplates(fsys fspath.FS) specerrors.ValidationErrors {
+	var errs specerrors.ValidationErrors
+
+	manifestPath := "manifest.yml"
+	data, err := fs.ReadFile(fsys, manifestPath)
+	if err != nil {
+		return specerrors.ValidationErrors{
+			specerrors.NewStructuredErrorf("file \"%s\" is invalid: %ww", fsys.Path(manifestPath), errFailedToReadManifest)}
+	}
+
+	var manifest inputPackageManifest
+	err = yaml.Unmarshal(data, &manifest)
+	if err != nil {
+		return specerrors.ValidationErrors{
+			specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToParseManifest)}
+	}
+
+	if manifest.Type != inputPackageType {
+		return specerrors.ValidationErrors{
+			specerrors.NewStructuredErrorf("file \"%s\" is invalid: expected package type \"%s\", got \"%s\": %w",
+				fsys.Path(manifestPath), inputPackageType, manifest.Type, errInvalidPackageType)}
+	}
+
+	for _, policyTemplate := range manifest.PolicyTemplates {
+		err := validateInputPackagePolicyTemplate(fsys, policyTemplate)
+		if err != nil {
+			errs = append(errs, specerrors.NewStructuredErrorf(
+				"file \"%s\" is invalid: policy template \"%s\" references template_path \"%s\": %w",
+				fsys.Path(manifestPath), policyTemplate.Name, policyTemplate.TemplatePath, err))
+		}
+	}
+
+	return errs
+}
+
+// validateInputPackagePolicyTemplate validates the template_path at the policy template level for input type packages
+// if the template_path is empty, it returns an error as this field is required for input type packages
+func validateInputPackagePolicyTemplate(fsys fspath.FS, policyTemplate inputPolicyTemplate) error {
+	if policyTemplate.TemplatePath == "" {
+		return errRequiredTemplatePath
+	}
+	return validateAgentInputTemplatePath(fsys, policyTemplate.TemplatePath)
+}
+
+func validateAgentInputTemplatePath(fsys fspath.FS, tmplPath string) error {
+	templatePath := path.Join("agent", "input", tmplPath)
+	_, err := fs.Stat(fsys, templatePath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return errTemplateNotFound
+		}
+		return fmt.Errorf("failed to stat template file %s: %w", fsys.Path(templatePath), err)
+	}
+
+	return nil
+}

code/go/internal/validator/semantic/validate_input_policy_template_template_path_test.go

@@ -0,0 +1,97 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+)
+
+func TestValidateInputPackagesPolicyTemplates(t *testing.T) {
+
+	t.Run("policy_templates_have_template_path", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.MkdirAll(filepath.Join(d, "agent", "input"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: input
+policy_templates:
+  - name: udp
+    template_path: udp.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "agent", "input", "udp.yml.hbs"), []byte("# UDP template"), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputPackagesPolicyTemplates(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+
+	})
+
+	t.Run("policy_templates_empty_template_path", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.MkdirAll(filepath.Join(d, "agent", "input"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: input
+policy_templates:
+  - name: udp
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputPackagesPolicyTemplates(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected no validation errors")
+
+		assert.Len(t, errs, 1)
+		assert.ErrorIs(t, errs[0], errRequiredTemplatePath)
+	})
+
+	t.Run("policy_templates_missing_template_path", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.MkdirAll(filepath.Join(d, "agent", "input"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: input
+policy_templates:
+  - name: udp
+    template_path: missing.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputPackagesPolicyTemplates(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected validation errors")
+		assert.Len(t, errs, 1)
+		assert.ErrorIs(t, errs[0], errTemplateNotFound)
+	})
+
+	t.Run("not_input_package_type", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.MkdirAll(filepath.Join(d, "agent", "input"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+policy_templates:
+  - name: udp
+    template_path: missing.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputPackagesPolicyTemplates(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected validation errors")
+		assert.Len(t, errs, 1)
+		assert.ErrorIs(t, errs[0], errInvalidPackageType)
+	})
+
+}

code/go/internal/validator/spec.go

@@ -217,6 +217,7 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules
 		{fn: semantic.ValidateDocsStructure},
 		{fn: semantic.ValidateDeploymentModes, types: []string{"integration"}},
 		{fn: semantic.ValidateDurationVariables, since: semver.MustParse("3.5.0")},
+		{fn: semantic.ValidateInputPackagesPolicyTemplates, types: []string{"input"}},
 		{fn: semantic.ValidateMinimumAgentVersion},
 	}
 

code/go/pkg/validator/validator_test.go

@@ -310,6 +310,13 @@ func TestValidateFile(t *testing.T) {
 				"field policy_templates.0.input: Must not be present",
 			},
 		},
+		"bad_input_template_path": {
+			"manifest.yml",
+			[]string{
+				"field policy_templates.0: template_path is required",
+				"policy template \"sql_query\" references template_path \"\": template_path is required for input type packages",
+			},
+		},
 	}
 
 	for pkgName, test := range tests {

spec/changelog.yml

@@ -23,6 +23,9 @@
     - description: Add support for script testing in data streams.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/985
+    - description: Input packages require to define template_path in manifest.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/1000
     - description: Allow to define agent version constrains in input and integration packages.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/999

spec/input/manifest.spec.yml

@@ -94,6 +94,7 @@ spec:
           - description
           - type
           - input
+          - template_path
     icons:
       $ref: "../integration/manifest.spec.yml#/definitions/icons"
     screenshots:

test/packages/bad_duplicated_fields_input/manifest.yml

@@ -27,6 +27,7 @@ policy_templates:
     type: logs
     input: log_file
     description: Collect sample logs
+    template_path: input.yml.hbs
     vars:
       - name: paths
         required: true