Fix template path including linked files

Author: teresaromero

code/go/internal/validator/semantic/validate_integration_policy_template_path.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"io/fs"
 	"path"
+	"path/filepath"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 
@@ -148,18 +150,18 @@ func validateInputWithStreams(fsys fspath.FS, input string, dsMap map[string]dat
 				stream.TemplatePath = defaultStreamTemplatePath
 			}
 
-			_, err := fs.ReadFile(fsys, path.Join(dsDir, "agent", "stream", stream.TemplatePath))
+			foundFile, err := findPathWithPattern(fsys, dsDir, stream.TemplatePath)
+			if err != nil {
+				return err
+			}
+			if foundFile == "" {
+				return errTemplateNotFound
+			}
+
+			_, err = fs.ReadFile(fsys, foundFile)
 			if err != nil {
 				if errors.Is(err, fs.ErrNotExist) {
-					// fallback to glob pattern matching in case the default template path is customized with a prefix
-					matches, err := fs.Glob(fsys, path.Join(dsDir, "agent", "stream", "*"+stream.TemplatePath))
-					if err != nil {
-						return err
-					}
-					if len(matches) == 0 {
-						return errTemplateNotFound
-					}
-					continue
+					return errTemplateNotFound
 				}
 				return err
 			}
@@ -168,3 +170,33 @@ func validateInputWithStreams(fsys fspath.FS, input string, dsMap map[string]dat
 
 	return nil
 }
+
+// findPathWithPattern looks for a file matching the templatePath in the given data stream directory
+// It checks for exact matches, files ending with the templatePath, or templatePath + ".link"
+func findPathWithPattern(fsys fspath.FS, dsDir, templatePath string) (string, error) {
+	// Check for exact match, files ending with stream.TemplatePath, or stream.TemplatePath + ".link"
+	pattern := path.Join(dsDir, "agent", "stream", "*"+templatePath+"*")
+	matches, err := fs.Glob(fsys, pattern)
+	if err != nil {
+		return "", err
+	}
+
+	// Filter matches to ensure they match our criteria:
+	// 1. Exact name match
+	// 2. Ends with stream.TemplatePath
+	// 3. Equals stream.TemplatePath + ".link"
+	var foundFile string
+	for _, match := range matches {
+		base := filepath.Base(match)
+		if base == templatePath || base == templatePath+".link" {
+			foundFile = match
+			break
+		}
+		// fallbak to check for suffix match, in case the path is prefixed
+		if strings.HasSuffix(match, templatePath) {
+			foundFile = match
+			break
+		}
+	}
+	return foundFile, nil
+}

code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go

@@ -179,3 +179,94 @@ streams:
 	errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
 	require.Empty(t, errs)
 }
+func TestFindPathWithPattern(t *testing.T) {
+	d := t.TempDir()
+	
+	dsDir := filepath.ToSlash(path.Join("data_stream", "logs"))
+	err := os.MkdirAll(filepath.Join(d, "data_stream", "logs", "agent", "stream"), 0o755)
+	require.NoError(t, err)
+
+	t.Run("exact match", func(t *testing.T) {
+		templatePath := "exact.yml.hbs"
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", templatePath), []byte("content"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", templatePath))
+
+		foundFile, err := findPathWithPattern(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, "agent", "stream", templatePath)), foundFile)
+	})
+
+	t.Run("match with .link extension", func(t *testing.T) {
+		templatePath := "linked.yml.hbs"
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", templatePath+".link"), []byte("content"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", templatePath+".link"))
+
+		foundFile, err := findPathWithPattern(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, "agent", "stream", templatePath+".link")), foundFile)
+	})
+
+	t.Run("match with prefix", func(t *testing.T) {
+		templatePath := "stream.yml.hbs"
+		prefixedFile := "prefixstream.yml.hbs"
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile), []byte("content"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile))
+
+		foundFile, err := findPathWithPattern(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, "agent", "stream", prefixedFile)), foundFile)
+	})
+
+	t.Run("no match found", func(t *testing.T) {
+		templatePath := "nonexistent.yml.hbs"
+		
+		foundFile, err := findPathWithPattern(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.Empty(t, foundFile)
+	})
+
+	t.Run("multiple matches - exact match takes precedence", func(t *testing.T) {
+		templatePath := "multi.yml.hbs"
+		exactFile := "multi.yml.hbs"
+		prefixedFile := "prefixmulti.yml.hbs"
+		
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", exactFile), []byte("exact"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", exactFile))
+		
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile), []byte("prefixed"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile))
+
+		foundFile, err := findPathWithPattern(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, "agent", "stream", exactFile)), foundFile)
+	})
+
+	t.Run("link file takes precedence over suffix match", func(t *testing.T) {
+		templatePath := "link.yml.hbs"
+		linkFile := "link.yml.hbs.link"
+		suffixFile := "prefixlink.yml.hbs"
+		
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", linkFile), []byte("link"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", linkFile))
+		
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", suffixFile), []byte("suffix"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", suffixFile))
+
+		foundFile, err := findPathWithPattern(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, "agent", "stream", linkFile)), foundFile)
+	})
+}
+

test/packages/with_links/data_stream/foo/agent/stream/s3.yml.hbs.link renamed to test/packages/with_links/data_stream/foo/agent/stream/stream.yml.hbs.link

@@ -1,7 +1,7 @@
 title: Nginx access logs
 type: logs
 streams:
-  - input: logfile
+  - input: logfile-link
     vars:
       - name: empty_array
         type: text

test/packages/with_links/data_stream/foo/manifest.yml

@@ -14,6 +14,20 @@ policy_templates:
     title: Apache logs and metrics
     description: Collect logs and metrics from Apache instances
     inputs:
+      - type: logfile-link
+        title: Collect logs from Apache instances
+        description: Collecting Apache access and error logs
+        multi: false
+        vars:
+          - name: paths
+            type: text
+            title: Paths
+            multi: true
+            required: true
+            show_user: true
+            default:
+              - /var/log/apache2/access.log
+              - /var/log/apache2/error.log
       - type: apache/metrics
         title: Collect metrics from Apache instances
         description: Collecting Apache status metrics