Fix documentation of ECS fields in linked files

cmd/benchmark.go

@@ -141,6 +141,7 @@ func pipelineCommandAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("locating repository root failed: %w", err)
 	}
+	defer repositoryRoot.Close()
 
 	packageRoot, err := packages.FindPackageRoot()
 	if err != nil {
@@ -306,6 +307,7 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("locating repository root failed: %w", err)
 	}
+	defer repositoryRoot.Close()
 
 	profile, err := cobraext.GetProfileFlag(cmd)
 	if err != nil {
@@ -480,6 +482,7 @@ func streamCommandAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("locating repository root failed: %w", err)
 	}
+	defer repositoryRoot.Close()
 
 	profile, err := cobraext.GetProfileFlag(cmd)
 	if err != nil {

cmd/install.go

@@ -85,6 +85,7 @@ func installCommandAction(cmd *cobra.Command, _ []string) error {
 	if err != nil {
 		return fmt.Errorf("locating repository root failed: %w", err)
 	}
+	defer repositoryRoot.Close()
 
 	installer, err := installer.NewForPackage(installer.Options{
 		Kibana:         kibanaClient,

cmd/testrunner.go

@@ -808,6 +808,7 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("locating repository root failed: %w", err)
 	}
+	defer repositoryRoot.Close()
 
 	dataStreams, err := getDataStreamsFlag(cmd, packageRoot)
 	if err != nil {

internal/builder/packages.go

@@ -369,6 +369,7 @@ func createBuildDirectory(dirs ...string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	defer root.Close()
 
 	p := []string{root.Name(), "build"}
 	if len(dirs) > 0 {

internal/docs/exported_fields.go

@@ -6,6 +6,7 @@ package docs
 
 import (
 	"fmt"
+	"os"
 	"sort"
 	"strings"
 
@@ -23,9 +24,8 @@ type fieldsTableRecord struct {
 
 var escaper = strings.NewReplacer("*", "\\*", "{", "\\{", "}", "\\}", "<", "\\<", ">", "\\>")
 
-// renderExportedFields renders the fields for a package or data stream, fieldsParentRoot must be
-// the path to the root directory of the package or data stream.
-func renderExportedFields(fieldsParentRoot string) (string, error) {
+// renderExportedFields renders the fields for a package or data stream.
+func renderExportedFields(repositoryRoot *os.Root, packageRoot, fieldsDir string) (string, error) {
 	injectOptions := fields.InjectFieldsOptions{
 		// Keep External parameter when rendering fields, so we can render
 		// documentation for empty groups imported from ECS, for backwards compatibility.
@@ -35,9 +35,9 @@ func renderExportedFields(fieldsParentRoot string) (string, error) {
 		// keep them to accept them for validation.
 		SkipEmptyFields: true,
 	}
-	validator, err := fields.CreateValidatorForDirectory(fieldsParentRoot, fields.WithInjectFieldsOptions(injectOptions))
+	validator, err := fields.CreateValidator(repositoryRoot, packageRoot, fieldsDir, fields.WithInjectFieldsOptions(injectOptions))
 	if err != nil {
-		return "", fmt.Errorf("can't create fields validator instance (path: %s): %w", fieldsParentRoot, err)
+		return "", fmt.Errorf("can't create fields validator instance (path: %s): %w", fieldsDir, err)
 	}
 
 	collected := collectFieldsFromDefinitions(validator)

internal/docs/readme.go

@@ -45,7 +45,7 @@ func AreReadmesUpToDate(repositoryRoot *os.Root, packageRoot string) ([]ReadmeFi
 	var readmeFiles []ReadmeFile
 	for _, filePath := range files {
 		fileName := filepath.Base(filePath)
-		ok, diff, err := isReadmeUpToDate(fileName, linksFilePath, packageRoot)
+		ok, diff, err := isReadmeUpToDate(repositoryRoot, fileName, linksFilePath, packageRoot)
 		if !ok || err != nil {
 			readmeFile := ReadmeFile{
 				FileName: fileName,
@@ -64,11 +64,11 @@ func AreReadmesUpToDate(repositoryRoot *os.Root, packageRoot string) ([]ReadmeFi
 }
 
 // isReadmeUpToDate function checks if a single readme file is up-to-date.
-func isReadmeUpToDate(fileName, linksFilePath, packageRoot string) (bool, string, error) {
+func isReadmeUpToDate(repositoryRoot *os.Root, fileName, linksFilePath, packageRoot string) (bool, string, error) {
 	logger.Debugf("Check if %s is up-to-date", fileName)
 
 	// the readme is generated within the package root, so source should be the packageRoot files too
-	rendered, shouldBeRendered, err := generateReadme(fileName, linksFilePath, packageRoot)
+	rendered, shouldBeRendered, err := generateReadme(repositoryRoot, fileName, linksFilePath, packageRoot)
 	if err != nil {
 		return false, "", fmt.Errorf("generating readme file failed: %w", err)
 	}
@@ -112,7 +112,7 @@ func UpdateReadmes(repositoryRoot *os.Root, packageRoot, buildPackageRoot string
 
 	for _, filePath := range readmeFiles {
 		fileName := filepath.Base(filePath)
-		target, err := updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot)
+		target, err := updateReadme(repositoryRoot, fileName, linksFilePath, packageRoot, buildPackageRoot)
 		if err != nil {
 			return fmt.Errorf("updating readme file %s failed: %w", fileName, err)
 		}
@@ -128,10 +128,10 @@ func UpdateReadmes(repositoryRoot *os.Root, packageRoot, buildPackageRoot string
 
 // updateReadme function updates a single readme file using a defined template file.
 // It writes the rendered file to both the package directory and the package build directory.
-func updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot string) (string, error) {
+func updateReadme(repositoryRoot *os.Root, fileName, linksFilePath, packageRoot, buildPackageRoot string) (string, error) {
 	logger.Debugf("Update the %s file", fileName)
 
-	rendered, shouldBeRendered, err := generateReadme(fileName, linksFilePath, packageRoot)
+	rendered, shouldBeRendered, err := generateReadme(repositoryRoot, fileName, linksFilePath, packageRoot)
 	if err != nil {
 		return "", err
 	}
@@ -154,7 +154,7 @@ func updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot string)
 // generateReadme function generates the readme file content
 // the readme takes a template that lives under the _dev/build/docs directory at the packageRoot.
 // the readme template reads data from the packageRoot directory.
-func generateReadme(fileName, linksFilePath, packageRoot string) ([]byte, bool, error) {
+func generateReadme(repositoryRoot *os.Root, fileName, linksFilePath, packageRoot string) ([]byte, bool, error) {
 	logger.Debugf("Generate %s file (package: %s)", fileName, packageRoot)
 	templatePath, found, err := findReadmeTemplatePath(fileName, packageRoot)
 	if err != nil {
@@ -173,7 +173,7 @@ func generateReadme(fileName, linksFilePath, packageRoot string) ([]byte, bool,
 
 	// templatePath lives under the _dev/build/docs directory at the package root.
 	// builtPackageRoot is the root directory of the built package.
-	rendered, err := renderReadme(fileName, packageRoot, templatePath, linksMap)
+	rendered, err := renderReadme(repositoryRoot, fileName, packageRoot, templatePath, linksMap)
 	if err != nil {
 		return nil, true, fmt.Errorf("rendering Readme failed: %w", err)
 	}
@@ -194,7 +194,7 @@ func findReadmeTemplatePath(fileName, packageRoot string) (string, bool, error)
 }
 
 // renderReadme function renders the readme file reading from
-func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap) ([]byte, error) {
+func renderReadme(repositoryRoot *os.Root, fileName, packageRoot, templatePath string, linksMap linkMap) ([]byte, error) {
 	logger.Debugf("Render %s file (package: %s, templatePath: %s)", fileName, packageRoot, templatePath)
 
 	t := template.New(fileName)
@@ -206,11 +206,11 @@ func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap)
 			return renderSampleEvent(packageRoot, "")
 		},
 		"fields": func(args ...string) (string, error) {
+			fieldsDir := filepath.Join(packageRoot, "fields")
 			if len(args) > 0 {
-				dataStreamRoot := filepath.Join(packageRoot, "data_stream", args[0])
-				return renderExportedFields(dataStreamRoot)
+				fieldsDir = filepath.Join(packageRoot, "data_stream", args[0], "fields")
 			}
-			return renderExportedFields(packageRoot)
+			return renderExportedFields(repositoryRoot, packageRoot, fieldsDir)
 		},
 		"url": func(args ...string) (string, error) {
 			options := linkOptions{}

internal/docs/readme_test.go

@@ -53,11 +53,14 @@ Introduction to the package`,
 	}
 	for _, c := range cases {
 		t.Run(c.title, func(t *testing.T) {
-
 			dir := t.TempDir()
 			createReadmeTemplateFile(t, dir, c.readmeTemplateContents)
 
-			rendered, isTemplate, err := generateReadme(c.filename, "", dir)
+			root, err := os.OpenRoot(dir)
+			require.NoError(t, err)
+			t.Cleanup(func() { root.Close() })
+
+			rendered, isTemplate, err := generateReadme(root, c.filename, "", dir)
 			require.NoError(t, err)
 
 			if c.readmeTemplateContents != "" {
@@ -113,7 +116,11 @@ http://www.example.com/bar
 
 			createReadmeTemplateFile(t, c.packageRoot, c.readmeTemplateContents)
 
-			rendered, err := renderReadme(filename, c.packageRoot, templatePath, c.linksMap)
+			root, err := os.OpenRoot(c.packageRoot)
+			require.NoError(t, err)
+			t.Cleanup(func() { root.Close() })
+
+			rendered, err := renderReadme(root, filename, c.packageRoot, templatePath, c.linksMap)
 			require.NoError(t, err)
 
 			renderedString := string(rendered)
@@ -167,7 +174,11 @@ An example event for ` + "`example`" + ` looks as following:
 			createSampleEventFile(t, c.packageRoot, c.dataStreamName, c.sampleEventJsonContents)
 			createManifestFile(t, c.packageRoot)
 
-			rendered, err := renderReadme(filename, c.packageRoot, templatePath, linksMap)
+			root, err := os.OpenRoot(c.packageRoot)
+			require.NoError(t, err)
+			t.Cleanup(func() { root.Close() })
+
+			rendered, err := renderReadme(root, filename, c.packageRoot, templatePath, linksMap)
 			require.NoError(t, err)
 
 			renderedString := string(rendered)
@@ -290,7 +301,11 @@ func TestRenderReadmeWithFields(t *testing.T) {
 			createReadmeTemplateFile(t, packageRoot, c.readmeTemplateContents)
 			createFieldsFile(t, packageRoot, c.dataStreamName, c.fieldsContents)
 
-			rendered, err := renderReadme(filename, packageRoot, templatePath, linksMap)
+			root, err := os.OpenRoot(packageRoot)
+			require.NoError(t, err)
+			t.Cleanup(func() { root.Close() })
+
+			rendered, err := renderReadme(root, filename, packageRoot, templatePath, linksMap)
 			require.NoError(t, err)
 
 			renderedString := string(rendered)
@@ -313,7 +328,11 @@ func TestUpdateReadmeWithFields(t *testing.T) {
 			buildPackageRoot := t.TempDir()
 			createManifestFile(t, buildPackageRoot)
 
-			readmePath, err := updateReadme(filename, "", packageRoot, buildPackageRoot)
+			root, err := os.OpenRoot(packageRoot)
+			require.NoError(t, err)
+			t.Cleanup(func() { root.Close() })
+
+			readmePath, err := updateReadme(root, filename, "", packageRoot, buildPackageRoot)
 			require.NoError(t, err)
 			require.NotEmpty(t, readmePath)
 			d, err := os.ReadFile(readmePath)

internal/fields/dependency_manager_test.go

@@ -787,10 +787,8 @@ func TestDependencyManagerWithECS(t *testing.T) {
 }
 
 func TestValidate_SetExternalECS(t *testing.T) {
-	finder := packageRootTestFinder{"../../test/packages/other/imported_mappings_tests"}
-
-	validator, err := createValidatorForDirectoryAndPackageRoot("../../test/packages/other/imported_mappings_tests/data_stream/first",
-		finder,
+	repositoryRoot, packageRoot, fieldsDir := pathsForValidator(t, "other", "imported_mappings_tests", "first")
+	validator, err := CreateValidator(repositoryRoot, packageRoot, fieldsDir,
 		WithSpecVersion("2.3.0"),
 		WithEnabledImportAllECSSChema(true))
 	require.NoError(t, err)

internal/fields/validate.go

@@ -10,9 +10,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/fs"
 	"net"
 	"os"
-	"path/filepath"
 	"regexp"
 	"slices"
 	"sort"
@@ -24,6 +24,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/elastic/elastic-package/internal/common"
+	"github.com/elastic/elastic-package/internal/files"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/multierror"
 	"github.com/elastic/elastic-package/internal/packages"
@@ -256,25 +257,8 @@ func WithOTelValidation(otelValidation bool) ValidatorOption {
 	}
 }
 
-type packageRootFinder interface {
-	FindPackageRoot() (string, error)
-}
-
-type packageRoot struct {
-	from string
-}
-
-func (p packageRoot) FindPackageRoot() (string, error) {
-	return packages.FindPackageRootFrom(p.from)
-}
-
-// CreateValidatorForDirectory function creates a validator for the directory.
-func CreateValidatorForDirectory(fieldsParentDir string, opts ...ValidatorOption) (v *Validator, err error) {
-	finder := packageRoot{from: fieldsParentDir}
-	return createValidatorForDirectoryAndPackageRoot(fieldsParentDir, finder, opts...)
-}
-
-func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder packageRootFinder, opts ...ValidatorOption) (v *Validator, err error) {
+// CreateValidator creates a validator for a given fields directory, contained under the indicated repository and package roots.
+func CreateValidator(repositoryRoot *os.Root, packageRoot string, fieldsDir string, opts ...ValidatorOption) (v *Validator, err error) {
 	v = new(Validator)
 	// In validator, inject fields with settings used for validation, such as `allowed_values`.
 	v.injectFieldsOptions.IncludeValidationSettings = true
@@ -286,30 +270,28 @@ func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder pa
 
 	v.allowedCIDRs = initializeAllowedCIDRsList()
 
-	fieldsDir := filepath.Join(fieldsParentDir, "fields")
-
-	var fdm *DependencyManager
-	if !v.disabledDependencyManagement {
-		packageRoot, err := finder.FindPackageRoot()
+	if _, err := os.Stat(fieldsDir); err == nil {
+		linksFS, err := files.CreateLinksFSFromPath(repositoryRoot, fieldsDir)
 		if err != nil {
-			if errors.Is(err, packages.ErrPackageRootNotFound) {
-				return nil, errors.New("package root not found and dependency management is enabled")
+			return nil, fmt.Errorf("can't create links filesystem: %w", err)
+
+		}
+
+		var fdm *DependencyManager
+		if !v.disabledDependencyManagement {
+			fdm, v.Schema, err = initDependencyManagement(packageRoot, v.specVersion, v.enabledImportAllECSSchema)
+			if err != nil {
+				return nil, fmt.Errorf("failed to initialize dependency management: %w", err)
 			}
-			return nil, fmt.Errorf("can't find package root: %w", err)
 		}
 
-		fdm, v.Schema, err = initDependencyManagement(packageRoot, v.specVersion, v.enabledImportAllECSSchema)
+		fields, err := loadFieldsFromDir(linksFS, fdm, v.injectFieldsOptions)
 		if err != nil {
-			return nil, fmt.Errorf("failed to initialize dependency management: %w", err)
+			return nil, fmt.Errorf("can't load fields from directory (path: %s): %w", fieldsDir, err)
 		}
+		v.Schema = append(fields, v.Schema...)
 	}
 
-	fields, err := loadFieldsFromDir(fieldsDir, fdm, v.injectFieldsOptions)
-	if err != nil {
-		return nil, fmt.Errorf("can't load fields from directory (path: %s): %w", fieldsDir, err)
-	}
-
-	v.Schema = append(fields, v.Schema...)
 	return v, nil
 }
 
@@ -513,15 +495,21 @@ func initializeAllowedCIDRsList() (cidrs []*net.IPNet) {
 	return cidrs
 }
 
-func loadFieldsFromDir(fieldsDir string, fdm *DependencyManager, injectOptions InjectFieldsOptions) ([]FieldDefinition, error) {
-	files, err := filepath.Glob(filepath.Join(fieldsDir, "*.yml"))
+// loadFieldsFromDir loads all the fields from a directory with fields files. The directory is passed as a filesystem.
+func loadFieldsFromDir(fieldsFS fs.FS, fdm *DependencyManager, injectOptions InjectFieldsOptions) ([]FieldDefinition, error) {
+	files, err := fs.Glob(fieldsFS, "*.yml")
+	if err != nil {
+		return nil, err
+	}
+	links, err := fs.Glob(fieldsFS, "*.yml.link")
 	if err != nil {
-		return nil, fmt.Errorf("reading directory with fields failed (path: %s): %w", fieldsDir, err)
+		return nil, err
 	}
+	files = append(files, links...)
 
 	var fields []FieldDefinition
 	for _, file := range files {
-		body, err := os.ReadFile(file)
+		body, err := fs.ReadFile(fieldsFS, file)
 		if err != nil {
 			return nil, fmt.Errorf("reading fields file failed: %w", err)
 		}