diff --git a/cmd/benchmark.go b/cmd/benchmark.go index 4a8fae5947..45cde94d25 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -141,6 +141,7 @@ func pipelineCommandAction(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("locating repository root failed: %w", err) } + defer repositoryRoot.Close() packageRoot, err := packages.FindPackageRoot() if err != nil { @@ -306,6 +307,7 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("locating repository root failed: %w", err) } + defer repositoryRoot.Close() profile, err := cobraext.GetProfileFlag(cmd) if err != nil { @@ -480,6 +482,7 @@ func streamCommandAction(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("locating repository root failed: %w", err) } + defer repositoryRoot.Close() profile, err := cobraext.GetProfileFlag(cmd) if err != nil { diff --git a/cmd/install.go b/cmd/install.go index 48f5d3a705..3acc21f628 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -85,6 +85,7 @@ func installCommandAction(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("locating repository root failed: %w", err) } + defer repositoryRoot.Close() installer, err := installer.NewForPackage(installer.Options{ Kibana: kibanaClient, diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 751049f92b..4ce403167d 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -165,6 +165,7 @@ func testRunnerAssetCommandAction(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("locating repository root failed: %w", err) } + defer repositoryRoot.Close() manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) if err != nil { @@ -365,6 +366,7 @@ func testRunnerPipelineCommandAction(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("locating repository root failed: %w", err) } + defer repositoryRoot.Close() packageRoot, err := packages.FindPackageRoot() if err != nil { @@ -525,6 +527,7 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("locating repository root failed: %w", err) } + defer repositoryRoot.Close() runSetup, err := cmd.Flags().GetBool(cobraext.SetupFlagName) if err != nil { @@ -808,6 +811,7 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("locating repository root failed: %w", err) } + defer repositoryRoot.Close() dataStreams, err := getDataStreamsFlag(cmd, packageRoot) if err != nil { diff --git a/internal/builder/packages.go b/internal/builder/packages.go index bce45095c2..e93b514b3f 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -369,6 +369,7 @@ func createBuildDirectory(dirs ...string) (string, error) { if err != nil { return "", err } + defer root.Close() p := []string{root.Name(), "build"} if len(dirs) > 0 { diff --git a/internal/docs/exported_fields.go b/internal/docs/exported_fields.go index 4f2896d2e3..cfa0799c03 100644 --- a/internal/docs/exported_fields.go +++ b/internal/docs/exported_fields.go @@ -6,6 +6,7 @@ package docs import ( "fmt" + "os" "sort" "strings" @@ -23,9 +24,8 @@ type fieldsTableRecord struct { var escaper = strings.NewReplacer("*", "\\*", "{", "\\{", "}", "\\}", "<", "\\<", ">", "\\>") -// renderExportedFields renders the fields for a package or data stream, fieldsParentRoot must be -// the path to the root directory of the package or data stream. -func renderExportedFields(fieldsParentRoot string) (string, error) { +// renderExportedFields renders the fields for a package or data stream. +func renderExportedFields(repositoryRoot *os.Root, packageRoot, fieldsDir string) (string, error) { injectOptions := fields.InjectFieldsOptions{ // Keep External parameter when rendering fields, so we can render // documentation for empty groups imported from ECS, for backwards compatibility. @@ -35,9 +35,9 @@ func renderExportedFields(fieldsParentRoot string) (string, error) { // keep them to accept them for validation. SkipEmptyFields: true, } - validator, err := fields.CreateValidatorForDirectory(fieldsParentRoot, fields.WithInjectFieldsOptions(injectOptions)) + validator, err := fields.CreateValidator(repositoryRoot, packageRoot, fieldsDir, fields.WithInjectFieldsOptions(injectOptions)) if err != nil { - return "", fmt.Errorf("can't create fields validator instance (path: %s): %w", fieldsParentRoot, err) + return "", fmt.Errorf("can't create fields validator instance (path: %s): %w", fieldsDir, err) } collected := collectFieldsFromDefinitions(validator) diff --git a/internal/docs/readme.go b/internal/docs/readme.go index 5ea9bf976c..c2c41795f5 100644 --- a/internal/docs/readme.go +++ b/internal/docs/readme.go @@ -45,7 +45,7 @@ func AreReadmesUpToDate(repositoryRoot *os.Root, packageRoot string) ([]ReadmeFi var readmeFiles []ReadmeFile for _, filePath := range files { fileName := filepath.Base(filePath) - ok, diff, err := isReadmeUpToDate(fileName, linksFilePath, packageRoot) + ok, diff, err := isReadmeUpToDate(repositoryRoot, fileName, linksFilePath, packageRoot) if !ok || err != nil { readmeFile := ReadmeFile{ FileName: fileName, @@ -64,11 +64,11 @@ func AreReadmesUpToDate(repositoryRoot *os.Root, packageRoot string) ([]ReadmeFi } // isReadmeUpToDate function checks if a single readme file is up-to-date. -func isReadmeUpToDate(fileName, linksFilePath, packageRoot string) (bool, string, error) { +func isReadmeUpToDate(repositoryRoot *os.Root, fileName, linksFilePath, packageRoot string) (bool, string, error) { logger.Debugf("Check if %s is up-to-date", fileName) // the readme is generated within the package root, so source should be the packageRoot files too - rendered, shouldBeRendered, err := generateReadme(fileName, linksFilePath, packageRoot) + rendered, shouldBeRendered, err := generateReadme(repositoryRoot, fileName, linksFilePath, packageRoot) if err != nil { return false, "", fmt.Errorf("generating readme file failed: %w", err) } @@ -112,7 +112,7 @@ func UpdateReadmes(repositoryRoot *os.Root, packageRoot, buildPackageRoot string for _, filePath := range readmeFiles { fileName := filepath.Base(filePath) - target, err := updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot) + target, err := updateReadme(repositoryRoot, fileName, linksFilePath, packageRoot, buildPackageRoot) if err != nil { return fmt.Errorf("updating readme file %s failed: %w", fileName, err) } @@ -128,10 +128,10 @@ func UpdateReadmes(repositoryRoot *os.Root, packageRoot, buildPackageRoot string // updateReadme function updates a single readme file using a defined template file. // It writes the rendered file to both the package directory and the package build directory. -func updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot string) (string, error) { +func updateReadme(repositoryRoot *os.Root, fileName, linksFilePath, packageRoot, buildPackageRoot string) (string, error) { logger.Debugf("Update the %s file", fileName) - rendered, shouldBeRendered, err := generateReadme(fileName, linksFilePath, packageRoot) + rendered, shouldBeRendered, err := generateReadme(repositoryRoot, fileName, linksFilePath, packageRoot) if err != nil { return "", err } @@ -154,7 +154,7 @@ func updateReadme(fileName, linksFilePath, packageRoot, buildPackageRoot string) // generateReadme function generates the readme file content // the readme takes a template that lives under the _dev/build/docs directory at the packageRoot. // the readme template reads data from the packageRoot directory. -func generateReadme(fileName, linksFilePath, packageRoot string) ([]byte, bool, error) { +func generateReadme(repositoryRoot *os.Root, fileName, linksFilePath, packageRoot string) ([]byte, bool, error) { logger.Debugf("Generate %s file (package: %s)", fileName, packageRoot) templatePath, found, err := findReadmeTemplatePath(fileName, packageRoot) if err != nil { @@ -173,7 +173,7 @@ func generateReadme(fileName, linksFilePath, packageRoot string) ([]byte, bool, // templatePath lives under the _dev/build/docs directory at the package root. // builtPackageRoot is the root directory of the built package. - rendered, err := renderReadme(fileName, packageRoot, templatePath, linksMap) + rendered, err := renderReadme(repositoryRoot, fileName, packageRoot, templatePath, linksMap) if err != nil { return nil, true, fmt.Errorf("rendering Readme failed: %w", err) } @@ -194,7 +194,7 @@ func findReadmeTemplatePath(fileName, packageRoot string) (string, bool, error) } // renderReadme function renders the readme file reading from -func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap) ([]byte, error) { +func renderReadme(repositoryRoot *os.Root, fileName, packageRoot, templatePath string, linksMap linkMap) ([]byte, error) { logger.Debugf("Render %s file (package: %s, templatePath: %s)", fileName, packageRoot, templatePath) t := template.New(fileName) @@ -206,11 +206,11 @@ func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap) return renderSampleEvent(packageRoot, "") }, "fields": func(args ...string) (string, error) { + fieldsDir := filepath.Join(packageRoot, "fields") if len(args) > 0 { - dataStreamRoot := filepath.Join(packageRoot, "data_stream", args[0]) - return renderExportedFields(dataStreamRoot) + fieldsDir = filepath.Join(packageRoot, "data_stream", args[0], "fields") } - return renderExportedFields(packageRoot) + return renderExportedFields(repositoryRoot, packageRoot, fieldsDir) }, "url": func(args ...string) (string, error) { options := linkOptions{} diff --git a/internal/docs/readme_test.go b/internal/docs/readme_test.go index eac8e4e410..8dd9908438 100644 --- a/internal/docs/readme_test.go +++ b/internal/docs/readme_test.go @@ -53,11 +53,14 @@ Introduction to the package`, } for _, c := range cases { t.Run(c.title, func(t *testing.T) { - dir := t.TempDir() createReadmeTemplateFile(t, dir, c.readmeTemplateContents) - rendered, isTemplate, err := generateReadme(c.filename, "", dir) + root, err := os.OpenRoot(dir) + require.NoError(t, err) + t.Cleanup(func() { root.Close() }) + + rendered, isTemplate, err := generateReadme(root, c.filename, "", dir) require.NoError(t, err) if c.readmeTemplateContents != "" { @@ -113,7 +116,11 @@ http://www.example.com/bar createReadmeTemplateFile(t, c.packageRoot, c.readmeTemplateContents) - rendered, err := renderReadme(filename, c.packageRoot, templatePath, c.linksMap) + root, err := os.OpenRoot(c.packageRoot) + require.NoError(t, err) + t.Cleanup(func() { root.Close() }) + + rendered, err := renderReadme(root, filename, c.packageRoot, templatePath, c.linksMap) require.NoError(t, err) renderedString := string(rendered) @@ -167,7 +174,11 @@ An example event for ` + "`example`" + ` looks as following: createSampleEventFile(t, c.packageRoot, c.dataStreamName, c.sampleEventJsonContents) createManifestFile(t, c.packageRoot) - rendered, err := renderReadme(filename, c.packageRoot, templatePath, linksMap) + root, err := os.OpenRoot(c.packageRoot) + require.NoError(t, err) + t.Cleanup(func() { root.Close() }) + + rendered, err := renderReadme(root, filename, c.packageRoot, templatePath, linksMap) require.NoError(t, err) renderedString := string(rendered) @@ -290,7 +301,11 @@ func TestRenderReadmeWithFields(t *testing.T) { createReadmeTemplateFile(t, packageRoot, c.readmeTemplateContents) createFieldsFile(t, packageRoot, c.dataStreamName, c.fieldsContents) - rendered, err := renderReadme(filename, packageRoot, templatePath, linksMap) + root, err := os.OpenRoot(packageRoot) + require.NoError(t, err) + t.Cleanup(func() { root.Close() }) + + rendered, err := renderReadme(root, filename, packageRoot, templatePath, linksMap) require.NoError(t, err) renderedString := string(rendered) @@ -313,7 +328,11 @@ func TestUpdateReadmeWithFields(t *testing.T) { buildPackageRoot := t.TempDir() createManifestFile(t, buildPackageRoot) - readmePath, err := updateReadme(filename, "", packageRoot, buildPackageRoot) + root, err := os.OpenRoot(packageRoot) + require.NoError(t, err) + t.Cleanup(func() { root.Close() }) + + readmePath, err := updateReadme(root, filename, "", packageRoot, buildPackageRoot) require.NoError(t, err) require.NotEmpty(t, readmePath) d, err := os.ReadFile(readmePath) diff --git a/internal/fields/dependency_manager_test.go b/internal/fields/dependency_manager_test.go index bf9c8b1f62..511419732d 100644 --- a/internal/fields/dependency_manager_test.go +++ b/internal/fields/dependency_manager_test.go @@ -787,10 +787,8 @@ func TestDependencyManagerWithECS(t *testing.T) { } func TestValidate_SetExternalECS(t *testing.T) { - finder := packageRootTestFinder{"../../test/packages/other/imported_mappings_tests"} - - validator, err := createValidatorForDirectoryAndPackageRoot("../../test/packages/other/imported_mappings_tests/data_stream/first", - finder, + repositoryRoot, packageRoot, fieldsDir := pathsForValidator(t, "other", "imported_mappings_tests", "first") + validator, err := CreateValidator(repositoryRoot, packageRoot, fieldsDir, WithSpecVersion("2.3.0"), WithEnabledImportAllECSSChema(true)) require.NoError(t, err) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 6bf0f11285..d9455310a5 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -10,9 +10,9 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "net" "os" - "path/filepath" "regexp" "slices" "sort" @@ -24,6 +24,7 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/common" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" @@ -256,25 +257,8 @@ func WithOTelValidation(otelValidation bool) ValidatorOption { } } -type packageRootFinder interface { - FindPackageRoot() (string, error) -} - -type packageRoot struct { - from string -} - -func (p packageRoot) FindPackageRoot() (string, error) { - return packages.FindPackageRootFrom(p.from) -} - -// CreateValidatorForDirectory function creates a validator for the directory. -func CreateValidatorForDirectory(fieldsParentDir string, opts ...ValidatorOption) (v *Validator, err error) { - finder := packageRoot{from: fieldsParentDir} - return createValidatorForDirectoryAndPackageRoot(fieldsParentDir, finder, opts...) -} - -func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder packageRootFinder, opts ...ValidatorOption) (v *Validator, err error) { +// CreateValidator creates a validator for a given fields directory, contained under the indicated repository and package roots. +func CreateValidator(repositoryRoot *os.Root, packageRoot string, fieldsDir string, opts ...ValidatorOption) (v *Validator, err error) { v = new(Validator) // In validator, inject fields with settings used for validation, such as `allowed_values`. v.injectFieldsOptions.IncludeValidationSettings = true @@ -286,30 +270,28 @@ func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder pa v.allowedCIDRs = initializeAllowedCIDRsList() - fieldsDir := filepath.Join(fieldsParentDir, "fields") - - var fdm *DependencyManager - if !v.disabledDependencyManagement { - packageRoot, err := finder.FindPackageRoot() + if _, err := os.Stat(fieldsDir); err == nil { + linksFS, err := files.CreateLinksFSFromPath(repositoryRoot, fieldsDir) if err != nil { - if errors.Is(err, packages.ErrPackageRootNotFound) { - return nil, errors.New("package root not found and dependency management is enabled") + return nil, fmt.Errorf("can't create links filesystem: %w", err) + + } + + var fdm *DependencyManager + if !v.disabledDependencyManagement { + fdm, v.Schema, err = initDependencyManagement(packageRoot, v.specVersion, v.enabledImportAllECSSchema) + if err != nil { + return nil, fmt.Errorf("failed to initialize dependency management: %w", err) } - return nil, fmt.Errorf("can't find package root: %w", err) } - fdm, v.Schema, err = initDependencyManagement(packageRoot, v.specVersion, v.enabledImportAllECSSchema) + fields, err := loadFieldsFromDir(linksFS, fdm, v.injectFieldsOptions) if err != nil { - return nil, fmt.Errorf("failed to initialize dependency management: %w", err) + return nil, fmt.Errorf("can't load fields from directory (path: %s): %w", fieldsDir, err) } + v.Schema = append(fields, v.Schema...) } - fields, err := loadFieldsFromDir(fieldsDir, fdm, v.injectFieldsOptions) - if err != nil { - return nil, fmt.Errorf("can't load fields from directory (path: %s): %w", fieldsDir, err) - } - - v.Schema = append(fields, v.Schema...) return v, nil } @@ -513,15 +495,21 @@ func initializeAllowedCIDRsList() (cidrs []*net.IPNet) { return cidrs } -func loadFieldsFromDir(fieldsDir string, fdm *DependencyManager, injectOptions InjectFieldsOptions) ([]FieldDefinition, error) { - files, err := filepath.Glob(filepath.Join(fieldsDir, "*.yml")) +// loadFieldsFromDir loads all the fields from a directory with fields files. The directory is passed as a filesystem. +func loadFieldsFromDir(fieldsFS fs.FS, fdm *DependencyManager, injectOptions InjectFieldsOptions) ([]FieldDefinition, error) { + files, err := fs.Glob(fieldsFS, "*.yml") + if err != nil { + return nil, err + } + links, err := fs.Glob(fieldsFS, "*.yml.link") if err != nil { - return nil, fmt.Errorf("reading directory with fields failed (path: %s): %w", fieldsDir, err) + return nil, err } + files = append(files, links...) var fields []FieldDefinition for _, file := range files { - body, err := os.ReadFile(file) + body, err := fs.ReadFile(fieldsFS, file) if err != nil { return nil, fmt.Errorf("reading fields file failed: %w", err) } diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index 1836f5ca32..9d2dbde8d7 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -24,16 +24,10 @@ type results struct { Expected []json.RawMessage } -type packageRootTestFinder struct { - packageRoot string -} - -func (p packageRootTestFinder) FindPackageRoot() (string, error) { - return p.packageRoot, nil -} - func TestValidate_NoWildcardFields(t *testing.T) { - validator, err := CreateValidatorForDirectory("../../test/packages/parallel/aws/data_stream/elb_logs", WithDisabledDependencyManagement()) + repositoryRoot, packageRoot, fieldsDir := pathsForValidator(t, "parallel", "aws", "elb_logs") + validator, err := CreateValidator(repositoryRoot, packageRoot, fieldsDir, + WithDisabledDependencyManagement()) require.NoError(t, err) require.NotNil(t, validator) @@ -45,7 +39,8 @@ func TestValidate_NoWildcardFields(t *testing.T) { } func TestValidate_WithWildcardFields(t *testing.T) { - validator, err := CreateValidatorForDirectory("../../test/packages/parallel/aws/data_stream/sns", WithDisabledDependencyManagement()) + repositoryRoot, packageRoot, fieldsDir := pathsForValidator(t, "parallel", "aws", "sns") + validator, err := CreateValidator(repositoryRoot, packageRoot, fieldsDir, WithDisabledDependencyManagement()) require.NoError(t, err) require.NotNil(t, validator) @@ -55,10 +50,7 @@ func TestValidate_WithWildcardFields(t *testing.T) { } func TestValidate_WithFlattenedFields(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", - WithDisabledDependencyManagement()) - require.NoError(t, err) - require.NotNil(t, validator) + validator := createValidatorForTestdata(t, WithDisabledDependencyManagement()) e := readSampleEvent(t, "testdata/flattened.json") errs := validator.ValidateDocumentBody(e) @@ -66,10 +58,7 @@ func TestValidate_WithFlattenedFields(t *testing.T) { } func TestValidate_ObjectTypeWithoutWildcard(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", - WithDisabledDependencyManagement()) - require.NoError(t, err) - require.NotNil(t, validator) + validator := createValidatorForTestdata(t, WithDisabledDependencyManagement()) t.Run("subobjects", func(t *testing.T) { e := readSampleEvent(t, "testdata/subobjects.json") @@ -85,10 +74,7 @@ func TestValidate_ObjectTypeWithoutWildcard(t *testing.T) { } func TestValidate_DisabledParent(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", - WithDisabledDependencyManagement()) - require.NoError(t, err) - require.NotNil(t, validator) + validator := createValidatorForTestdata(t, WithDisabledDependencyManagement()) t.Run("disabled", func(t *testing.T) { e := readSampleEvent(t, "testdata/disabled.json") @@ -98,10 +84,7 @@ func TestValidate_DisabledParent(t *testing.T) { } func TestValidate_EnabledNotMappedError(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", - WithDisabledDependencyManagement()) - require.NoError(t, err) - require.NotNil(t, validator) + validator := createValidatorForTestdata(t, WithDisabledDependencyManagement()) t.Run("enabled", func(t *testing.T) { e := readSampleEvent(t, "testdata/enabled_not_mapped.json") @@ -115,7 +98,7 @@ func TestValidate_EnabledNotMappedError(t *testing.T) { } func TestValidate_WithNumericKeywordFields(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", + validator := createValidatorForTestdata(t, WithNumericKeywordFields([]string{ "foo.code", // Contains a number. "foo.pid", // Contains an array of numbers. @@ -124,8 +107,6 @@ func TestValidate_WithNumericKeywordFields(t *testing.T) { }), WithSpecVersion("2.3.0"), // Needed to validate normalization. WithDisabledDependencyManagement()) - require.NoError(t, err) - require.NotNil(t, validator) e := readSampleEvent(t, "testdata/numeric.json") errs := validator.ValidateDocumentBody(e) @@ -133,15 +114,13 @@ func TestValidate_WithNumericKeywordFields(t *testing.T) { } func TestValidate_WithStringNumberFields(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", + validator := createValidatorForTestdata(t, WithStringNumberFields([]string{ "foo.count", // Contains a number as string. "foo.metric", // Contains a floating number as string. }), WithSpecVersion("2.3.0"), // Needed to validate normalization. WithDisabledDependencyManagement()) - require.NoError(t, err) - require.NotNil(t, validator) e := readSampleEvent(t, "testdata/stringnumbers.json") errs := validator.ValidateDocumentBody(e) @@ -149,10 +128,8 @@ func TestValidate_WithStringNumberFields(t *testing.T) { } func TestValidate_WithEnabledImportAllECSSchema(t *testing.T) { - finder := packageRootTestFinder{"../../test/packages/other/imported_mappings_tests"} - - validator, err := createValidatorForDirectoryAndPackageRoot("../../test/packages/other/imported_mappings_tests/data_stream/first", - finder, + repositoryRoot, packageRoot, fieldsDir := pathsForValidator(t, "other", "imported_mappings_tests", "first") + validator, err := CreateValidator(repositoryRoot, packageRoot, fieldsDir, WithSpecVersion("2.3.0"), WithEnabledImportAllECSSChema(true)) require.NoError(t, err) @@ -164,10 +141,8 @@ func TestValidate_WithEnabledImportAllECSSchema(t *testing.T) { } func TestValidate_WithDisabledImportAllECSSchema(t *testing.T) { - finder := packageRootTestFinder{"../../test/packages/other/imported_mappings_tests"} - - validator, err := createValidatorForDirectoryAndPackageRoot("../../test/packages/other/imported_mappings_tests/data_stream/first", - finder, + repositoryRoot, packageRoot, fieldsDir := pathsForValidator(t, "other", "imported_mappings_tests", "first") + validator, err := CreateValidator(repositoryRoot, packageRoot, fieldsDir, WithSpecVersion("2.3.0"), WithEnabledImportAllECSSChema(false)) require.NoError(t, err) @@ -186,9 +161,7 @@ func TestValidate_WithDisabledImportAllECSSchema(t *testing.T) { } func TestValidate_constantKeyword(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", WithDisabledDependencyManagement()) - require.NoError(t, err) - require.NotNil(t, validator) + validator := createValidatorForTestdata(t, WithDisabledDependencyManagement()) e := readSampleEvent(t, "testdata/constant-keyword-invalid.json") errs := validator.ValidateDocumentBody(e) @@ -200,9 +173,9 @@ func TestValidate_constantKeyword(t *testing.T) { } func TestValidate_ipAddress(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", WithEnabledAllowedIPCheck(), WithDisabledDependencyManagement()) - require.NoError(t, err) - require.NotNil(t, validator) + validator := createValidatorForTestdata(t, + WithEnabledAllowedIPCheck(), + WithDisabledDependencyManagement()) e := readSampleEvent(t, "testdata/ip-address-forbidden.json") errs := validator.ValidateDocumentBody(e) @@ -215,9 +188,9 @@ func TestValidate_ipAddress(t *testing.T) { } func TestValidate_undefinedArrayOfObjects(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", WithSpecVersion("2.0.0"), WithDisabledDependencyManagement()) - require.NoError(t, err) - require.NotNil(t, validator) + validator := createValidatorForTestdata(t, + WithSpecVersion("2.0.0"), + WithDisabledDependencyManagement()) e := readSampleEvent(t, "testdata/undefined-array-of-objects.json") errs := validator.ValidateDocumentBody(e) @@ -226,8 +199,9 @@ func TestValidate_undefinedArrayOfObjects(t *testing.T) { } func TestValidate_WithSpecVersion(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", WithSpecVersion("2.0.0"), WithDisabledDependencyManagement()) - require.NoError(t, err) + validator := createValidatorForTestdata(t, + WithSpecVersion("2.0.0"), + WithDisabledDependencyManagement()) e := readSampleEvent(t, "testdata/invalid-array-normalization.json") errs := validator.ValidateDocumentBody(e) @@ -239,8 +213,9 @@ func TestValidate_WithSpecVersion(t *testing.T) { require.Empty(t, errs) // Check now that this validation was only enabled for 2.0.0. - validator, err = CreateValidatorForDirectory("testdata", WithSpecVersion("1.99.99"), WithDisabledDependencyManagement()) - require.NoError(t, err) + validator = createValidatorForTestdata(t, + WithSpecVersion("1.99.99"), + WithDisabledDependencyManagement()) e = readSampleEvent(t, "testdata/invalid-array-normalization.json") errs = validator.ValidateDocumentBody(e) @@ -248,9 +223,9 @@ func TestValidate_WithSpecVersion(t *testing.T) { } func TestValidate_ExpectedEventType(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", WithSpecVersion("2.0.0"), WithDisabledDependencyManagement()) - require.NoError(t, err) - require.NotNil(t, validator) + validator := createValidatorForTestdata(t, + WithSpecVersion("2.0.0"), + WithDisabledDependencyManagement()) cases := []struct { title string @@ -337,13 +312,11 @@ func TestValidate_ExpectedEventType(t *testing.T) { } func TestValidate_ExpectedDatasets(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata", + validator := createValidatorForTestdata(t, WithSpecVersion("2.0.0"), WithExpectedDatasets([]string{"apache.status"}), WithDisabledDependencyManagement(), ) - require.NoError(t, err) - require.NotNil(t, validator) cases := []struct { title string @@ -992,8 +965,8 @@ func TestCompareKeys(t *testing.T) { } func TestValidateGeoPoint(t *testing.T) { - validator, err := CreateValidatorForDirectory("../../test/packages/other/fields_tests/data_stream/first", WithDisabledDependencyManagement()) - + repositoryRoot, packageRoot, fieldsDir := pathsForValidator(t, "other", "fields_tests", "first") + validator, err := CreateValidator(repositoryRoot, packageRoot, fieldsDir, WithDisabledDependencyManagement()) require.NoError(t, err) require.NotNil(t, validator) @@ -1003,15 +976,13 @@ func TestValidateGeoPoint(t *testing.T) { } func TestValidateExternalMultiField(t *testing.T) { - packageRoot := "../../test/packages/parallel/mongodb" - dataStreamRoot := filepath.Join(packageRoot, "data_stream", "status") - - validator, err := createValidatorForDirectoryAndPackageRoot(dataStreamRoot, - packageRootTestFinder{packageRoot}) + repositoryRoot, packageRoot, fieldsDir := pathsForValidator(t, "parallel", "mongodb", "status") + validator, err := CreateValidator(repositoryRoot, packageRoot, fieldsDir) require.NoError(t, err) require.NotNil(t, validator) def := FindElementDefinition("process.name", validator.Schema) + require.NotNil(t, def) require.NotEmpty(t, def.MultiFields, "expected to test with a data stream with a field with multifields") e := readSampleEvent(t, "testdata/mongodb-multi-fields.json") @@ -1287,3 +1258,28 @@ func Test_IsAllowedIPValue(t *testing.T) { } } + +func pathsForValidator(t *testing.T, packagesDir, packageName, dataStream string) (*os.Root, string, string) { + t.Helper() + repositoryRoot, err := os.OpenRoot(filepath.Join("..", "..", "test")) + require.NoError(t, err) + t.Cleanup(func() { repositoryRoot.Close() }) + packageRoot := filepath.Join(repositoryRoot.Name(), "packages", packagesDir, packageName) + fieldsDirParent := packageRoot + if dataStream != "" { + fieldsDirParent = filepath.Join(fieldsDirParent, "data_stream", dataStream) + } + fieldsDir := filepath.Join(fieldsDirParent, "fields") + return repositoryRoot, packageRoot, fieldsDir +} + +func createValidatorForTestdata(t *testing.T, opts ...ValidatorOption) *Validator { + t.Helper() + repositoryRoot, err := os.OpenRoot("testdata") + require.NoError(t, err) + t.Cleanup(func() { repositoryRoot.Close() }) + v, err := CreateValidator(repositoryRoot, "testdata", filepath.Join("testdata", "fields"), opts...) + require.NoError(t, err) + require.NotNil(t, v) + return v +} diff --git a/internal/files/linkedfiles.go b/internal/files/linkedfiles.go index ad9c25fa29..53b4f838e5 100644 --- a/internal/files/linkedfiles.go +++ b/internal/files/linkedfiles.go @@ -36,25 +36,16 @@ type PackageLinks struct { } // CreateLinksFSFromPath creates a LinksFS for the given directory within the repository. -// -// - workDir can be an absolute path or a path relative to the repository root. -// in both cases, it must point to a directory within the repository. +// workDir must be a path to a directory, relative to the repository root. func CreateLinksFSFromPath(repositoryRoot *os.Root, workDir string) (*LinksFS, error) { if workDir == "" { return nil, errEmptyWorkDir } - var relWorkDir string - if filepath.IsAbs(workDir) { - var err error - relWorkDir, err = filepath.Rel(repositoryRoot.Name(), workDir) - if err != nil { - return nil, fmt.Errorf("unable to find rel path for %s: %w: %w", workDir, errInvalidWorkDir, err) - } - } else { - relWorkDir = workDir + relWorkDir, err := filepath.Rel(repositoryRoot.Name(), workDir) + if err != nil { + return nil, fmt.Errorf("unable to find rel path for %s: %w: %w", workDir, errInvalidWorkDir, err) } - info, err := repositoryRoot.Stat(relWorkDir) if err != nil { return nil, fmt.Errorf("unable to stat %s: %w: %w", relWorkDir, errInvalidWorkDir, err) @@ -249,10 +240,8 @@ func newLinkedFile(repositoryRoot *os.Root, linkFilePath string) (*Link, error) var includedPackageName string includedPackageRoot, err := packages.FindPackageRootFrom(filepath.Dir(includedFilePath)) - if err != nil { - return nil, fmt.Errorf("could not find package root for included file %s: %w", includedFilePath, err) - } - if includedPackageRoot != "" { + if err == nil && includedPackageRoot != "" { + // Linked file doesn't need to be in a package, so ignore failures looking for it. includedPackageName = filepath.Base(includedPackageRoot) } diff --git a/internal/files/linkedfiles_test.go b/internal/files/linkedfiles_test.go index 3900a425ab..7dba3d0c17 100644 --- a/internal/files/linkedfiles_test.go +++ b/internal/files/linkedfiles_test.go @@ -238,11 +238,8 @@ func TestLinkedFilesByPackageFrom(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = root.Close() }) - fromDir, err := filepath.Rel(root.Name(), basePath) - require.NoError(t, err) - // Create LinksFS - linksFS, err := CreateLinksFSFromPath(root, fromDir) + linksFS, err := CreateLinksFSFromPath(root, basePath) require.NoError(t, err) // Get linked files organized by package @@ -539,11 +536,7 @@ func TestLinksFSSecurityIsolation(t *testing.T) { require.NoError(t, err) defer root.Close() - // Get the relative path from repo root to work directory - relWorkDir, err := filepath.Rel(repoDir, workDir) - require.NoError(t, err) - - lfs, err := CreateLinksFSFromPath(root, relWorkDir) + lfs, err := CreateLinksFSFromPath(root, workDir) require.NoError(t, err) // Test opening the linked file - this should work and use the repository root @@ -693,83 +686,6 @@ func TestLinksFS_Open(t *testing.T) { } } -// TestLinksFS_RelativeWorkDir tests LinksFS with relative workDir paths. -func TestLinksFS_RelativeWorkDir(t *testing.T) { - tempDir := t.TempDir() - - // Create repository structure - repoDir := filepath.Join(tempDir, "repo") - err := os.MkdirAll(repoDir, 0755) - require.NoError(t, err) - - workDir := createPackageStructure(t, repoDir, "work") - - // Create test files - regularFile := filepath.Join(workDir, "regular.txt") - regularContent := "regular file content" - err = os.WriteFile(regularFile, []byte(regularContent), 0644) - require.NoError(t, err) - - includedFile := filepath.Join(workDir, "included.txt") - includedContent := "included file content" - err = os.WriteFile(includedFile, []byte(includedContent), 0644) - require.NoError(t, err) - - // Create link file with correct checksum - linkFile := filepath.Join(workDir, "linked.txt.link") - hash := sha256.Sum256([]byte(includedContent)) - checksum := hex.EncodeToString(hash[:]) - linkContent := fmt.Sprintf("./included.txt %s", checksum) - err = os.WriteFile(linkFile, []byte(linkContent), 0644) - require.NoError(t, err) - - // Setup LinksFS with relative workDir - root, err := os.OpenRoot(repoDir) - require.NoError(t, err) - t.Cleanup(func() { _ = root.Close() }) - - // Use relative path "work" instead of absolute path - lfs, err := CreateLinksFSFromPath(root, "work") - require.NoError(t, err) - - tests := []struct { - name string - fileName string - expectedContent string - expectError bool - }{ - { - name: "read regular file", - fileName: "regular.txt", - expectedContent: regularContent, - }, - { - name: "read linked file returns included content", - fileName: "linked.txt.link", - expectedContent: includedContent, - }, - { - name: "read non-existent file should fail", - fileName: "nonexistent.txt", - expectError: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - content, err := lfs.ReadFile(tc.fileName) - - if tc.expectError { - assert.Error(t, err) - assert.Nil(t, content) - } else { - assert.NoError(t, err) - assert.Equal(t, tc.expectedContent, string(content)) - } - }) - } -} - // TestLinksFS_ErrorConditions tests various error conditions in LinksFS. func TestLinksFS_ErrorConditions(t *testing.T) { tempDir := t.TempDir() @@ -872,7 +788,7 @@ func TestLinksFS_WorkDirValidation(t *testing.T) { }{ { name: "valid relative workDir", - workDir: "inside", + workDir: filepath.Join(repoDir, "inside"), }, { name: "valid absolute workDir", @@ -886,7 +802,7 @@ func TestLinksFS_WorkDirValidation(t *testing.T) { }, { name: "invalid relative workDir escaping repo", - workDir: "../outside", + workDir: filepath.Join(repoDir, "..", "outside"), expectError: true, errorMsg: "path escapes from parent", }, diff --git a/internal/files/repository.go b/internal/files/repository.go index 8c3ab0bd31..83a4b5573f 100644 --- a/internal/files/repository.go +++ b/internal/files/repository.go @@ -14,7 +14,15 @@ import ( ) func FindRepositoryRoot() (*os.Root, error) { - rootPath, err := findRepositoryRootDirectory() + workDir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("locating working directory failed: %w", err) + } + return FindRepositoryRootFrom(workDir) +} + +func FindRepositoryRootFrom(workDir string) (*os.Root, error) { + rootPath, err := findRepositoryRootDirectoryFrom(workDir) if err != nil { return nil, fmt.Errorf("root not found: %w", err) } @@ -28,14 +36,6 @@ func FindRepositoryRoot() (*os.Root, error) { return dirRoot, nil } -func findRepositoryRootDirectory() (string, error) { - workDir, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("locating working directory failed: %w", err) - } - return findRepositoryRootDirectoryFrom(workDir) -} - func findRepositoryRootDirectoryFrom(workDir string) (string, error) { // VolumeName() will return something like "C:" in Windows, and "" in other OSs // rootDir will be something like "C:\" in Windows, and "/" everywhere else. diff --git a/internal/testrunner/coverage.go b/internal/testrunner/coverage.go index dffb93638f..4e98b3ed81 100644 --- a/internal/testrunner/coverage.go +++ b/internal/testrunner/coverage.go @@ -25,6 +25,7 @@ func GenerateBasePackageCoverageReport(packageName, packageRoot, format string) if err != nil { return nil, fmt.Errorf("failed to find repository root directory: %w", err) } + defer root.Close() var coverage CoverageReport err = filepath.WalkDir(packageRoot, func(match string, d fs.DirEntry, err error) error { @@ -73,6 +74,7 @@ func GenerateBaseFileCoverageReport(packageName, path, format string, covered bo if err != nil { return nil, fmt.Errorf("failed to find repository root directory: %w", err) } + defer root.Close() return generateBaseFileCoverageReport(root, packageName, path, format, covered) } @@ -84,6 +86,7 @@ func GenerateBaseFileCoverageReportGlob(packageName string, patterns []string, f if err != nil { return nil, fmt.Errorf("failed to find repository root directory: %w", err) } + defer root.Close() var coverage CoverageReport for _, pattern := range patterns { diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index 2503ee1754..7acdd13a3b 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -44,6 +44,7 @@ func getPipelineCoverage(pkgName string, options PipelineTesterOptions, pipeline if err != nil { return nil, err } + defer repositoryRoot.Close() if options.CoverageType == "cobertura" { pkg := &testrunner.CoberturaPackage{ diff --git a/internal/testrunner/runners/pipeline/tester.go b/internal/testrunner/runners/pipeline/tester.go index c3daa3847a..571b28117a 100644 --- a/internal/testrunner/runners/pipeline/tester.go +++ b/internal/testrunner/runners/pipeline/tester.go @@ -347,7 +347,8 @@ func (r *tester) runTestCase(ctx context.Context, testCaseFile string, dsPath st fields.WithNumericKeywordFields(tc.config.NumericKeywordFields), fields.WithStringNumberFields(tc.config.StringNumberFields), ) - fieldsValidator, err := fields.CreateValidatorForDirectory(dsPath, validatorOptions...) + fieldsDir := filepath.Join(dsPath, "fields") + fieldsValidator, err := fields.CreateValidator(r.repositoryRoot, r.packageRoot, fieldsDir, validatorOptions...) if err != nil { return rc.WithErrorf("creating fields validator for data stream failed (path: %s, test case file: %s): %w", dsPath, testCaseFile, err) } diff --git a/internal/testrunner/runners/static/tester.go b/internal/testrunner/runners/static/tester.go index 3aa0c8972e..1cd7c0001f 100644 --- a/internal/testrunner/runners/static/tester.go +++ b/internal/testrunner/runners/static/tester.go @@ -14,6 +14,7 @@ import ( "github.com/elastic/elastic-package/internal/benchrunner/runners/stream" "github.com/elastic/elastic-package/internal/fields" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/signal" @@ -162,7 +163,14 @@ func (r tester) verifySampleEvent(pkgManifest *packages.PackageManifest) []testr results, _ := resultComposer.WithError(err) return results } - fieldsValidator, err := fields.CreateValidatorForDirectory(filepath.Dir(sampleEventPath), + repositoryRoot, err := files.FindRepositoryRootFrom(r.packageRoot) + if err != nil { + results, _ := resultComposer.WithErrorf("cannot find repository root from %s: %w", r.packageRoot, err) + return results + } + defer repositoryRoot.Close() + fieldsDir := filepath.Join(filepath.Dir(sampleEventPath), "fields") + fieldsValidator, err := fields.CreateValidator(repositoryRoot, r.packageRoot, fieldsDir, fields.WithSpecVersion(pkgManifest.SpecVersion), fields.WithDefaultNumericConversion(), fields.WithExpectedDatasets(expectedDatasets), diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 24146fdf6c..1ede2f5088 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/elastic-package/internal/elasticsearch/ingest" "github.com/elastic/elastic-package/internal/environment" "github.com/elastic/elastic-package/internal/fields" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/formatter" "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" @@ -1660,7 +1661,16 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re logger.Warn("Validation for packages using OpenTelemetry Collector input is experimental") } - fieldsValidator, err := fields.CreateValidatorForDirectory(r.dataStream, + repositoryRoot, err := files.FindRepositoryRootFrom(r.packageRoot) + if err != nil { + return result.WithErrorf("cannot find repository root from %s: %w", r.packageRoot, err) + } + defer repositoryRoot.Close() + fieldsDir := filepath.Join(r.packageRoot, "fields") + if r.dataStream != "" { + fieldsDir = filepath.Join(r.dataStream, "fields") + } + fieldsValidator, err := fields.CreateValidator(repositoryRoot, r.packageRoot, fieldsDir, fields.WithSpecVersion(r.pkgManifest.SpecVersion), fields.WithNumericKeywordFields(config.NumericKeywordFields), fields.WithStringNumberFields(config.StringNumberFields), @@ -1671,7 +1681,7 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re fields.WithOTelValidation(r.isTestUsingOTelCollectorInput(scenario.policyTemplate.Input)), ) if err != nil { - return result.WithErrorf("creating fields validator for data stream failed (path: %s): %w", r.dataStream, err) + return result.WithErrorf("creating fields validator for data stream failed (path: %s): %w", fieldsDir, err) } if errs := validateFields(scenario.docs, fieldsValidator); len(errs) > 0 { @@ -2266,8 +2276,14 @@ func (r *tester) checkTransforms(ctx context.Context, config *testConfig, pkgMan return fmt.Errorf("no documents found in preview for transform %q", transformId) } + repositoryRoot, err := files.FindRepositoryRootFrom(r.packageRoot) + if err != nil { + return fmt.Errorf("cannot find repository root from %s: %w", r.packageRoot, err) + } + defer repositoryRoot.Close() transformRoot := filepath.Dir(transform.Path) - fieldsValidator, err := fields.CreateValidatorForDirectory(transformRoot, + fieldsDir := filepath.Join(transformRoot, "fields") + fieldsValidator, err := fields.CreateValidator(repositoryRoot, r.packageRoot, fieldsDir, fields.WithSpecVersion(pkgManifest.SpecVersion), fields.WithNumericKeywordFields(config.NumericKeywordFields), fields.WithEnabledImportAllECSSChema(true), diff --git a/test/packages/parallel/apache/data_stream/access/fields/agent.yml b/test/packages/parallel/apache/data_stream/access/fields/agent.yml deleted file mode 100644 index 3c8ad89f03..0000000000 --- a/test/packages/parallel/apache/data_stream/access/fields/agent.yml +++ /dev/null @@ -1,200 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/test/packages/parallel/apache/data_stream/access/fields/agent.yml.link b/test/packages/parallel/apache/data_stream/access/fields/agent.yml.link new file mode 100644 index 0000000000..44474c3f7d --- /dev/null +++ b/test/packages/parallel/apache/data_stream/access/fields/agent.yml.link @@ -0,0 +1 @@ +../../../../../shared/agent.yml d8fa2d1b28a9d9832ac2d4012c7154b484071c71ecc8c3225b7bbc5515fe286a \ No newline at end of file diff --git a/test/packages/parallel/apache/docs/README.md b/test/packages/parallel/apache/docs/README.md index da4a12b2eb..c10210e022 100644 --- a/test/packages/parallel/apache/docs/README.md +++ b/test/packages/parallel/apache/docs/README.md @@ -121,14 +121,13 @@ For more information on architectures that can be used for scaling this integrat | apache.access.ssl.cipher | SSL cipher name. | keyword | | apache.access.ssl.protocol | SSL protocol version. | keyword | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | | cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -149,19 +148,16 @@ For more information on architectures that can be used for scaling this integrat | file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | | file.path.text | Multi-field of `file.path`. | match_only_text | | host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | host.os.kernel | Operating system kernel version as a raw string. | keyword | | host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | diff --git a/test/packages/parallel/nginx/data_stream/access/fields/agent.yml b/test/packages/parallel/nginx/data_stream/access/fields/agent.yml deleted file mode 100644 index 3c8ad89f03..0000000000 --- a/test/packages/parallel/nginx/data_stream/access/fields/agent.yml +++ /dev/null @@ -1,200 +0,0 @@ -- name: cloud - title: Cloud - group: 2 - description: Fields related to the cloud or infrastructure the events are coming from. - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - type: group - fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - - name: image.id - type: keyword - description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - - name: containerized - type: boolean - description: > - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - -- name: input.type - type: keyword - description: Input type -- name: log.offset - type: long - description: Log offset diff --git a/test/packages/parallel/nginx/data_stream/access/fields/agent.yml.link b/test/packages/parallel/nginx/data_stream/access/fields/agent.yml.link new file mode 100644 index 0000000000..44474c3f7d --- /dev/null +++ b/test/packages/parallel/nginx/data_stream/access/fields/agent.yml.link @@ -0,0 +1 @@ +../../../../../shared/agent.yml d8fa2d1b28a9d9832ac2d4012c7154b484071c71ecc8c3225b7bbc5515fe286a \ No newline at end of file diff --git a/test/packages/parallel/nginx/docs/README.md b/test/packages/parallel/nginx/docs/README.md index 872c8ce225..3bd28fa010 100644 --- a/test/packages/parallel/nginx/docs/README.md +++ b/test/packages/parallel/nginx/docs/README.md @@ -145,14 +145,13 @@ An example event for `access` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | | cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -168,19 +167,16 @@ An example event for `access` looks as following: | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | +| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | host.os.kernel | Operating system kernel version as a raw string. | keyword | | host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | diff --git a/test/packages/shared/agent.yml b/test/packages/shared/agent.yml new file mode 100644 index 0000000000..3d0e2d2df5 --- /dev/null +++ b/test/packages/shared/agent.yml @@ -0,0 +1,55 @@ +- name: cloud.account.id + external: ecs +- name: cloud.availability_zone + external: ecs +- name: cloud.instance.id + external: ecs +- name: cloud.instance.name + external: ecs +- name: cloud.machine.type + external: ecs +- name: cloud.provider + external: ecs +- name: cloud.region + external: ecs +- name: cloud.project.id + external: ecs +- name: container.id + external: ecs +- name: container.image.name + external: ecs +- name: container.labels + external: ecs +- name: container.name + external: ecs +- name: host.architecture + external: ecs +- name: host.domain + external: ecs +- name: host.hostname + external: ecs +- name: host.id + external: ecs +- name: host.mac + external: ecs +- name: host.name + external: ecs +- name: host.os.family + external: ecs +- name: host.os.kernel + external: ecs +- name: host.os.name + external: ecs +- name: host.os.platform + external: ecs +- name: host.os.version + external: ecs +- name: host.type + external: ecs + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset