Skip to content

WIP Add processors groups to Observability Pipelines #3450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

WIP Add processors groups to Observability Pipelines #3450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
275 changes: 210 additions & 65 deletions .generator/schemas/v2/openapi.yaml
View file
Open in desktop

Large diffs are not rendered by default.

70 changes: 54 additions & 16 deletions api/datadogV2/model_observability_pipeline_add_env_vars_processor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ import (

// ObservabilityPipelineAddEnvVarsProcessor The `add_env_vars` processor adds environment variable values to log events.
type ObservabilityPipelineAddEnvVarsProcessor struct {
// Whether this processor is enabled.
Enabled *bool `json:"enabled,omitempty"`
// The unique identifier for this component. Used to reference this processor in the pipeline.
Id string `json:"id"`
// A Datadog search query used to determine which logs this processor targets.
Include string `json:"include"`
// A list of component IDs whose output is used as the input for this processor.
Inputs []string `json:"inputs"`
// A list of component IDs whose output is used as input for this processor. Required when used as a standalone processor, omit when used within a processor group.
Inputs []string `json:"inputs,omitempty"`
// The processor type. The value should always be `add_env_vars`.
Type ObservabilityPipelineAddEnvVarsProcessorType `json:"type"`
// A list of environment variable mappings to apply to log fields.
Expand All @@ -31,11 +33,10 @@ type ObservabilityPipelineAddEnvVarsProcessor struct {
// This constructor will assign default values to properties that have it defined,
// and makes sure properties required by API are set, but the set of arguments
// will change when the set of required properties is changed.
func NewObservabilityPipelineAddEnvVarsProcessor(id string, include string, inputs []string, typeVar ObservabilityPipelineAddEnvVarsProcessorType, variables []ObservabilityPipelineAddEnvVarsProcessorVariable) *ObservabilityPipelineAddEnvVarsProcessor {
func NewObservabilityPipelineAddEnvVarsProcessor(id string, include string, typeVar ObservabilityPipelineAddEnvVarsProcessorType, variables []ObservabilityPipelineAddEnvVarsProcessorVariable) *ObservabilityPipelineAddEnvVarsProcessor {
this := ObservabilityPipelineAddEnvVarsProcessor{}
this.Id = id
this.Include = include
this.Inputs = inputs
this.Type = typeVar
this.Variables = variables
return &this
Expand All @@ -51,6 +52,34 @@ func NewObservabilityPipelineAddEnvVarsProcessorWithDefaults() *ObservabilityPip
return &this
}

// GetEnabled returns the Enabled field value if set, zero value otherwise.
func (o *ObservabilityPipelineAddEnvVarsProcessor) GetEnabled() bool {
if o == nil || o.Enabled == nil {
var ret bool
return ret
}
return *o.Enabled
}

// GetEnabledOk returns a tuple with the Enabled field value if set, nil otherwise
// and a boolean to check if the value has been set.
func (o *ObservabilityPipelineAddEnvVarsProcessor) GetEnabledOk() (*bool, bool) {
if o == nil || o.Enabled == nil {
return nil, false
}
return o.Enabled, true
}

// HasEnabled returns a boolean if a field has been set.
func (o *ObservabilityPipelineAddEnvVarsProcessor) HasEnabled() bool {
return o != nil && o.Enabled != nil
}

// SetEnabled gets a reference to the given bool and assigns it to the Enabled field.
func (o *ObservabilityPipelineAddEnvVarsProcessor) SetEnabled(v bool) {
o.Enabled = &v
}

// GetId returns the Id field value.
func (o *ObservabilityPipelineAddEnvVarsProcessor) GetId() string {
if o == nil {
Expand Down Expand Up @@ -97,25 +126,30 @@ func (o *ObservabilityPipelineAddEnvVarsProcessor) SetInclude(v string) {
o.Include = v
}

// GetInputs returns the Inputs field value.
// GetInputs returns the Inputs field value if set, zero value otherwise.
func (o *ObservabilityPipelineAddEnvVarsProcessor) GetInputs() []string {
if o == nil {
if o == nil || o.Inputs == nil {
var ret []string
return ret
}
return o.Inputs
}

// GetInputsOk returns a tuple with the Inputs field value
// GetInputsOk returns a tuple with the Inputs field value if set, nil otherwise
// and a boolean to check if the value has been set.
func (o *ObservabilityPipelineAddEnvVarsProcessor) GetInputsOk() (*[]string, bool) {
if o == nil {
if o == nil || o.Inputs == nil {
return nil, false
}
return &o.Inputs, true
}

// SetInputs sets field value.
// HasInputs returns a boolean if a field has been set.
func (o *ObservabilityPipelineAddEnvVarsProcessor) HasInputs() bool {
return o != nil && o.Inputs != nil
}

// SetInputs gets a reference to the given []string and assigns it to the Inputs field.
func (o *ObservabilityPipelineAddEnvVarsProcessor) SetInputs(v []string) {
o.Inputs = v
}
Expand Down Expand Up @@ -172,9 +206,14 @@ func (o ObservabilityPipelineAddEnvVarsProcessor) MarshalJSON() ([]byte, error)
if o.UnparsedObject != nil {
return datadog.Marshal(o.UnparsedObject)
}
if o.Enabled != nil {
toSerialize["enabled"] = o.Enabled
}
toSerialize["id"] = o.Id
toSerialize["include"] = o.Include
toSerialize["inputs"] = o.Inputs
if o.Inputs != nil {
toSerialize["inputs"] = o.Inputs
}
toSerialize["type"] = o.Type
toSerialize["variables"] = o.Variables

Expand All @@ -187,9 +226,10 @@ func (o ObservabilityPipelineAddEnvVarsProcessor) MarshalJSON() ([]byte, error)
// UnmarshalJSON deserializes the given payload.
func (o *ObservabilityPipelineAddEnvVarsProcessor) UnmarshalJSON(bytes []byte) (err error) {
all := struct {
Enabled *bool `json:"enabled,omitempty"`
Id *string `json:"id"`
Include *string `json:"include"`
Inputs *[]string `json:"inputs"`
Inputs []string `json:"inputs,omitempty"`
Type *ObservabilityPipelineAddEnvVarsProcessorType `json:"type"`
Variables *[]ObservabilityPipelineAddEnvVarsProcessorVariable `json:"variables"`
}{}
Expand All @@ -202,9 +242,6 @@ func (o *ObservabilityPipelineAddEnvVarsProcessor) UnmarshalJSON(bytes []byte) (
if all.Include == nil {
return fmt.Errorf("required field include missing")
}
if all.Inputs == nil {
return fmt.Errorf("required field inputs missing")
}
if all.Type == nil {
return fmt.Errorf("required field type missing")
}
Expand All @@ -213,15 +250,16 @@ func (o *ObservabilityPipelineAddEnvVarsProcessor) UnmarshalJSON(bytes []byte) (
}
additionalProperties := make(map[string]interface{})
if err = datadog.Unmarshal(bytes, &additionalProperties); err == nil {
datadog.DeleteKeys(additionalProperties, &[]string{"id", "include", "inputs", "type", "variables"})
datadog.DeleteKeys(additionalProperties, &[]string{"enabled", "id", "include", "inputs", "type", "variables"})
} else {
return err
}

hasInvalidField := false
o.Enabled = all.Enabled
o.Id = *all.Id
o.Include = *all.Include
o.Inputs = *all.Inputs
o.Inputs = all.Inputs
if !all.Type.IsValid() {
hasInvalidField = true
} else {
Expand Down
70 changes: 54 additions & 16 deletions api/datadogV2/model_observability_pipeline_add_fields_processor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,16 @@ import (

// ObservabilityPipelineAddFieldsProcessor The `add_fields` processor adds static key-value fields to logs.
type ObservabilityPipelineAddFieldsProcessor struct {
// Whether this processor is enabled.
Enabled *bool `json:"enabled,omitempty"`
// A list of static fields (key-value pairs) that is added to each log event processed by this component.
Fields []ObservabilityPipelineFieldValue `json:"fields"`
// The unique identifier for this component. Used to reference this component in other parts of the pipeline (for example, as the `input` to downstream components).
Id string `json:"id"`
// A Datadog search query used to determine which logs this processor targets.
Include string `json:"include"`
// A list of component IDs whose output is used as the `input` for this component.
Inputs []string `json:"inputs"`
// A list of component IDs whose output is used as input for this processor. Required when used as a standalone processor, omit when used within a processor group.
Inputs []string `json:"inputs,omitempty"`
// The processor type. The value should always be `add_fields`.
Type ObservabilityPipelineAddFieldsProcessorType `json:"type"`
// UnparsedObject contains the raw value of the object if there was an error when deserializing into the struct
Expand All @@ -31,12 +33,11 @@ type ObservabilityPipelineAddFieldsProcessor struct {
// This constructor will assign default values to properties that have it defined,
// and makes sure properties required by API are set, but the set of arguments
// will change when the set of required properties is changed.
func NewObservabilityPipelineAddFieldsProcessor(fields []ObservabilityPipelineFieldValue, id string, include string, inputs []string, typeVar ObservabilityPipelineAddFieldsProcessorType) *ObservabilityPipelineAddFieldsProcessor {
func NewObservabilityPipelineAddFieldsProcessor(fields []ObservabilityPipelineFieldValue, id string, include string, typeVar ObservabilityPipelineAddFieldsProcessorType) *ObservabilityPipelineAddFieldsProcessor {
this := ObservabilityPipelineAddFieldsProcessor{}
this.Fields = fields
this.Id = id
this.Include = include
this.Inputs = inputs
this.Type = typeVar
return &this
}
Expand All @@ -51,6 +52,34 @@ func NewObservabilityPipelineAddFieldsProcessorWithDefaults() *ObservabilityPipe
return &this
}

// GetEnabled returns the Enabled field value if set, zero value otherwise.
func (o *ObservabilityPipelineAddFieldsProcessor) GetEnabled() bool {
if o == nil || o.Enabled == nil {
var ret bool
return ret
}
return *o.Enabled
}

// GetEnabledOk returns a tuple with the Enabled field value if set, nil otherwise
// and a boolean to check if the value has been set.
func (o *ObservabilityPipelineAddFieldsProcessor) GetEnabledOk() (*bool, bool) {
if o == nil || o.Enabled == nil {
return nil, false
}
return o.Enabled, true
}

// HasEnabled returns a boolean if a field has been set.
func (o *ObservabilityPipelineAddFieldsProcessor) HasEnabled() bool {
return o != nil && o.Enabled != nil
}

// SetEnabled gets a reference to the given bool and assigns it to the Enabled field.
func (o *ObservabilityPipelineAddFieldsProcessor) SetEnabled(v bool) {
o.Enabled = &v
}

// GetFields returns the Fields field value.
func (o *ObservabilityPipelineAddFieldsProcessor) GetFields() []ObservabilityPipelineFieldValue {
if o == nil {
Expand Down Expand Up @@ -120,25 +149,30 @@ func (o *ObservabilityPipelineAddFieldsProcessor) SetInclude(v string) {
o.Include = v
}

// GetInputs returns the Inputs field value.
// GetInputs returns the Inputs field value if set, zero value otherwise.
func (o *ObservabilityPipelineAddFieldsProcessor) GetInputs() []string {
if o == nil {
if o == nil || o.Inputs == nil {
var ret []string
return ret
}
return o.Inputs
}

// GetInputsOk returns a tuple with the Inputs field value
// GetInputsOk returns a tuple with the Inputs field value if set, nil otherwise
// and a boolean to check if the value has been set.
func (o *ObservabilityPipelineAddFieldsProcessor) GetInputsOk() (*[]string, bool) {
if o == nil {
if o == nil || o.Inputs == nil {
return nil, false
}
return &o.Inputs, true
}

// SetInputs sets field value.
// HasInputs returns a boolean if a field has been set.
func (o *ObservabilityPipelineAddFieldsProcessor) HasInputs() bool {
return o != nil && o.Inputs != nil
}

// SetInputs gets a reference to the given []string and assigns it to the Inputs field.
func (o *ObservabilityPipelineAddFieldsProcessor) SetInputs(v []string) {
o.Inputs = v
}
Expand Down Expand Up @@ -172,10 +206,15 @@ func (o ObservabilityPipelineAddFieldsProcessor) MarshalJSON() ([]byte, error) {
if o.UnparsedObject != nil {
return datadog.Marshal(o.UnparsedObject)
}
if o.Enabled != nil {
toSerialize["enabled"] = o.Enabled
}
toSerialize["fields"] = o.Fields
toSerialize["id"] = o.Id
toSerialize["include"] = o.Include
toSerialize["inputs"] = o.Inputs
if o.Inputs != nil {
toSerialize["inputs"] = o.Inputs
}
toSerialize["type"] = o.Type

for key, value := range o.AdditionalProperties {
Expand All @@ -187,10 +226,11 @@ func (o ObservabilityPipelineAddFieldsProcessor) MarshalJSON() ([]byte, error) {
// UnmarshalJSON deserializes the given payload.
func (o *ObservabilityPipelineAddFieldsProcessor) UnmarshalJSON(bytes []byte) (err error) {
all := struct {
Enabled *bool `json:"enabled,omitempty"`
Fields *[]ObservabilityPipelineFieldValue `json:"fields"`
Id *string `json:"id"`
Include *string `json:"include"`
Inputs *[]string `json:"inputs"`
Inputs []string `json:"inputs,omitempty"`
Type *ObservabilityPipelineAddFieldsProcessorType `json:"type"`
}{}
if err = datadog.Unmarshal(bytes, &all); err != nil {
Expand All @@ -205,24 +245,22 @@ func (o *ObservabilityPipelineAddFieldsProcessor) UnmarshalJSON(bytes []byte) (e
if all.Include == nil {
return fmt.Errorf("required field include missing")
}
if all.Inputs == nil {
return fmt.Errorf("required field inputs missing")
}
if all.Type == nil {
return fmt.Errorf("required field type missing")
}
additionalProperties := make(map[string]interface{})
if err = datadog.Unmarshal(bytes, &additionalProperties); err == nil {
datadog.DeleteKeys(additionalProperties, &[]string{"fields", "id", "include", "inputs", "type"})
datadog.DeleteKeys(additionalProperties, &[]string{"enabled", "fields", "id", "include", "inputs", "type"})
} else {
return err
}

hasInvalidField := false
o.Enabled = all.Enabled
o.Fields = *all.Fields
o.Id = *all.Id
o.Include = *all.Include
o.Inputs = *all.Inputs
o.Inputs = all.Inputs
if !all.Type.IsValid() {
hasInvalidField = true
} else {
Expand Down
22 changes: 11 additions & 11 deletions api/datadogV2/model_observability_pipeline_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ import (
type ObservabilityPipelineConfig struct {
// A list of destination components where processed logs are sent.
Destinations []ObservabilityPipelineConfigDestinationItem `json:"destinations"`
// A list of processors that transform or enrich log data.
Processors []ObservabilityPipelineConfigProcessorItem `json:"processors,omitempty"`
// A list of processors that transform or enrich log data, or a list of grouped processor configurations.
Processors *ObservabilityPipelineConfigProcessors `json:"processors,omitempty"`
// A list of configured data sources for the pipeline.
Sources []ObservabilityPipelineConfigSourceItem `json:"sources"`
// UnparsedObject contains the raw value of the object if there was an error when deserializing into the struct
Expand Down Expand Up @@ -66,31 +66,31 @@ func (o *ObservabilityPipelineConfig) SetDestinations(v []ObservabilityPipelineC
}

// GetProcessors returns the Processors field value if set, zero value otherwise.
func (o *ObservabilityPipelineConfig) GetProcessors() []ObservabilityPipelineConfigProcessorItem {
func (o *ObservabilityPipelineConfig) GetProcessors() ObservabilityPipelineConfigProcessors {
if o == nil || o.Processors == nil {
var ret []ObservabilityPipelineConfigProcessorItem
var ret ObservabilityPipelineConfigProcessors
return ret
}
return o.Processors
return *o.Processors
}

// GetProcessorsOk returns a tuple with the Processors field value if set, nil otherwise
// and a boolean to check if the value has been set.
func (o *ObservabilityPipelineConfig) GetProcessorsOk() (*[]ObservabilityPipelineConfigProcessorItem, bool) {
func (o *ObservabilityPipelineConfig) GetProcessorsOk() (*ObservabilityPipelineConfigProcessors, bool) {
if o == nil || o.Processors == nil {
return nil, false
}
return &o.Processors, true
return o.Processors, true
}

// HasProcessors returns a boolean if a field has been set.
func (o *ObservabilityPipelineConfig) HasProcessors() bool {
return o != nil && o.Processors != nil
}

// SetProcessors gets a reference to the given []ObservabilityPipelineConfigProcessorItem and assigns it to the Processors field.
func (o *ObservabilityPipelineConfig) SetProcessors(v []ObservabilityPipelineConfigProcessorItem) {
o.Processors = v
// SetProcessors gets a reference to the given ObservabilityPipelineConfigProcessors and assigns it to the Processors field.
func (o *ObservabilityPipelineConfig) SetProcessors(v ObservabilityPipelineConfigProcessors) {
o.Processors = &v
}

// GetSources returns the Sources field value.
Expand Down Expand Up @@ -138,7 +138,7 @@ func (o ObservabilityPipelineConfig) MarshalJSON() ([]byte, error) {
func (o *ObservabilityPipelineConfig) UnmarshalJSON(bytes []byte) (err error) {
all := struct {
Destinations *[]ObservabilityPipelineConfigDestinationItem `json:"destinations"`
Processors []ObservabilityPipelineConfigProcessorItem `json:"processors,omitempty"`
Processors *ObservabilityPipelineConfigProcessors `json:"processors,omitempty"`
Sources *[]ObservabilityPipelineConfigSourceItem `json:"sources"`
}{}
if err = datadog.Unmarshal(bytes, &all); err != nil {
Expand Down
Loading