Skip to content

Commit 92115e8

Browse files
PaamickyPaamicky
authored
Add windows event regex filtering to CWAgent (#1764)
1 parent 86cdb5e commit 92115e8

File tree

16 files changed

+544
-25
lines changed

16 files changed

+544
-25
lines changed

‎.github/workflows/test-artifacts.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1361,4 +1361,4 @@ jobs:
13611361
else
13621362
cd terraform/eks/addon/gpu
13631363
fi
1364-
terraform destroy -auto-approve
1364+
terraform destroy -auto-approve

‎cmd/config-translator/translator_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,17 +111,18 @@ func TestLogWindowsEventConfig(t *testing.T) {
111111
expectedErrorMap3 := map[string]int{}
112112
expectedErrorMap3["enum"] = 1
113113
checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventFormatType.json", false, expectedErrorMap3)
114-
115-
//New tests for event_ids feature
116114
checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventFormatType.json", false, expectedErrorMap3)
117115
expectedErrorMap4 := map[string]int{}
118116
expectedErrorMap4["invalid_type"] = 1
119117
checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidEventIdsType.json", false, expectedErrorMap4)
120-
121118
expectedErrorMap5 := map[string]int{}
122119
expectedErrorMap5["required"] = 1
123120
expectedErrorMap5["number_any_of"] = 1
124121
checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithMissingEventIdsAndEventLevels.json", false, expectedErrorMap5)
122+
expectedErrorMap6 := map[string]int{}
123+
expectedErrorMap6["invalid_type"] = 1
124+
expectedErrorMap6["enum"] = 1
125+
checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidFilterType.json", false, expectedErrorMap6)
125126

126127
}
127128

‎plugins/inputs/windows_event_log/windows_event_log.go‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,17 @@ const (
2929
var startOnlyOnce sync.Once
3030

3131
type EventConfig struct {
32-
Name string `toml:"event_name"`
33-
Levels []string `toml:"event_levels"`
34-
EventIDs []int `toml:"event_ids"`
35-
RenderFormat string `toml:"event_format"`
36-
BatchReadSize int `toml:"batch_read_size"`
37-
LogGroupName string `toml:"log_group_name"`
38-
LogStreamName string `toml:"log_stream_name"`
39-
LogGroupClass string `toml:"log_group_class"`
40-
Destination string `toml:"destination"`
41-
Retention int `toml:"retention_in_days"`
32+
Name string `toml:"event_name"`
33+
Levels []string `toml:"event_levels"`
34+
EventIDs []int `toml:"event_ids"`
35+
Filters []*wineventlog.EventFilter `toml:"filters"`
36+
RenderFormat string `toml:"event_format"`
37+
BatchReadSize int `toml:"batch_read_size"`
38+
LogGroupName string `toml:"log_group_name"`
39+
LogStreamName string `toml:"log_stream_name"`
40+
LogGroupClass string `toml:"log_group_class"`
41+
Destination string `toml:"destination"`
42+
Retention int `toml:"retention_in_days"`
4243
}
4344
type Plugin struct {
4445
FileStateFolder string `toml:"file_state_folder"`
@@ -108,6 +109,7 @@ func (s *Plugin) Start(acc telegraf.Accumulator) error {
108109
eventConfig.Name,
109110
eventConfig.Levels,
110111
eventConfig.EventIDs,
112+
eventConfig.Filters,
111113
eventConfig.LogGroupName,
112114
eventConfig.LogStreamName,
113115
eventConfig.RenderFormat,

‎plugins/inputs/windows_event_log/wineventlog/eventFilter_test.go‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package wineventlog
5+
6+
import (
7+
"testing"
8+
9+
"github.com/stretchr/testify/assert"
10+
)
11+
12+
func TestEventLogFilterInit(t *testing.T) {
13+
exp := "(foo|bar|baz)"
14+
filter, err := initEventLogFilter(includeFilterType, exp)
15+
assert.NoError(t, err)
16+
assert.NotNil(t, filter.expressionP)
17+
filter, err = initEventLogFilter(excludeFilterType, exp)
18+
assert.NoError(t, err)
19+
assert.NotNil(t, filter.expressionP)
20+
}
21+
22+
func TestLogEventFilterInitInvalidType(t *testing.T) {
23+
_, err := initEventLogFilter("something wrong", "(foo|bar|baz)")
24+
assert.Error(t, err)
25+
}
26+
27+
func TestLogEventFilterInitInvalidRegex(t *testing.T) {
28+
_, err := initEventLogFilter(excludeFilterType, "abc)")
29+
assert.Error(t, err)
30+
}
31+
32+
func TestLogEventFilterShouldPublishInclude(t *testing.T) {
33+
exp := "(foo|bar|baz)"
34+
filter, err := initEventLogFilter(includeFilterType, exp)
35+
assert.NoError(t, err)
36+
37+
assertShouldPublish(t, filter, "foo bar baz")
38+
assertShouldNotPublish(t, filter, "something else")
39+
}
40+
41+
func TestEventLogFilterShouldPublishExclude(t *testing.T) {
42+
exp := "(foo|bar|baz)"
43+
filter, err := initEventLogFilter(excludeFilterType, exp)
44+
assert.NoError(t, err)
45+
46+
assertShouldNotPublish(t, filter, "foo bar baz")
47+
assertShouldPublish(t, filter, "something else")
48+
}
49+
50+
func BenchmarkEventLogFilterShouldPublish(b *testing.B) {
51+
exp := "(foo|bar|baz)"
52+
filter, err := initEventLogFilter(excludeFilterType, exp)
53+
assert.NoError(b, err)
54+
b.ResetTimer()
55+
56+
msg := "foo bar baz"
57+
58+
for i := 0; i < b.N; i++ {
59+
filter.ShouldPublish(msg)
60+
}
61+
}
62+
63+
func BenchmarkEventLogFilterShouldNotPublish(b *testing.B) {
64+
exp := "(foo|bar|baz)"
65+
filter, err := initEventLogFilter(excludeFilterType, exp)
66+
assert.NoError(b, err)
67+
b.ResetTimer()
68+
69+
msg := "something else"
70+
71+
for i := 0; i < b.N; i++ {
72+
filter.ShouldPublish(msg)
73+
}
74+
}
75+
76+
func initEventLogFilter(filterType, expressionStr string) (EventFilter, error) {
77+
filter := EventFilter{
78+
Type: filterType,
79+
Expression: expressionStr,
80+
}
81+
err := filter.init()
82+
return filter, err
83+
}
84+
85+
func assertShouldPublish(t *testing.T, filter EventFilter, msg string) {
86+
res := filter.ShouldPublish(msg)
87+
assert.True(t, res)
88+
}
89+
90+
func assertShouldNotPublish(t *testing.T, filter EventFilter, msg string) {
91+
res := filter.ShouldPublish(msg)
92+
assert.False(t, res)
93+
}

‎plugins/inputs/windows_event_log/wineventlog/eventfilter.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package wineventlog
5+
6+
import (
7+
"fmt"
8+
"log"
9+
"regexp"
10+
)
11+
12+
const (
13+
includeFilterType = "include"
14+
excludeFilterType = "exclude"
15+
)
16+
17+
var (
18+
validFilterTypes = []string{includeFilterType, excludeFilterType}
19+
validFilterTypesSet = map[string]struct{}{
20+
includeFilterType: {},
21+
excludeFilterType: {},
22+
}
23+
)
24+
25+
type EventFilter struct {
26+
Type string `toml:"type"`
27+
Expression string `toml:"expression"`
28+
expressionP *regexp.Regexp
29+
}
30+
31+
func (filter *EventFilter) init() error {
32+
if _, ok := validFilterTypesSet[filter.Type]; !ok {
33+
return fmt.Errorf("filter type %s is incorrect, valid types are: %v", filter.Type, validFilterTypes)
34+
}
35+
36+
var err error
37+
if filter.expressionP, err = regexp.Compile(filter.Expression); err != nil {
38+
return fmt.Errorf("filter regex has issue, regexp: Compile( %v ): %v", filter.Expression, err.Error())
39+
}
40+
return nil
41+
}
42+
43+
func (filter *EventFilter) ShouldPublish(message string) bool {
44+
if filter.expressionP == nil {
45+
log.Printf("E! [wineventlog] Filter regex is invalid, expression: %s", filter.Expression)
46+
return false
47+
}
48+
match := filter.expressionP.MatchString(message)
49+
return (filter.Type == includeFilterType) == match
50+
}

‎plugins/inputs/windows_event_log/wineventlog/wineventlog.go‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ type windowsEventLog struct {
5353
name string
5454
levels []string
5555
eventIDs []int
56+
filters []*EventFilter
5657
logGroupName string
5758
logStreamName string
5859
logGroupClass string
@@ -71,11 +72,12 @@ type windowsEventLog struct {
7172
resubscribeCh chan struct{}
7273
}
7374

74-
func NewEventLog(name string, levels []string, eventIDs []int, logGroupName, logStreamName, renderFormat, destination string, stateManager state.FileRangeManager, maximumToRead int, retention int, logGroupClass string) *windowsEventLog {
75+
func NewEventLog(name string, levels []string, eventIDs []int, filters []*EventFilter, logGroupName, logStreamName, renderFormat, destination string, stateManager state.FileRangeManager, maximumToRead int, retention int, logGroupClass string) *windowsEventLog {
7576
eventLog := &windowsEventLog{
7677
name: name,
7778
levels: levels,
7879
eventIDs: eventIDs,
80+
filters: filters,
7981
logGroupName: logGroupName,
8082
logStreamName: logStreamName,
8183
logGroupClass: logGroupClass,
@@ -90,20 +92,26 @@ func NewEventLog(name string, levels []string, eventIDs []int, logGroupName, log
9092
done: make(chan struct{}),
9193
resubscribeCh: make(chan struct{}),
9294
}
95+
9396
return eventLog
9497
}
9598

9699
func (w *windowsEventLog) Init() error {
100+
97101
const (
98102
minEventID = 0
99103
maxEventID = 65535
100104
)
101-
102105
for _, eventID := range w.eventIDs {
103106
if eventID < minEventID || eventID > maxEventID {
104107
return fmt.Errorf("invalid event ID: %d, event IDs must be between %d and %d", eventID, minEventID, maxEventID)
105108
}
106109
}
110+
for _, filter := range w.filters {
111+
if err := filter.init(); err != nil {
112+
return err
113+
}
114+
}
107115

108116
go w.stateManager.Run(state.Notification{Done: w.done})
109117
restored, _ := w.stateManager.Restore()
@@ -201,6 +209,19 @@ func (w *windowsEventLog) run() {
201209
log.Printf("E! [wineventlog] Error happened when collecting windows events: %v", err)
202210
continue
203211
}
212+
213+
shouldPublish := true
214+
for _, filter := range w.filters {
215+
if !filter.ShouldPublish(value) {
216+
shouldPublish = false
217+
break
218+
}
219+
}
220+
221+
if !shouldPublish {
222+
continue
223+
}
224+
204225
recordNumber, _ := strconv.ParseUint(record.System.EventRecordID, 10, 64)
205226
r.Shift(recordNumber)
206227
evt := &LogEvent{

‎plugins/inputs/windows_event_log/wineventlog/wineventlog_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ var (
3232
// 2 is ERROR
3333
LEVELS = []string{"2"}
3434
EVENTIDS = []int{777}
35+
FILTERS = []*EventFilter{}
3536
GROUP_NAME = "fake"
3637
STREAM_NAME = "fake"
3738
RENDER_FMT = FormatPlainText
@@ -393,7 +394,7 @@ func newTestEventLogWithState(t *testing.T, name string, levels []string, rl sta
393394
Name: filepath.Base(file.Name()),
394395
MaxPersistedItems: 10,
395396
})
396-
return NewEventLog(name, levels, EVENTIDS, GROUP_NAME, STREAM_NAME, RENDER_FMT, DEST,
397+
return NewEventLog(name, levels, EVENTIDS, FILTERS, GROUP_NAME, STREAM_NAME, RENDER_FMT, DEST,
397398
manager, BATCH_SIZE, RETENTION, LOG_GROUP_CLASS), file.Name()
398399
}
399400

@@ -404,7 +405,7 @@ func newTestEventLog(t *testing.T, name string, levels []string, eventids []int)
404405
StateFilePrefix: logscommon.WindowsEventLogPrefix,
405406
Name: GROUP_NAME + "_" + STREAM_NAME + "_" + name,
406407
})
407-
return NewEventLog(name, levels, eventids, GROUP_NAME, STREAM_NAME, RENDER_FMT, DEST,
408+
return NewEventLog(name, levels, eventids, FILTERS, GROUP_NAME, STREAM_NAME, RENDER_FMT, DEST,
408409
manager, BATCH_SIZE, RETENTION, LOG_GROUP_CLASS)
409410
}
410411

‎translator/cmdutil/translatorutil.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,11 @@ func checkSchema(inputJsonMap map[string]interface{}) {
9999
} else {
100100
errorDetails := result.Errors()
101101
for _, errorDetail := range errorDetails {
102+
if errorDetail.Type() == "number_any_of" || errorDetail.Type() == "any_of" {
103+
errDescription := "E! At least one of event_levels, event_ids, or filters is required"
104+
translator.AddErrorMessages(config.GetFormattedPath(errorDetail.Context().String()), errDescription)
105+
log.Panic("E! Invalid Json input schema.")
106+
}
102107
translator.AddErrorMessages(config.GetFormattedPath(errorDetail.Context().String()), errorDetail.Description())
103108
}
104109
log.Panic("E! Invalid Json input schema.")

‎translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidFilterType.json‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"logs": {
3+
"logs_collected": {
4+
"windows_events": {
5+
"collect_list": [
6+
{
7+
"event_name": "System",
8+
"event_ids": [2300],
9+
"log_group_name": "System",
10+
"log_stream_name": "System",
11+
"filters": [
12+
{
13+
"type": "invalid",
14+
"expression": "(TRACE|DEBUG)"
15+
}
16+
]
17+
},
18+
{
19+
"event_name": "Application",
20+
"event_levels": [
21+
"ERROR"
22+
],
23+
"log_group_name": "Application",
24+
"log_stream_name": "Application",
25+
"filters": [
26+
{
27+
"type": "include",
28+
"expression": 40
29+
}
30+
]
31+
}
32+
]
33+
}
34+
},
35+
"log_stream_name": "LOG_STREAM_NAME"
36+
}
37+
}

‎translator/config/sampleSchema/validLogWindowsEvents.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,26 @@
1313
"log_stream_name": "System",
1414
"event_format": "xml"
1515
},
16+
{
17+
"event_name": "Security",
18+
"event_ids": [
19+
4624,
20+
4625
21+
],
22+
"log_group_name": "Security",
23+
"log_stream_name": "Security",
24+
"event_format": "text"
25+
},
1626
{
1727
"event_name": "Application",
1828
"event_levels": [
1929
"INFORMATION",
2030
"ERROR"
2131
],
32+
"event_ids": [
33+
1001,
34+
2225
35+
],
2236
"log_group_name": "Application",
2337
"log_stream_name": "Application",
2438
"event_format": "text"

0 commit comments

Comments
 (0)