Skip to content

Commit 94fcd20

Browse files
committed
Add windows event regex filtering
1 parent 10f9f0a commit 94fcd20

File tree

15 files changed

+545
-21
lines changed

15 files changed

+545
-21
lines changed

cmd/config-translator/translator_test.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,11 @@ func TestLogWindowsEventConfig(t *testing.T) {
123123
expectedErrorMap5["number_any_of"] = 1
124124
checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithMissingEventIdsAndEventLevels.json", false, expectedErrorMap5)
125125

126+
expectedErrorMap6 := map[string]int{}
127+
expectedErrorMap6["invalid_type"] = 1
128+
expectedErrorMap6["enum"] = 1
129+
checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidLogWindowsEventsWithInvalidFilterType.json", false, expectedErrorMap6)
130+
126131
}
127132

128133
func TestMetricsConfig(t *testing.T) {

plugins/inputs/windows_event_log/windows_event_log.go

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,17 @@ const (
2929
var startOnlyOnce sync.Once
3030

3131
type EventConfig struct {
32-
Name string `toml:"event_name"`
33-
Levels []string `toml:"event_levels"`
34-
EventIDs []int `toml:"event_ids"`
35-
RenderFormat string `toml:"event_format"`
36-
BatchReadSize int `toml:"batch_read_size"`
37-
LogGroupName string `toml:"log_group_name"`
38-
LogStreamName string `toml:"log_stream_name"`
39-
LogGroupClass string `toml:"log_group_class"`
40-
Destination string `toml:"destination"`
41-
Retention int `toml:"retention_in_days"`
32+
Name string `toml:"event_name"`
33+
Levels []string `toml:"event_levels"`
34+
EventIDs []int `toml:"event_ids"`
35+
Filters []*wineventlog.EventFilter `toml:"filters"`
36+
RenderFormat string `toml:"event_format"`
37+
BatchReadSize int `toml:"batch_read_size"`
38+
LogGroupName string `toml:"log_group_name"`
39+
LogStreamName string `toml:"log_stream_name"`
40+
LogGroupClass string `toml:"log_group_class"`
41+
Destination string `toml:"destination"`
42+
Retention int `toml:"retention_in_days"`
4243
}
4344
type Plugin struct {
4445
FileStateFolder string `toml:"file_state_folder"`
@@ -108,6 +109,7 @@ func (s *Plugin) Start(acc telegraf.Accumulator) error {
108109
eventConfig.Name,
109110
eventConfig.Levels,
110111
eventConfig.EventIDs,
112+
eventConfig.Filters,
111113
eventConfig.LogGroupName,
112114
eventConfig.LogStreamName,
113115
eventConfig.RenderFormat,
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package wineventlog
5+
6+
import (
7+
"testing"
8+
9+
"github.com/stretchr/testify/assert"
10+
)
11+
12+
func TestEventLogFilterInit(t *testing.T) {
13+
exp := "(foo|bar|baz)"
14+
filter, err := initEventLogFilter(includeFilterType, exp)
15+
assert.NoError(t, err)
16+
assert.NotNil(t, filter.expressionP)
17+
filter, err = initEventLogFilter(excludeFilterType, exp)
18+
assert.NoError(t, err)
19+
assert.NotNil(t, filter.expressionP)
20+
}
21+
22+
func TestLogEventFilterInitInvalidType(t *testing.T) {
23+
_, err := initEventLogFilter("something wrong", "(foo|bar|baz)")
24+
assert.Error(t, err)
25+
}
26+
27+
func TestLogEventFilterInitInvalidRegex(t *testing.T) {
28+
_, err := initEventLogFilter(excludeFilterType, "abc)")
29+
assert.Error(t, err)
30+
}
31+
32+
func TestLogEventFilterShouldPublishInclude(t *testing.T) {
33+
exp := "(foo|bar|baz)"
34+
filter, err := initEventLogFilter(includeFilterType, exp)
35+
assert.NoError(t, err)
36+
37+
assertShouldPublish(t, filter, "foo bar baz")
38+
assertShouldNotPublish(t, filter, "something else")
39+
}
40+
41+
func TestEventLogFilterShouldPublishExclude(t *testing.T) {
42+
exp := "(foo|bar|baz)"
43+
filter, err := initEventLogFilter(excludeFilterType, exp)
44+
assert.NoError(t, err)
45+
46+
assertShouldNotPublish(t, filter, "foo bar baz")
47+
assertShouldPublish(t, filter, "something else")
48+
}
49+
50+
func BenchmarkEventLogFilterShouldPublish(b *testing.B) {
51+
exp := "(foo|bar|baz)"
52+
filter, err := initEventLogFilter(excludeFilterType, exp)
53+
assert.NoError(b, err)
54+
b.ResetTimer()
55+
56+
msg := "foo bar baz"
57+
58+
for i := 0; i < b.N; i++ {
59+
filter.ShouldPublish(msg)
60+
}
61+
}
62+
63+
func BenchmarkEventLogFilterShouldNotPublish(b *testing.B) {
64+
exp := "(foo|bar|baz)"
65+
filter, err := initEventLogFilter(excludeFilterType, exp)
66+
assert.NoError(b, err)
67+
b.ResetTimer()
68+
69+
msg := "something else"
70+
71+
for i := 0; i < b.N; i++ {
72+
filter.ShouldPublish(msg)
73+
}
74+
}
75+
76+
func initEventLogFilter(filterType, expressionStr string) (EventFilter, error) {
77+
filter := EventFilter{
78+
Type: filterType,
79+
Expression: expressionStr,
80+
}
81+
err := filter.init()
82+
return filter, err
83+
}
84+
85+
func assertShouldPublish(t *testing.T, filter EventFilter, msg string) {
86+
res := filter.ShouldPublish(msg)
87+
assert.True(t, res)
88+
}
89+
90+
func assertShouldNotPublish(t *testing.T, filter EventFilter, msg string) {
91+
res := filter.ShouldPublish(msg)
92+
assert.False(t, res)
93+
}
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package wineventlog
5+
6+
import (
7+
"fmt"
8+
"log"
9+
"regexp"
10+
)
11+
12+
const (
13+
includeFilterType = "include"
14+
excludeFilterType = "exclude"
15+
)
16+
17+
var (
18+
validFilterTypes = []string{includeFilterType, excludeFilterType}
19+
validFilterTypesSet = map[string]struct{}{
20+
includeFilterType: {},
21+
excludeFilterType: {},
22+
}
23+
)
24+
25+
type EventFilter struct {
26+
Type string `toml:"type"`
27+
Expression string `toml:"expression"`
28+
expressionP *regexp.Regexp
29+
}
30+
31+
func (filter *EventFilter) init() error {
32+
if _, ok := validFilterTypesSet[filter.Type]; !ok {
33+
return fmt.Errorf("filter type %s is incorrect, valid types are: %v", filter.Type, validFilterTypes)
34+
}
35+
36+
var err error
37+
if filter.expressionP, err = regexp.Compile(filter.Expression); err != nil {
38+
return fmt.Errorf("filter regex has issue, regexp: Compile( %v ): %v", filter.Expression, err.Error())
39+
}
40+
return nil
41+
}
42+
43+
func (filter *EventFilter) ShouldPublish(message string) bool {
44+
if filter.expressionP == nil {
45+
log.Printf("E! [wineventlog] Filter regex is invalid, expression: %s", filter.Expression)
46+
return false
47+
}
48+
match := filter.expressionP.MatchString(message)
49+
return (filter.Type == includeFilterType) == match
50+
}

plugins/inputs/windows_event_log/wineventlog/wineventlog.go

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ type windowsEventLog struct {
5050
name string
5151
levels []string
5252
eventIDs []int
53+
filters []*EventFilter
5354
logGroupName string
5455
logStreamName string
5556
logGroupClass string
@@ -67,11 +68,12 @@ type windowsEventLog struct {
6768
resubscribeCh chan struct{}
6869
}
6970

70-
func NewEventLog(name string, levels []string, eventIDs []int, logGroupName, logStreamName, renderFormat, destination string, stateManager state.FileRangeManager, maximumToRead int, retention int, logGroupClass string) *windowsEventLog {
71+
func NewEventLog(name string, levels []string, eventIDs []int, filters []*EventFilter, logGroupName, logStreamName, renderFormat, destination string, stateManager state.FileRangeManager, maximumToRead int, retention int, logGroupClass string) *windowsEventLog {
7172
eventLog := &windowsEventLog{
7273
name: name,
7374
levels: levels,
7475
eventIDs: eventIDs,
76+
filters: filters,
7577
logGroupName: logGroupName,
7678
logStreamName: logStreamName,
7779
logGroupClass: logGroupClass,
@@ -84,6 +86,7 @@ func NewEventLog(name string, levels []string, eventIDs []int, logGroupName, log
8486
done: make(chan struct{}),
8587
resubscribeCh: make(chan struct{}),
8688
}
89+
8790
return eventLog
8891
}
8992

@@ -94,6 +97,12 @@ func (w *windowsEventLog) Init() error {
9497
}
9598
}
9699

100+
for _, filter := range w.filters {
101+
if err := filter.init(); err != nil {
102+
return err
103+
}
104+
}
105+
97106
go w.stateManager.Run(state.Notification{Done: w.done})
98107
restored, _ := w.stateManager.Restore()
99108
w.eventOffset = restored.Last().EndOffset()
@@ -175,6 +184,19 @@ func (w *windowsEventLog) run() {
175184
log.Printf("E! [wineventlog] Error happened when collecting windows events: %v", err)
176185
continue
177186
}
187+
188+
shouldPublish := true
189+
for _, filter := range w.filters {
190+
if !filter.ShouldPublish(value) {
191+
shouldPublish = false
192+
break
193+
}
194+
}
195+
196+
if !shouldPublish {
197+
continue
198+
}
199+
178200
recordNumber, _ := strconv.ParseUint(record.System.EventRecordID, 10, 64)
179201
r.Shift(recordNumber)
180202
evt := &LogEvent{

plugins/inputs/windows_event_log/wineventlog/wineventlog_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ var (
2525
// 2 is ERROR
2626
LEVELS = []string{"2"}
2727
EVENTIDS = []int{777}
28+
FILTERS = []*EventFilter{}
2829
GROUP_NAME = "fake"
2930
STREAM_NAME = "fake"
3031
RENDER_FMT = FormatPlainText
@@ -184,6 +185,6 @@ func newTestEventLog(t *testing.T, name string, levels []string, eventids []int)
184185
StateFilePrefix: logscommon.WindowsEventLogPrefix,
185186
Name: GROUP_NAME + "_" + STREAM_NAME + "_" + name,
186187
})
187-
return NewEventLog(name, levels, eventids, GROUP_NAME, STREAM_NAME, RENDER_FMT, DEST,
188+
return NewEventLog(name, levels, eventids, FILTERS, GROUP_NAME, STREAM_NAME, RENDER_FMT, DEST,
188189
manager, BATCH_SIZE, RETENTION, LOG_GROUP_CLASS)
189190
}

translator/cmdutil/translatorutil.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,13 @@ func checkSchema(inputJsonMap map[string]interface{}) {
9999
} else {
100100
errorDetails := result.Errors()
101101
for _, errorDetail := range errorDetails {
102-
translator.AddErrorMessages(config.GetFormattedPath(errorDetail.Context().String()), errorDetail.Description())
102+
if errorDetail.Type() == "number_any_of" || errorDetail.Type() == "any_of" {
103+
errDescription := "E! At least one of event_levels, event_ids, or filters is required"
104+
translator.AddErrorMessages(config.GetFormattedPath(errorDetail.Context().String()), errDescription)
105+
log.Panic("E! Invalid Json input schema.")
106+
} else {
107+
translator.AddErrorMessages(config.GetFormattedPath(errorDetail.Context().String()), errorDetail.Description())
108+
}
103109
}
104110
log.Panic("E! Invalid Json input schema.")
105111
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"logs": {
3+
"logs_collected": {
4+
"windows_events": {
5+
"collect_list": [
6+
{
7+
"event_name": "System",
8+
"event_ids": [2300],
9+
"log_group_name": "System",
10+
"log_stream_name": "System",
11+
"filters": [
12+
{
13+
"type": "invalid",
14+
"expression": "(TRACE|DEBUG)"
15+
}
16+
]
17+
},
18+
{
19+
"event_name": "Application",
20+
"event_levels": [
21+
"ERROR"
22+
],
23+
"log_group_name": "Application",
24+
"log_stream_name": "Application",
25+
"filters": [
26+
{
27+
"type": "include",
28+
"expression": 40
29+
}
30+
]
31+
}
32+
]
33+
}
34+
},
35+
"log_stream_name": "LOG_STREAM_NAME"
36+
}
37+
}

translator/config/sampleSchema/validLogWindowsEvents.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,26 @@
1313
"log_stream_name": "System",
1414
"event_format": "xml"
1515
},
16+
{
17+
"event_name": "Security",
18+
"event_ids": [
19+
4624,
20+
4625
21+
],
22+
"log_group_name": "Security",
23+
"log_stream_name": "Security",
24+
"event_format": "text"
25+
},
1626
{
1727
"event_name": "Application",
1828
"event_levels": [
1929
"INFORMATION",
2030
"ERROR"
2131
],
32+
"event_ids": [
33+
1001,
34+
2225
35+
],
2236
"log_group_name": "Application",
2337
"log_stream_name": "Application",
2438
"event_format": "text"

0 commit comments

Comments
 (0)