From 2a84dfdfe3bce7a7363f46e0e1ac9146b4b0a7c8 Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Mon, 14 Jul 2025 11:37:54 +0100 Subject: [PATCH 01/14] multiple syslog server support --- internal/collector/otel_collector_plugin.go | 19 +-- .../collector/otel_collector_plugin_test.go | 9 +- .../datasource/config/nginx_config_parser.go | 63 ++++++-- .../config/nginx_config_parser_test.go | 136 +++++++++++++++--- internal/model/config.go | 18 +-- internal/resource/resource_service_test.go | 14 +- .../nginx-with-multiple-syslog-servers.conf | 50 +++++++ test/config/nginx_config.go | 20 +++ test/model/config.go | 18 +-- 9 files changed, 275 insertions(+), 72 deletions(-) create mode 100644 test/config/nginx/nginx-with-multiple-syslog-servers.conf diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 18c101818..c28acefca 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -543,17 +543,12 @@ func (oc *Collector) updateExistingNginxOSSReceiver( func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfigContext) bool { newTcplogReceiverAdded := false - if nginxConfigContext.NAPSysLogServers != nil { - napLoop: - for _, napSysLogServer := range nginxConfigContext.NAPSysLogServers { - if oc.doesTcplogReceiverAlreadyExist(napSysLogServer) { - continue napLoop - } - + if nginxConfigContext.NAPSysLogServer != "" { + if !oc.doesTcplogReceiverAlreadyExist(nginxConfigContext.NAPSysLogServer) { oc.config.Collector.Receivers.TcplogReceivers = append( oc.config.Collector.Receivers.TcplogReceivers, config.TcplogReceiver{ - ListenAddress: napSysLogServer, + ListenAddress: nginxConfigContext.NAPSysLogServer, Operators: []config.Operator{ { Type: "add", @@ -621,12 +616,10 @@ func (oc *Collector) configDeletedNapReceivers(nginxConfigContext *model.NginxCo elements[tcplogReceiver.ListenAddress] = true } - if nginxConfigContext.NAPSysLogServers != nil { + if nginxConfigContext.NAPSysLogServer != "" { addressesToDelete := make(map[string]bool) - for _, napAddress := range nginxConfigContext.NAPSysLogServers { - if !elements[napAddress] { - addressesToDelete[napAddress] = true - } + if !elements[nginxConfigContext.NAPSysLogServer] { + addressesToDelete[nginxConfigContext.NAPSysLogServer] = true } return addressesToDelete diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index a767c62cc..201262dce 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -734,9 +734,7 @@ func TestCollector_updateTcplogReceivers(t *testing.T) { require.NoError(t, err) nginxConfigContext := &model.NginxConfigContext{ - NAPSysLogServers: []string{ - "localhost:151", - }, + NAPSysLogServer: "localhost:151", } assert.Empty(t, conf.Collector.Receivers.TcplogReceivers) @@ -767,9 +765,8 @@ func TestCollector_updateTcplogReceivers(t *testing.T) { }) t.Run("Test 4: New tcplogReceiver added and deleted another", func(tt *testing.T) { - tcplogReceiverDeleted := collector.updateTcplogReceivers(&model.NginxConfigContext{NAPSysLogServers: []string{ - "localhost:152", - }}) + tcplogReceiverDeleted := collector. + updateTcplogReceivers(&model.NginxConfigContext{NAPSysLogServer: "localhost:152"}) assert.True(t, tcplogReceiverDeleted) assert.Len(t, conf.Collector.Receivers.TcplogReceivers, 1) assert.Equal(t, "localhost:152", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) diff --git a/internal/datasource/config/nginx_config_parser.go b/internal/datasource/config/nginx_config_parser.go index 949deb781..00bac0e28 100644 --- a/internal/datasource/config/nginx_config_parser.go +++ b/internal/datasource/config/nginx_config_parser.go @@ -173,18 +173,16 @@ func (ncp *NginxConfigParser) createNginxConfigContext( } case "app_protect_security_log": if len(directive.Args) > 1 { - syslogArg := directive.Args[1] - re := regexp.MustCompile(`syslog:server=([\S]+)`) - matches := re.FindStringSubmatch(syslogArg) - if len(matches) > 1 { - syslogServer := matches[1] - if !napSyslogServersFound[syslogServer] { - nginxConfigContext.NAPSysLogServers = append( - nginxConfigContext.NAPSysLogServers, - syslogServer, - ) - napSyslogServersFound[syslogServer] = true - slog.DebugContext(ctx, "Found NAP syslog server", "address", syslogServer) + slog.Info("args", "", directive.Args) + sysLogServers := ncp.findValidSysLogServers(directive.Args) + if len(sysLogServers) == 0 { + slog.WarnContext(ctx, "Could not find valid Nap Syslog server") + } + for i := range sysLogServers { + sysLogServer := sysLogServers[i] + if !napSyslogServersFound[sysLogServer] { + napSyslogServersFound[sysLogServer] = true + slog.DebugContext(ctx, "Found NAP syslog server", "address", sysLogServer) } } } @@ -207,6 +205,13 @@ func (ncp *NginxConfigParser) createNginxConfigContext( nginxConfigContext.PlusAPI = plusAPI } + if len(napSyslogServersFound) > 0 { + syslogServer := ncp.parseSyslogDirective(ctx, napSyslogServersFound) + if syslogServer != "" { + nginxConfigContext.NAPSysLogServer = syslogServer + } + } + fileMeta, err := files.FileMeta(conf.File) if err != nil { slog.WarnContext(ctx, "Unable to get file metadata", "file_name", conf.File, "error", err) @@ -218,6 +223,40 @@ func (ncp *NginxConfigParser) createNginxConfigContext( return nginxConfigContext, nil } +func (ncp *NginxConfigParser) parseSyslogDirective(ctx context.Context, napSyslogServers map[string]bool) string { + for napSyslogServer := range napSyslogServers { + ln, err := net.Listen("tcp", napSyslogServer) + if err != nil { + slog.WarnContext(ctx, "NAP syslog server is not reachable", "address", napSyslogServer, + "error", err) + + continue + } + ln.Close() + slog.DebugContext(ctx, "Found valid NAP syslog server", "address", napSyslogServer) + + return napSyslogServer + } + slog.WarnContext(ctx, "Could not find usable NAP syslog server") + + return "" +} + +func (ncp *NginxConfigParser) findValidSysLogServers(sysLogServers []string) []string { + re := regexp.MustCompile(`syslog:server=([\S]+)`) + var servers []string + for i := range sysLogServers { + matches := re.FindStringSubmatch(sysLogServers[i]) + if len(matches) > 1 { + if strings.HasPrefix(matches[1], "localhost") || strings.HasPrefix(matches[1], "127.0.0.1") { + servers = append(servers, matches[1]) + } + } + } + + return servers +} + func (ncp *NginxConfigParser) parseIncludeDirective(directive *crossplane.Directive) string { var include string if filepath.IsAbs(directive.Args[0]) { diff --git a/internal/datasource/config/nginx_config_parser_test.go b/internal/datasource/config/nginx_config_parser_test.go index 348d69de7..7088660fe 100644 --- a/internal/datasource/config/nginx_config_parser_test.go +++ b/internal/datasource/config/nginx_config_parser_test.go @@ -9,6 +9,7 @@ import ( "bytes" "context" "fmt" + "net" "net/http" "net/http/httptest" "os" @@ -357,7 +358,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { ltsvAccessLog.Name(), errorLog.Name(), protos.NginxOssInstance([]string{}).GetInstanceMeta().GetInstanceId(), - []string{"127.0.0.1:1515"}, + "127.0.0.1:1515", ), expectedLog: "", allowedDirectories: []string{dir}, @@ -377,7 +378,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { ltsvAccessLog.Name(), errorLog.Name(), protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), - []string{"127.0.0.1:1515"}, + "127.0.0.1:1515", ), expectedLog: "", allowedDirectories: []string{dir}, @@ -392,7 +393,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { errorLog.Name(), []*mpi.File{&allowedFileWithMetas}, protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), - nil, + "", ), expectedLog: "", allowedDirectories: []string{dir}, @@ -402,13 +403,13 @@ func TestNginxConfigParser_Parse(t *testing.T) { instance: protos.NginxPlusInstance([]string{}), content: testconfig.NginxConfWithSSLCertsWithVariables(), expectedConfigContext: &model.NginxConfigContext{ - StubStatus: &model.APIDetails{}, - PlusAPI: &model.APIDetails{}, - InstanceID: protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), - Files: []*mpi.File{}, - AccessLogs: []*model.AccessLog{}, - ErrorLogs: []*model.ErrorLog{}, - NAPSysLogServers: nil, + StubStatus: &model.APIDetails{}, + PlusAPI: &model.APIDetails{}, + InstanceID: protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), + Files: []*mpi.File{}, + AccessLogs: []*model.AccessLog{}, + ErrorLogs: []*model.ErrorLog{}, + NAPSysLogServer: "", }, allowedDirectories: []string{dir}, }, @@ -426,7 +427,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { combinedAccessLog.Name(), ltsvAccessLog.Name(), protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), - []string{"127.0.0.1:1515"}, + "127.0.0.1:1515", ), expectedLog: "Currently error log outputs to stderr. Log monitoring is disabled while applying a " + "config; log errors to file to enable error monitoring", @@ -446,7 +447,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { combinedAccessLog.Name(), ltsvAccessLog.Name(), protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), - []string{"127.0.0.1:1515"}, + "127.0.0.1:1515", ), expectedLog: "Currently error log outputs to stdout. Log monitoring is disabled while applying a " + "config; log errors to file to enable error monitoring", @@ -465,7 +466,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { errorLog.Name(), []*mpi.File{&certFileWithMetas}, protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), - nil, + "", ), allowedDirectories: []string{dir}, }, @@ -483,7 +484,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { errorLog.Name(), []*mpi.File{&diffCertFileWithMetas, &certFileWithMetas}, protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), - nil, + "", ), allowedDirectories: []string{dir}, }, @@ -501,7 +502,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { errorLog.Name(), []*mpi.File{&certFileWithMetas}, protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), - nil, + "", ), allowedDirectories: []string{dir}, }, @@ -532,6 +533,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { result, parseError := nginxConfig.Parse(ctx, test.instance) require.NoError(t, parseError) + t.Logf("Log: %s", logBuf.String()) helpers.ValidateLog(t, test.expectedLog, logBuf) logBuf.Reset() @@ -548,7 +550,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { assert.Truef(t, protoListEqual(test.expectedConfigContext.Files, result.Files), "Expect %s Got %s", test.expectedConfigContext.Files, result.Files) - assert.Equal(t, test.expectedConfigContext.NAPSysLogServers, result.NAPSysLogServers) + assert.Equal(t, test.expectedConfigContext.NAPSysLogServer, result.NAPSysLogServer) assert.Equal(t, test.expectedConfigContext.PlusAPI, result.PlusAPI) assert.ElementsMatch(t, test.expectedConfigContext.AccessLogs, result.AccessLogs) assert.ElementsMatch(t, test.expectedConfigContext.ErrorLogs, result.ErrorLogs) @@ -582,6 +584,107 @@ func TestNginxConfigParser_sslCert(t *testing.T) { assert.Equal(t, certFile, sslCert.GetFileMeta().GetName()) } +func TestNginxConfigParser_SyslogServerParse(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + + file := helpers.CreateFileWithErrorCheck(t, dir, "nginx-parse-config.conf") + defer helpers.RemoveFileWithErrorCheck(t, file.Name()) + + errorLog := helpers.CreateFileWithErrorCheck(t, dir, "error.log") + defer helpers.RemoveFileWithErrorCheck(t, errorLog.Name()) + + accessLog := helpers.CreateFileWithErrorCheck(t, dir, "access.log") + defer helpers.RemoveFileWithErrorCheck(t, accessLog.Name()) + + instance := protos.NginxOssInstance([]string{}) + instance.InstanceRuntime.ConfigPath = file.Name() + + tests := []struct { + name string + content string + expectedLog string + expectedSyslogServers string + portInUse bool + }{ + { + name: "Test 1: Valid port", + expectedSyslogServers: "127.0.0.1:1515", + content: testconfig.NginxConfigWithMultipleSysLogs(errorLog.Name(), accessLog.Name(), + "192.168.12.34:1517", "my.domain.com:1517", "127.0.0.1:1515"), + expectedLog: "Found valid NAP syslog server", + portInUse: false, + }, + { + name: "Test 2: No valid server", + expectedSyslogServers: "", + content: testconfig.NginxConfigWithMultipleSysLogs(errorLog.Name(), accessLog.Name(), + "random.domain:1515", "192.168.12.34:1517", "my.domain.com:1517"), + expectedLog: "Could not find valid Nap Syslog server", + portInUse: false, + }, + { + name: "Test 3: Port unavailable, use next valid sever", + expectedSyslogServers: "localhost:1516", + content: testconfig.NginxConfigWithMultipleSysLogs(errorLog.Name(), accessLog.Name(), + "192.168.12.34:1517", "127.0.0.1:1515", "localhost:1516"), + expectedLog: "NAP syslog server is not reachable", + portInUse: true, + }, + { + name: "Test 4: Port unavailable, no server available", + expectedSyslogServers: "", + content: testconfig.NginxConfigWithMultipleSysLogs(errorLog.Name(), accessLog.Name(), + "my.domain.com:1517", "127.0.0.1:1515", "my.domain.com:1517"), + expectedLog: "Could not find usable NAP syslog server", + portInUse: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + logBuf := &bytes.Buffer{} + stub.StubLoggerWith(logBuf) + + agentConfig := types.AgentConfig() + agentConfig.AllowedDirectories = []string{dir} + nginxConfig := NewNginxConfigParser(agentConfig) + + writeErr := os.WriteFile(file.Name(), []byte(test.content), 0o600) + require.NoError(t, writeErr) + + if test.portInUse { + ln, err := net.Listen("tcp", "127.0.0.1:1515") + require.NoError(t, err) + defer ln.Close() + } + + result, parseError := nginxConfig.Parse(ctx, instance) + require.NoError(t, parseError) + + t.Logf("Log: %s", logBuf.String()) + helpers.ValidateLog(t, test.expectedLog, logBuf) + logBuf.Reset() + + assert.Equal(t, test.expectedSyslogServers, result.NAPSysLogServer) + }) + } +} + +func TestNginxConfigParser_findValidSysLogServers(t *testing.T) { + servers := []string{ + "/etc/app_protect/conf/log_default.json", "syslog:server=192.168.12.34:1517", + "app_protect_security_log", "/etc/app_protect/conf/log_default1.json", "syslog:server=my.domain.com:1517", + "app_protect_security_log", "/etc/app_protect/conf/log_default2.json", "syslog:server=127.0.0.1:1515", + "app_protect_security_log", "/etc/app_protect/conf/log_default3.json", "syslog:server=localhost:1516", + } + expected := []string{"127.0.0.1:1515", "localhost:1516"} + ncp := NewNginxConfigParser(types.AgentConfig()) + result := ncp.findValidSysLogServers(servers) + + assert.Equal(t, expected, result) +} + func TestNginxConfigParser_checkLog(t *testing.T) { logBuf := &bytes.Buffer{} stub.StubLoggerWith(logBuf) @@ -1067,6 +1170,7 @@ func TestNginxConfigParser_pingAPIEndpoint_PlusAPI(t *testing.T) { }) fakeServer := httptest.NewServer(handler) + t.Logf(fakeServer.URL) defer fakeServer.Close() nginxConfigParser := NewNginxConfigParser(types.AgentConfig()) diff --git a/internal/model/config.go b/internal/model/config.go index ed1d92487..67dea7449 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -12,14 +12,14 @@ import ( ) type NginxConfigContext struct { - StubStatus *APIDetails - PlusAPI *APIDetails - InstanceID string - Files []*v1.File - AccessLogs []*AccessLog - ErrorLogs []*ErrorLog - NAPSysLogServers []string - Includes []string + StubStatus *APIDetails + PlusAPI *APIDetails + InstanceID string + Files []*v1.File + AccessLogs []*AccessLog + ErrorLogs []*ErrorLog + NAPSysLogServer string + Includes []string } type APIDetails struct { @@ -113,7 +113,7 @@ func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext return false } - if !reflect.DeepEqual(ncc.NAPSysLogServers, otherNginxConfigContext.NAPSysLogServers) { + if !reflect.DeepEqual(ncc.NAPSysLogServer, otherNginxConfigContext.NAPSysLogServer) { return false } diff --git a/internal/resource/resource_service_test.go b/internal/resource/resource_service_test.go index b60af6d23..e9651013a 100644 --- a/internal/resource/resource_service_test.go +++ b/internal/resource/resource_service_test.go @@ -336,13 +336,13 @@ func TestResourceService_ApplyConfig(t *testing.T) { nginxParser := instancefakes.FakeNginxConfigParser{} nginxParser.ParseReturns(&model.NginxConfigContext{ - StubStatus: &model.APIDetails{}, - PlusAPI: &model.APIDetails{}, - InstanceID: test.instanceID, - Files: nil, - AccessLogs: nil, - ErrorLogs: nil, - NAPSysLogServers: nil, + StubStatus: &model.APIDetails{}, + PlusAPI: &model.APIDetails{}, + InstanceID: test.instanceID, + Files: nil, + AccessLogs: nil, + ErrorLogs: nil, + NAPSysLogServer: "", }, nil) resourceService := NewResourceService(ctx, types.AgentConfig()) diff --git a/test/config/nginx/nginx-with-multiple-syslog-servers.conf b/test/config/nginx/nginx-with-multiple-syslog-servers.conf new file mode 100644 index 000000000..aaccfe109 --- /dev/null +++ b/test/config/nginx/nginx-with-multiple-syslog-servers.conf @@ -0,0 +1,50 @@ +user nginx; +worker_processes auto; + +error_log %s notice; +pid /var/run/nginx.pid; + +load_module modules/ngx_http_app_protect_module.so; + +events { + worker_connections 1024; +} + +http { + log_format upstream_time '$remote_addr - $remote_user [$time_local]'; + + server { + access_log %s upstream_time; + } +} + +http { + log_format ltsv "time:$time_local" + "\thost:$remote_addr" + "\tmethod:$request_method" + "\turi:$request_uri" + "\tprotocol:$server_protocol" + "\tstatus:$status" + "\tsize:$body_bytes_sent" + "\treferer:$http_referer" + "\tua:$http_user_agent" + "\treqtime:$request_time" + "\tapptime:$upstream_response_time"; + + server { + listen 9093; + server_name lua.example.com; + + ssl_certificate_by_lua_block { + print("Test lua block") + } + } + + server { + + app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=%s + app_protect_security_log "/etc/app_protect/conf/log_default1.json" syslog:server=%s + app_protect_security_log "/etc/app_protect/conf/log_default2.json" syslog:server=%s + app_protect_security_log "/etc/app_protect/conf/log_default3.json" syslog:server=%s + } +} diff --git a/test/config/nginx_config.go b/test/config/nginx_config.go index e5b3299e4..8eba52461 100644 --- a/test/config/nginx_config.go +++ b/test/config/nginx_config.go @@ -13,6 +13,9 @@ import ( //go:embed nginx/nginx-with-multiple-access-logs.conf var embedNginxConfWithMultipleAccessLogs string +//go:embed nginx/nginx-with-multiple-syslog-servers.conf +var embedNginxConfWithMultipleSysLogs string + //go:embed nginx/nginx-not-allowed-dir.conf var embedNginxConfWithNotAllowedDir string @@ -46,6 +49,23 @@ func NginxConfigWithMultipleAccessLogs( ) } +func NginxConfigWithMultipleSysLogs( + errorLogName, + accessLogName, + syslogServer, + syslogServer3, + syslogServer4 string, +) string { + return fmt.Sprintf( + embedNginxConfWithMultipleSysLogs, + errorLogName, + accessLogName, + syslogServer, + syslogServer3, + syslogServer4, + ) +} + func NginxConfigWithNotAllowedDir(errorLogFile, notAllowedFile, allowedFileDir, accessLogFile string) string { return fmt.Sprintf(embedNginxConfWithNotAllowedDir, errorLogFile, notAllowedFile, allowedFileDir, accessLogFile) } diff --git a/test/model/config.go b/test/model/config.go index 07c6c53c3..4b5e69dc1 100644 --- a/test/model/config.go +++ b/test/model/config.go @@ -29,7 +29,7 @@ func ConfigContextWithNames( ltsvAccessLogName, errorLogName string, instanceID string, - syslogServers []string, + syslogServers string, ) *model.NginxConfigContext { return &model.NginxConfigContext{ StubStatus: &model.APIDetails{ @@ -71,8 +71,8 @@ func ConfigContextWithNames( Permissions: "0600", }, }, - InstanceID: instanceID, - NAPSysLogServers: syslogServers, + InstanceID: instanceID, + NAPSysLogServer: syslogServers, } } @@ -81,7 +81,7 @@ func ConfigContextWithoutErrorLog( combinedAccessLogName, ltsvAccessLogName, instanceID string, - syslogServers []string, + syslogServers string, ) *model.NginxConfigContext { return &model.NginxConfigContext{ StubStatus: &model.APIDetails{ @@ -115,8 +115,8 @@ func ConfigContextWithoutErrorLog( Permissions: "0600", }, }, - InstanceID: instanceID, - NAPSysLogServers: syslogServers, + InstanceID: instanceID, + NAPSysLogServer: syslogServers, } } @@ -125,7 +125,7 @@ func ConfigContextWithFiles( errorLogName string, files []*mpi.File, instanceID string, - syslogServers []string, + syslogServers string, ) *model.NginxConfigContext { return &model.NginxConfigContext{ StubStatus: &model.APIDetails{ @@ -156,7 +156,7 @@ func ConfigContextWithFiles( Permissions: "0600", }, }, - InstanceID: instanceID, - NAPSysLogServers: syslogServers, + InstanceID: instanceID, + NAPSysLogServer: syslogServers, } } From 814884c154348dad39b96abd7db893f758713cc9 Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Mon, 14 Jul 2025 11:48:16 +0100 Subject: [PATCH 02/14] log message --- internal/datasource/config/nginx_config_parser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/datasource/config/nginx_config_parser.go b/internal/datasource/config/nginx_config_parser.go index 00bac0e28..16bf04df7 100644 --- a/internal/datasource/config/nginx_config_parser.go +++ b/internal/datasource/config/nginx_config_parser.go @@ -176,7 +176,7 @@ func (ncp *NginxConfigParser) createNginxConfigContext( slog.Info("args", "", directive.Args) sysLogServers := ncp.findValidSysLogServers(directive.Args) if len(sysLogServers) == 0 { - slog.WarnContext(ctx, "Could not find valid Nap Syslog server") + slog.WarnContext(ctx, "Could not find usable NAP syslog server, security violations will be unavailable") } for i := range sysLogServers { sysLogServer := sysLogServers[i] @@ -237,7 +237,7 @@ func (ncp *NginxConfigParser) parseSyslogDirective(ctx context.Context, napSyslo return napSyslogServer } - slog.WarnContext(ctx, "Could not find usable NAP syslog server") + slog.WarnContext(ctx, "Could not find usable NAP syslog server, security violations will be unavailable") return "" } From 26fcdfa390c11db2f2a3a515d2863ec4f6e80b8e Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Mon, 14 Jul 2025 14:22:45 +0100 Subject: [PATCH 03/14] fix port already in use --- .../datasource/config/nginx_config_parser.go | 16 +++++++++++--- .../config/nginx_config_parser_test.go | 21 ++++++++++++++----- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/internal/datasource/config/nginx_config_parser.go b/internal/datasource/config/nginx_config_parser.go index 16bf04df7..fab24aa0d 100644 --- a/internal/datasource/config/nginx_config_parser.go +++ b/internal/datasource/config/nginx_config_parser.go @@ -47,7 +47,8 @@ const ( type ( NginxConfigParser struct { - agentConfig *config.Config + agentConfig *config.Config + previousNAPSysLogServer string } ) @@ -65,7 +66,8 @@ type ( func NewNginxConfigParser(agentConfig *config.Config) *NginxConfigParser { return &NginxConfigParser{ - agentConfig: agentConfig, + agentConfig: agentConfig, + previousNAPSysLogServer: "", } } @@ -176,7 +178,8 @@ func (ncp *NginxConfigParser) createNginxConfigContext( slog.Info("args", "", directive.Args) sysLogServers := ncp.findValidSysLogServers(directive.Args) if len(sysLogServers) == 0 { - slog.WarnContext(ctx, "Could not find usable NAP syslog server, security violations will be unavailable") + slog.WarnContext(ctx, "Could not find usable NAP syslog server, "+ + "security violations will be unavailable") } for i := range sysLogServers { sysLogServer := sysLogServers[i] @@ -209,6 +212,7 @@ func (ncp *NginxConfigParser) createNginxConfigContext( syslogServer := ncp.parseSyslogDirective(ctx, napSyslogServersFound) if syslogServer != "" { nginxConfigContext.NAPSysLogServer = syslogServer + ncp.previousNAPSysLogServer = syslogServer } } @@ -224,6 +228,12 @@ func (ncp *NginxConfigParser) createNginxConfigContext( } func (ncp *NginxConfigParser) parseSyslogDirective(ctx context.Context, napSyslogServers map[string]bool) string { + if ncp.previousNAPSysLogServer != "" { + if _, ok := napSyslogServers[ncp.previousNAPSysLogServer]; ok { + return ncp.previousNAPSysLogServer + } + } + for napSyslogServer := range napSyslogServers { ln, err := net.Listen("tcp", napSyslogServer) if err != nil { diff --git a/internal/datasource/config/nginx_config_parser_test.go b/internal/datasource/config/nginx_config_parser_test.go index 7088660fe..33eb3b9b2 100644 --- a/internal/datasource/config/nginx_config_parser_test.go +++ b/internal/datasource/config/nginx_config_parser_test.go @@ -601,11 +601,12 @@ func TestNginxConfigParser_SyslogServerParse(t *testing.T) { instance.InstanceRuntime.ConfigPath = file.Name() tests := []struct { - name string - content string - expectedLog string - expectedSyslogServers string - portInUse bool + name string + content string + expectedLog string + expectedSyslogServers string + previousNAPSysLogServer string + portInUse bool }{ { name: "Test 1: Valid port", @@ -639,6 +640,15 @@ func TestNginxConfigParser_SyslogServerParse(t *testing.T) { expectedLog: "Could not find usable NAP syslog server", portInUse: true, }, + { + name: "Test 5: Server hasn't changed", + expectedSyslogServers: "127.0.0.1:1515", + content: testconfig.NginxConfigWithMultipleSysLogs(errorLog.Name(), accessLog.Name(), + "my.domain.com:1517", "127.0.0.1:1515", "my.domain.com:1517"), + expectedLog: "Found NAP syslog server", + portInUse: true, + previousNAPSysLogServer: "127.0.0.1:1515", + }, } for _, test := range tests { @@ -649,6 +659,7 @@ func TestNginxConfigParser_SyslogServerParse(t *testing.T) { agentConfig := types.AgentConfig() agentConfig.AllowedDirectories = []string{dir} nginxConfig := NewNginxConfigParser(agentConfig) + nginxConfig.previousNAPSysLogServer = test.previousNAPSysLogServer writeErr := os.WriteFile(file.Name(), []byte(test.content), 0o600) require.NoError(t, writeErr) From fe8b9cb161aa381bf772be4217a425abbce12aaf Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Mon, 14 Jul 2025 14:56:10 +0100 Subject: [PATCH 04/14] fix linter --- internal/datasource/config/nginx_config_parser_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/datasource/config/nginx_config_parser_test.go b/internal/datasource/config/nginx_config_parser_test.go index f41cb3e0c..5afba0941 100644 --- a/internal/datasource/config/nginx_config_parser_test.go +++ b/internal/datasource/config/nginx_config_parser_test.go @@ -1181,7 +1181,6 @@ func TestNginxConfigParser_pingAPIEndpoint_PlusAPI(t *testing.T) { }) fakeServer := httptest.NewServer(handler) - t.Logf(fakeServer.URL) defer fakeServer.Close() nginxConfigParser := NewNginxConfigParser(types.AgentConfig()) From 4cc242f686c6a7f749f0df1ca32122ed781df757 Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Mon, 14 Jul 2025 15:18:14 +0100 Subject: [PATCH 05/14] fix test --- internal/datasource/config/nginx_config_parser.go | 1 - internal/datasource/config/nginx_config_parser_test.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/datasource/config/nginx_config_parser.go b/internal/datasource/config/nginx_config_parser.go index db627fabf..ec5037030 100644 --- a/internal/datasource/config/nginx_config_parser.go +++ b/internal/datasource/config/nginx_config_parser.go @@ -175,7 +175,6 @@ func (ncp *NginxConfigParser) createNginxConfigContext( } case "app_protect_security_log": if len(directive.Args) > 1 { - slog.Info("args", "", directive.Args) sysLogServers := ncp.findValidSysLogServers(directive.Args) if len(sysLogServers) == 0 { slog.WarnContext(ctx, "Could not find usable NAP syslog server, "+ diff --git a/internal/datasource/config/nginx_config_parser_test.go b/internal/datasource/config/nginx_config_parser_test.go index 5afba0941..450facfcb 100644 --- a/internal/datasource/config/nginx_config_parser_test.go +++ b/internal/datasource/config/nginx_config_parser_test.go @@ -621,7 +621,7 @@ func TestNginxConfigParser_SyslogServerParse(t *testing.T) { expectedSyslogServers: "", content: testconfig.NginxConfigWithMultipleSysLogs(errorLog.Name(), accessLog.Name(), "random.domain:1515", "192.168.12.34:1517", "my.domain.com:1517"), - expectedLog: "Could not find valid Nap Syslog server", + expectedLog: "Could not find usable NAP syslog server, security violations will be unavailable", portInUse: false, }, { @@ -629,7 +629,7 @@ func TestNginxConfigParser_SyslogServerParse(t *testing.T) { expectedSyslogServers: "localhost:1516", content: testconfig.NginxConfigWithMultipleSysLogs(errorLog.Name(), accessLog.Name(), "192.168.12.34:1517", "127.0.0.1:1515", "localhost:1516"), - expectedLog: "NAP syslog server is not reachable", + expectedLog: "\"Found valid NAP syslog server\" address=localhost:1516", portInUse: true, }, { From 4a38ea9ed119186c5c7cf9355f728cbe0ba60a91 Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Wed, 16 Jul 2025 10:53:32 +0100 Subject: [PATCH 06/14] fix checking host --- internal/datasource/config/nginx_config_parser.go | 8 +++++++- internal/datasource/config/nginx_config_parser_test.go | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/datasource/config/nginx_config_parser.go b/internal/datasource/config/nginx_config_parser.go index ec5037030..4d04ad39f 100644 --- a/internal/datasource/config/nginx_config_parser.go +++ b/internal/datasource/config/nginx_config_parser.go @@ -257,7 +257,13 @@ func (ncp *NginxConfigParser) findValidSysLogServers(sysLogServers []string) []s for i := range sysLogServers { matches := re.FindStringSubmatch(sysLogServers[i]) if len(matches) > 1 { - if strings.HasPrefix(matches[1], "localhost") || strings.HasPrefix(matches[1], "127.0.0.1") { + host, _, err := net.SplitHostPort(matches[1]) + if err != nil { + continue + } + + ip := net.ParseIP(host) + if ip.IsLoopback() || strings.EqualFold(host, "localhost") { servers = append(servers, matches[1]) } } diff --git a/internal/datasource/config/nginx_config_parser_test.go b/internal/datasource/config/nginx_config_parser_test.go index 450facfcb..bac696f6f 100644 --- a/internal/datasource/config/nginx_config_parser_test.go +++ b/internal/datasource/config/nginx_config_parser_test.go @@ -688,8 +688,9 @@ func TestNginxConfigParser_findValidSysLogServers(t *testing.T) { "app_protect_security_log", "/etc/app_protect/conf/log_default1.json", "syslog:server=my.domain.com:1517", "app_protect_security_log", "/etc/app_protect/conf/log_default2.json", "syslog:server=127.0.0.1:1515", "app_protect_security_log", "/etc/app_protect/conf/log_default3.json", "syslog:server=localhost:1516", + "app_protect_security_log\", \"/etc/app_protect/conf/log_default3.json\", \"syslog:server=127.255.255.255:1517", } - expected := []string{"127.0.0.1:1515", "localhost:1516"} + expected := []string{"127.0.0.1:1515", "localhost:1516", "127.255.255.255:1517"} ncp := NewNginxConfigParser(types.AgentConfig()) result := ncp.findValidSysLogServers(servers) From e59640d87cab9488a3b3167762b519911a0959cf Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Wed, 16 Jul 2025 14:17:57 +0100 Subject: [PATCH 07/14] add filter --- internal/collector/otel_collector_plugin.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 1a8cac572..6f0a725dc 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -563,6 +563,15 @@ func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfig "protocol": "rfc3164", }, }, + // filter drops all logs that have a severity above 4 + // https://docs.secureauth.com/0902/en/how-to-read-a-syslog-message.html#severity-code-table + { + Type: "filter", + Fields: map[string]string{ + "expr": "'attributes.priority % 8 > 4'", + "drop_ratio": "1.0", + }, + }, { Type: "remove", Fields: map[string]string{ From 3e99cf779fafde0c2419979a199b9c3d47cad330 Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Wed, 16 Jul 2025 14:29:08 +0100 Subject: [PATCH 08/14] fix test --- internal/collector/otel_collector_plugin_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 6073951a1..e020fd830 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -744,7 +744,7 @@ func TestCollector_updateTcplogReceivers(t *testing.T) { assert.True(tt, tcplogReceiverAdded) assert.Len(tt, conf.Collector.Receivers.TcplogReceivers, 1) assert.Equal(tt, "localhost:151", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) - assert.Len(tt, conf.Collector.Receivers.TcplogReceivers[0].Operators, 4) + assert.Len(tt, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) }) // Calling updateTcplogReceivers shouldn't update the TcplogReceivers slice @@ -754,7 +754,7 @@ func TestCollector_updateTcplogReceivers(t *testing.T) { assert.False(t, tcplogReceiverAdded) assert.Len(t, conf.Collector.Receivers.TcplogReceivers, 1) assert.Equal(t, "localhost:151", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) - assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 4) + assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) }) t.Run("Test 3: TcplogReceiver deleted", func(tt *testing.T) { @@ -769,7 +769,7 @@ func TestCollector_updateTcplogReceivers(t *testing.T) { assert.True(t, tcplogReceiverDeleted) assert.Len(t, conf.Collector.Receivers.TcplogReceivers, 1) assert.Equal(t, "localhost:152", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) - assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 4) + assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) }) } From 7ccfa4951320f70c31efbca4b74d109fdc88fb43 Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Wed, 16 Jul 2025 15:27:19 +0100 Subject: [PATCH 09/14] fix logs --- .../datasource/config/nginx_config_parser.go | 49 +++++++++---------- .../config/nginx_config_parser_test.go | 46 +++++++++++++---- .../nginx-with-multiple-syslog-servers.conf | 19 ++----- test/model/config.go | 38 ++++++++++++++ 4 files changed, 101 insertions(+), 51 deletions(-) diff --git a/internal/datasource/config/nginx_config_parser.go b/internal/datasource/config/nginx_config_parser.go index 4d04ad39f..b0ff48146 100644 --- a/internal/datasource/config/nginx_config_parser.go +++ b/internal/datasource/config/nginx_config_parser.go @@ -109,6 +109,7 @@ func (ncp *NginxConfigParser) createNginxConfigContext( payload *crossplane.Payload, ) (*model.NginxConfigContext, error) { napSyslogServersFound := make(map[string]bool) + napEnabled := false nginxConfigContext := &model.NginxConfigContext{ InstanceID: instance.GetInstanceMeta().GetInstanceId(), @@ -175,17 +176,11 @@ func (ncp *NginxConfigParser) createNginxConfigContext( } case "app_protect_security_log": if len(directive.Args) > 1 { - sysLogServers := ncp.findValidSysLogServers(directive.Args) - if len(sysLogServers) == 0 { - slog.WarnContext(ctx, "Could not find usable NAP syslog server, "+ - "security violations will be unavailable") - } - for i := range sysLogServers { - sysLogServer := sysLogServers[i] - if !napSyslogServersFound[sysLogServer] { - napSyslogServersFound[sysLogServer] = true - slog.DebugContext(ctx, "Found NAP syslog server", "address", sysLogServer) - } + napEnabled = true + sysLogServer := ncp.findValidSysLogServers(directive.Args[1]) + if sysLogServer != "" && !napSyslogServersFound[sysLogServer] { + napSyslogServersFound[sysLogServer] = true + slog.DebugContext(ctx, "Found NAP syslog server", "address", sysLogServer) } } } @@ -213,6 +208,9 @@ func (ncp *NginxConfigParser) createNginxConfigContext( nginxConfigContext.NAPSysLogServer = syslogServer ncp.previousNAPSysLogServer = syslogServer } + } else if napEnabled { + slog.WarnContext(ctx, "Could not find usable NAP syslog server, "+ + "security violations will be unavailable") } fileMeta, err := files.FileMeta(conf.File) @@ -236,40 +234,37 @@ func (ncp *NginxConfigParser) parseSyslogDirective(ctx context.Context, napSyslo for napSyslogServer := range napSyslogServers { ln, err := net.Listen("tcp", napSyslogServer) if err != nil { - slog.WarnContext(ctx, "NAP syslog server is not reachable", "address", napSyslogServer, + slog.DebugContext(ctx, "NAP syslog server is not reachable", "address", napSyslogServer, "error", err) continue } ln.Close() + slog.DebugContext(ctx, "Found valid NAP syslog server", "address", napSyslogServer) return napSyslogServer } - slog.WarnContext(ctx, "Could not find usable NAP syslog server, security violations will be unavailable") return "" } -func (ncp *NginxConfigParser) findValidSysLogServers(sysLogServers []string) []string { +func (ncp *NginxConfigParser) findValidSysLogServers(sysLogServer string) string { re := regexp.MustCompile(`syslog:server=([\S]+)`) - var servers []string - for i := range sysLogServers { - matches := re.FindStringSubmatch(sysLogServers[i]) - if len(matches) > 1 { - host, _, err := net.SplitHostPort(matches[1]) - if err != nil { - continue - } + matches := re.FindStringSubmatch(sysLogServer) + if len(matches) > 1 { + host, _, err := net.SplitHostPort(matches[1]) + if err != nil { + return "" + } - ip := net.ParseIP(host) - if ip.IsLoopback() || strings.EqualFold(host, "localhost") { - servers = append(servers, matches[1]) - } + ip := net.ParseIP(host) + if ip.IsLoopback() || strings.EqualFold(host, "localhost") { + return matches[1] } } - return servers + return "" } func (ncp *NginxConfigParser) parseIncludeDirective(directive *crossplane.Directive) string { diff --git a/internal/datasource/config/nginx_config_parser_test.go b/internal/datasource/config/nginx_config_parser_test.go index bac696f6f..aed6fc946 100644 --- a/internal/datasource/config/nginx_config_parser_test.go +++ b/internal/datasource/config/nginx_config_parser_test.go @@ -506,6 +506,34 @@ func TestNginxConfigParser_Parse(t *testing.T) { ), allowedDirectories: []string{dir}, }, + { + name: "Test 10: Check with multiple syslog servers", + instance: protos.NginxPlusInstance([]string{}), + content: testconfig.NginxConfigWithMultipleSysLogs(errorLog.Name(), accessLog.Name(), + "192.168.12.34:1517", "my.domain.com:1517", "127.0.0.1:1515"), + expectedConfigContext: modelHelpers.ConfigContextWithSysLog( + accessLog.Name(), + errorLog.Name(), + protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), + "127.0.0.1:1515", + ), + allowedDirectories: []string{dir}, + expectedLog: "Found valid NAP syslog server", + }, + { + name: "Test 10: Check with multiple syslog servers", + instance: protos.NginxPlusInstance([]string{}), + content: testconfig.NginxConfigWithMultipleSysLogs(errorLog.Name(), accessLog.Name(), + "192.168.12.34:1517", "my.domain.com:1517", "not.allowed:1515"), + expectedConfigContext: modelHelpers.ConfigContextWithSysLog( + accessLog.Name(), + errorLog.Name(), + protos.NginxPlusInstance([]string{}).GetInstanceMeta().GetInstanceId(), + "", + ), + allowedDirectories: []string{dir}, + expectedLog: "Could not find usable NAP syslog server, security violations will be unavailable", + }, } for _, test := range tests { @@ -637,7 +665,7 @@ func TestNginxConfigParser_SyslogServerParse(t *testing.T) { expectedSyslogServers: "", content: testconfig.NginxConfigWithMultipleSysLogs(errorLog.Name(), accessLog.Name(), "my.domain.com:1517", "127.0.0.1:1515", "my.domain.com:1517"), - expectedLog: "Could not find usable NAP syslog server", + expectedLog: "NAP syslog server is not reachable", portInUse: true, }, { @@ -684,17 +712,17 @@ func TestNginxConfigParser_SyslogServerParse(t *testing.T) { func TestNginxConfigParser_findValidSysLogServers(t *testing.T) { servers := []string{ - "/etc/app_protect/conf/log_default.json", "syslog:server=192.168.12.34:1517", - "app_protect_security_log", "/etc/app_protect/conf/log_default1.json", "syslog:server=my.domain.com:1517", - "app_protect_security_log", "/etc/app_protect/conf/log_default2.json", "syslog:server=127.0.0.1:1515", - "app_protect_security_log", "/etc/app_protect/conf/log_default3.json", "syslog:server=localhost:1516", - "app_protect_security_log\", \"/etc/app_protect/conf/log_default3.json\", \"syslog:server=127.255.255.255:1517", + "syslog:server=192.168.12.34:1517", "syslog:server=my.domain.com:1517", "syslog:server=127.0.0.1:1515", + "syslog:server=localhost:1516", "syslog:server=127.255.255.255:1517", } - expected := []string{"127.0.0.1:1515", "localhost:1516", "127.255.255.255:1517"} + expected := []string{"", "", "127.0.0.1:1515", "localhost:1516", "127.255.255.255:1517"} ncp := NewNginxConfigParser(types.AgentConfig()) - result := ncp.findValidSysLogServers(servers) - assert.Equal(t, expected, result) + for i, server := range servers { + result := ncp.findValidSysLogServers(server) + + assert.Equal(t, expected[i], result) + } } func TestNginxConfigParser_checkLog(t *testing.T) { diff --git a/test/config/nginx/nginx-with-multiple-syslog-servers.conf b/test/config/nginx/nginx-with-multiple-syslog-servers.conf index aaccfe109..01f04b026 100644 --- a/test/config/nginx/nginx-with-multiple-syslog-servers.conf +++ b/test/config/nginx/nginx-with-multiple-syslog-servers.conf @@ -19,17 +19,6 @@ http { } http { - log_format ltsv "time:$time_local" - "\thost:$remote_addr" - "\tmethod:$request_method" - "\turi:$request_uri" - "\tprotocol:$server_protocol" - "\tstatus:$status" - "\tsize:$body_bytes_sent" - "\treferer:$http_referer" - "\tua:$http_user_agent" - "\treqtime:$request_time" - "\tapptime:$upstream_response_time"; server { listen 9093; @@ -42,9 +31,9 @@ http { server { - app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=%s - app_protect_security_log "/etc/app_protect/conf/log_default1.json" syslog:server=%s - app_protect_security_log "/etc/app_protect/conf/log_default2.json" syslog:server=%s - app_protect_security_log "/etc/app_protect/conf/log_default3.json" syslog:server=%s + app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=%s; + app_protect_security_log "/etc/app_protect/conf/log_default1.json" syslog:server=%s; + app_protect_security_log "/etc/app_protect/conf/log_default2.json" syslog:server=%s; + app_protect_security_log "/etc/app_protect/conf/log_default3.json" syslog:server=%s; } } diff --git a/test/model/config.go b/test/model/config.go index 4b5e69dc1..661bb2635 100644 --- a/test/model/config.go +++ b/test/model/config.go @@ -160,3 +160,41 @@ func ConfigContextWithFiles( NAPSysLogServer: syslogServers, } } + +func ConfigContextWithSysLog( + accessLogName, + errorLogName string, + instanceID string, + syslogServers string, +) *model.NginxConfigContext { + return &model.NginxConfigContext{ + StubStatus: &model.APIDetails{ + URL: "", + Listen: "", + Location: "", + }, + PlusAPI: &model.APIDetails{ + URL: "", + Listen: "", + Location: "", + }, + AccessLogs: []*model.AccessLog{ + { + Name: accessLogName, + Format: "$remote_addr - $remote_user [$time_local]", + Readable: true, + Permissions: "0600", + }, + }, + ErrorLogs: []*model.ErrorLog{ + { + Name: errorLogName, + Readable: true, + LogLevel: "notice", + Permissions: "0600", + }, + }, + InstanceID: instanceID, + NAPSysLogServer: syslogServers, + } +} From 6a4c1971852e4dc4e4ae382012399d5870c393bc Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Wed, 16 Jul 2025 16:22:25 +0100 Subject: [PATCH 10/14] add regex to filter logs --- internal/collector/otel_collector_plugin.go | 27 ++++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 6f0a725dc..a101f7c6f 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -550,6 +550,24 @@ func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfig config.TcplogReceiver{ ListenAddress: nginxConfigContext.NAPSysLogServer, Operators: []config.Operator{ + // regex captures the priority number from the log line + { + Type: "regex_parser", + Fields: map[string]string{ + "regex": "^<(?P\\d+)>", + "parse_from": "body", + "parse_to": "attributes", + }, + }, + // filter drops all logs that have a severity above 4 + // https://docs.secureauth.com/0902/en/how-to-read-a-syslog-message.html#severity-code-table + { + Type: "filter", + Fields: map[string]string{ + "expr": "'int(attributes.priority) % 8 > 4'", + "drop_ratio": "1.0", + }, + }, { Type: "add", Fields: map[string]string{ @@ -563,15 +581,6 @@ func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfig "protocol": "rfc3164", }, }, - // filter drops all logs that have a severity above 4 - // https://docs.secureauth.com/0902/en/how-to-read-a-syslog-message.html#severity-code-table - { - Type: "filter", - Fields: map[string]string{ - "expr": "'attributes.priority % 8 > 4'", - "drop_ratio": "1.0", - }, - }, { Type: "remove", Fields: map[string]string{ From 10e8d31b264f1495a8c99281551870feddabcbfe Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Wed, 16 Jul 2025 16:29:18 +0100 Subject: [PATCH 11/14] add regex to filter logs --- internal/collector/otel_collector_plugin_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index e020fd830..1f43e38f6 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -744,7 +744,7 @@ func TestCollector_updateTcplogReceivers(t *testing.T) { assert.True(tt, tcplogReceiverAdded) assert.Len(tt, conf.Collector.Receivers.TcplogReceivers, 1) assert.Equal(tt, "localhost:151", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) - assert.Len(tt, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) + assert.Len(tt, conf.Collector.Receivers.TcplogReceivers[0].Operators, 6) }) // Calling updateTcplogReceivers shouldn't update the TcplogReceivers slice @@ -754,7 +754,7 @@ func TestCollector_updateTcplogReceivers(t *testing.T) { assert.False(t, tcplogReceiverAdded) assert.Len(t, conf.Collector.Receivers.TcplogReceivers, 1) assert.Equal(t, "localhost:151", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) - assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) + assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 6) }) t.Run("Test 3: TcplogReceiver deleted", func(tt *testing.T) { @@ -769,7 +769,7 @@ func TestCollector_updateTcplogReceivers(t *testing.T) { assert.True(t, tcplogReceiverDeleted) assert.Len(t, conf.Collector.Receivers.TcplogReceivers, 1) assert.Equal(t, "localhost:152", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) - assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) + assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 6) }) } From 39df77f01a6dccd699d19b6de3734e5f30540630 Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Tue, 22 Jul 2025 14:48:11 +0100 Subject: [PATCH 12/14] merge main --- internal/collector/otel_collector_plugin.go | 32 ++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 203524f88..953f1838d 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -554,24 +554,24 @@ func (oc *Collector) updateNginxAppProtectTcplogReceivers(nginxConfigContext *mo oc.config.Collector.Receivers.TcplogReceivers["nginx_app_protect"] = &config.TcplogReceiver{ ListenAddress: nginxConfigContext.NAPSysLogServer, Operators: []config.Operator{ - // regex captures the priority number from the log line - { - Type: "regex_parser", - Fields: map[string]string{ - "regex": "^<(?P\\d+)>", - "parse_from": "body", - "parse_to": "attributes", + // regex captures the priority number from the log line + { + Type: "regex_parser", + Fields: map[string]string{ + "regex": "^<(?P\\d+)>", + "parse_from": "body", + "parse_to": "attributes", + }, }, - }, - // filter drops all logs that have a severity above 4 - // https://docs.secureauth.com/0902/en/how-to-read-a-syslog-message.html#severity-code-table - { - Type: "filter", - Fields: map[string]string{ - "expr": "'int(attributes.priority) % 8 > 4'", - "drop_ratio": "1.0", + // filter drops all logs that have a severity above 4 + // https://docs.secureauth.com/0902/en/how-to-read-a-syslog-message.html#severity-code-table + { + Type: "filter", + Fields: map[string]string{ + "expr": "'int(attributes.priority) % 8 > 4'", + "drop_ratio": "1.0", + }, }, - }, { Type: "add", Fields: map[string]string{ From e7361cc5a3cc72d2fe370b88ac3fe060403d0efe Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Tue, 22 Jul 2025 16:01:34 +0100 Subject: [PATCH 13/14] fix unit test --- internal/collector/otel_collector_plugin_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 5a1471e0d..5623d2eef 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -173,8 +173,9 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, receivers: config.Receivers{ - HostMetrics: nil, - OtlpReceivers: nil, + HostMetrics: nil, + OtlpReceivers: nil, + TcplogReceivers: make(map[string]*config.TcplogReceiver), NginxPlusReceivers: []config.NginxPlusReceiver{ { InstanceID: "123", @@ -213,8 +214,9 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, receivers: config.Receivers{ - HostMetrics: nil, - OtlpReceivers: nil, + HostMetrics: nil, + OtlpReceivers: nil, + TcplogReceivers: make(map[string]*config.TcplogReceiver), NginxReceivers: []config.NginxReceiver{ { InstanceID: "123", From 95f0ab13cdffc1e0fee5ee03855ecee69c0573f2 Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Wed, 23 Jul 2025 09:23:28 +0100 Subject: [PATCH 14/14] fix otel template --- internal/collector/otelcol.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index b331152c0..572e8c222 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -296,7 +296,7 @@ service: receivers: {{- range $receiver := $pipeline.Receivers }} {{- if eq $receiver "tcplog/nginx_app_protect" }} - - tcplog/nginx_app_protect: + - tcplog/nginx_app_protect {{- else }} - {{ $receiver }} {{- end }}