feat: initial RFC3164 implementation

README.md

@@ -73,6 +73,10 @@ The following environment variables can be used to configure the application:
 - `SLOGGO_TCP_PORT`: Port for the TCP Syslog listener (default: `6514`).
 - `SLOGGO_API_PORT`: Port for the API (default: `8080`).
 - `SLOGGO_LOG_RETENTION_MINUTES`: Duration in minutes to keep logs before deletion (default: `43200` - 30 days).
+- `SLOGGO_LOG_FORMAT`: Log parsing format (default: `auto`). Supported values:
+   - `auto`: Try RFC 5424 first, then fall back to RFC 3164.
+   - `RFC5424`: Only parse messages as RFC 5424.
+   - `RFC3164`: Only parse messages as RFC 3164.
 
 ## What Sloggo is
 

backend/formats/rfc3164.go

@@ -0,0 +1,92 @@
+package formats
+
+import (
+    "errors"
+    "regexp"
+    "sloggo/models"
+    "strconv"
+    "strings"
+    "time"
+)
+
+var (
+    // Example: <34>Oct 11 22:14:15 mymachine su[123]: 'su root' failed
+    rfc3164Regex = regexp.MustCompile(`^<(?P<pri>\d{1,3})>(?P<ts>[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+(?P<host>\S+)\s+(?P<tag>[A-Za-z0-9_.\-\/]+)(?:\[(?P<pid>[^\]]+)\])?:\s*(?P<msg>[\s\S]*)$`)
+)
+
+// ParseRFC3164ToLogEntry parses an RFC3164 (BSD) syslog line into a LogEntry
+// Best-effort: fills missing fields with defaults compatible with the DB schema
+func ParseRFC3164ToLogEntry(line string) (*models.LogEntry, error) {
+    line = strings.TrimSpace(line)
+    if line == "" {
+        return nil, errors.New("empty message")
+    }
+
+    m := rfc3164Regex.FindStringSubmatch(line)
+    if m == nil {
+        return nil, errors.New("not rfc3164 format")
+    }
+
+    // Extract named groups
+    groups := make(map[string]string)
+    for i, name := range rfc3164Regex.SubexpNames() {
+        if i != 0 && name != "" {
+            groups[name] = m[i]
+        }
+    }
+
+    // Priority -> facility/severity
+    pri, err := strconv.Atoi(groups["pri"])
+    if err != nil {
+        return nil, err
+    }
+    facility := uint8(pri / 8)
+    severity := uint8(pri % 8)
+
+    // Timestamp (no year) e.g. "Oct 11 22:14:15"
+    // Parse with current year in local time, then convert to time.Now() location
+    now := time.Now()
+    tsStr := groups["ts"]
+    // time layout with optional leading space in day
+    // Jan _2 15:04:05 handles single-digit days
+    tsParsed, err := time.ParseInLocation("Jan _2 15:04:05", tsStr, now.Location())
+    if err != nil {
+        // Fallback: try RFC822-like without seconds? keep robust
+        // If still failing, use current time
+        tsParsed = now
+    }
+    // Inject current year (RFC3164 has no year)
+    ts := time.Date(now.Year(), tsParsed.Month(), tsParsed.Day(), tsParsed.Hour(), tsParsed.Minute(), tsParsed.Second(), 0, now.Location())
+
+    hostname := groups["host"]
+    if hostname == "" {
+        hostname = "-"
+    }
+
+    appName := groups["tag"]
+    if appName == "" {
+        appName = "-"
+    }
+
+    procID := groups["pid"]
+    if procID == "" {
+        procID = "-"
+    }
+
+    msg := groups["msg"]
+
+    entry := &models.LogEntry{
+        Severity:       severity,
+        Facility:       facility,
+        Version:        1,
+        Timestamp:      ts,
+        Hostname:       hostname,
+        AppName:        appName,
+        ProcID:         procID,
+        MsgID:          "-",
+        StructuredData: "-",
+        Message:        msg,
+    }
+
+    return entry, nil
+}

backend/formats/rfc3164_test.go

@@ -0,0 +1,75 @@
+package formats
+
+import (
+	"testing"
+)
+
+func TestParseRFC3164ToLogEntry_Basic(t *testing.T) {
+	line := "<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8"
+	entry, err := ParseRFC3164ToLogEntry(line)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if entry.Facility != 4 || entry.Severity != 2 { // 34 / 8 = 4, 34 % 8 = 2
+		t.Errorf("facility/severity mismatch: got (%d,%d)", entry.Facility, entry.Severity)
+	}
+	if entry.Hostname != "mymachine" {
+		t.Errorf("hostname: got %q", entry.Hostname)
+	}
+	if entry.AppName != "su" {
+		t.Errorf("appname: got %q", entry.AppName)
+	}
+	if entry.ProcID != "-" {
+		t.Errorf("procid: got %q", entry.ProcID)
+	}
+	if entry.Message != "'su root' failed for lonvick on /dev/pts/8" {
+		t.Errorf("message: got %q", entry.Message)
+	}
+	if entry.Timestamp.IsZero() {
+		t.Error("timestamp should not be zero")
+	}
+}
+
+func TestParseRFC3164ToLogEntry_WithPID(t *testing.T) {
+	line := "<190>Nov  6 09:01:02 esphome-device esphome[1234]: Sensor reading: 42"
+	entry, err := ParseRFC3164ToLogEntry(line)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if entry.Facility != 23 || entry.Severity != 6 { // 190 / 8 = 23, 190 % 8 = 6
+		t.Errorf("facility/severity mismatch: got (%d,%d)", entry.Facility, entry.Severity)
+	}
+	if entry.Hostname != "esphome-device" {
+		t.Errorf("hostname: got %q", entry.Hostname)
+	}
+	if entry.AppName != "esphome" {
+		t.Errorf("appname: got %q", entry.AppName)
+	}
+	if entry.ProcID != "1234" {
+		t.Errorf("procid: got %q", entry.ProcID)
+	}
+	if entry.Message != "Sensor reading: 42" {
+		t.Errorf("message: got %q", entry.Message)
+	}
+}
+
+func TestParseRFC3164ToLogEntry_MultilineMessage(t *testing.T) {
+	line := "<134>Feb  1 11:37:00 modbus-ble-bridge mdns: [C][mdns:124]: mDNS:\n\n  Hostname: modbus-ble-bridge"
+	entry, err := ParseRFC3164ToLogEntry(line)
+	if err != nil {
+		t.Fatalf("unexpected error parsing multiline: %v", err)
+	}
+	if entry.Facility != 16 || entry.Severity != 6 { // 134 / 8 = 16, 134 % 8 = 6
+		t.Errorf("facility/severity mismatch: got (%d,%d)", entry.Facility, entry.Severity)
+	}
+	if entry.Hostname != "modbus-ble-bridge" {
+		t.Errorf("hostname: got %q", entry.Hostname)
+	}
+	if entry.AppName != "mdns" {
+		t.Errorf("appname: got %q", entry.AppName)
+	}
+	expectedMsg := "[C][mdns:124]: mDNS:\n\n  Hostname: modbus-ble-bridge"
+	if entry.Message != expectedMsg {
+		t.Errorf("message mismatch:\nexpected: %q\n     got: %q", expectedMsg, entry.Message)
+	}
+}

backend/listener/tcp.go

@@ -69,7 +69,6 @@ func handleTCPConnection(conn net.Conn) {
 	defer conn.Close()
 
 	scanner := bufio.NewScanner(conn)
-	parser := rfc5424.NewParser(rfc5424.WithBestEffort())
 
 	// Configure scanner with a larger buffer for bigger messages
 	const maxScanSize = 1024 * 1024 // 1MB max message size
@@ -103,30 +102,41 @@ func handleTCPConnection(conn net.Conn) {
 			continue
 		}
 
-		// Parse the message
-		syslogMsg, err := parser.Parse([]byte(message))
-		if err != nil {
-			log.Printf("Failed to parse message: %v: %s", err, message)
-			continue
-		}
-
-		// Convert to RFC5424 syslog message
-		rfc5424Msg, ok := syslogMsg.(*rfc5424.SyslogMessage)
-		if !ok {
-			log.Printf("Parsed message is not a valid RFC5424 message: %s", message)
-			continue
+		parsed := false
+		var lastErr error
+
+		// Try RFC5424 if enabled
+		if utils.LogFormat == "rfc5424" || utils.LogFormat == "auto" {
+			parser := rfc5424.NewParser(rfc5424.WithBestEffort())
+			if syslogMsg, err := parser.Parse([]byte(message)); err == nil {
+				if rfc5424Msg, ok := syslogMsg.(*rfc5424.SyslogMessage); ok {
+					logEntry := formats.SyslogMessageToLogEntry(rfc5424Msg)
+					if logEntry != nil {
+						if err := db.StoreLog(*logEntry); err != nil {
+							log.Printf("Error storing log: %v", err)
+						}
+						parsed = true
+					}
+				}
+			} else {
+				lastErr = err
+			}
 		}
 
-		// Convert directly to LogEntry for efficient DuckDB insertion
-		logEntry := formats.SyslogMessageToLogEntry(rfc5424Msg)
-
-		if logEntry == nil {
-			log.Printf("Failed to convert message to LogEntry: %s", message)
+		// Try RFC3164 if enabled and not yet parsed
+		if !parsed && (utils.LogFormat == "rfc3164" || utils.LogFormat == "auto") {
+			if logEntry, err := formats.ParseRFC3164ToLogEntry(message); err == nil {
+				if err := db.StoreLog(*logEntry); err != nil {
+					log.Printf("Error storing log: %v", err)
+				}
+				parsed = true
+			} else {
+				lastErr = err
+			}
 		}
 
-		// Store log without blocking if possible
-		if err := db.StoreLog(*logEntry); err != nil {
-			log.Printf("Error storing log: %v", err)
+		if !parsed {
+			log.Printf("Failed to parse message with format %s: %v: %s", utils.LogFormat, lastErr, message)
 		}
 	}
 }

backend/listener/tcp_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"sloggo/db"
+	"sloggo/utils"
 	"strings"
 	"testing"
 	"time"
@@ -59,12 +60,17 @@ func TestTCPListener(t *testing.T) {
 	}
 	defer conn.Close()
 
-	// Run test cases sequentially on the same connection
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			sendTCPMessage(t, conn, tc.message)
-			// No need to explicitly force batch processing - handled in verifyLogEntry
-			verifyLogEntry(t, tc)
-		})
+	// Run test cases sequentially on the same connection for different log formats
+	formats := []string{"auto", "rfc5424", "rfc3164"}
+	for _, format := range formats {
+		utils.LogFormat = format
+		for _, tc := range testCases {
+			name := fmt.Sprintf("%s_%s", format, tc.name)
+			t.Run(name, func(t *testing.T) {
+				sendTCPMessage(t, conn, tc.message)
+				// No need to explicitly force batch processing - handled in verifyLogEntry
+				verifyLogEntry(t, tc)
+			})
+		}
 	}
 }

backend/listener/test_helpers.go

@@ -167,5 +167,35 @@ func getTestCases() []testCase {
 				shouldError:    false,
 			},
 		},
+		{
+			name:    "RFC3164 basic without pid",
+			message: "<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8",
+			expected: expectedResult{
+				facility:       4,
+				severity:       2,
+				hostname:       "mymachine",
+				appName:        "su",
+				procid:         "-",
+				msgid:          "-",
+				structuredData: "-",
+				msg:            "'su root' failed for lonvick on /dev/pts/8",
+				shouldError:    false,
+			},
+		},
+		{
+			name:    "RFC3164 with pid typical esphome",
+			message: "<190>Nov  6 09:01:02 esphome-device esphome[1234]: Sensor reading: 42",
+			expected: expectedResult{
+				facility:       23,
+				severity:       6,
+				hostname:       "esphome-device",
+				appName:        "esphome",
+				procid:         "1234",
+				msgid:          "-",
+				structuredData: "-",
+				msg:            "Sensor reading: 42",
+				shouldError:    false,
+			},
+		},
 	}
 }