Ship botlog URI with query params intact (#9)

* Ship botlog URI with query params intact

- Add ParsedLine.RawURI (path + query string) alongside RawPath
  so observers see the full hit URL; populated in all three parse
  paths (text/$request, text/$uri, JSON request_uri)
- Switch botlog Observer to prefer RawURI with fallback chain
  RawURI -> RawPath -> URI; gatesrv-side HasQueryParams now works
  on bot-log events instead of always returning false
- Bump DefaultBatchInterval 5s -> 30s; quiet hosts were flushing
  ~50-event payloads every tick, larger batches compress better
- Cover the new field across both parsers including
  percent-encoded query strings, plus the observer fallback chain

* Drop redundant ParsedLine.RawPath

- RawPath was only read by botlog Observer as the middle of a
  three-level fallback (RawURI -> RawPath -> URI). All three parser
  paths populate RawPath and RawURI together, so RawPath was never
  the only field set — middle fallback was unreachable
- Remove rawPathFromRequest and rawPathFromJSON; this eliminates a
  duplicate strings.SplitN per parsed line on the hot path
- Collapse observer fallback to RawURI -> URI; drop the synthetic
  TestObserver_FallsBackToRawPathWhenRawURIEmpty that exercised
  the unreachable state
- Rename TestParsedLine_RawPathUnnormalized to *_RawURIUnnormalized
  and drop wantRawPath assertions; recordedLine.rawPath gone too

* Cap botlog Event.URI at URITruncate

- Add Config.URITruncate (default 2048) symmetrical to UATruncate;
  IE's legacy 2083-char URL limit covers >99% of legitimate traffic
- Observer truncates Event.URI before BuildEvent so pathological
  bot probes (4-8KB base64/SQLi payloads in query strings) cannot
  bloat WAL writes or the gatesrv POST body
- Pre-fix RawPath stripped queries kept URIs naturally short; with
  RawURI carrying the query verbatim we need an explicit cap

* Update README botlog defaults

- BatchInterval 5s -> 30s
- Add URITruncate row (default 2048)

Author: sergeyfast

README.md

@@ -149,10 +149,11 @@ Channel  = "stable"         # stable / beta
 # Token         = ""        # required — issued by topsrv.io per project
 # Endpoint      = ""        # default: [Push].Endpoint with /v1/bot-logs path
 # BatchSize     = 5000      # events per batch
-# BatchInterval = "5s"      # flush interval
+# BatchInterval = "30s"     # flush interval
 # SpoolDir      = ""        # default: [Push].SpoolDir; a "botlog/" subdir is created inside
 # MaxSpoolMB    = 200       # WAL disk budget
 # UATruncate    = 1024      # truncate user-agent at this length
+# URITruncate   = 2048      # truncate request URI at this length
 # ExtraUAPatterns = ["MyCustomCrawler/"]  # local additions to the bot list
 ```
 
@@ -183,10 +184,11 @@ Channel  = "stable"         # stable / beta
 | `BotLogs.Token` | — | Bot-logs ingest bearer token (separate from `Push.Token`) |
 | `BotLogs.Endpoint` | derived | Ingest URL; defaults to `[Push].Endpoint` with `/v1/bot-logs` path |
 | `BotLogs.BatchSize` | `5000` | Events per batch |
-| `BotLogs.BatchInterval` | `5s` | Flush interval |
+| `BotLogs.BatchInterval` | `30s` | Flush interval |
 | `BotLogs.SpoolDir` | derived | Parent dir for WAL spool; `botlog/` subdir is created inside. Defaults to `[Push].SpoolDir` |
 | `BotLogs.MaxSpoolMB` | `200` | Disk budget for spool subdir |
 | `BotLogs.UATruncate` | `1024` | Max UA length per event |
+| `BotLogs.URITruncate` | `2048` | Max URI length per event |
 | `BotLogs.ExtraUAPatterns` | `[]` | Local additions to known-bots UA patterns |
 
 ### Environment variables

cfg/local.toml.dist

@@ -45,8 +45,9 @@ Channel  = "stable"      # stable / beta
 # Token         = ""        # required — issued by topsrv.io per project
 # Endpoint      = ""        # default: derived from [Push].Endpoint
 # BatchSize     = 5000      # events per batch
-# BatchInterval = "5s"      # flush interval
+# BatchInterval = "30s"     # flush interval
 # SpoolDir      = ""        # default: [Push].SpoolDir
 # MaxSpoolMB    = 200       # WAL disk budget
 # UATruncate    = 1024      # truncate user-agent at this length
+# URITruncate   = 2048      # truncate request URI at this length
 # ExtraUAPatterns = ["MyCustomCrawler/"]  # local additions to the bot list

internal/topsrv/botlog/config.go

@@ -15,10 +15,16 @@ import (
 )
 
 const (
-	DefaultBatchSize     = 5000
-	DefaultBatchInterval = 5 * time.Second
+	DefaultBatchSize = 5000
+	// 30s gives the receiver larger batches on light bot traffic; a 5s
+	// interval was flushing ~50-event payloads every tick on quiet hosts.
+	DefaultBatchInterval = 30 * time.Second
 	DefaultMaxSpoolMB    = 200
 	DefaultUATruncate    = 1024
+	// 2048 covers >99% of legitimate URLs (IE legacy limit was 2083). Bot
+	// probes routinely send 4-8KB URIs with base64/SQLi payloads — capping
+	// keeps WAL writes and gatesrv payload size bounded.
+	DefaultURITruncate = 2048
 
 	// ingestPath is the control-plane handler that accepts ndjson bot-log batches.
 	ingestPath = "/v1/bot-logs"
@@ -32,10 +38,11 @@ type Config struct {
 	Endpoint        string   // ingest URL; default: [Push].Endpoint with path replaced by /v1/bot-logs
 	Token           string   // Bearer token; required when Enabled
 	BatchSize       int      // events per batch; default 5000
-	BatchInterval   string   // flush interval as Go duration; default "5s"
+	BatchInterval   string   // flush interval as Go duration; default "30s"
 	SpoolDir        string   // parent directory; subdir "botlog" is created inside; default = [Push].SpoolDir
 	MaxSpoolMB      int      // disk budget for spool subdir; default 200
 	UATruncate      int      // max UA length stored per event; default 1024
+	URITruncate     int      // max URI length stored per event; default 2048
 	ExtraUAPatterns []string // local additions to knownBots (substring, case-sensitive)
 
 	parsedBatchInterval time.Duration // populated by Validate
@@ -89,6 +96,9 @@ func (c *Config) Validate(push topsrv.PushConfig) error {
 	if c.UATruncate <= 0 {
 		c.UATruncate = DefaultUATruncate
 	}
+	if c.URITruncate <= 0 {
+		c.URITruncate = DefaultURITruncate
+	}
 	return nil
 }
 

internal/topsrv/botlog/observer.go

@@ -53,6 +53,7 @@ type Observer struct {
 	pusher        *Pusher
 	hostname      string
 	uaTruncate    int
+	uriTruncate   int
 	extraPatterns []string
 
 	// Resolved at construction from the LogCollector's ExtractFields. -1 when
@@ -68,6 +69,7 @@ func NewObserver(p *Pusher, cfg Config, hostname string, extractFields []string)
 		pusher:        p,
 		hostname:      hostname,
 		uaTruncate:    cfg.UATruncate,
+		uriTruncate:   cfg.URITruncate,
 		extraPatterns: cfg.ExtraUAPatterns,
 		idxUA:         slices.Index(extractFields, fieldUserAgent),
 		idxHost:       slices.Index(extractFields, fieldHost),
@@ -91,15 +93,16 @@ func (o *Observer) OnLogLine(p *nginx.ParsedLine, _ string) {
 	}
 
 	// Bot-log UI groups by actual URL, not the nginx-metrics-normalized form.
-	// Fall back to URI if the log format doesn't yield a raw path (legacy
-	// $uri after rewrite — less precise but still useful).
-	uri := p.RawPath
+	// RawURI keeps the query string so HasQueryParams on gatesrv works; the
+	// normalized URI is the last-resort fallback for log formats that yield
+	// neither $request nor $uri.
+	uri := p.RawURI
 	if uri == "" {
 		uri = p.URI
 	}
 	ev := BuildEvent(time.Now(), o.hostname, Fields{
 		Status:               p.Status,
-		URI:                  uri,
+		URI:                  truncate(uri, o.uriTruncate),
 		BodyBytesSent:        p.BodyBytesSent,
 		RequestTime:          p.RequestTime,
 		UpstreamResponseTime: p.UpstreamResponseTime,

internal/topsrv/botlog/observer_test.go

@@ -75,31 +75,49 @@ func TestObserver_EnqueuesBotEvent(t *testing.T) {
 	}
 }
 
-// Verifies the bot-logs UI fix: Event.URI must carry RawPath (un-normalized
-// hit URL), not the cardinality-collapsed ParsedLine.URI.
-func TestObserver_UsesRawPathOverURI(t *testing.T) {
+// Verifies the bot-logs UI fix: Event.URI must carry RawURI (un-normalized
+// hit URL with query string), not the cardinality-collapsed ParsedLine.URI.
+func TestObserver_UsesRawURIOverURI(t *testing.T) {
 	o, p := newObserverPair(t)
 	pl := &nginx.ParsedLine{
 		Status:  "200",
 		URI:     "/news/:id/:rest",
-		RawPath: "/news/12345/some-title",
+		RawURI:  "/news/12345/some-title?utm=x",
 		Extras:  [nginx.MaxExtras]string{"GPTBot/1.0", "", "", ""},
 		NExtras: 1,
 	}
 	o.OnLogLine(pl, "")
 	require.Len(t, p.queue, 1)
 	ev := <-p.queue
-	assert.Equal(t, "/news/12345/some-title", ev.URI)
+	assert.Equal(t, "/news/12345/some-title?utm=x", ev.URI)
 }
 
-// When the log format yields no RawPath (legacy $uri after rewrite), Observer
+// Pathological bot probes regularly send 4-8KB URIs with base64/SQLi payloads.
+// Observer must cap Event.URI at Config.URITruncate before shipping.
+func TestObserver_TruncatesURI(t *testing.T) {
+	o, p := newObserverPair(t)
+	long := "/probe?p=" + strings.Repeat("A", 5000)
+	pl := &nginx.ParsedLine{
+		Status:  "200",
+		URI:     "/probe",
+		RawURI:  long,
+		Extras:  [nginx.MaxExtras]string{"GPTBot/1.0", "", "", ""},
+		NExtras: 1,
+	}
+	o.OnLogLine(pl, "")
+	require.Len(t, p.queue, 1)
+	ev := <-p.queue
+	assert.Len(t, ev.URI, DefaultURITruncate)
+	assert.Equal(t, long[:DefaultURITruncate], ev.URI)
+}
+
+// When the log format yields no RawURI (legacy $uri after rewrite), Observer
 // must fall back to ParsedLine.URI rather than emit an empty event URI.
-func TestObserver_FallsBackToURIWhenRawPathEmpty(t *testing.T) {
+func TestObserver_FallsBackToURIWhenRawURIEmpty(t *testing.T) {
 	o, p := newObserverPair(t)
 	pl := &nginx.ParsedLine{
 		Status:  "200",
 		URI:     "/news/:id",
-		RawPath: "",
 		Extras:  [nginx.MaxExtras]string{"GPTBot/1.0", "", "", ""},
 		NExtras: 1,
 	}

internal/topsrv/botlog/pusher.go

@@ -103,7 +103,7 @@ func isPermanentFailure(err error) bool {
 }
 
 // gzipPool reuses gzip writers across flushes — gzip.NewWriter allocates ~256 KB
-// of internal buffers, which adds up at 5s flush cadence.
+// of internal buffers, which adds up at the default flush cadence.
 var gzipPool = sync.Pool{
 	New: func() any { return gzip.NewWriter(io.Discard) },
 }

internal/topsrv/nginx/log.go

@@ -347,7 +347,7 @@ func (c *LogCollector) parseLine(line string) {
 type ParsedLine struct {
 	Status               string
 	URI                  string // path normalized for nginx-metrics cardinality (/:id, /:rest)
-	RawPath              string // un-normalized request path (querystring stripped) — for observers that need the actual URL (e.g. botlog)
+	RawURI               string // un-normalized request URI (path + query) — for observers that need the full URL (e.g. botlog)
 	BodyBytesSent        string
 	RequestTime          string
 	UpstreamResponseTime string
@@ -371,10 +371,10 @@ func (c *LogCollector) parseLineWith(parser *gonx.Parser, line, path string) {
 
 	if req, err := entry.Field("request"); err == nil {
 		p.URI = normalizeURI(req)
-		p.RawPath = rawPathFromRequest(req)
+		p.RawURI = rawURIFromRequest(req)
 	} else if u, err := entry.Field("uri"); err == nil {
 		p.URI = normalizePath(u)
-		p.RawPath = stripQuery(u)
+		p.RawURI = stripQuery(u)
 	}
 
 	for i, f := range c.extractFields {
@@ -408,7 +408,7 @@ func (c *LogCollector) parseJSONLine(line, path string) {
 		p.UpstreamResponseTime = m["upstream_response_time"]
 		p.UpstreamCacheStatus = m["upstream_cache_status"]
 		p.URI = normalizeRequestURI(m["request_uri"], m["request"])
-		p.RawPath = rawPathFromJSON(m["request_uri"], m["request"])
+		p.RawURI = rawURIFromJSON(m["request_uri"], m["request"])
 
 		for i, f := range c.extractFields {
 			if i >= len(p.Extras) {
@@ -434,7 +434,7 @@ func (c *LogCollector) parseJSONLine(line, path string) {
 		UpstreamResponseTime: entry.UpstreamResponseTime,
 		UpstreamCacheStatus:  entry.UpstreamCacheStatus,
 		URI:                  normalizeRequestURI(entry.RequestURI, entry.Request),
-		RawPath:              rawPathFromJSON(entry.RequestURI, entry.Request),
+		RawURI:               rawURIFromJSON(entry.RequestURI, entry.Request),
 	}
 
 	c.finishLine(&p, path)
@@ -475,24 +475,23 @@ func normalizeRequestURI(requestURI, request string) string {
 	return ""
 }
 
-// rawPathFromRequest extracts the un-normalized path from "$request" (e.g.
-// "GET /news/12345/title HTTP/1.1") with the querystring stripped. Returns ""
-// if request is malformed.
-func rawPathFromRequest(request string) string {
+// rawURIFromRequest extracts the un-normalized request URI (path + query) from
+// "$request" (e.g. "GET /news/12345?utm=x HTTP/1.1"). Returns "" if malformed.
+func rawURIFromRequest(request string) string {
 	parts := strings.SplitN(request, " ", 3)
 	if len(parts) < 2 {
 		return ""
 	}
-	return stripQuery(parts[1])
+	return parts[1]
 }
 
-// rawPathFromJSON picks request_uri (preferred — already path-only in most
-// log_formats) and falls back to parsing $request. Querystring is stripped.
-func rawPathFromJSON(requestURI, request string) string {
+// rawURIFromJSON picks request_uri (already carries query string from nginx)
+// and falls back to parsing $request. Querystring is preserved.
+func rawURIFromJSON(requestURI, request string) string {
 	if requestURI != "" {
-		return stripQuery(requestURI)
+		return requestURI
 	}
-	return rawPathFromRequest(request)
+	return rawURIFromRequest(request)
 }
 
 func stripQuery(p string) string {

internal/topsrv/nginx/nginx_test.go

@@ -115,15 +115,15 @@ type recordingObserver struct {
 }
 
 type recordedLine struct {
-	status  string
-	uri     string
-	rawPath string
-	ua      string
-	path    string
+	status string
+	uri    string
+	rawURI string
+	ua     string
+	path   string
 }
 
 func (r *recordingObserver) OnLogLine(p *ParsedLine, path string) {
-	rl := recordedLine{status: p.Status, uri: p.URI, rawPath: p.RawPath, path: path}
+	rl := recordedLine{status: p.Status, uri: p.URI, rawURI: p.RawURI, path: path}
 	if r.uaIdx >= 0 && r.uaIdx < p.NExtras {
 		rl.ua = p.Extras[r.uaIdx]
 	}
@@ -164,15 +164,16 @@ func TestLogCollectorObserver(t *testing.T) {
 	assert.EqualValues(t, 0, c.reqCount, "no timing field → no histogram update")
 }
 
-// Verifies the URI vs RawPath split: nginx-metrics keep the normalized form
-// (cardinality control), observers like botlog get the actual hit URL.
-func TestParsedLine_RawPathUnnormalized(t *testing.T) {
+// Verifies the URI / RawURI split: nginx-metrics keep the normalized form
+// (cardinality control), observers like botlog get the full hit URL with
+// query string from RawURI.
+func TestParsedLine_RawURIUnnormalized(t *testing.T) {
 	cases := []struct {
-		name        string
-		setup       func(c *LogCollector)
-		feed        func(c *LogCollector)
-		wantURI     string
-		wantRawPath string
+		name       string
+		setup      func(c *LogCollector)
+		feed       func(c *LogCollector)
+		wantURI    string
+		wantRawURI string
 	}{
 		{
 			name: "text/$request with numeric segment",
@@ -184,8 +185,8 @@ func TestParsedLine_RawPathUnnormalized(t *testing.T) {
 					`1.2.3.4 [11/Apr/2026:17:15:23 +0300] "GET /news/12345/title?utm=x HTTP/1.1" 200 1234 "Bot/1"`,
 					"/var/log/nginx/access.log")
 			},
-			wantURI:     "/news/:id/:rest",
-			wantRawPath: "/news/12345/title",
+			wantURI:    "/news/:id/:rest",
+			wantRawURI: "/news/12345/title?utm=x",
 		},
 		{
 			name: "JSON request_uri with querystring",
@@ -196,8 +197,8 @@ func TestParsedLine_RawPathUnnormalized(t *testing.T) {
 				c.ParseJSONLine(`{"status":"200","body_bytes_sent":"100","request_time":"0.1",` +
 					`"request_uri":"/series/777/episodes?page=2","upstream_response_time":""}`)
 			},
-			wantURI:     "/series/:id/:rest",
-			wantRawPath: "/series/777/episodes",
+			wantURI:    "/series/:id/:rest",
+			wantRawURI: "/series/777/episodes?page=2",
 		},
 		{
 			name: "JSON with extra-labels path (map unmarshal)",
@@ -209,8 +210,33 @@ func TestParsedLine_RawPathUnnormalized(t *testing.T) {
 				c.ParseJSONLine(`{"status":"200","body_bytes_sent":"100","request_time":"0.1",` +
 					`"request_uri":"/user/42/comments","http_user_agent":"Bot/1"}`)
 			},
-			wantURI:     "/user/:id/:rest",
-			wantRawPath: "/user/42/comments",
+			wantURI:    "/user/:id/:rest",
+			wantRawURI: "/user/42/comments",
+		},
+		{
+			// Percent-encoded chars in query must survive verbatim so the
+			// gatesrv side can decode params (e.g. utm_term=hello%20world).
+			name: "text/$request with percent-encoded query",
+			setup: func(c *LogCollector) {
+				c.AddObserver(&recordingObserver{})
+			},
+			feed: func(c *LogCollector) {
+				c.parseLine(`10.0.0.1 [01/Jan/2026:00:00:00 +0000] "GET /search?q=hello%20world&p=a%2Fb HTTP/1.1" 200 100 "Bot/1"`)
+			},
+			wantURI:    "/search",
+			wantRawURI: "/search?q=hello%20world&p=a%2Fb",
+		},
+		{
+			name: "JSON request_uri preserves percent-encoded query",
+			setup: func(c *LogCollector) {
+				c.AddObserver(&recordingObserver{})
+			},
+			feed: func(c *LogCollector) {
+				c.ParseJSONLine(`{"status":"200","body_bytes_sent":"100","request_time":"0.1",` +
+					`"request_uri":"/api?token=ab%3Dcd%26ef"}`)
+			},
+			wantURI:    "/api",
+			wantRawURI: "/api?token=ab%3Dcd%26ef",
 		},
 	}
 	for _, tc := range cases {
@@ -224,7 +250,7 @@ func TestParsedLine_RawPathUnnormalized(t *testing.T) {
 			rec := c.observers[0].(*recordingObserver)
 			require.Len(t, rec.lines, 1)
 			assert.Equal(t, tc.wantURI, rec.lines[0].uri, "URI must be normalized for nginx-metrics")
-			assert.Equal(t, tc.wantRawPath, rec.lines[0].rawPath, "RawPath must be the raw request path (querystring stripped)")
+			assert.Equal(t, tc.wantRawURI, rec.lines[0].rawURI, "RawURI must be the raw request URI (path + query)")
 		})
 	}
 }