Ship request Host as a dedicated botlog Event field (#8)

- Add Event.Host (JSON: host) carrying the request \$host header
  alongside the existing Event.ServerName (\$server_name vhost
  config). Both ship side by side so receiver-side joins can pick
  the right one — no silent semantic shift on serverName
- Add `host` to botlog.RequiredFields(); app.registerLogCollector
  merges it into ExtractFields; Observer resolves idxHost at startup
  like the other field indices
- New normalizeHost: lowercase + strip optional `:port`, handling
  bracketed IPv6 literals correctly via net.SplitHostPort.
  256-byte length cap applied first so a hostile 8 KB Host header
  can't make SplitHostPort scan the full payload
- BuildEvent runs normalizeHost so Fields.Host can stay raw —
  single source of truth; non-Observer callers don't have to know
  the contract
- Reuse the existing truncate() helper instead of duplicating the
  length-cap branch
- Tests: TestNormalizeHost (9 cases incl. bracketed IPv6 with/
  without port and the 256-byte cap), TestObserver_HostAndServer
  NameIndependent (Host normalized, ServerName raw),
  TestObserver_HostMissingShipsEmpty (log_format without \$host)
- Fix stale "4 RequiredFields" wording in nginx/log.go;
  docs/metrics.md gets an Event payload notes section documenting
  the host vs serverName distinction and the normalization rules

Author: sergeyfast

docs/metrics.md

@@ -271,6 +271,12 @@ Metrics from Angie JSON API (`/status/`). Requires `api /status/;` directive in
 
 Opt-in. Emitted when `[BotLogs].Enabled = true`. The agent matches every parsed nginx access-log line against a built-in UA fingerprint table (38 families) and ships matched events as gzipped ndjson to the topsrv.io `/v1/bot-logs` endpoint, with disk-backed WAL spool for retry on transient send failures.
 
+Event payload notes:
+
+- `host` — the request `$host` header, normalized (lowercased, optional `:port` stripped, bracketed IPv6 preserved). Client-controlled. Use this when grouping by the actual domain the client requested.
+- `serverName` — nginx `$server_name` of the matched virtual host (config-controlled). Use this when grouping by the operator-configured vhost.
+- Both ship side by side; downstream dashboards/joins should pick the one that fits the question.
+
 | Metric | Type | Labels | Description |
 |--------|------|--------|-------------|
 | `topsrv_botlog_events_total` | counter | state, reason | Event lifecycle counts. `state=enqueued` (entered queue), `sent` (acked by ingest), `spooled` (written to WAL after transient failure), `dropped` (`reason` splits cause). For non-dropped states `reason=""`. For `state=dropped` the `reason` label is one of `queue_full` (Enqueue while queue full → raise `BatchSize`), `permanent` (4xx, payload bad), `spool_write` (mkdir/write/missing SpoolDir), `spool_evict` (trim by `MaxSpoolMB` or foreign-owned file). |

internal/topsrv/botlog/event.go

@@ -13,6 +13,7 @@ import (
 // wire and save bytes.
 type Event struct {
 	TS                     time.Time `json:"ts"`
+	Host                   string    `json:"host,omitempty"`
 	ServerName             string    `json:"serverName,omitempty"`
 	AgentHostname          string    `json:"agentHostname"`
 	RemoteAddr             string    `json:"remoteAddr,omitempty"`
@@ -40,7 +41,8 @@ type Fields struct {
 	UpstreamResponseTime string // may be a comma-separated chain on retries
 	UpstreamCacheStatus  string
 	UserAgent            string
-	ServerName           string
+	Host                 string // raw request Host header — BuildEvent runs normalizeHost
+	ServerName           string // matched nginx vhost config name
 	RemoteAddr           string
 	Referer              string
 	Method               string
@@ -62,6 +64,7 @@ func NewEvent(now time.Time, agentHostname string, f Fields, extraUAPatterns []s
 func BuildEvent(now time.Time, agentHostname string, f Fields, family, name string, uaTruncate int) Event {
 	return Event{
 		TS:                     now,
+		Host:                   normalizeHost(f.Host),
 		ServerName:             f.ServerName,
 		AgentHostname:          agentHostname,
 		RemoteAddr:             dashToEmpty(f.RemoteAddr),

internal/topsrv/botlog/observer.go

@@ -1,7 +1,9 @@
 package botlog
 
 import (
+	"net"
 	"slices"
+	"strings"
 	"time"
 
 	"github.com/vmkteam/topsrv/internal/topsrv/nginx"
@@ -10,6 +12,7 @@ import (
 // nginx variable names botlog needs from the log_format.
 const (
 	fieldUserAgent  = "http_user_agent"
+	fieldHost       = "host"
 	fieldServerName = "server_name"
 	fieldRemoteAddr = "remote_addr"
 	fieldReferer    = "http_referer"
@@ -21,7 +24,25 @@ const (
 // LogConfig.ExtractFields, leaving operator-supplied ExtraLabels (low
 // cardinality) as the only Prometheus label set.
 func RequiredFields() []string {
-	return []string{fieldUserAgent, fieldServerName, fieldRemoteAddr, fieldReferer}
+	return []string{fieldUserAgent, fieldHost, fieldServerName, fieldRemoteAddr, fieldReferer}
+}
+
+// maxHostLen caps the request Host header value retained on an Event. Picked
+// to fit any legal FQDN (DNS RFC 1035 = 253 chars) with headroom for IDN
+// punycode plus a bracketed IPv6 literal — anything longer is hostile input.
+// Operator-tunable UA truncation lives separately as Config.UATruncate.
+const maxHostLen = 256
+
+// normalizeHost lowercases the host and strips an optional `:port` suffix,
+// handling bracketed IPv6 literals (`[::1]:80` → `[::1]`). The length cap is
+// applied first so SplitHostPort doesn't scan a hostile 8 KB Host header
+// (nginx accepts up to large_client_header_buffers without escaping).
+func normalizeHost(s string) string {
+	s = truncate(s, maxHostLen)
+	if h, _, err := net.SplitHostPort(s); err == nil {
+		s = h
+	}
+	return strings.ToLower(s)
 }
 
 // Observer implements nginx.LogObserver. On every parsed access log line it
@@ -36,7 +57,7 @@ type Observer struct {
 
 	// Resolved at construction from the LogCollector's ExtractFields. -1 when
 	// the operator's log_format doesn't carry that variable.
-	idxUA, idxServerName, idxRemoteAddr, idxReferer int
+	idxUA, idxHost, idxServerName, idxRemoteAddr, idxReferer int
 }
 
 // NewObserver wires an Observer against an already-constructed Pusher.
@@ -49,6 +70,7 @@ func NewObserver(p *Pusher, cfg Config, hostname string, extractFields []string)
 		uaTruncate:    cfg.UATruncate,
 		extraPatterns: cfg.ExtraUAPatterns,
 		idxUA:         slices.Index(extractFields, fieldUserAgent),
+		idxHost:       slices.Index(extractFields, fieldHost),
 		idxServerName: slices.Index(extractFields, fieldServerName),
 		idxRemoteAddr: slices.Index(extractFields, fieldRemoteAddr),
 		idxReferer:    slices.Index(extractFields, fieldReferer),
@@ -83,6 +105,7 @@ func (o *Observer) OnLogLine(p *nginx.ParsedLine, _ string) {
 		UpstreamResponseTime: p.UpstreamResponseTime,
 		UpstreamCacheStatus:  p.UpstreamCacheStatus,
 		UserAgent:            ua,
+		Host:                 o.field(p, o.idxHost),
 		ServerName:           o.field(p, o.idxServerName),
 		RemoteAddr:           o.field(p, o.idxRemoteAddr),
 		Referer:              o.field(p, o.idxReferer),

internal/topsrv/botlog/observer_test.go

@@ -3,6 +3,7 @@ package botlog
 import (
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 	"time"
 
@@ -17,11 +18,11 @@ import (
 )
 
 func TestRequiredFieldsContents(t *testing.T) {
-	// RequiredFields contract: the four nginx variables botlog needs read into
+	// RequiredFields contract: nginx variables botlog needs read into
 	// ParsedLine.Extras. Order is no longer load-bearing (Observer resolves
 	// indices at runtime), but the set must stay stable across versions.
 	assert.ElementsMatch(t,
-		[]string{fieldUserAgent, fieldServerName, fieldRemoteAddr, fieldReferer},
+		[]string{fieldUserAgent, fieldHost, fieldServerName, fieldRemoteAddr, fieldReferer},
 		RequiredFields())
 }
 
@@ -35,37 +36,37 @@ func newObserverPair(t *testing.T) (*Observer, *Pusher) {
 	}
 	require.NoError(t, cfg.Validate(topsrv.PushConfig{}))
 	p := NewPusher(embedlog.Logger{}, "topsrv-test", "test", cfg, prometheus.NewRegistry())
-	// Default order matches botParsedLine's Extras layout (ua, serverName,
-	// remoteAddr, referer) so existing tests' positional assumptions hold.
+	// Tests use the canonical RequiredFields() order: ua, host, server_name,
+	// remote_addr, referer (see botParsedLine).
 	o := NewObserver(p, cfg, "web01", RequiredFields())
 	return o, p
 }
 
-func botParsedLine(ua, serverName, remoteAddr, referer string) *nginx.ParsedLine {
+func botParsedLine(ua, host, serverName, remoteAddr, referer string) *nginx.ParsedLine {
 	return &nginx.ParsedLine{
 		Status:        "200",
 		URI:           "/api",
 		BodyBytesSent: "1234",
 		RequestTime:   "0.150",
-		Extras:        [nginx.MaxExtras]string{ua, serverName, remoteAddr, referer},
-		NExtras:       4,
+		Extras:        [nginx.MaxExtras]string{ua, host, serverName, remoteAddr, referer},
+		NExtras:       5,
 	}
 }
 
 func TestObserver_EnqueuesBotEvent(t *testing.T) {
 	o, p := newObserverPair(t)
 
-	o.OnLogLine(botParsedLine("Mozilla/5.0 GPTBot/1.0", "api.example.com", "203.0.113.5", "-"), "")
+	o.OnLogLine(botParsedLine("Mozilla/5.0 GPTBot/1.0", "api.example.com", "vhost_cfg", "203.0.113.5", "-"), "")
 
 	assert.InDelta(t, 1, testutil.ToFloat64(p.eventsTotal.WithLabelValues(stateEnqueued, "")), 0.01)
 	assert.InDelta(t, 1, testutil.ToFloat64(p.matchTotal.WithLabelValues("openai")), 0.01)
 
-	// Pull the event off the queue and inspect.
 	select {
 	case ev := <-p.queue:
 		assert.Equal(t, "openai", ev.BotFamily)
 		assert.Equal(t, "gptbot", ev.BotName)
-		assert.Equal(t, "api.example.com", ev.ServerName)
+		assert.Equal(t, "api.example.com", ev.Host, "Host carries the request $host header")
+		assert.Equal(t, "vhost_cfg", ev.ServerName, "ServerName carries the matched $server_name")
 		assert.Equal(t, "203.0.113.5", ev.RemoteAddr)
 		assert.Empty(t, ev.Referer, "dash referer dropped")
 		assert.Equal(t, "web01", ev.AgentHostname)
@@ -111,7 +112,7 @@ func TestObserver_FallsBackToURIWhenRawPathEmpty(t *testing.T) {
 func TestObserver_NonBotIgnored(t *testing.T) {
 	o, p := newObserverPair(t)
 
-	o.OnLogLine(botParsedLine("Mozilla/5.0 (Macintosh) Safari/605", "x.example.com", "1.2.3.4", "-"), "")
+	o.OnLogLine(botParsedLine("Mozilla/5.0 (Macintosh) Safari/605", "x.example.com", "", "1.2.3.4", "-"), "")
 
 	assert.InDelta(t, 0, testutil.ToFloat64(p.eventsTotal.WithLabelValues(stateEnqueued, "")), 0.01)
 	assert.InDelta(t, 0, testutil.ToFloat64(p.matchTotal.WithLabelValues("openai")), 0.01)
@@ -120,7 +121,7 @@ func TestObserver_NonBotIgnored(t *testing.T) {
 func TestObserver_EmptyUAIgnored(t *testing.T) {
 	o, p := newObserverPair(t)
 
-	o.OnLogLine(botParsedLine("", "x.example.com", "1.2.3.4", "-"), "")
+	o.OnLogLine(botParsedLine("", "x.example.com", "", "1.2.3.4", "-"), "")
 
 	assert.InDelta(t, 0, testutil.ToFloat64(p.eventsTotal.WithLabelValues(stateEnqueued, "")), 0.01)
 }
@@ -238,7 +239,56 @@ func TestObserver_MissingFieldsSafe(t *testing.T) {
 	require.Len(t, p.queue, 1)
 	ev := <-p.queue
 	assert.Equal(t, "openai", ev.BotFamily)
+	assert.Empty(t, ev.Host)
 	assert.Empty(t, ev.ServerName)
 	assert.Empty(t, ev.RemoteAddr)
 	assert.Empty(t, ev.Referer)
 }
+
+func TestNormalizeHost(t *testing.T) {
+	cases := []struct {
+		name, in, want string
+	}{
+		{"empty", "", ""},
+		{"lowercase", "Example.COM", "example.com"},
+		{"strip port", "example.com:8080", "example.com"},
+		{"strip port + lowercase", "API.Example.com:443", "api.example.com"},
+		{"no port", "example.com", "example.com"},
+		{"trailing colon — SplitHostPort accepts empty port", "example.com:", "example.com"},
+		{"IPv6 bracketed with port", "[::1]:8080", "::1"},
+		// SplitHostPort needs a port to unwrap brackets; bare bracketed
+		// literals pass through verbatim. Not a design choice — artifact.
+		{"IPv6 bracketed no port", "[2001:db8::1]", "[2001:db8::1]"},
+		{"truncate over maxHostLen", strings.Repeat("a", 300), strings.Repeat("a", 256)},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, normalizeHost(tc.in))
+		})
+	}
+}
+
+// Host and ServerName are independent: Host carries the (normalized) request
+// $host, ServerName carries the matched nginx $server_name. Either may be
+// empty if the log_format doesn't include it.
+func TestObserver_HostAndServerNameIndependent(t *testing.T) {
+	o, p := newObserverPair(t)
+	o.OnLogLine(botParsedLine("GPTBot/1.0", "Real.Example.COM:443", "vhost_cfg", "1.2.3.4", "https://prev.example.com/page"), "")
+
+	require.Len(t, p.queue, 1)
+	ev := <-p.queue
+	assert.Equal(t, "real.example.com", ev.Host, "Host = normalizeHost($host)")
+	assert.Equal(t, "vhost_cfg", ev.ServerName, "ServerName = raw $server_name")
+	assert.Equal(t, "https://prev.example.com/page", ev.Referer, "non-dash referer passes through")
+}
+
+// log_format without $host: Event.Host is empty, ServerName still ships.
+func TestObserver_HostMissingShipsEmpty(t *testing.T) {
+	o, p := newObserverPair(t)
+	o.OnLogLine(botParsedLine("GPTBot/1.0", "", "vhost_cfg", "1.2.3.4", "-"), "")
+
+	require.Len(t, p.queue, 1)
+	ev := <-p.queue
+	assert.Empty(t, ev.Host)
+	assert.Equal(t, "vhost_cfg", ev.ServerName)
+}

internal/topsrv/nginx/log.go

@@ -114,7 +114,7 @@ type statusURI struct {
 }
 
 // MaxExtras caps the per-line Extras slot count. Sized to fit operator
-// ExtraLabels (typically ≤3) plus botlog's 4 RequiredFields() with room.
+// ExtraLabels (typically ≤3) plus botlog's RequiredFields() with room.
 const MaxExtras = 8
 
 type taggedStatusKey struct {