diff --git a/gateway/api_loader.go b/gateway/api_loader.go index 5a670777101..253f76bab8b 100644 --- a/gateway/api_loader.go +++ b/gateway/api_loader.go @@ -1065,6 +1065,7 @@ func (gw *Gateway) loadApps(specs []*APISpec) { muxer := &proxyMux{ track404Logs: gwConf.Track404Logs, + gw: gw, } router := mux.NewRouter() router.NotFoundHandler = http.HandlerFunc(muxer.handle404) diff --git a/gateway/proxy_muxer.go b/gateway/proxy_muxer.go index 3b37482dc2f..e7172170ef0 100644 --- a/gateway/proxy_muxer.go +++ b/gateway/proxy_muxer.go @@ -21,6 +21,7 @@ import ( "github.com/TykTechnologies/again" "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/internal/httputil" + "github.com/TykTechnologies/tyk/request" "github.com/TykTechnologies/tyk/tcp" "github.com/gorilla/mux" @@ -134,6 +135,7 @@ type proxyMux struct { proxies []*proxy again again.Again track404Logs bool + gw *Gateway } func (m *proxyMux) getProxy(listenPort int, conf config.Config) *proxy { @@ -208,9 +210,71 @@ func (m *proxyMux) setRouter(port int, protocol string, router *mux.Router, conf func (m *proxyMux) handle404(w http.ResponseWriter, r *http.Request) { if m.track404Logs { + // Existing logging functionality requestMeta := fmt.Sprintf("%s %s %s", r.Method, r.URL.Path, r.Proto) log.WithField("request", requestMeta).WithField("origin", r.RemoteAddr). Error(http.StatusText(http.StatusNotFound)) + + // NEW: Also create analytics record if gateway and analytics are available + if m.gw != nil && m.gw.Analytics.Store != nil { + clientIP := request.RealIP(r) + + // Check if analytics should be recorded (respects enable_analytics and ignored_ips) + gwConfig := m.gw.GetConfig() + if gwConfig.StoreAnalytics(clientIP) { + t := time.Now() + + host := r.URL.Host + if host == "" { + host = r.Host + } + + record := analytics.AnalyticsRecord{ + Method: r.Method, + Host: host, + Path: r.URL.Path, + RawPath: r.URL.Path, + RawRequest: r.URL.RawQuery, + ContentLength: r.ContentLength, + UserAgent: r.Header.Get("User-Agent"), + Day: t.Day(), + Month: t.Month(), + Year: t.Year(), + Hour: t.Hour(), + ResponseCode: http.StatusNotFound, + APIKey: "", + TimeStamp: t, + APIVersion: "", + APIName: "", + APIID: "", + OrgID: "", + OauthID: "", + RequestTime: 0, + IPAddress: clientIP, + Geo: analytics.GeoData{}, + Network: analytics.NetworkStats{}, + Latency: analytics.Latency{ + Total: 0, + Upstream: 0, + }, + Tags: []string{"404", "path-not-found"}, + Alias: "", + TrackPath: false, + ExpireAt: t.Add(time.Duration(gwConfig.AnalyticsConfig.StorageExpirationTime) * time.Second), + } + + // Enable GeoIP if configured + if gwConfig.AnalyticsConfig.EnableGeoIP && m.gw.Analytics.GeoIPDB != nil { + record.GetGeo(clientIP, m.gw.Analytics.GeoIPDB) + } + + // Record the hit + err := m.gw.Analytics.RecordHit(&record) + if err != nil { + mainLog.Errorf("Failed to record 404 analytics for stream on path '%s %s', %v", r.Method, r.URL.Path, err) + } + } + } } w.WriteHeader(http.StatusNotFound) diff --git a/gateway/proxy_muxer_test.go b/gateway/proxy_muxer_test.go index 661056e1887..02c88c94e59 100644 --- a/gateway/proxy_muxer_test.go +++ b/gateway/proxy_muxer_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/TykTechnologies/tyk-pump/analytics" "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/test" ) @@ -365,6 +366,115 @@ func TestHandle404(t *testing.T) { }...) } +func TestHandle404WithAnalytics(t *testing.T) { + t.Run("track_404_logs disabled - no analytics record", func(t *testing.T) { + ts := StartTest(func(c *config.Config) { + c.Track404Logs = false + c.EnableAnalytics = true + }) + defer ts.Close() + + redisAnalyticsKeyName := analyticsKeyName + ts.Gw.Analytics.analyticsSerializer.GetSuffix() + + ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/api/" + spec.UseKeylessAccess = true + }) + + // Cleanup before test + ts.Gw.Analytics.Store.GetAndDeleteSet(redisAnalyticsKeyName) + + // Make request to non-existent path + _, _ = ts.Run(t, test.TestCase{ + Path: "/nonexistent", + Code: http.StatusNotFound, + }) + + // Flush analytics to Redis + ts.Gw.Analytics.Flush() + + // Verify NO analytics record was created + results := ts.Gw.Analytics.Store.GetAndDeleteSet(redisAnalyticsKeyName) + assert.Equal(t, 0, len(results), "No analytics record should be created when track_404_logs is disabled") + }) + + t.Run("track_404_logs enabled + analytics enabled - record created", func(t *testing.T) { + ts := StartTest(func(c *config.Config) { + c.Track404Logs = true + c.EnableAnalytics = true + }) + defer ts.Close() + + redisAnalyticsKeyName := analyticsKeyName + ts.Gw.Analytics.analyticsSerializer.GetSuffix() + + ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/api/" + spec.UseKeylessAccess = true + }) + + // Cleanup before test + ts.Gw.Analytics.Store.GetAndDeleteSet(redisAnalyticsKeyName) + + // Make request to non-existent path + _, _ = ts.Run(t, test.TestCase{ + Path: "/nonexistent", + Code: http.StatusNotFound, + }) + + // Flush analytics to Redis + ts.Gw.Analytics.Flush() + + // Verify analytics record was created + results := ts.Gw.Analytics.Store.GetAndDeleteSet(redisAnalyticsKeyName) + assert.True(t, len(results) > 0, "Analytics record should be created when track_404_logs and enable_analytics are true") + + // Verify the record has correct fields + if len(results) > 0 { + var record analytics.AnalyticsRecord + err := ts.Gw.Analytics.analyticsSerializer.Decode([]byte(results[0].(string)), &record) + if err != nil { + t.Fatal("Error decoding analytics:", err) + } + assert.Equal(t, http.StatusNotFound, record.ResponseCode, "Response code should be 404") + assert.Contains(t, record.Tags, "404", "Tags should contain '404'") + assert.Contains(t, record.Tags, "path-not-found", "Tags should contain 'path-not-found'") + assert.Equal(t, "/nonexistent", record.Path, "Path should be '/nonexistent'") + } + }) + + t.Run("IP in ignored_ips - no record", func(t *testing.T) { + ts := StartTest(func(c *config.Config) { + c.Track404Logs = true + c.EnableAnalytics = true + c.AnalyticsConfig.IgnoredIPs = []string{"127.0.0.1"} + }) + defer ts.Close() + + redisAnalyticsKeyName := analyticsKeyName + ts.Gw.Analytics.analyticsSerializer.GetSuffix() + + ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/api/" + spec.UseKeylessAccess = true + }) + + // Cleanup before test + ts.Gw.Analytics.Store.GetAndDeleteSet(redisAnalyticsKeyName) + + // Make request to non-existent path from localhost (127.0.0.1) + _, _ = ts.Run(t, test.TestCase{ + Path: "/nonexistent", + Code: http.StatusNotFound, + }) + + // Flush analytics to Redis + ts.Gw.Analytics.Flush() + + // Verify NO analytics record (IP is ignored) + results := ts.Gw.Analytics.Store.GetAndDeleteSet(redisAnalyticsKeyName) + assert.Equal(t, 0, len(results), "No analytics record should be created when IP is in ignored_ips list") + }) +} + func TestHandleSubroutes(t *testing.T) { ts := StartTest(func(globalConf *config.Config) { globalConf.HttpServerOptions.EnableStrictRoutes = true