Skip to content

Commit 898b30e

Browse files
sallyomsallyom
committed
Add minimal tracing instrumentation for auto-instrumentation
1 parent 025dc57 commit 898b30e

File tree

6 files changed

+49
-16
lines changed

6 files changed

+49
-16
lines changed

‎cmd/llm-d-routing-sidecar/main.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@ func main() {
5353
ctx := signals.SetupSignalHandler(context.Background())
5454
logger := klog.FromContext(ctx)
5555

56-
5756
if *connector != proxy.ConnectorNIXLV1 && *connector != proxy.ConnectorNIXLV2 && *connector != proxy.ConnectorLMCache {
5857
logger.Info("Error: --connector must either be 'nixl', 'nixlv2' or 'lmcache'")
5958
return

‎go.mod‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ require (
88
github.com/hashicorp/golang-lru/v2 v2.0.7
99
github.com/onsi/ginkgo/v2 v2.23.4
1010
github.com/onsi/gomega v1.37.0
11+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0
12+
go.opentelemetry.io/otel v1.36.0
1113
k8s.io/apimachinery v0.31.3
1214
k8s.io/client-go v0.31.3
1315
k8s.io/klog/v2 v2.130.1
@@ -17,7 +19,9 @@ require (
1719
require (
1820
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
1921
github.com/emicklei/go-restful/v3 v3.12.0 // indirect
22+
github.com/felixge/httpsnoop v1.0.4 // indirect
2023
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
24+
github.com/go-logr/stdr v1.2.2 // indirect
2125
github.com/go-openapi/jsonreference v0.21.0 // indirect
2226
github.com/go-openapi/swag v0.23.0 // indirect
2327
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
@@ -29,19 +33,19 @@ require (
2933
github.com/json-iterator/go v1.1.12 // indirect
3034
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
3135
github.com/modern-go/reflect2 v1.0.2 // indirect
32-
github.com/rogpeppe/go-internal v1.13.1 // indirect
3336
github.com/spf13/pflag v1.0.5 // indirect
34-
github.com/stretchr/testify v1.10.0 // indirect
3537
github.com/x448/float16 v0.8.4 // indirect
38+
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
39+
go.opentelemetry.io/otel/metric v1.36.0 // indirect
40+
go.opentelemetry.io/otel/trace v1.36.0 // indirect
3641
go.uber.org/automaxprocs v1.6.0 // indirect
3742
golang.org/x/net v0.38.0 // indirect
3843
golang.org/x/oauth2 v0.27.0 // indirect
39-
golang.org/x/sys v0.32.0 // indirect
44+
golang.org/x/sys v0.33.0 // indirect
4045
golang.org/x/term v0.30.0 // indirect
4146
golang.org/x/text v0.23.0 // indirect
4247
golang.org/x/time v0.5.0 // indirect
4348
golang.org/x/tools v0.31.0 // indirect
44-
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
4549
gopkg.in/inf.v0 v0.9.1 // indirect
4650
gopkg.in/yaml.v2 v2.4.0 // indirect
4751
gopkg.in/yaml.v3 v3.0.1 // indirect

‎go.sum‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,15 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
44
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
55
github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk=
66
github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
7+
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
8+
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
79
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
810
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
11+
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
912
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
1013
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
14+
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
15+
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
1116
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
1217
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
1318
github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
@@ -42,11 +47,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
4247
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
4348
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
4449
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
45-
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
4650
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
4751
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
48-
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
49-
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
5052
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
5153
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
5254
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
@@ -79,6 +81,16 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
7981
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
8082
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
8183
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
84+
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
85+
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
86+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw=
87+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94=
88+
go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
89+
go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
90+
go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
91+
go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
92+
go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
93+
go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
8294
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
8395
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
8496
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -100,8 +112,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
100112
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
101113
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
102114
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
103-
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
104-
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
115+
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
116+
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
105117
golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
106118
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
107119
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

‎internal/proxy/chat_completions.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ package proxy
1818

1919
import (
2020
"net/http"
21+
22+
"go.opentelemetry.io/otel"
23+
"go.opentelemetry.io/otel/attribute"
2124
)
2225

2326
var (
@@ -29,15 +32,26 @@ var (
2932
)
3033

3134
func (s *Server) chatCompletionsHandler(w http.ResponseWriter, r *http.Request) {
32-
prefillPodHostPort := r.Header.Get(requestHeaderPrefillHostPort)
35+
tracer := otel.GetTracerProvider().Tracer("llm-d-routing")
36+
ctx, span := tracer.Start(r.Context(), "routing_proxy.request")
37+
defer span.End()
38+
39+
// Add component attribute to distinguish this part of the system
40+
span.SetAttributes(
41+
attribute.String("component", "llm-d-routing-sidecar"),
42+
attribute.String("operation", "route_request"),
43+
)
3344

45+
prefillPodHostPort := r.Header.Get(requestHeaderPrefillHostPort)
3446
if prefillPodHostPort == "" {
3547
// backward compatible behavior: to remove in next release
3648
prefillPodHostPort = r.Header.Get(requestHeaderPrefillURL)
3749
}
3850

3951
if prefillPodHostPort == "" {
4052
s.logger.V(4).Info("skip disaggregated prefill")
53+
// Update the request context for downstream handlers
54+
r = r.WithContext(ctx)
4155
s.decoderProxy.ServeHTTP(w, r)
4256
return
4357
}
@@ -54,5 +68,7 @@ func (s *Server) chatCompletionsHandler(w http.ResponseWriter, r *http.Request)
5468
}
5569

5670
s.logger.V(4).Info("SSRF protection: prefill target allowed", "target", prefillPodHostPort)
71+
72+
r = r.WithContext(ctx)
5773
s.runConnectorProtocol(w, r, prefillPodHostPort)
5874
}

‎internal/proxy/connector_nixlv2.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,7 @@ func (s *Server) runNIXLProtocolV2(w http.ResponseWriter, r *http.Request, prefi
5959
// Prefill Stage
6060

6161
// 1. Prepare prefill request
62-
ctx := r.Context()
63-
preq := r.Clone(ctx)
62+
preq := r.Clone(r.Context())
6463

6564
preq.Header.Add(requestHeaderRequestID, uuidStr)
6665

@@ -131,7 +130,7 @@ func (s *Server) runNIXLProtocolV2(w http.ResponseWriter, r *http.Request, prefi
131130
// Decode Stage
132131

133132
// 1. Prepare decode request
134-
dreq := r.Clone(ctx)
133+
dreq := r.Clone(r.Context())
135134

136135
dreq.Header.Add(requestHeaderRequestID, uuidStr)
137136

@@ -159,7 +158,6 @@ func (s *Server) runNIXLProtocolV2(w http.ResponseWriter, r *http.Request, prefi
159158
dreq.ContentLength = int64(len(dbody))
160159

161160
// 2. Forward to local decoder.
162-
163161
s.logger.V(5).Info("sending request to decoder", "body", string(dbody))
164162
s.decoderProxy.ServeHTTP(w, dreq)
165163
}

‎internal/proxy/proxy.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131

3232
"github.com/go-logr/logr"
3333
lru "github.com/hashicorp/golang-lru/v2"
34+
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
3435
"k8s.io/klog/v2"
3536
)
3637

@@ -165,8 +166,11 @@ func (s *Server) Start(ctx context.Context) error {
165166
// Configure handlers
166167
mux := s.createRoutes()
167168

169+
// Wrap the server with OpenTelemetry HTTP instrumentation
170+
handler := otelhttp.NewHandler(mux, "routing-proxy-server")
171+
168172
server := &http.Server{
169-
Handler: mux,
173+
Handler: handler,
170174
// No ReadTimeout/WriteTimeout for LLM inference - can take hours for large contexts
171175
IdleTimeout: 300 * time.Second, // 5 minutes for keep-alive connections
172176
ReadHeaderTimeout: 30 * time.Second, // Reasonable for headers only

0 commit comments

Comments
 (0)