@@ -80,7 +80,7 @@ type Director struct {
80
80
// 1. Parses request details.
81
81
// 2. Calls admitRequest for admission control.
82
82
// 3. Calls Scheduler.Schedule if request is approved.
83
- // 4. Calls prepareRequest to populate RequestContext with results and call PreRequest plugins.
83
+ // 4. Calls prepareRequest to populate RequestContext with result and call PreRequest plugins.
84
84
//
85
85
// It always returns the requestContext even in the error case, as the request context is used in error handling.
86
86
func (d * Director ) HandleRequest (ctx context.Context , reqCtx * handlers.RequestContext ) (* handlers.RequestContext , error ) {
@@ -147,15 +147,15 @@ func (d *Director) HandleRequest(ctx context.Context, reqCtx *handlers.RequestCo
147
147
if len (candidatePods ) == 0 {
148
148
return reqCtx , errutil.Error {Code : errutil .ServiceUnavailable , Msg : "failed to find candidate pods for serving the request" }
149
149
}
150
- results , err := d .scheduler .Schedule (ctx , reqCtx .SchedulingRequest , candidatePods )
150
+ result , err := d .scheduler .Schedule (ctx , reqCtx .SchedulingRequest , candidatePods )
151
151
if err != nil {
152
152
return reqCtx , errutil.Error {Code : errutil .InferencePoolResourceExhausted , Msg : fmt .Errorf ("failed to find target pod: %w" , err ).Error ()}
153
153
}
154
154
155
155
// --- 4. Prepare Request (Populates RequestContext and call PreRequest plugins) ---
156
156
// Insert target endpoint to instruct Envoy to route requests to the specified target pod and attach the port number.
157
157
// Invoke PreRequest registered plugins.
158
- reqCtx , err = d .prepareRequest (ctx , reqCtx , results )
158
+ reqCtx , err = d .prepareRequest (ctx , reqCtx , result )
159
159
if err != nil {
160
160
return reqCtx , err
161
161
}
@@ -231,7 +231,7 @@ func (d *Director) getCandidatePodsForScheduling(ctx context.Context, requestMet
231
231
}
232
232
233
233
// prepareRequest populates the RequestContext and calls the registered PreRequest plugins
234
- // for allowing plugging customized logic based on the scheduling results .
234
+ // for allowing plugging customized logic based on the scheduling result .
235
235
func (d * Director ) prepareRequest (ctx context.Context , reqCtx * handlers.RequestContext , result * schedulingtypes.SchedulingResult ) (* handlers.RequestContext , error ) {
236
236
logger := log .FromContext (ctx )
237
237
if result == nil || len (result .ProfileResults ) == 0 {
@@ -320,7 +320,8 @@ func RandomWeightedDraw(logger logr.Logger, model *v1alpha2.InferenceModel, seed
320
320
}
321
321
322
322
func (d * Director ) runPreRequestPlugins (ctx context.Context , request * schedulingtypes.LLMRequest , schedulingResult * schedulingtypes.SchedulingResult ,
323
- targetPort int ) {
323
+ targetPort int ,
324
+ ) {
324
325
for _ , plugin := range d .preRequestPlugins {
325
326
log .FromContext (ctx ).V (logutil .DEBUG ).Info ("Running pre-request plugin" , "plugin" , plugin .TypedName ().Type )
326
327
before := time .Now ()
0 commit comments