Skip to content

Commit 9a84146

Browse files
knfreemLDknfreemLD
authored
fix: Update AI tracker to include model & provider name for metrics generation (#302)
**Requirements** - [X] I have added test coverage for new or changed functionality - [ ] I have followed the repository's [pull request submission guidelines](../blob/v5/CONTRIBUTING.md#submitting-pull-requests) - [ ] I have validated my changes against all supported platform versions **Related issues** Provide links to any issues in this repository or elsewhere relating to this pull request. **Describe the solution you've provided** Provide a clear and concise description of what you expect to happen. **Describe alternatives you've considered** Provide a clear and concise description of any alternative solutions or features you've considered. **Additional context** For https://launchdarkly.atlassian.net/wiki/spaces/PD/pages/3849420877/Tech+Spec+-+AI+Config+Insights
1 parent be50cc1 commit 9a84146

File tree

2 files changed

+37
-25
lines changed

2 files changed

+37
-25
lines changed

‎ldai/tracker.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,8 @@ func newTrackerWithStopwatch(
172172
Set("variationKey", ldvalue.String(variationKey)).
173173
Set("configKey", ldvalue.String(key)).
174174
Set("version", ldvalue.Int(version)).
175+
Set("providerName", ldvalue.String(config.ProviderName())).
176+
Set("modelName", ldvalue.String(config.ModelName())).
175177
Build()
176178

177179
return &Tracker{

‎ldai/tracker_test.go‎

Lines changed: 35 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -47,31 +47,34 @@ func TestTracker_NewDoesNotPanicWithConfig(t *testing.T) {
4747
})
4848
}
4949

50-
func makeTrackData(configKey, variationKey string, version int) ldvalue.Value {
50+
func makeTrackData(configKey, variationKey string, version int, config *Config) ldvalue.Value {
5151
return ldvalue.ObjectBuild().
5252
Set("variationKey", ldvalue.String(variationKey)).
5353
Set("configKey", ldvalue.String(configKey)).
5454
Set("version", ldvalue.Int(version)).
55+
Set("providerName", ldvalue.String(config.ProviderName())).
56+
Set("modelName", ldvalue.String(config.ModelName())).
5557
Build()
5658
}
5759

5860
func TestTracker_TrackSuccess(t *testing.T) {
5961
events := newMockEvents()
60-
tracker := newTracker("key", "variationKey", 1, events, &Config{}, ldcontext.New("key"), nil)
62+
config := &Config{}
63+
tracker := newTracker("key", "variationKey", 1, events, config, ldcontext.New("key"), nil)
6164
assert.NoError(t, tracker.TrackSuccess())
6265

6366
expectedEvents := []trackEvent{
6467
{
6568
name: "$ld:ai:generation",
6669
context: ldcontext.New("key"),
6770
metricValue: 1.0,
68-
data: makeTrackData("key", "variationKey", 1),
71+
data: makeTrackData("key", "variationKey", 1, config),
6972
},
7073
{
7174
name: "$ld:ai:generation:success",
7275
context: ldcontext.New("key"),
7376
metricValue: 1.0,
74-
data: makeTrackData("key", "variationKey", 1),
77+
data: makeTrackData("key", "variationKey", 1, config),
7578
},
7679
}
7780

@@ -80,21 +83,22 @@ func TestTracker_TrackSuccess(t *testing.T) {
8083

8184
func TestTracker_TrackError(t *testing.T) {
8285
events := newMockEvents()
83-
tracker := newTracker("key", "variationKey", 2, events, &Config{}, ldcontext.New("key"), nil)
86+
config := &Config{}
87+
tracker := newTracker("key", "variationKey", 2, events, config, ldcontext.New("key"), nil)
8488
assert.NoError(t, tracker.TrackError())
8589

8690
expectedEvents := []trackEvent{
8791
{
8892
name: "$ld:ai:generation",
8993
context: ldcontext.New("key"),
9094
metricValue: 1.0,
91-
data: makeTrackData("key", "variationKey", 2),
95+
data: makeTrackData("key", "variationKey", 2, config),
9296
},
9397
{
9498
name: "$ld:ai:generation:error",
9599
context: ldcontext.New("key"),
96100
metricValue: 1.0,
97-
data: makeTrackData("key", "variationKey", 2),
101+
data: makeTrackData("key", "variationKey", 2, config),
98102
},
99103
}
100104

@@ -103,7 +107,8 @@ func TestTracker_TrackError(t *testing.T) {
103107

104108
func TestTracker_TrackRequest(t *testing.T) {
105109
events := newMockEvents()
106-
tracker := newTracker("key", "variationKey", 3, events, &Config{}, ldcontext.New("key"), nil)
110+
config := &Config{}
111+
tracker := newTracker("key", "variationKey", 3, events, config, ldcontext.New("key"), nil)
107112

108113
expectedResponse := ProviderResponse{
109114
Usage: TokenUsage{
@@ -127,31 +132,31 @@ func TestTracker_TrackRequest(t *testing.T) {
127132
name: "$ld:ai:generation",
128133
context: ldcontext.New("key"),
129134
metricValue: 1,
130-
data: makeTrackData("key", "variationKey", 3),
135+
data: makeTrackData("key", "variationKey", 3, config),
131136
},
132137
{
133138
name: "$ld:ai:generation:success",
134139
context: ldcontext.New("key"),
135140
metricValue: 1,
136-
data: makeTrackData("key", "variationKey", 3),
141+
data: makeTrackData("key", "variationKey", 3, config),
137142
},
138143
{
139144
name: "$ld:ai:duration:total",
140145
context: ldcontext.New("key"),
141146
metricValue: 10.0,
142-
data: makeTrackData("key", "variationKey", 3),
147+
data: makeTrackData("key", "variationKey", 3, config),
143148
},
144149
{
145150
name: "$ld:ai:tokens:total",
146151
context: ldcontext.New("key"),
147152
metricValue: 1,
148-
data: makeTrackData("key", "variationKey", 3),
153+
data: makeTrackData("key", "variationKey", 3, config),
149154
},
150155
{
151156
name: "$ld:ai:tokens:ttf",
152157
context: ldcontext.New("key"),
153158
metricValue: 42.0,
154-
data: makeTrackData("key", "variationKey", 3),
159+
data: makeTrackData("key", "variationKey", 3, config),
155160
},
156161
}
157162

@@ -191,9 +196,10 @@ func (m mockStopwatch) Stop() time.Duration {
191196

192197
func TestTracker_LatencyMeasuredIfNotProvided(t *testing.T) {
193198
events := newMockEvents()
199+
config := &Config{}
194200

195201
tracker := newTrackerWithStopwatch(
196-
"key", "variationKey", 5, events, &Config{}, ldcontext.New("key"), nil, mockStopwatch(42*time.Millisecond))
202+
"key", "variationKey", 5, events, config, ldcontext.New("key"), nil, mockStopwatch(42*time.Millisecond))
197203

198204
expectedResponse := ProviderResponse{
199205
Usage: TokenUsage{
@@ -216,23 +222,25 @@ func TestTracker_LatencyMeasuredIfNotProvided(t *testing.T) {
216222

217223
func TestTracker_TrackDuration(t *testing.T) {
218224
events := newMockEvents()
219-
tracker := newTracker("key", "variationKey", 6, events, &Config{}, ldcontext.New("key"), nil)
225+
config := &Config{}
226+
tracker := newTracker("key", "variationKey", 6, events, config, ldcontext.New("key"), nil)
220227

221228
assert.NoError(t, tracker.TrackDuration(time.Millisecond*10))
222229

223230
expectedEvent := trackEvent{
224231
name: "$ld:ai:duration:total",
225232
context: ldcontext.New("key"),
226233
metricValue: 10.0,
227-
data: makeTrackData("key", "variationKey", 6),
234+
data: makeTrackData("key", "variationKey", 6, config),
228235
}
229236

230237
assert.ElementsMatch(t, []trackEvent{expectedEvent}, events.events)
231238
}
232239

233240
func TestTracker_TrackFeedback(t *testing.T) {
234241
events := newMockEvents()
235-
tracker := newTracker("key", "variationKey", 7, events, &Config{}, ldcontext.New("key"), nil)
242+
config := &Config{}
243+
tracker := newTracker("key", "variationKey", 7, events, config, ldcontext.New("key"), nil)
236244

237245
assert.NoError(t, tracker.TrackFeedback(FeedbackPositive))
238246
assert.NoError(t, tracker.TrackFeedback(FeedbackNegative))
@@ -242,14 +250,14 @@ func TestTracker_TrackFeedback(t *testing.T) {
242250
name: "$ld:ai:feedback:user:positive",
243251
context: ldcontext.New("key"),
244252
metricValue: 1.0,
245-
data: makeTrackData("key", "variationKey", 7),
253+
data: makeTrackData("key", "variationKey", 7, config),
246254
}
247255

248256
expectedNegativeEvent := trackEvent{
249257
name: "$ld:ai:feedback:user:negative",
250258
context: ldcontext.New("key"),
251259
metricValue: 1.0,
252-
data: makeTrackData("key", "variationKey", 7),
260+
data: makeTrackData("key", "variationKey", 7, config),
253261
}
254262

255263
assert.ElementsMatch(t, []trackEvent{expectedPositiveEvent, expectedNegativeEvent}, events.events)
@@ -258,7 +266,8 @@ func TestTracker_TrackFeedback(t *testing.T) {
258266
func TestTracker_TrackUsage(t *testing.T) {
259267
t.Run("only one field set, only one event", func(t *testing.T) {
260268
events := newMockEvents()
261-
tracker := newTracker("key", "variationKey", 8, events, &Config{}, ldcontext.New("key"), nil)
269+
config := &Config{}
270+
tracker := newTracker("key", "variationKey", 8, events, config, ldcontext.New("key"), nil)
262271

263272
assert.NoError(t, tracker.TrackUsage(TokenUsage{
264273
Total: 42,
@@ -268,15 +277,16 @@ func TestTracker_TrackUsage(t *testing.T) {
268277
name: "$ld:ai:tokens:total",
269278
context: ldcontext.New("key"),
270279
metricValue: 42.0,
271-
data: makeTrackData("key", "variationKey", 8),
280+
data: makeTrackData("key", "variationKey", 8, config),
272281
}
273282

274283
assert.ElementsMatch(t, []trackEvent{expectedEvent}, events.events)
275284
})
276285

277286
t.Run("all fields set, all events", func(t *testing.T) {
278287
events := newMockEvents()
279-
tracker := newTracker("key", "variationKey", 9, events, &Config{}, ldcontext.New("key"), nil)
288+
config := &Config{}
289+
tracker := newTracker("key", "variationKey", 9, events, config, ldcontext.New("key"), nil)
280290

281291
assert.NoError(t, tracker.TrackUsage(TokenUsage{
282292
Total: 42,
@@ -288,21 +298,21 @@ func TestTracker_TrackUsage(t *testing.T) {
288298
name: "$ld:ai:tokens:total",
289299
context: ldcontext.New("key"),
290300
metricValue: 42.0,
291-
data: makeTrackData("key", "variationKey", 9),
301+
data: makeTrackData("key", "variationKey", 9, config),
292302
}
293303

294304
expectedInput := trackEvent{
295305
name: "$ld:ai:tokens:input",
296306
context: ldcontext.New("key"),
297307
metricValue: 20.0,
298-
data: makeTrackData("key", "variationKey", 9),
308+
data: makeTrackData("key", "variationKey", 9, config),
299309
}
300310

301311
expectedOutput := trackEvent{
302312
name: "$ld:ai:tokens:output",
303313
context: ldcontext.New("key"),
304314
metricValue: 22.0,
305-
data: makeTrackData("key", "variationKey", 9),
315+
data: makeTrackData("key", "variationKey", 9, config),
306316
}
307317

308318
assert.ElementsMatch(t, []trackEvent{expectedTotal, expectedInput, expectedOutput}, events.events)

0 commit comments

Comments
 (0)