Skip to content

Commit 6429125

Browse files
committed
Add Open Telemetry instrumentation.
This commit instruments various operations with Open Telemetry spans that abide by the (still nascent) semantic conventions for Generative AI clients [0]. These conventions classify `ellmer` chatbots as "agents" due to their ability to run tool calls, so in fact there are three types of span: (1) a top-level `invoke_agent` span for each chat interaction; (2) `chat` spans that wrap model API calls; and (3) `execute_tool` spans that wrap tool calls on our end. There's currently no community concensus for how to attach turns to spans, so I've left that out for now. Example code: library(otelsdk) Sys.setenv(OTEL_TRACES_EXPORTER = "stderr") chat <- ellmer::chat_databricks(model = "databricks-claude-3-7-sonnet") chat$chat("Tell me a joke in the form of an SQL query.") Unit tests are included. [0]: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/ Signed-off-by: Aaron Jacobs <[email protected]>
1 parent 1964c37 commit 6429125

File tree

5 files changed

+383
-1
lines changed

5 files changed

+383
-1
lines changed

DESCRIPTION

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ Suggests:
4040
knitr,
4141
magick,
4242
openssl,
43+
otel (>= 0.0.0.9000),
44+
otelsdk (>= 0.0.0.9000),
4345
paws.common,
4446
rmarkdown,
4547
shiny,
@@ -78,6 +80,7 @@ Collate:
7880
'import-standalone-purrr.R'
7981
'import-standalone-types-check.R'
8082
'interpolate.R'
83+
'otel.R'
8184
'params.R'
8285
'provider-openai.R'
8386
'provider-azure.R'
@@ -108,3 +111,6 @@ Collate:
108111
'utils-prettytime.R'
109112
'utils.R'
110113
'zzz.R'
114+
Remotes:
115+
r-lib/otel,
116+
r-lib/otelsdk

R/chat.R

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -492,6 +492,7 @@ Chat <- R6::R6Class(
492492
) {
493493
tool_errors <- list()
494494
withr::defer(warn_tool_errors(tool_errors))
495+
start_agent_span(private$provider)
495496

496497
while (!is.null(user_turn)) {
497498
assistant_chunks <- private$submit_turns(
@@ -551,6 +552,8 @@ Chat <- R6::R6Class(
551552
) {
552553
tool_errors <- list()
553554
withr::defer(warn_tool_errors(tool_errors))
555+
span <- start_agent_span(private$provider, active = FALSE)
556+
withr::defer(span$end())
554557

555558
while (!is.null(user_turn)) {
556559
assistant_chunks <- private$submit_turns_async(
@@ -627,7 +630,7 @@ Chat <- R6::R6Class(
627630
if (echo == "all") {
628631
cat_line(format(user_turn), prefix = "> ")
629632
}
630-
633+
span <- start_chat_span(private$provider)
631634
response <- chat_perform(
632635
provider = private$provider,
633636
mode = if (stream) "stream" else "value",
@@ -654,9 +657,11 @@ Chat <- R6::R6Class(
654657

655658
result <- stream_merge_chunks(private$provider, result, chunk)
656659
}
660+
record_chat_span_status(span, result)
657661
turn <- value_turn(private$provider, result, has_type = !is.null(type))
658662
turn <- match_tools(turn, private$tools)
659663
} else {
664+
record_chat_span_status(span, response)
660665
turn <- value_turn(
661666
private$provider,
662667
response,
@@ -709,6 +714,8 @@ Chat <- R6::R6Class(
709714
type = NULL,
710715
yield_as_content = FALSE
711716
) {
717+
span <- start_chat_span(private$provider, active = FALSE)
718+
withr::defer(span$end())
712719
response <- chat_perform(
713720
provider = private$provider,
714721
mode = if (stream) "async-stream" else "async-value",
@@ -735,10 +742,12 @@ Chat <- R6::R6Class(
735742

736743
result <- stream_merge_chunks(private$provider, result, chunk)
737744
}
745+
record_chat_span_status(span, result)
738746
turn <- value_turn(private$provider, result, has_type = !is.null(type))
739747
} else {
740748
result <- await(response)
741749

750+
record_chat_span_status(span, result)
742751
turn <- value_turn(private$provider, result, has_type = !is.null(type))
743752
text <- turn@text
744753
if (!is.null(text)) {

R/content-tools.R

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,12 +145,14 @@ invoke_tool <- function(request) {
145145
return(args)
146146
}
147147

148+
span <- start_tool_span(request)
148149
tryCatch(
149150
{
150151
result <- do.call(request@tool@fun, args)
151152
new_tool_result(request, result)
152153
},
153154
error = function(e) {
155+
record_tool_error(span, e)
154156
new_tool_result(request, error = e)
155157
}
156158
)
@@ -168,12 +170,15 @@ on_load(
168170
return(args)
169171
}
170172

173+
span <- start_tool_span(request, active = FALSE)
174+
withr::defer(span$end())
171175
tryCatch(
172176
{
173177
result <- await(do.call(request@tool@fun, args))
174178
new_tool_result(request, result)
175179
},
176180
error = function(e) {
181+
record_tool_error(span, e)
177182
new_tool_result(request, error = e)
178183
}
179184
)

R/otel.R

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# Starts an Open Telemetry span that abides by the semantic conventions for
2+
# Generative AI completions.
3+
#
4+
# See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#inference
5+
start_chat_span <- function(
6+
provider,
7+
tracer = default_tracer(),
8+
scope = parent.frame(),
9+
active = TRUE
10+
) {
11+
if (is.null(tracer) || !tracer$is_enabled()) {
12+
return(NULL)
13+
}
14+
# Ensure we set attributes relevant to sampling at span creation time.
15+
attributes <- list(
16+
"gen_ai.operation.name" = "chat",
17+
"gen_ai.system" = tolower(provider@name),
18+
"gen_ai.request.model" = provider@model
19+
)
20+
if (active) {
21+
tracer$start_span(
22+
name = sprintf("chat %s", provider@model),
23+
options = list(kind = "CLIENT"),
24+
attributes = attributes,
25+
scope = scope
26+
)
27+
} else {
28+
tracer$start_session(
29+
name = sprintf("chat %s", provider@model),
30+
options = list(kind = "CLIENT"),
31+
attributes = attributes,
32+
session_scope = scope
33+
)
34+
}
35+
}
36+
37+
record_chat_span_status <- function(span, result) {
38+
if (is.null(span) || !span$is_recording()) {
39+
return(invisible(span))
40+
}
41+
if (!is.null(result$model)) {
42+
span$set_attribute("gen_ai.response.model", result$model)
43+
}
44+
if (!is.null(result$id)) {
45+
span$set_attribute("gen_ai.response.id", result$id)
46+
}
47+
if (!is.null(result$usage)) {
48+
span$set_attribute("gen_ai.usage.input_tokens", result$usage$prompt_tokens)
49+
span$set_attribute(
50+
"gen_ai.usage.output_tokens",
51+
result$usage$completion_tokens
52+
)
53+
}
54+
# TODO: Consider setting gen_ai.response.finish_reasons.
55+
span$set_status("ok")
56+
}
57+
58+
# Starts an Open Telemetry span that abides by the semantic conventions for
59+
# Generative AI tool calls.
60+
#
61+
# See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span
62+
start_tool_span <- function(
63+
request,
64+
tracer = default_tracer(),
65+
scope = parent.frame(),
66+
active = TRUE
67+
) {
68+
if (is.null(tracer) || !tracer$is_enabled()) {
69+
return(NULL)
70+
}
71+
attributes <- compact(list(
72+
"gen_ai.operation.name" = "execute_tool",
73+
"gen_ai.tool.name" = request@tool@name,
74+
"gen_ai.tool.description" = request@tool@description,
75+
"gen_ai.tool.call.id" = request@id
76+
))
77+
if (active) {
78+
tracer$start_span(
79+
name = sprintf("execute_tool %s", request@tool@name),
80+
options = list(kind = "INTERNAL"),
81+
attributes = attributes,
82+
scope = scope
83+
)
84+
} else {
85+
tracer$start_session(
86+
name = sprintf("execute_tool %s", request@tool@name),
87+
options = list(kind = "INTERNAL"),
88+
attributes = attributes,
89+
session_scope = scope
90+
)
91+
}
92+
}
93+
94+
record_tool_error <- function(span, error) {
95+
if (is.null(span) || !span$is_recording()) {
96+
return()
97+
}
98+
span$record_exception(error)
99+
span$set_status("error")
100+
span$set_attribute("error.type", class(error)[1])
101+
}
102+
103+
# Starts an Open Telemetry span that abides by the semantic conventions for
104+
# Generative AI "agents".
105+
#
106+
# See: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#inference
107+
start_agent_span <- function(
108+
provider,
109+
tracer = default_tracer(),
110+
scope = parent.frame(),
111+
active = TRUE
112+
) {
113+
if (is.null(tracer) || !tracer$is_enabled()) {
114+
return(NULL)
115+
}
116+
attributes <- list(
117+
"gen_ai.operation.name" = "chat",
118+
"gen_ai.system" = tolower(provider@name)
119+
)
120+
if (active) {
121+
tracer$start_span(
122+
name = "invoke_agent",
123+
options = list(kind = "CLIENT"),
124+
attributes = attributes,
125+
scope = scope
126+
)
127+
} else {
128+
tracer$start_session(
129+
name = "invoke_agent",
130+
options = list(kind = "CLIENT"),
131+
attributes = attributes,
132+
session_scope = scope
133+
)
134+
}
135+
}
136+
137+
default_tracer <- function() {
138+
if (!is_installed("otel")) {
139+
return(NULL)
140+
}
141+
otel::get_tracer("ellmer")
142+
}

0 commit comments

Comments
 (0)