From cd72321b38d3dea736286a7692f07e0195c6b981 Mon Sep 17 00:00:00 2001 From: Nayeem Kamal Date: Tue, 29 Jul 2025 04:47:28 -0400 Subject: [PATCH 1/4] added retrieval and embedding span methods --- .../java/datadog/trace/llmobs/LLMObsSystem.java | 15 +++++++++++++++ .../java/datadog/trace/api/llmobs/LLMObs.java | 16 ++++++++++++++++ .../api/llmobs/noop/NoOpLLMObsSpanFactory.java | 10 ++++++++++ .../bootstrap/instrumentation/api/Tags.java | 2 ++ 4 files changed, 43 insertions(+) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index e184a645f71..003de3ec8b8 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -189,6 +189,21 @@ public LLMObsSpan startWorkflowSpan( Tags.LLMOBS_WORKFLOW_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); } + @Override + public LLMObsSpan startEmbeddingSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return new DDLLMObsSpan( + Tags.LLMOBS_EMBEDDING_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); + } + + public LLMObsSpan startRetrievalSpan( + String spanName, + @javax.annotation.Nullable String mlApp, + @javax.annotation.Nullable String sessionId) { + return new DDLLMObsSpan( + Tags.LLMOBS_RETRIEVAL_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); + } + private String getMLApp(String mlApp) { if (mlApp == null || mlApp.isEmpty()) { return defaultMLApp; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index 392e76a4a82..3feb885e57d 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -46,6 +46,16 @@ public static LLMObsSpan startWorkflowSpan( return SPAN_FACTORY.startWorkflowSpan(spanName, mlApp, sessionId); } + public LLMObsSpan startEmbeddingSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return SPAN_FACTORY.startEmbeddingSpan(spanName, mlApp, sessionId); + } + + public LLMObsSpan startRetrievalSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return SPAN_FACTORY.startRetrievalSpan(spanName, mlApp, sessionId); + } + public static void SubmitEvaluation( LLMObsSpan llmObsSpan, String label, String categoricalValue, Map tags) { EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, categoricalValue, tags); @@ -90,6 +100,12 @@ LLMObsSpan startLLMSpan( LLMObsSpan startWorkflowSpan( String spanName, @Nullable String mlApp, @Nullable String sessionId); + + LLMObsSpan startEmbeddingSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId); + + LLMObsSpan startRetrievalSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId); } public interface LLMObsEvalProcessor { diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java index 080aa41bd82..a402b17e3b1 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java @@ -35,4 +35,14 @@ public LLMObsSpan startWorkflowSpan( String spanName, @Nullable String mlApp, @Nullable String sessionId) { return NoOpLLMObsSpan.INSTANCE; } + + public LLMObsSpan startEmbeddingSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } + + public LLMObsSpan startRetrievalSpan( + String spanName, @Nullable String mlApp, @Nullable String sessionId) { + return NoOpLLMObsSpan.INSTANCE; + } } diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java index 63aec166076..8a75bb12068 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java @@ -166,4 +166,6 @@ public class Tags { public static final String LLMOBS_TASK_SPAN_KIND = "task"; public static final String LLMOBS_AGENT_SPAN_KIND = "agent"; public static final String LLMOBS_TOOL_SPAN_KIND = "tool"; + public static final String LLMOBS_EMBEDDING_SPAN_KIND = "embedding"; + public static final String LLMOBS_RETRIEVAL_SPAN_KIND = "retrieval"; } From e07929090c59c6e505056a9cd44797f90c3d64d3 Mon Sep 17 00:00:00 2001 From: Nayeem Kamal Date: Wed, 30 Jul 2025 01:09:21 -0400 Subject: [PATCH 2/4] applied fixes from comments --- .../datadog/trace/llmobs/LLMObsSystem.java | 21 ++++++++++++------- .../java/datadog/trace/api/llmobs/LLMObs.java | 12 ++++++++--- .../llmobs/noop/NoOpLLMObsSpanFactory.java | 5 ++++- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 003de3ec8b8..3321232631a 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -12,7 +12,7 @@ import java.lang.instrument.Instrumentation; import java.util.Map; import java.util.concurrent.TimeUnit; -import org.jetbrains.annotations.Nullable; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -191,15 +191,22 @@ public LLMObsSpan startWorkflowSpan( @Override public LLMObsSpan startEmbeddingSpan( - String spanName, @Nullable String mlApp, @Nullable String sessionId) { - return new DDLLMObsSpan( - Tags.LLMOBS_EMBEDDING_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); + String spanName, + @Nullable String mlApp, + @Nullable String mlProvider, + @Nullable String sessionId) { + if (mlProvider == null) { + mlProvider = "custom"; + } + DDLLMObsSpan embeddingSpan = + new DDLLMObsSpan( + Tags.LLMOBS_EMBEDDING_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); + embeddingSpan.setTag("ml.provider", mlProvider); + return embeddingSpan; } public LLMObsSpan startRetrievalSpan( - String spanName, - @javax.annotation.Nullable String mlApp, - @javax.annotation.Nullable String sessionId) { + String spanName, @Nullable String mlApp, @Nullable String sessionId) { return new DDLLMObsSpan( Tags.LLMOBS_RETRIEVAL_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index 3feb885e57d..a9a87c5d3a1 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -47,8 +47,11 @@ public static LLMObsSpan startWorkflowSpan( } public LLMObsSpan startEmbeddingSpan( - String spanName, @Nullable String mlApp, @Nullable String sessionId) { - return SPAN_FACTORY.startEmbeddingSpan(spanName, mlApp, sessionId); + String spanName, + @Nullable String mlApp, + @Nullable String mlProvider, + @Nullable String sessionId) { + return SPAN_FACTORY.startEmbeddingSpan(spanName, mlApp, mlProvider, sessionId); } public LLMObsSpan startRetrievalSpan( @@ -102,7 +105,10 @@ LLMObsSpan startWorkflowSpan( String spanName, @Nullable String mlApp, @Nullable String sessionId); LLMObsSpan startEmbeddingSpan( - String spanName, @Nullable String mlApp, @Nullable String sessionId); + String spanName, + @Nullable String mlApp, + @Nullable String mlProvider, + @Nullable String sessionId); LLMObsSpan startRetrievalSpan( String spanName, @Nullable String mlApp, @Nullable String sessionId); diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java index a402b17e3b1..838566b3609 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java @@ -37,7 +37,10 @@ public LLMObsSpan startWorkflowSpan( } public LLMObsSpan startEmbeddingSpan( - String spanName, @Nullable String mlApp, @Nullable String sessionId) { + String spanName, + @Nullable String mlApp, + @Nullable String mlProvider, + @Nullable String sessionId) { return NoOpLLMObsSpan.INSTANCE; } From ee6d7226e4a773651d15fbb58ab67a8458311603 Mon Sep 17 00:00:00 2001 From: Nayeem Kamal Date: Wed, 30 Jul 2025 02:11:08 -0400 Subject: [PATCH 3/4] applied fixes from comments Signed-off-by: Nayeem Kamal --- .../src/main/java/datadog/trace/llmobs/LLMObsSystem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 3321232631a..f03a617331c 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -201,7 +201,7 @@ public LLMObsSpan startEmbeddingSpan( DDLLMObsSpan embeddingSpan = new DDLLMObsSpan( Tags.LLMOBS_EMBEDDING_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); - embeddingSpan.setTag("ml.provider", mlProvider); + embeddingSpan.setTag(LLMObsTags.MODEL_PROVIDER, mlProvider); return embeddingSpan; } From e0b3dca1e495aac46b42887cee09852f756c33f4 Mon Sep 17 00:00:00 2001 From: Nayeem Kamal Date: Mon, 4 Aug 2025 00:55:48 -0400 Subject: [PATCH 4/4] addressed comments Signed-off-by: Nayeem Kamal --- .../main/java/datadog/trace/llmobs/LLMObsSystem.java | 10 ++++++---- .../src/main/java/datadog/trace/api/llmobs/LLMObs.java | 8 +++++--- .../trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java | 3 ++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index f03a617331c..00a04e5e59d 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -193,15 +193,17 @@ public LLMObsSpan startWorkflowSpan( public LLMObsSpan startEmbeddingSpan( String spanName, @Nullable String mlApp, - @Nullable String mlProvider, + @Nullable String modelProvider, + @Nullable String modelName, @Nullable String sessionId) { - if (mlProvider == null) { - mlProvider = "custom"; + if (modelProvider == null) { + modelProvider = "custom"; } DDLLMObsSpan embeddingSpan = new DDLLMObsSpan( Tags.LLMOBS_EMBEDDING_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); - embeddingSpan.setTag(LLMObsTags.MODEL_PROVIDER, mlProvider); + embeddingSpan.setTag(LLMObsTags.MODEL_PROVIDER, modelProvider); + embeddingSpan.setTag(LLMObsTags.MODEL_NAME, modelName); return embeddingSpan; } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index a9a87c5d3a1..6f3224dfcaa 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -49,9 +49,10 @@ public static LLMObsSpan startWorkflowSpan( public LLMObsSpan startEmbeddingSpan( String spanName, @Nullable String mlApp, - @Nullable String mlProvider, + @Nullable String modelProvider, + @Nullable String modelName, @Nullable String sessionId) { - return SPAN_FACTORY.startEmbeddingSpan(spanName, mlApp, mlProvider, sessionId); + return SPAN_FACTORY.startEmbeddingSpan(spanName, mlApp, modelProvider, modelName, sessionId); } public LLMObsSpan startRetrievalSpan( @@ -107,7 +108,8 @@ LLMObsSpan startWorkflowSpan( LLMObsSpan startEmbeddingSpan( String spanName, @Nullable String mlApp, - @Nullable String mlProvider, + @Nullable String modelProvider, + @Nullable String modelName, @Nullable String sessionId); LLMObsSpan startRetrievalSpan( diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java index 838566b3609..b142ee43313 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java @@ -39,7 +39,8 @@ public LLMObsSpan startWorkflowSpan( public LLMObsSpan startEmbeddingSpan( String spanName, @Nullable String mlApp, - @Nullable String mlProvider, + @Nullable String modelProvider, + @Nullable String modelName, @Nullable String sessionId) { return NoOpLLMObsSpan.INSTANCE; }