Skip to content

fix(apm): Enhance Synthetic Span Service Representation #751

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Aug 7, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions bottlecap/src/config/env.rs
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,11 @@ pub struct EnvConfig {
/// <https://docs.datadoghq.com/agent/configuration/dual-shipping/?tab=helm#environment-variable-configuration-1>
#[serde(deserialize_with = "deserialize_additional_endpoints")]
pub apm_additional_endpoints: HashMap<String, Vec<String>>,
/// @env `DD_TRACE_AWS_SERVICE_REPRESENTATION_ENABLED`
///
/// Enable the new AWS-resource naming logic in the tracer.
#[serde(deserialize_with = "deserialize_optional_bool_from_anything")]
pub trace_aws_service_representation_enabled: Option<bool>,
//
// Trace Propagation
/// @env `DD_TRACE_PROPAGATION_STYLE`
Expand Down Expand Up @@ -362,6 +367,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
merge_option_to_value!(config, env_config, apm_config_compression_level);
merge_vec!(config, env_config, apm_features);
merge_hashmap!(config, env_config, apm_additional_endpoints);
merge_option_to_value!(config, env_config, trace_aws_service_representation_enabled);

// Trace Propagation
merge_vec!(config, env_config, trace_propagation_style);
Expand Down Expand Up @@ -560,6 +566,7 @@ mod tests {
jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "b3");
jail.set_env("DD_TRACE_PROPAGATION_EXTRACT_FIRST", "true");
jail.set_env("DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED", "true");
jail.set_env("DD_TRACE_AWS_SERVICE_REPRESENTATION_ENABLED", "true");

// OTLP
jail.set_env("DD_OTLP_CONFIG_TRACES_ENABLED", "false");
Expand Down Expand Up @@ -709,6 +716,7 @@ mod tests {
trace_propagation_style_extract: vec![TracePropagationStyle::B3],
trace_propagation_extract_first: true,
trace_propagation_http_baggage_enabled: true,
trace_aws_service_representation_enabled: true,
otlp_config_traces_enabled: false,
otlp_config_traces_span_name_as_resource_name: true,
otlp_config_traces_span_name_remappings: HashMap::from([(
Expand Down
2 changes: 2 additions & 0 deletions bottlecap/src/config/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,7 @@ pub struct Config {
pub trace_propagation_style_extract: Vec<TracePropagationStyle>,
pub trace_propagation_extract_first: bool,
pub trace_propagation_http_baggage_enabled: bool,
pub trace_aws_service_representation_enabled: bool,

// OTLP
//
Expand Down Expand Up @@ -371,6 +372,7 @@ impl Default for Config {
apm_config_compression_level: 6,
apm_features: vec![],
apm_additional_endpoints: HashMap::new(),
trace_aws_service_representation_enabled: true,
trace_propagation_style: vec![
TracePropagationStyle::Datadog,
TracePropagationStyle::TraceContext,
Expand Down
9 changes: 9 additions & 0 deletions bottlecap/src/config/yaml.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ pub struct YamlConfig {
pub apm_config: ApmConfig,
#[serde(deserialize_with = "deserialize_service_mapping")]
pub service_mapping: HashMap<String, String>,
#[serde(deserialize_with = "deserialize_optional_bool_from_anything")]
pub trace_aws_service_representation_enabled: Option<bool>,
// Trace Propagation
#[serde(deserialize_with = "deserialize_trace_propagation_style")]
pub trace_propagation_style: Vec<TracePropagationStyle>,
Expand Down Expand Up @@ -451,6 +453,11 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) {
merge_vec!(config, yaml_config, trace_propagation_style_extract);
merge_option_to_value!(config, yaml_config, trace_propagation_extract_first);
merge_option_to_value!(config, yaml_config, trace_propagation_http_baggage_enabled);
merge_option_to_value!(
config,
yaml_config,
trace_aws_service_representation_enabled
);

// OTLP
if let Some(otlp_config) = &yaml_config.otlp_config {
Expand Down Expand Up @@ -711,6 +718,7 @@ trace_propagation_style: "datadog"
trace_propagation_style_extract: "b3"
trace_propagation_extract_first: true
trace_propagation_http_baggage_enabled: true
trace_aws_service_representation_enabled: true

# OTLP
otlp_config:
Expand Down Expand Up @@ -839,6 +847,7 @@ extension_version: "compatibility"
trace_propagation_style_extract: vec![TracePropagationStyle::B3],
trace_propagation_extract_first: true,
trace_propagation_http_baggage_enabled: true,
trace_aws_service_representation_enabled: true,
otlp_config_traces_enabled: false,
otlp_config_traces_span_name_as_resource_name: true,
otlp_config_traces_span_name_remappings: HashMap::from([(
Expand Down
10 changes: 8 additions & 2 deletions bottlecap/src/lifecycle/invocation/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,19 @@ pub fn base64_to_string(base64_string: &str) -> Result<String, DecodeError> {
}

fn create_empty_span(name: String, resource: &str, service: &str) -> Span {
Span {
let mut span = Span {
name,
resource: resource.to_string(),
service: service.to_string(),
r#type: String::from("serverless"),
..Default::default()
}
};

// Add span.kind to the span to enable other server based features for serverless
span.meta
.insert("span.kind".to_string(), "server".to_string());

span
}

#[must_use]
Expand Down
15 changes: 12 additions & 3 deletions bottlecap/src/lifecycle/invocation/processor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ use crate::{
},
};

use crate::lifecycle::invocation::triggers::get_default_service_name;

pub const MS_TO_NS: f64 = 1_000_000.0;
pub const S_TO_MS: u64 = 1_000;
pub const S_TO_NS: f64 = 1_000_000_000.0;
Expand Down Expand Up @@ -89,16 +91,23 @@ impl Processor {
aws_config: Arc<AwsConfig>,
metrics_aggregator: Arc<Mutex<MetricsAggregator>>,
) -> Self {
let service = config.service.clone().unwrap_or(String::from("aws.lambda"));
let resource = tags_provider
.get_canonical_resource_name()
.unwrap_or(String::from("aws.lambda"));

let service = get_default_service_name(
&config.service.clone().unwrap_or(resource.clone()),
"aws.lambda",
config.trace_aws_service_representation_enabled,
);
let propagator = DatadogCompositePropagator::new(Arc::clone(&config));

Processor {
context_buffer: ContextBuffer::default(),
inferrer: SpanInferrer::new(config.service_mapping.clone()),
inferrer: SpanInferrer::new(
config.service_mapping.clone(),
config.trace_aws_service_representation_enabled,
),
propagator,
enhanced_metrics: EnhancedMetrics::new(metrics_aggregator, Arc::clone(&config)),
aws_config,
Expand Down Expand Up @@ -195,6 +204,7 @@ impl Processor {

self.dynamic_tags
.insert(String::from("cold_start"), cold_start.to_string());

if proactive_initialization {
self.dynamic_tags.insert(
String::from("proactive_initialization"),
Expand Down Expand Up @@ -238,7 +248,6 @@ impl Processor {
);
cold_start_span.span_id = generate_span_id();
cold_start_span.start = start_time;

context.cold_start_span = Some(cold_start_span);
}

Expand Down
94 changes: 77 additions & 17 deletions bottlecap/src/lifecycle/invocation/span_inferrer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ use crate::{
#[derive(Default)]
pub struct SpanInferrer {
service_mapping: HashMap<String, String>,
aws_service_representation_enabled: bool,
// Span inferred from the Lambda incoming request payload
pub inferred_span: Option<Span>,
// Nested span inferred from the Lambda incoming request payload
Expand All @@ -50,9 +51,13 @@ pub struct SpanInferrer {

impl SpanInferrer {
#[must_use]
pub fn new(service_mapping: HashMap<String, String>) -> Self {
pub fn new(
service_mapping: HashMap<String, String>,
aws_service_representation_enabled: bool,
) -> Self {
Self {
service_mapping,
aws_service_representation_enabled,
inferred_span: None,
wrapped_inferred_span: None,
is_async_span: false,
Expand Down Expand Up @@ -87,19 +92,31 @@ impl SpanInferrer {

if APIGatewayHttpEvent::is_match(payload_value) {
if let Some(t) = APIGatewayHttpEvent::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);

trigger = Some(Box::new(t));
}
} else if APIGatewayRestEvent::is_match(payload_value) {
if let Some(t) = APIGatewayRestEvent::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);

trigger = Some(Box::new(t));
}
} else if APIGatewayWebSocketEvent::is_match(payload_value) {
if let Some(t) = APIGatewayWebSocketEvent::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);

trigger = Some(Box::new(t));
}
Expand All @@ -110,19 +127,31 @@ impl SpanInferrer {
}
} else if LambdaFunctionUrlEvent::is_match(payload_value) {
if let Some(t) = LambdaFunctionUrlEvent::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);

trigger = Some(Box::new(t));
}
} else if MSKEvent::is_match(payload_value) {
if let Some(t) = MSKEvent::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);

trigger = Some(Box::new(t));
}
} else if SqsRecord::is_match(payload_value) {
if let Some(t) = SqsRecord::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);

self.generated_span_context = extract_trace_context_from_aws_trace_header(
t.attributes.aws_trace_header.clone(),
Expand All @@ -140,7 +169,11 @@ impl SpanInferrer {
sns: sns_entity,
event_subscription_arn: None,
};
wt.enrich_span(&mut wrapped_inferred_span, &self.service_mapping);
wt.enrich_span(
&mut wrapped_inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);
inferred_span.meta.extend(wt.get_tags());

wrapped_inferred_span.duration =
Expand All @@ -155,8 +188,11 @@ impl SpanInferrer {
..Default::default()
};

event_bridge_entity
.enrich_span(&mut wrapped_inferred_span, &self.service_mapping);
event_bridge_entity.enrich_span(
&mut wrapped_inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);
inferred_span.meta.extend(event_bridge_entity.get_tags());

wrapped_inferred_span.duration =
Expand All @@ -169,7 +205,11 @@ impl SpanInferrer {
}
} else if SnsRecord::is_match(payload_value) {
if let Some(t) = SnsRecord::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);

if let Some(message) = &t.sns.message {
if let Ok(event_bridge_wrapper_message) =
Expand All @@ -180,8 +220,11 @@ impl SpanInferrer {
..Default::default()
};

event_bridge_wrapper_message
.enrich_span(&mut wrapped_inferred_span, &self.service_mapping);
event_bridge_wrapper_message.enrich_span(
&mut wrapped_inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);
inferred_span
.meta
.extend(event_bridge_wrapper_message.get_tags());
Expand All @@ -197,27 +240,43 @@ impl SpanInferrer {
}
} else if DynamoDbRecord::is_match(payload_value) {
if let Some(t) = DynamoDbRecord::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);
self.span_pointers = t.get_span_pointers();

trigger = Some(Box::new(t));
}
} else if S3Record::is_match(payload_value) {
if let Some(t) = S3Record::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);
self.span_pointers = t.get_span_pointers();

trigger = Some(Box::new(t));
}
} else if EventBridgeEvent::is_match(payload_value) {
if let Some(t) = EventBridgeEvent::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);

trigger = Some(Box::new(t));
}
} else if KinesisRecord::is_match(payload_value) {
if let Some(t) = KinesisRecord::new(payload_value.clone()) {
t.enrich_span(&mut inferred_span, &self.service_mapping);
t.enrich_span(
&mut inferred_span,
&self.service_mapping,
self.aws_service_representation_enabled,
);

trigger = Some(Box::new(t));
}
Expand Down Expand Up @@ -282,6 +341,7 @@ impl SpanInferrer {
String::from("peer.service"),
invocation_span.service.clone(),
);
s.meta.insert("span.kind".to_string(), "server".to_string());

if let Some(ws) = &mut self.wrapped_inferred_span {
ws.trace_id = invocation_span.trace_id;
Expand Down
Loading
Loading