diff --git a/core/io/http_command.hxx b/core/io/http_command.hxx index e490cf914..135cc0d76 100644 --- a/core/io/http_command.hxx +++ b/core/io/http_command.hxx @@ -120,7 +120,7 @@ struct http_command : public std::enable_shared_from_this> if (span_->uses_tags()) { span_->add_tag(tracing::attributes::service, tracing::service_name_for_http_service(request.type)); - span_->add_tag(tracing::attributes::operation_id, client_context_id_); + span_->add_tag(tracing::attributes::dispatch::operation_id, client_context_id_); } handler_ = std::move(handler); @@ -209,7 +209,7 @@ struct http_command : public std::enable_shared_from_this> return; } if (span_->uses_tags()) { - span_->add_tag(tracing::attributes::local_id, session_->id()); + span_->add_tag(tracing::attributes::dispatch::local_id, session_->id()); } send(); } diff --git a/core/io/mcbp_command.hxx b/core/io/mcbp_command.hxx index d4b61ff06..aa4c236b7 100644 --- a/core/io/mcbp_command.hxx +++ b/core/io/mcbp_command.hxx @@ -139,12 +139,6 @@ struct mcbp_command : public std::enable_shared_from_this(protocol::parse_server_duration_us(msg.value())); - span_->add_tag(tracing::attributes::server_duration, server_duration_us); - server_durations_.emplace_back(std::chrono::microseconds(server_duration_us)); - } span_->end(); span_ = nullptr; } @@ -234,9 +228,6 @@ struct mcbp_command : public std::enable_shared_from_thisnext_opaque(); request.opaque = *opaque_; - if (span_->uses_tags()) { - span_->add_tag(tracing::attributes::operation_id, fmt::format("0x{:x}", request.opaque)); - } if (request.id.use_collections() && !request.id.is_collection_resolved()) { if (session_->supports_feature(protocol::hello_feature::collections)) { auto collection_id = session_->get_collection_uid(request.id.collection_path()); @@ -269,14 +260,23 @@ struct mcbp_command : public std::enable_shared_from_thiswrite_and_subscribe( request.opaque, encoded.data(session_->supports_feature(protocol::hello_feature::snappy)), - [self = this->shared_from_this(), start = std::chrono::steady_clock::now()]( + [self = this->shared_from_this(), + start = std::chrono::steady_clock::now(), + dispatch_span = std::move(dispatch_span)]( std::error_code ec, retry_reason reason, io::mcbp_message&& msg, std::optional /* error_info */) mutable { + { + const auto server_duration_us = + static_cast(protocol::parse_server_duration_us(msg)); + self->server_durations_.emplace_back(std::chrono::microseconds(server_duration_us)); + self->close_dispatch_span(dispatch_span, server_duration_us); + } { auto latency = std::chrono::duration_cast( std::chrono::steady_clock::now() - start); @@ -306,15 +306,17 @@ struct mcbp_command : public std::enable_shared_from_thisrecord_latency(category, latency); } - metrics::metric_attributes attrs{ - service_type::key_value, - self->request.observability_identifier, - ec, - self->request.id.bucket(), - self->request.id.scope(), - self->request.id.collection(), - }; - self->manager_->meter()->record_value(std::move(attrs), start); + { + metrics::metric_attributes attrs{ + service_type::key_value, + self->request.observability_identifier, + ec, + self->request.id.bucket(), + self->request.id.scope(), + self->request.id.collection(), + }; + self->manager_->meter()->record_value(std::move(attrs), start); + } self->retry_backoff.cancel(); if (ec == asio::error::operation_aborted) { @@ -397,12 +399,6 @@ struct mcbp_command : public std::enable_shared_from_thisuses_tags()) - span_->add_tag(tracing::attributes::remote_socket, session_->remote_address()); - if (span_->uses_tags()) - span_->add_tag(tracing::attributes::local_socket, session_->local_address()); - if (span_->uses_tags()) - span_->add_tag(tracing::attributes::local_id, session_->id()); send(); } @@ -427,6 +423,35 @@ private: return attrs; } + + auto create_dispatch_span() const -> std::shared_ptr + { + std::shared_ptr dispatch_span = + manager_->tracer()->create_span(tracing::operation::step_dispatch, span_); + if (dispatch_span->uses_tags()) { + dispatch_span->add_tag(tracing::attributes::dispatch::network_transport, "tcp"); + dispatch_span->add_tag(tracing::attributes::dispatch::operation_id, + fmt::format("0x{:x}", request.opaque)); + dispatch_span->add_tag(tracing::attributes::dispatch::local_id, session_->id()); + } + return dispatch_span; + } + + void close_dispatch_span(const std::shared_ptr& dispatch_span, + const std::uint64_t server_duration_us) const + { + if (dispatch_span->uses_tags()) { + dispatch_span->add_tag(tracing::attributes::dispatch::server_duration, server_duration_us); + dispatch_span->add_tag(tracing::attributes::dispatch::server_address, + session_->canonical_hostname()); + dispatch_span->add_tag(tracing::attributes::dispatch::server_port, + session_->canonical_port_number()); + dispatch_span->add_tag(tracing::attributes::dispatch::peer_address, + session_->remote_hostname()); + dispatch_span->add_tag(tracing::attributes::dispatch::peer_port, session_->remote_port()); + } + dispatch_span->end(); + } }; } // namespace couchbase::core::operations diff --git a/core/io/mcbp_session.cxx b/core/io/mcbp_session.cxx index a4dc7e71c..4b785c0b3 100644 --- a/core/io/mcbp_session.cxx +++ b/core/io/mcbp_session.cxx @@ -943,6 +943,16 @@ class mcbp_session_impl return connection_endpoints_.remote_address_with_port; } + auto remote_hostname() const -> std::string + { + return connection_endpoints_.remote_address; + } + + auto remote_port() const -> std::uint16_t + { + return connection_endpoints_.remote.port(); + } + auto local_address() const -> std::string { return connection_endpoints_.local_address_with_port; @@ -1533,6 +1543,16 @@ class mcbp_session_impl return bootstrap_port_number_; } + [[nodiscard]] auto canonical_hostname() const -> const std::string& + { + return canonical_hostname_; + } + + [[nodiscard]] auto canonical_port_number() const -> std::uint16_t + { + return canonical_port_number_; + } + [[nodiscard]] auto next_opaque() -> std::uint32_t { return ++opaque_; @@ -1743,10 +1763,18 @@ class mcbp_session_impl if (ec) { return stop(retry_reason::node_not_available); } - if (node_uuid_.empty() && config_.has_value()) { + if (config_.has_value()) { for (const auto& node : config_.value().nodes) { if (node.this_node) { - node_uuid_ = node.node_uuid; + if (node_uuid_.empty()) { + node_uuid_ = node.node_uuid; + } + if (canonical_hostname_.empty()) { + canonical_hostname_ = node.hostname; + } + if (canonical_port_number_ == 0) { + canonical_port_number_ = node.port_or(service_type::key_value, is_tls_, 0); + } } } } @@ -2112,6 +2140,10 @@ class mcbp_session_impl std::optional error_map_; collection_cache collection_cache_; + // Only used for tracing & metrics. They represent the address of the node as given in the config. + std::string canonical_hostname_{}; + std::uint16_t canonical_port_number_{}; + const bool is_tls_; std::shared_ptr state_listener_{ nullptr }; @@ -2241,6 +2273,18 @@ mcbp_session::remote_address() const -> std::string return impl_->remote_address(); } +auto +mcbp_session::remote_hostname() const -> std::string +{ + return impl_->remote_hostname(); +} + +auto +mcbp_session::remote_port() const -> std::uint16_t +{ + return impl_->remote_port(); +} + auto mcbp_session::local_address() const -> std::string { @@ -2277,6 +2321,18 @@ mcbp_session::last_bootstrap_error() const& -> const std::optionallast_bootstrap_error(); } +auto +mcbp_session::canonical_hostname() const -> const std::string& +{ + return impl_->canonical_hostname(); +} + +auto +mcbp_session::canonical_port_number() const -> std::uint16_t +{ + return impl_->canonical_port_number(); +} + void mcbp_session::write_and_subscribe(std::uint32_t opaque, std::vector&& data, diff --git a/core/io/mcbp_session.hxx b/core/io/mcbp_session.hxx index 69c649018..ff7ef4d64 100644 --- a/core/io/mcbp_session.hxx +++ b/core/io/mcbp_session.hxx @@ -119,6 +119,8 @@ public: [[nodiscard]] auto id() const -> const std::string&; [[nodiscard]] auto node_uuid() const -> const std::string&; [[nodiscard]] auto remote_address() const -> std::string; + [[nodiscard]] auto remote_hostname() const -> std::string; + [[nodiscard]] auto remote_port() const -> std::uint16_t; [[nodiscard]] auto local_address() const -> std::string; [[nodiscard]] auto bootstrap_address() const -> const std::string&; [[nodiscard]] auto bootstrap_hostname() const -> const std::string&; @@ -126,6 +128,8 @@ public: [[nodiscard]] auto bootstrap_port_number() const -> std::uint16_t; [[nodiscard]] auto last_bootstrap_error() && -> std::optional; [[nodiscard]] auto last_bootstrap_error() const& -> const std::optional&; + [[nodiscard]] auto canonical_hostname() const -> const std::string&; + [[nodiscard]] auto canonical_port_number() const -> std::uint16_t; void write_and_flush(std::vector&& buffer); void write_and_subscribe(const std::shared_ptr&, const std::shared_ptr& handler); diff --git a/core/tracing/constants.hxx b/core/tracing/constants.hxx index 83c0e96df..66d83da6f 100644 --- a/core/tracing/constants.hxx +++ b/core/tracing/constants.hxx @@ -27,8 +27,9 @@ namespace couchbase::core::tracing { namespace operation { -constexpr auto step_dispatch = "cb.dispatch_to_server"; -constexpr auto step_request_encoding = "cb.request_encoding"; +constexpr auto step_dispatch = "dispatch_to_server"; +constexpr auto step_request_encoding = "request_encoding"; + constexpr auto http_query = "cb.query"; constexpr auto http_analytics = "cb.analytics"; constexpr auto http_search = "cb.search"; @@ -77,7 +78,7 @@ constexpr auto mcbp_internal = "cb.internal"; namespace attributes { -constexpr auto system = "db.system"; +constexpr auto system = "db.system.name"; constexpr auto cluster_name = "db.couchbase.cluster_name"; constexpr auto cluster_uuid = "db.couchbase.cluster_uuid"; @@ -86,12 +87,20 @@ constexpr auto component = "db.couchbase.component"; constexpr auto instance = "db.instance"; constexpr auto service = "cb.service"; -constexpr auto operation_id = "cb.operation_id"; - -constexpr auto server_duration = "cb.server_duration"; -constexpr auto local_id = "cb.local_id"; constexpr auto local_socket = "cb.local_socket"; constexpr auto remote_socket = "cb.remote_socket"; + +namespace dispatch +{ +constexpr auto server_duration = "db.couchbase.server_duration"; +constexpr auto local_id = "db.couchbase.local_id"; +constexpr auto server_address = "server.address"; +constexpr auto server_port = "server.port"; +constexpr auto peer_address = "network.peer.address"; +constexpr auto peer_port = "network.peer.port"; +constexpr auto network_transport = "network.transport"; +constexpr auto operation_id = "db.couchbase.operation_id"; +} // namespace dispatch } // namespace attributes namespace service diff --git a/core/tracing/threshold_logging_tracer.cxx b/core/tracing/threshold_logging_tracer.cxx index 92a200c61..849cf1e00 100644 --- a/core/tracing/threshold_logging_tracer.cxx +++ b/core/tracing/threshold_logging_tracer.cxx @@ -57,16 +57,18 @@ class threshold_logging_span { private: std::chrono::system_clock::time_point start_{ std::chrono::system_clock::now() }; - std::string id_{ uuid::to_string(uuid::random()) }; - std::map integer_tags_{}; - std::map string_tags_{ - { attributes::system, "couchbase" }, - { attributes::span_kind, "client" }, - { attributes::component, couchbase::core::meta::sdk_id() }, - }; - std::chrono::microseconds duration_{ 0 }; + std::chrono::microseconds total_duration_{ 0 }; + std::uint64_t last_server_duration_us_{ 0 }; std::uint64_t total_server_duration_us_{ 0 }; + std::optional operation_id_{}; + std::optional last_local_id_{}; + std::optional service_{}; + std::optional peer_hostname_{}; + std::optional peer_port_{}; + + // Used by legacy top-level spans. TODO(CXXCBC-738): Remove once HTTP dispatch spans are added. + std::optional last_remote_socket_{}; std::shared_ptr tracer_{}; @@ -79,30 +81,64 @@ class threshold_logging_span { } - void add_tag(const std::string& name, std::uint64_t value) override + void add_tag(const std::string& tag_name, std::uint64_t value) override { - if (name == tracing::attributes::server_duration) { + if (tag_name == tracing::attributes::dispatch::server_duration) { last_server_duration_us_ = value; - total_server_duration_us_ += value; + if (name() != tracing::operation::step_dispatch) { + total_server_duration_us_ += value; + } + } + if (tag_name == tracing::attributes::dispatch::peer_port) { + peer_port_ = static_cast(value); } - integer_tags_.try_emplace(name, value); } - void add_tag(const std::string& name, const std::string& value) override + void add_tag(const std::string& tag_name, const std::string& value) override { - string_tags_.try_emplace(name, value); + if (tag_name == tracing::attributes::service) { + service_ = value; + } + if (tag_name == tracing::attributes::remote_socket) { + last_remote_socket_ = value; + } + if (tag_name == tracing::attributes::dispatch::local_id) { + last_local_id_ = value; + } + if (tag_name == tracing::attributes::dispatch::operation_id) { + operation_id_ = value; + } + if (tag_name == tracing::attributes::dispatch::peer_address) { + peer_hostname_ = value; + } } void end() override; - [[nodiscard]] auto string_tags() const -> const auto& + void set_last_remote_socket(const std::string& socket) { - return string_tags_; + last_remote_socket_ = socket; } - [[nodiscard]] auto duration() const -> std::chrono::microseconds + void set_last_local_id(const std::string& id) { - return duration_; + last_local_id_ = id; + } + + void set_operation_id(const std::string& id) + { + operation_id_ = id; + } + + void add_server_duration(const std::uint64_t duration_us) + { + last_server_duration_us_ = duration_us; + total_server_duration_us_ += duration_us; + } + + [[nodiscard]] auto total_duration() const -> std::chrono::microseconds + { + return total_duration_; } [[nodiscard]] auto last_server_duration_us() const -> std::uint64_t @@ -115,22 +151,32 @@ class threshold_logging_span return total_server_duration_us_; } + [[nodiscard]] auto operation_id() const -> std::optional + { + return operation_id_; + } + + [[nodiscard]] auto last_remote_socket() const -> std::optional + { + return last_remote_socket_; + } + + [[nodiscard]] auto last_local_id() const -> std::optional + { + return last_local_id_; + } + [[nodiscard]] auto is_key_value() const -> bool { - auto service_tag = string_tags_.find(tracing::attributes::service); - if (service_tag == string_tags_.end()) { - return false; - } - return service_tag->second == tracing::service::key_value; + return service_.has_value() && service_.value() == tracing::service::key_value; } [[nodiscard]] auto service() const -> std::optional { - auto service_tag = string_tags_.find(tracing::attributes::service); - if (service_tag == string_tags_.end()) { + if (!service_.has_value()) { return {}; } - const auto& service_name = service_tag->second; + const auto& service_name = service_.value(); if (service_name == tracing::service::key_value) { return service_type::key_value; } @@ -161,35 +207,26 @@ convert(const std::shared_ptr& span) -> reported_span tao::json::value entry{ { "operation_name", span->name() }, { "total_duration_us", - std::chrono::duration_cast(span->duration()).count() } + std::chrono::duration_cast(span->total_duration()).count() } }; if (span->is_key_value()) { entry["last_server_duration_us"] = span->last_server_duration_us(); entry["total_server_duration_us"] = span->total_server_duration_us(); } - const auto& tags = span->string_tags(); - auto pair = tags.find(attributes::operation_id); - if (pair != tags.end()) { - entry["last_operation_id"] = pair->second; + if (span->operation_id().has_value()) { + entry["last_operation_id"] = span->operation_id().value(); } - pair = tags.find(attributes::local_id); - if (pair != tags.end()) { - entry["last_local_id"] = pair->second; + if (span->last_local_id().has_value()) { + entry["last_local_id"] = span->last_local_id().value(); } - pair = tags.find(attributes::local_socket); - if (pair != tags.end()) { - entry["last_local_socket"] = pair->second; + if (span->last_remote_socket().has_value()) { + entry["last_remote_socket"] = span->last_remote_socket().value(); } - pair = tags.find(attributes::remote_socket); - if (pair != tags.end()) { - entry["last_remote_socket"] = pair->second; - } - - return { span->duration(), std::move(entry) }; + return { span->total_duration(), std::move(entry) }; } class threshold_logging_tracer_impl @@ -232,13 +269,13 @@ class threshold_logging_tracer_impl void check_threshold(const std::shared_ptr& span) { - auto service = span->service(); + const auto service = span->service(); if (!service.has_value()) { return; } - if (span->duration() > options_.threshold_for_service(service.value())) { - auto queue = threshold_queues_.find(service.value()); - if (queue != threshold_queues_.end()) { + if (span->total_duration() > options_.threshold_for_service(service.value())) { + if (const auto queue = threshold_queues_.find(service.value()); + queue != threshold_queues_.end()) { queue->second.emplace(convert(span)); } } @@ -328,9 +365,27 @@ threshold_logging_tracer::stop() void threshold_logging_span::end() { - duration_ = std::chrono::duration_cast( + total_duration_ = std::chrono::duration_cast( std::chrono::system_clock::now() - start_); - tracer_->report(shared_from_this()); + if (service_.has_value()) { + tracer_->report(shared_from_this()); + } + if (name() == tracing::operation::step_dispatch) { + // Transfer the relevant attributes to the operation-level span + if (const auto p = std::dynamic_pointer_cast(parent()); p) { + if (last_local_id_.has_value()) { + p->set_last_local_id(last_local_id_.value()); + } + if (operation_id_.has_value()) { + p->set_operation_id(operation_id_.value()); + } + if (peer_hostname_.has_value() && peer_port_.has_value()) { + p->set_last_remote_socket(fmt::format("{}:{}", peer_hostname_.value(), peer_port_.value())); + } + if (last_server_duration_us_ > 0) { + p->add_server_duration(last_server_duration_us_); + } + } + } } - } // namespace couchbase::core::tracing diff --git a/couchbase/tracing/request_span.hxx b/couchbase/tracing/request_span.hxx index ab96b157f..4466a1f9f 100644 --- a/couchbase/tracing/request_span.hxx +++ b/couchbase/tracing/request_span.hxx @@ -56,7 +56,7 @@ public: return parent_; } - virtual auto uses_tags() const -> bool + [[nodiscard]] virtual auto uses_tags() const -> bool { return true; } diff --git a/test/test_integration_tracer.cxx b/test/test_integration_tracer.cxx index ed2cd7bb3..051efe1c1 100644 --- a/test/test_integration_tracer.cxx +++ b/test/test_integration_tracer.cxx @@ -39,7 +39,9 @@ #include #include +#include #include +#include class test_span : public couchbase::tracing::request_span { @@ -48,42 +50,65 @@ class test_span : public couchbase::tracing::request_span : test_span(name, nullptr) { } + test_span(const std::string& name, std::shared_ptr parent) : request_span(name, parent) { start_ = std::chrono::steady_clock::now(); id_ = test::utils::uniq_id("span"); } + void add_tag(const std::string& name, std::uint64_t value) override { int_tags_[name] = value; } + void add_tag(const std::string& name, const std::string& value) override { string_tags_[name] = value; } + void end() override { duration_ = std::chrono::duration_cast( std::chrono::steady_clock::now() - start_); } - std::map string_tags() + + void add_child_span(const std::shared_ptr& child) + { + const auto child_span_name = child->name(); + if (child_spans_.count(child_span_name) == 0) { + child_spans_.insert({ child_span_name, {} }); + } + child_spans_[child_span_name].emplace_back(child); + } + + auto child_spans() -> const std::map>>& + { + return child_spans_; + } + + auto string_tags() -> std::map { return string_tags_; } - std::map int_tags() + + auto int_tags() -> std::map { return int_tags_; } - std::chrono::nanoseconds duration() + + auto duration() -> std::chrono::nanoseconds { return duration_; } - std::chrono::time_point start() + + auto start() -> std::chrono::time_point { return start_; } - std::string id() + + auto id() -> std::string { return id_; } @@ -94,26 +119,34 @@ class test_span : public couchbase::tracing::request_span std::chrono::nanoseconds duration_{ 0 }; std::map string_tags_; std::map int_tags_; + std::map>> child_spans_{}; }; class test_tracer : public couchbase::tracing::request_tracer { public: - std::shared_ptr start_span( - std::string name, - std::shared_ptr parent = {}) + auto start_span(std::string name, std::shared_ptr parent = {}) + -> std::shared_ptr { - std::lock_guard lock(mutex_); + const std::lock_guard lock(mutex_); spans_.push_back(std::make_shared(name, parent)); + + if (parent != nullptr) { + const auto parent_test_span = std::dynamic_pointer_cast(parent); + parent_test_span->add_child_span(spans_.back()); + } + return spans_.back(); } - std::vector> spans() + + auto spans() -> std::vector> { return spans_; } + void reset() { - std::lock_guard lock(mutex_); + const std::lock_guard lock(mutex_); spans_.clear(); } @@ -122,8 +155,8 @@ class test_tracer : public couchbase::tracing::request_tracer std::mutex mutex_; }; -couchbase::core::document_id -make_id(const test::utils::test_context& ctx, std::string key = "") +auto +make_id(const test::utils::test_context& ctx, std::string key = "") -> couchbase::core::document_id { if (key.empty()) { key = test::utils::uniq_id("tracer"); @@ -134,18 +167,19 @@ make_id(const test::utils::test_context& ctx, std::string key = "") void assert_span_ok(test::utils::integration_test_guard& guard, const std::shared_ptr& span, + bool is_top_level_op_span, std::shared_ptr parent = nullptr) { REQUIRE(span->parent() == parent); - if (parent) { - // the parent span should not be closed yet + if (parent && is_top_level_op_span) { + // the parent span that was given to the operation's options should not be closed yet REQUIRE(parent->duration().count() == 0); } const auto& tags = span->string_tags(); - REQUIRE(tags.at("db.system") == "couchbase"); + REQUIRE(tags.at("db.system.name") == "couchbase"); if (guard.cluster_version().supports_cluster_labels()) { REQUIRE_FALSE(tags.at("db.couchbase.cluster_name").empty()); REQUIRE_FALSE(tags.at("db.couchbase.cluster_uuid").empty()); @@ -155,23 +189,51 @@ assert_span_ok(test::utils::integration_test_guard& guard, } } +void +assert_kv_dispatch_span_ok(test::utils::integration_test_guard& guard, + const std::shared_ptr& span, + std::shared_ptr parent) +{ + assert_span_ok(guard, span, false, parent); + + INFO(fmt::format("Dispatch span string tags: {}", fmt::join(span->string_tags(), ", "))); + INFO(fmt::format("Dispatch span integer tags: {}", fmt::join(span->int_tags(), ", "))); + + std::size_t expected_tag_count = (guard.cluster_version().supports_cluster_labels()) ? 11 : 9; + REQUIRE(span->string_tags().size() + span->int_tags().size() == expected_tag_count); + REQUIRE("dispatch_to_server" == span->name()); + + REQUIRE(static_cast(span->duration().count()) >= + span->int_tags()["db.couchbase.server_duration"]); + REQUIRE_FALSE(span->string_tags()["db.couchbase.local_id"].empty()); + REQUIRE_FALSE(span->string_tags()["server.address"].empty()); + REQUIRE(span->int_tags()["server.port"] != 0); + REQUIRE_FALSE(span->string_tags()["network.peer.address"].empty()); + REQUIRE(span->int_tags()["network.peer.port"] != 0); + REQUIRE(span->string_tags()["network.transport"] == "tcp"); + REQUIRE_FALSE(span->string_tags()["db.couchbase.operation_id"].empty()); +} + void assert_kv_op_span_ok(test::utils::integration_test_guard& guard, const std::shared_ptr& span, const std::string& op, std::shared_ptr parent = nullptr) { - assert_span_ok(guard, span, parent); + assert_span_ok(guard, span, true, parent); - auto server_duration = span->int_tags()["cb.server_duration"]; REQUIRE(op == span->name()); - REQUIRE(static_cast(span->duration().count()) >= server_duration); REQUIRE(span->string_tags()["cb.service"] == "kv"); - REQUIRE_FALSE(span->string_tags()["cb.local_id"].empty()); - REQUIRE_FALSE(span->string_tags()["cb.local_socket"].empty()); - REQUIRE_FALSE(span->string_tags()["cb.remote_socket"].empty()); - REQUIRE_FALSE(span->string_tags()["cb.operation_id"].empty()); REQUIRE(span->string_tags()["db.instance"] == guard.ctx.bucket); + + // There must be at least one dispatch span + auto dispatch_spans = span->child_spans().find("dispatch_to_server"); + REQUIRE(dispatch_spans != span->child_spans().end()); + REQUIRE_FALSE(dispatch_spans->second.empty()); + + for (const auto& dispatch_span : dispatch_spans->second) { + assert_kv_dispatch_span_ok(guard, dispatch_span.lock(), span); + } } void @@ -180,12 +242,12 @@ assert_http_op_span_ok(test::utils::integration_test_guard& guard, const std::string& op, std::shared_ptr parent = nullptr) { - assert_span_ok(guard, span, parent); + assert_span_ok(guard, span, true, parent); REQUIRE(span->name().find(op) != std::string::npos); - REQUIRE_FALSE(span->string_tags()["cb.local_id"].empty()); + REQUIRE_FALSE(span->string_tags()["db.couchbase.local_id"].empty()); REQUIRE_FALSE(span->string_tags()["cb.local_socket"].empty()); - REQUIRE_FALSE(span->string_tags()["cb.operation_id"].empty()); + REQUIRE_FALSE(span->string_tags()["db.couchbase.operation_id"].empty()); REQUIRE_FALSE(span->string_tags()["cb.remote_socket"].empty()); REQUIRE(span->string_tags()["cb.service"] == op); REQUIRE(span->duration().count() > 0); @@ -350,4 +412,6 @@ TEST_CASE("integration: enable external tracer", "[integration]") } } } + + tracer->reset(); }