From cab1bf21e18bfbdba467aec5409f787a699d4c34 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 17 Mar 2025 10:29:28 +0500 Subject: [PATCH 01/11] update gossip (part) Signed-off-by: turuslan --- .../scheduler/manual_scheduler_backend.hpp | 2 +- include/libp2p/connection/stream_pair.hpp | 20 + include/libp2p/peer/protocol.hpp | 4 + .../libp2p/protocol/gossip/explicit_peers.hpp | 18 + include/libp2p/protocol/gossip/gossip.hpp | 49 +- include/libp2p/protocol/gossip/peer_kind.hpp | 19 + include/libp2p/protocol/gossip/score.hpp | 18 + .../libp2p/protocol/gossip/score_config.hpp | 15 + include/libp2p/protocol/gossip/time_cache.hpp | 81 ++++ src/basic/message_read_writer_uvarint.cpp | 4 +- src/connection/CMakeLists.txt | 7 + src/connection/stream_pair.cpp | 168 +++++++ src/protocol/gossip/impl/choose_peers.hpp | 46 ++ src/protocol/gossip/impl/connectivity.cpp | 91 ++-- src/protocol/gossip/impl/connectivity.hpp | 33 +- src/protocol/gossip/impl/gossip_core.cpp | 71 +-- src/protocol/gossip/impl/gossip_core.hpp | 11 +- src/protocol/gossip/impl/message_builder.cpp | 25 +- src/protocol/gossip/impl/message_builder.hpp | 5 +- src/protocol/gossip/impl/message_parser.cpp | 6 +- src/protocol/gossip/impl/message_receiver.hpp | 2 +- src/protocol/gossip/impl/peer_context.cpp | 16 + src/protocol/gossip/impl/peer_context.hpp | 8 + src/protocol/gossip/impl/peer_set.cpp | 6 +- src/protocol/gossip/impl/peer_set.hpp | 13 + .../gossip/impl/remote_subscriptions.cpp | 78 +-- .../gossip/impl/remote_subscriptions.hpp | 14 +- src/protocol/gossip/impl/stream.cpp | 11 +- src/protocol/gossip/impl/stream.hpp | 4 +- .../gossip/impl/topic_subscriptions.cpp | 401 ++++++++++----- .../gossip/impl/topic_subscriptions.hpp | 50 +- src/protocol/gossip/protobuf/rpc.proto | 5 + test/libp2p/protocol/gossip/CMakeLists.txt | 12 + .../protocol/gossip/gossip_mock_test.cpp | 459 ++++++++++++++++++ test/mock/libp2p/crypto/crypto_provider.hpp | 45 ++ 35 files changed, 1509 insertions(+), 308 deletions(-) create mode 100644 include/libp2p/connection/stream_pair.hpp create mode 100644 include/libp2p/protocol/gossip/explicit_peers.hpp create mode 100644 include/libp2p/protocol/gossip/peer_kind.hpp create mode 100644 include/libp2p/protocol/gossip/score.hpp create mode 100644 include/libp2p/protocol/gossip/score_config.hpp create mode 100644 include/libp2p/protocol/gossip/time_cache.hpp create mode 100644 src/connection/stream_pair.cpp create mode 100644 src/protocol/gossip/impl/choose_peers.hpp create mode 100644 test/libp2p/protocol/gossip/gossip_mock_test.cpp create mode 100644 test/mock/libp2p/crypto/crypto_provider.hpp diff --git a/include/libp2p/basic/scheduler/manual_scheduler_backend.hpp b/include/libp2p/basic/scheduler/manual_scheduler_backend.hpp index 1a1ece46e..8fe9deff9 100644 --- a/include/libp2p/basic/scheduler/manual_scheduler_backend.hpp +++ b/include/libp2p/basic/scheduler/manual_scheduler_backend.hpp @@ -66,9 +66,9 @@ namespace libp2p::basic { } } - private: void callDeferred(); + private: /// Current time, set manually std::chrono::milliseconds current_clock_; diff --git a/include/libp2p/connection/stream_pair.hpp b/include/libp2p/connection/stream_pair.hpp new file mode 100644 index 000000000..52af49e40 --- /dev/null +++ b/include/libp2p/connection/stream_pair.hpp @@ -0,0 +1,20 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace libp2p::basic { + class Scheduler; +} // namespace libp2p::basic + +namespace libp2p::connection { + struct Stream; + + std::pair, std::shared_ptr> streamPair( + std::shared_ptr post, PeerId peer1, PeerId peer2); +} // namespace libp2p::connection diff --git a/include/libp2p/peer/protocol.hpp b/include/libp2p/peer/protocol.hpp index 67fae1f25..b8b87561f 100644 --- a/include/libp2p/peer/protocol.hpp +++ b/include/libp2p/peer/protocol.hpp @@ -17,3 +17,7 @@ namespace libp2p::peer { std::string; } // namespace libp2p::peer + +namespace libp2p { + using peer::ProtocolName; +} // namespace libp2p diff --git a/include/libp2p/protocol/gossip/explicit_peers.hpp b/include/libp2p/protocol/gossip/explicit_peers.hpp new file mode 100644 index 000000000..43212ae04 --- /dev/null +++ b/include/libp2p/protocol/gossip/explicit_peers.hpp @@ -0,0 +1,18 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace libp2p::protocol::gossip { + class ExplicitPeers { + public: + bool contains(const PeerId &) const { + return false; + } + }; +} // namespace libp2p::protocol::gossip diff --git a/include/libp2p/protocol/gossip/gossip.hpp b/include/libp2p/protocol/gossip/gossip.hpp index 0b289db4d..a05ef8eb8 100644 --- a/include/libp2p/protocol/gossip/gossip.hpp +++ b/include/libp2p/protocol/gossip/gossip.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -17,7 +18,10 @@ #include #include #include +#include #include +#include +#include namespace libp2p { struct Host; @@ -41,6 +45,7 @@ namespace libp2p::protocol::gossip { struct Config { /// Network density factors for gossip meshes size_t D_min = 5; + size_t D = 6; size_t D_max = 10; /// Ideal number of connected peers to support the network @@ -50,27 +55,12 @@ namespace libp2p::protocol::gossip { /// incoming peers will be rejected size_t max_connections_num = 1000; - /// Forward messages to all subscribers not in mesh - /// (floodsub mode compatibility) - bool floodsub_forward_mode = false; - /// Forward local message to local subscribers bool echo_forward_mode = false; /// Read or write timeout per whole network operation std::chrono::milliseconds rw_timeout_msec{std::chrono::seconds(10)}; - /// Lifetime of a message in message cache - std::chrono::milliseconds message_cache_lifetime_msec{ - std::chrono::minutes(2)}; - - /// Topic's message seen cache lifetime - std::chrono::milliseconds seen_cache_lifetime_msec{ - message_cache_lifetime_msec * 3 / 4}; - - /// Topic's seen cache limit - unsigned seen_cache_limit = 100; - /// Heartbeat interval std::chrono::milliseconds heartbeat_interval_msec{1000}; @@ -86,11 +76,36 @@ namespace libp2p::protocol::gossip { /// Max RPC message size size_t max_message_size = 1 << 24; - /// Protocol version - std::string protocol_version = "/meshsub/1.0.0"; + /// Protocol versions + std::unordered_map protocol_versions{ + {"/floodsub/1.0.0", PeerKind::Floodsub}, + {"/meshsub/1.0.0", PeerKind::Gossipsub}, + {"/meshsub/1.1.0", PeerKind::Gossipsubv1_1}, + {"/meshsub/1.2.0", PeerKind::Gossipsubv1_2}, + }; /// Sign published messages bool sign_messages = false; + + size_t history_length{5}; + + size_t history_gossip{3}; + + std::chrono::seconds fanout_ttl{60}; + + std::chrono::seconds duplicate_cache_time{60}; + + std::chrono::seconds prune_backoff{60}; + + std::chrono::seconds unsubscribe_backoff{10}; + + size_t backoff_slack = 1; + + bool flood_publish = true; + + size_t max_ihave_length = 5000; + + ScoreConfig score; }; using TopicId = std::string; diff --git a/include/libp2p/protocol/gossip/peer_kind.hpp b/include/libp2p/protocol/gossip/peer_kind.hpp new file mode 100644 index 000000000..d0416f0de --- /dev/null +++ b/include/libp2p/protocol/gossip/peer_kind.hpp @@ -0,0 +1,19 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace libp2p::protocol::gossip { + enum class PeerKind : uint8_t { + NotSupported, + Floodsub, + Gossipsub, + Gossipsubv1_1, + Gossipsubv1_2, + }; +} // namespace libp2p::protocol::gossip diff --git a/include/libp2p/protocol/gossip/score.hpp b/include/libp2p/protocol/gossip/score.hpp new file mode 100644 index 000000000..8923b4db9 --- /dev/null +++ b/include/libp2p/protocol/gossip/score.hpp @@ -0,0 +1,18 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace libp2p::protocol::gossip { + class Score { + public: + bool below(const PeerId &peer_id, double threshold) { + return false; + } + }; +} // namespace libp2p::protocol::gossip diff --git a/include/libp2p/protocol/gossip/score_config.hpp b/include/libp2p/protocol/gossip/score_config.hpp new file mode 100644 index 000000000..258385e34 --- /dev/null +++ b/include/libp2p/protocol/gossip/score_config.hpp @@ -0,0 +1,15 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +namespace libp2p::protocol::gossip { + struct ScoreConfig { + double zero = 0; + double gossip_threshold = -10; + double publish_threshold = -50; + }; +} // namespace libp2p::protocol::gossip diff --git a/include/libp2p/protocol/gossip/time_cache.hpp b/include/libp2p/protocol/gossip/time_cache.hpp new file mode 100644 index 000000000..60b1d27d7 --- /dev/null +++ b/include/libp2p/protocol/gossip/time_cache.hpp @@ -0,0 +1,81 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace libp2p::protocol::gossip::time_cache { + using Ttl = std::chrono::milliseconds; + using Clock = std::chrono::steady_clock; + using Time = Clock::time_point; + + template + class TimeCache { + public: + TimeCache(Ttl ttl) : ttl_{ttl} {} + + bool contains(const K &key) const { + return map_.contains(key); + } + + void clearExpired(Time now = Clock::now()) { + while (not expirations_.empty() and expirations_.front().first <= now) { + map_.erase(expirations_.front().second); + expirations_.pop_front(); + } + } + + V &getOrDefault(const K &key, Time now = Clock::now()) { + clearExpired(now); + auto it = map_.find(key); + if (it == map_.end()) { + it = map_.emplace(key, V()).first; + expirations_.emplace_back(now + ttl_, it); + } + return it->second; + } + + private: + using Map = std::unordered_map; + + Ttl ttl_; + Map map_; + std::deque> + expirations_; + }; + + template + class DuplicateCache { + public: + DuplicateCache(Ttl ttl) : cache_{ttl} {} + + bool contains(const K &key) const { + return cache_.contains(key); + } + + bool insert(const K &key, Time now = Clock::now()) { + cache_.clearExpired(now); + if (cache_.contains(key)) { + return false; + } + cache_.getOrDefault(key); + return true; + } + + private: + TimeCache cache_; + }; +} // namespace libp2p::protocol::gossip::time_cache + +namespace libp2p::protocol::gossip { + using time_cache::DuplicateCache; + using time_cache::TimeCache; +} // namespace libp2p::protocol::gossip diff --git a/src/basic/message_read_writer_uvarint.cpp b/src/basic/message_read_writer_uvarint.cpp index 26871ca36..6bc2b2b02 100644 --- a/src/basic/message_read_writer_uvarint.cpp +++ b/src/basic/message_read_writer_uvarint.cpp @@ -32,8 +32,8 @@ namespace libp2p::basic { } auto msg_len = varint_res.value().toUInt64(); + auto buffer = std::make_shared>(msg_len, 0); if (0 != msg_len) { - auto buffer = std::make_shared>(msg_len, 0); self->conn_->read( *buffer, msg_len, @@ -44,7 +44,7 @@ namespace libp2p::basic { cb(std::move(buffer)); }); } else { - cb(ResultType{}); + cb(buffer); } }); } diff --git a/src/connection/CMakeLists.txt b/src/connection/CMakeLists.txt index da0ca9d1a..59a0563ac 100644 --- a/src/connection/CMakeLists.txt +++ b/src/connection/CMakeLists.txt @@ -19,5 +19,12 @@ target_link_libraries(p2p_loopback_stream p2p_peer_id ) +libp2p_add_library(p2p_stream_pair + stream_pair.cpp + ) +target_link_libraries(p2p_stream_pair + p2p_peer_id + ) + libp2p_install(p2p_loopback_stream) diff --git a/src/connection/stream_pair.cpp b/src/connection/stream_pair.cpp new file mode 100644 index 000000000..8f768edd9 --- /dev/null +++ b/src/connection/stream_pair.cpp @@ -0,0 +1,168 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include +#include +#include +#include +#include +#include + +namespace libp2p::connection { + class StreamPair : public std::enable_shared_from_this, + public Stream { + friend std::pair, std::shared_ptr> + streamPair(std::shared_ptr post, + PeerId peer1, + PeerId peer2); + + public: + StreamPair(std::shared_ptr post, + bool is_initiator, + PeerId peer_id) + : post_{std::move(post)}, + is_initiator_{is_initiator}, + peer_id_{std::move(peer_id)} {} + + ~StreamPair() { + reset(); + } + + void read(BytesOut out, size_t bytes, ReadCallbackFunc cb) override { + ambigousSize(out, bytes); + libp2p::readReturnSize(shared_from_this(), out, std::move(cb)); + } + void readSome(BytesOut out, size_t bytes, ReadCallbackFunc cb) override { + ambigousSize(out, bytes); + if (out.empty()) { + throw std::logic_error{"StreamPair::readSome zero bytes"}; + } + if (read_buffer_.size() == 0) { + if (read_closed_) { + post(std::move(cb), make_error_code(boost::asio::error::eof)); + return; + } + reading_ = Reading{out, std::move(cb)}; + return; + } + auto n = std::min(read_buffer_.size(), out.size()); + memcpy(out.data(), read_buffer_.data().data(), n); + read_buffer_.consume(n); + post(std::move(cb), n); + } + void deferReadCallback(outcome::result res, + ReadCallbackFunc cb) override { + post(std::move(cb), res); + } + + void writeSome(BytesIn in, size_t bytes, WriteCallbackFunc cb) override { + ambigousSize(in, bytes); + if (in.empty()) { + throw std::logic_error{"StreamPair::writeSome zero bytes"}; + } + if (auto writer = writer_.lock()) { + writer->onRead(in); + post(std::move(cb), in.size()); + } else { + post(std::move(cb), std::errc::broken_pipe); + } + } + void deferWriteCallback(std::error_code ec, WriteCallbackFunc cb) override { + post(std::move(cb), ec); + } + + bool isClosedForRead() const override { + return read_buffer_.size() == 0 and read_closed_; + } + bool isClosedForWrite() const override { + return writer_.expired(); + } + bool isClosed() const override { + return isClosedForRead() and isClosedForWrite(); + } + void close(VoidResultHandlerFunc cb) override { + if (auto writer = writer_.lock()) { + writer->onReadClose(); + } + onWriteClose(); + post(std::move(cb), outcome::success()); + } + void reset() override { + if (auto writer = writer_.lock()) { + writer->onReadClose(); + writer->onWriteClose(); + } + onReadClose(); + onWriteClose(); + } + void adjustWindowSize(uint32_t, VoidResultHandlerFunc) override {} + outcome::result isInitiator() const override { + return is_initiator_; + } + outcome::result remotePeerId() const override { + return peer_id_; + } + outcome::result localMultiaddr() const override { + throw std::logic_error{"StreamPair::localMultiaddr"}; + } + outcome::result remoteMultiaddr() const override { + return Multiaddress::create(fmt::format("/p2p/{}", peer_id_.toBase58())) + .value(); + } + + private: + void post(auto cb, auto arg) { + post_->schedule([cb{std::move(cb)}, arg{std::move(arg)}]() mutable { + cb(std::move(arg)); + }); + } + void onRead(BytesIn in) { + if (auto reading = qtils::optionTake(reading_)) { + auto n = std::min(in.size(), reading->out.size()); + memcpy(reading->out.data(), in.data(), n); + in = in.subspan(n); + post(std::move(reading->cb), n); + } + if (not in.empty()) { + auto n = in.size(); + memcpy(read_buffer_.prepare(n).data(), in.data(), n); + read_buffer_.commit(n); + } + } + void onReadClose() { + read_closed_ = true; + if (auto reading = qtils::optionTake(reading_)) { + post(std::move(reading->cb), make_error_code(boost::asio::error::eof)); + } + } + void onWriteClose() { + writer_.reset(); + } + + struct Reading { + BytesOut out; + ReadCallbackFunc cb; + }; + std::shared_ptr post_; + bool is_initiator_; + PeerId peer_id_; + std::weak_ptr writer_; + std::optional reading_; + boost::asio::streambuf read_buffer_; + bool read_closed_ = false; + }; + + std::pair, std::shared_ptr> streamPair( + std::shared_ptr post, PeerId peer1, PeerId peer2) { + auto stream1 = std::make_shared(post, true, peer2); + auto stream2 = std::make_shared(post, false, peer1); + stream1->writer_ = stream2; + stream2->writer_ = stream1; + return {stream1, stream2}; + } +} // namespace libp2p::connection diff --git a/src/protocol/gossip/impl/choose_peers.hpp b/src/protocol/gossip/impl/choose_peers.hpp new file mode 100644 index 000000000..2a0ee8451 --- /dev/null +++ b/src/protocol/gossip/impl/choose_peers.hpp @@ -0,0 +1,46 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include "peer_context.hpp" + +namespace libp2p::protocol::gossip { + class ChoosePeers { + public: + std::deque choose( + auto &&all, + const std::invocable auto &predicate, + const std::invocable auto &get_count) { + std::deque chosen; + for (auto &ctx : all) { + if (ctx->isGossipsub() and predicate(ctx)) { + chosen.emplace_back(ctx); + } + } + std::ranges::shuffle(chosen, random_); + auto count = get_count(chosen.size()); + if (chosen.size() > count) { + chosen.resize(count); + } + return chosen; + } + std::deque choose( + auto &&all, + const std::invocable auto &predicate, + size_t count) { + return choose(std::forward(all), predicate, [&](size_t) { + return count; + }); + } + + private: + std::default_random_engine random_; + }; +} // namespace libp2p::protocol::gossip diff --git a/src/protocol/gossip/impl/connectivity.cpp b/src/protocol/gossip/impl/connectivity.cpp index 8a308d64d..08d165044 100644 --- a/src/protocol/gossip/impl/connectivity.cpp +++ b/src/protocol/gossip/impl/connectivity.cpp @@ -29,7 +29,7 @@ namespace libp2p::protocol::gossip { Connectivity::Connectivity(Config config, std::shared_ptr scheduler, std::shared_ptr host, - std::shared_ptr msg_receiver, + std::weak_ptr msg_receiver, ConnectionStatusFeedback on_connected) : config_(std::move(config)), scheduler_(std::move(scheduler)), @@ -38,7 +38,16 @@ namespace libp2p::protocol::gossip { connected_cb_(std::move(on_connected)), log_("gossip", "Connectivity", - host_->getPeerInfo().id.toBase58().substr(46)) {} + host_->getPeerInfo().id.toBase58().substr(46)) { + for (auto &p : config_.protocol_versions) { + protocols_.emplace_back(p.first); + } + std::ranges::sort(protocols_, + [&](const ProtocolName &l, const ProtocolName &r) { + return config_.protocol_versions.at(l) + > config_.protocol_versions.at(r); + }); + } Connectivity::~Connectivity() { stop(); @@ -58,7 +67,7 @@ namespace libp2p::protocol::gossip { }; host_->setProtocolHandler( - {config_.protocol_version}, + protocols_, [self_wptr=weak_from_this()] (StreamAndProtocol stream) { auto h = self_wptr.lock(); @@ -134,10 +143,6 @@ namespace libp2p::protocol::gossip { ctx->outbound_stream->write(std::move(serialized)); } - peer::ProtocolName Connectivity::getProtocolId() const { - return config_.protocol_version; - } - void Connectivity::handle(StreamAndProtocol stream_and_protocol) { auto &stream = stream_and_protocol.stream; @@ -178,6 +183,7 @@ namespace libp2p::protocol::gossip { unban(ctx); } } + updatePeerKind(*ctx, stream_and_protocol); size_t stream_id = 0; bool is_new_connection = false; @@ -189,7 +195,7 @@ namespace libp2p::protocol::gossip { config_, *scheduler_, on_stream_event_, - *msg_receiver_, + msg_receiver_, std::move(stream), ctx); @@ -232,7 +238,7 @@ namespace libp2p::protocol::gossip { // clang-format off host_->newStream( pi, - {config_.protocol_version}, + protocols_, [wptr = weak_from_this(), this, ctx=ctx] (auto &&rstream) mutable { auto self = wptr.lock(); if (self) { @@ -253,7 +259,7 @@ namespace libp2p::protocol::gossip { // clang-format off host_->newStream( ctx->peer_id, - {config_.protocol_version}, + protocols_, [wptr = weak_from_this(), this, ctx=ctx] (auto &&rstream) mutable { auto self = wptr.lock(); if (self) { @@ -279,6 +285,7 @@ namespace libp2p::protocol::gossip { } auto &stream = rstream.value().stream; + updatePeerKind(*ctx, rstream.value()); if (!started_) { stream->reset(); @@ -306,7 +313,7 @@ namespace libp2p::protocol::gossip { config_, *scheduler_, on_stream_event_, - *msg_receiver_, + msg_receiver_, std::move(stream), ctx); @@ -395,26 +402,31 @@ namespace libp2p::protocol::gossip { banOrForget(from); } - void Connectivity::peerIsWritable(const PeerContextPtr &ctx, - bool low_latency) { + void Connectivity::peerIsWritable(const PeerContextPtr &ctx) { if (ctx->message_builder->empty()) { return; } - if (low_latency) { - writable_peers_low_latency_.insert(ctx); - } else { - writable_peers_on_heartbeat_.insert(ctx); + const auto was_empty = writable_peers_.empty(); + writable_peers_.insert(ctx); + if (was_empty) { + scheduler_->schedule([weak_self{weak_from_this()}] { + auto self = weak_self.lock(); + if (not self) { + return; + } + self->flush(); + }); } } void Connectivity::flush() { - writable_peers_low_latency_.selectAll( + writable_peers_.selectAll( [this](const PeerContextPtr &ctx) { flush(ctx); }); - writable_peers_low_latency_.clear(); + writable_peers_.clear(); } - void Connectivity::onHeartbeat(const std::map &local_changes) { + void Connectivity::onHeartbeat() { if (!started_) { return; } @@ -448,32 +460,6 @@ namespace libp2p::protocol::gossip { } } - if (!local_changes.empty()) { - // we have something to say to all connected peers - - std::vector> flat_changes; - flat_changes.reserve(local_changes.size()); - boost::for_each(local_changes, [&flat_changes](auto &&p) { - flat_changes.emplace_back(p.second, std::move(p.first)); - }); - - connected_peers_.selectAll( - [&flat_changes, this](const PeerContextPtr &ctx) { - boost::for_each(flat_changes, [&ctx](auto &&p) { - ctx->message_builder->addSubscription(p.first, p.second); - }); - flush(ctx); - }); - - } else { - flush(); - writable_peers_on_heartbeat_.selectAll( - [this](const PeerContextPtr &ctx) { flush(ctx); }); - } - - writable_peers_low_latency_.clear(); - writable_peers_on_heartbeat_.clear(); - if (ts >= addresses_renewal_time_) { addresses_renewal_time_ = scheduler_->now() + config_.address_expiration_msec * 9 / 10; @@ -489,4 +475,17 @@ namespace libp2p::protocol::gossip { return connected_peers_; } + void Connectivity::subscribe(const TopicId &topic, bool subscribe) { + for (auto &ctx : connected_peers_) { + ctx->message_builder->addSubscription(subscribe, topic); + peerIsWritable(ctx); + } + } + + void Connectivity::updatePeerKind(PeerContext &ctx, + const StreamAndProtocol &stream) const { + if (not ctx.peer_kind) { + ctx.peer_kind = config_.protocol_versions.at(stream.protocol); + } + } } // namespace libp2p::protocol::gossip diff --git a/src/protocol/gossip/impl/connectivity.hpp b/src/protocol/gossip/impl/connectivity.hpp index fe7a603a2..932ccdba0 100644 --- a/src/protocol/gossip/impl/connectivity.hpp +++ b/src/protocol/gossip/impl/connectivity.hpp @@ -21,8 +21,7 @@ namespace libp2p::protocol::gossip { class MessageReceiver; /// Part of GossipCore: Protocol server and network connections manager - class Connectivity : public protocol::BaseProtocol, - public std::enable_shared_from_this { + class Connectivity : public std::enable_shared_from_this { public: Connectivity(const Connectivity &) = delete; Connectivity &operator=(const Connectivity &) = delete; @@ -35,10 +34,10 @@ namespace libp2p::protocol::gossip { Connectivity(Config config, std::shared_ptr scheduler, std::shared_ptr host, - std::shared_ptr msg_receiver, + std::weak_ptr msg_receiver, ConnectionStatusFeedback on_connected); - ~Connectivity() override; + ~Connectivity(); void start(); @@ -52,26 +51,25 @@ namespace libp2p::protocol::gossip { /// Add peer to writable set, actual writes occur on flush() (piggybacking) /// The idea behind writable set and flush() is a compromise between /// latency and message rate - void peerIsWritable(const PeerContextPtr &ctx, bool low_latency); + void peerIsWritable(const PeerContextPtr &ctx); /// Flushes all pending writes for peers in writable set void flush(); /// Performs periodic tasks and broadcasts heartbeat message to /// all connected peers. The changes are subscribe/unsubscribe events - void onHeartbeat(const std::map &local_changes); + void onHeartbeat(); /// Returns connected peers const PeerSet &getConnectedPeers() const; + void subscribe(const TopicId &topic, bool subscribe); + private: using BannedPeers = std::set>; - /// BaseProtocol override - peer::ProtocolName getProtocolId() const override; - - /// BaseProtocol override, on new inbound stream - void handle(StreamAndProtocol stream) override; + /// On new inbound stream + void handle(StreamAndProtocol stream); /// Tries to connect to peer void dial(const PeerContextPtr &peer); @@ -99,10 +97,14 @@ namespace libp2p::protocol::gossip { /// Flushes outgoing messages into wire for a given peer, if connected void flush(const PeerContextPtr &ctx) const; + void updatePeerKind(PeerContext &ctx, + const StreamAndProtocol &stream) const; + const Config config_; + StreamProtocols protocols_; std::shared_ptr scheduler_; std::shared_ptr host_; - std::shared_ptr msg_receiver_; + std::weak_ptr msg_receiver_; ConnectionStatusFeedback connected_cb_; Stream::Feedback on_stream_event_; bool started_ = false; @@ -120,11 +122,8 @@ namespace libp2p::protocol::gossip { /// Writable peers PeerSet connected_peers_; - /// Peers with pending write operation before the next heartbeat - PeerSet writable_peers_low_latency_; - - /// Peers to be flushed on next heartbeat - PeerSet writable_peers_on_heartbeat_; + /// Peers with pending write operation + PeerSet writable_peers_; /// Renew addresses in address repo periodically within heartbeat timer std::chrono::milliseconds addresses_renewal_time_{0}; diff --git a/src/protocol/gossip/impl/gossip_core.cpp b/src/protocol/gossip/impl/gossip_core.cpp index 2ee17239e..8be418a72 100644 --- a/src/protocol/gossip/impl/gossip_core.cpp +++ b/src/protocol/gossip/impl/gossip_core.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "connectivity.hpp" @@ -54,9 +55,11 @@ namespace libp2p::protocol::gossip { key_marshaller_(std::move(key_marshaller)), local_peer_id_(host_->getPeerInfo().id), msg_cache_( - config_.message_cache_lifetime_msec, + config_.history_length * config_.heartbeat_interval_msec, [sch = scheduler_] { return sch->now(); } ), + score_{std::make_shared()}, + duplicate_cache_{config.duplicate_cache_time}, local_subscriptions_(std::make_shared( [this](bool subscribe, const TopicId &topic) { onLocalSubscriptionChanged(subscribe, topic); @@ -109,7 +112,7 @@ namespace libp2p::protocol::gossip { } remote_subscriptions_ = std::make_shared( - config_, *connectivity_, *scheduler_, log_); + config_, *connectivity_, score_, scheduler_, log_); started_ = true; @@ -177,6 +180,10 @@ namespace libp2p::protocol::gossip { MessageId msg_id = create_message_id_(msg->from, msg->seq_no, msg->data); + if (not duplicate_cache_.insert(msg_id)) { + return false; + } + [[maybe_unused]] bool inserted = msg_cache_.insert(msg, msg_id); assert(inserted); @@ -223,25 +230,39 @@ namespace libp2p::protocol::gossip { const MessageId &msg_id) { assert(started_); + if (not from->isGossipsub()) { + return; + } + if (score_->below(from->peer_id, config_.score.gossip_threshold)) { + return; + } + log_.debug("peer {} has msg for topic {}", from->str, topic); - if (remote_subscriptions_->hasTopic(topic) - && !msg_cache_.contains(msg_id)) { + if (remote_subscriptions_->isSubscribed(topic) + and not duplicate_cache_.contains(msg_id)) { log_.debug("requesting msg id {:x}", msg_id); from->message_builder->addIWant(msg_id); - connectivity_->peerIsWritable(from, false); + connectivity_->peerIsWritable(from); } } void GossipCore::onIWant(const PeerContextPtr &from, const MessageId &msg_id) { + if (not from->isGossipsub()) { + return; + } + if (score_->below(from->peer_id, config_.score.gossip_threshold)) { + return; + } + log_.debug("peer {} wants message {:x}", from->str, msg_id); auto msg_found = msg_cache_.getMessage(msg_id); if (msg_found) { from->message_builder->addMessage(*msg_found.value(), msg_id); - connectivity_->peerIsWritable(from, true); + connectivity_->peerIsWritable(from); } else { log_.debug("wanted message not in cache"); } @@ -250,6 +271,10 @@ namespace libp2p::protocol::gossip { void GossipCore::onGraft(const PeerContextPtr &from, const TopicId &topic) { assert(started_); + if (not from->isGossipsub()) { + return; + } + log_.debug("graft from peer {} for topic {}", from->str, topic); remote_subscriptions_->onGraft(from, topic); @@ -257,9 +282,13 @@ namespace libp2p::protocol::gossip { void GossipCore::onPrune(const PeerContextPtr &from, const TopicId &topic, - uint64_t backoff_time) { + std::optional backoff_time) { assert(started_); + if (not from->isGossipsub()) { + return; + } + log_.debug("prune from peer {} for topic {}", from->str, topic); remote_subscriptions_->onPrune(from, topic, backoff_time); @@ -270,21 +299,19 @@ namespace libp2p::protocol::gossip { assert(started_); // do we need this message? - auto subscribed = remote_subscriptions_->hasTopic(msg->topic); + auto subscribed = remote_subscriptions_->isSubscribed(msg->topic); if (!subscribed) { // ignore this message return; } MessageId msg_id = create_message_id_(msg->from, msg->seq_no, msg->data); - log_.debug("message arrived, msg id={:x}", msg_id); - - if (msg_cache_.contains(msg_id)) { - // already there, ignore - log_.debug("ignoring message, already in cache"); + if (not duplicate_cache_.insert(msg_id)) { return; } + log_.debug("message arrived, msg id={:x}", msg_id); + // validate message. If no validator is set then we // suppose that the message is valid (we might not know topic details) bool valid = true; @@ -316,9 +343,6 @@ namespace libp2p::protocol::gossip { assert(started_); log_.debug("finished dispatching message from peer {}", from->str); - - // Apply immediate send operation to affected peers - connectivity_->flush(); } void GossipCore::onHeartbeat() { @@ -331,8 +355,7 @@ namespace libp2p::protocol::gossip { remote_subscriptions_->onHeartbeat(); // send changes to peers - connectivity_->onHeartbeat(broadcast_on_heartbeat_); - broadcast_on_heartbeat_.clear(); + connectivity_->onHeartbeat(); setTimerHeartbeat(); } @@ -347,8 +370,7 @@ namespace libp2p::protocol::gossip { for (const auto &local_sub : local_subscriptions_->subscribedTo()) { ctx->message_builder->addSubscription(true, local_sub.first); } - connectivity_->peerIsWritable(ctx, true); - connectivity_->flush(); + connectivity_->peerIsWritable(ctx); } } else { log_.debug("peer {} disconnected", ctx->str); @@ -362,14 +384,7 @@ namespace libp2p::protocol::gossip { return; } - // send this notification on next heartbeat to all connected peers - auto it = broadcast_on_heartbeat_.find(topic); - if (it == broadcast_on_heartbeat_.end()) { - broadcast_on_heartbeat_.emplace(topic, subscribe); - } else if (it->second != subscribe) { - // save traffic - broadcast_on_heartbeat_.erase(it); - } + connectivity_->subscribe(topic, subscribe); // update meshes per topic remote_subscriptions_->onSelfSubscribed(subscribe, topic); diff --git a/src/protocol/gossip/impl/gossip_core.hpp b/src/protocol/gossip/impl/gossip_core.hpp index 0d7e42509..bc133e09a 100644 --- a/src/protocol/gossip/impl/gossip_core.hpp +++ b/src/protocol/gossip/impl/gossip_core.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include "message_cache.hpp" #include "message_receiver.hpp" @@ -23,6 +24,7 @@ namespace libp2p::protocol::gossip { class LocalSubscriptions; class RemoteSubscriptions; class Connectivity; + class Score; /// Central component in gossip protocol impl, manages pub-sub logic itself class GossipCore : public Gossip, @@ -71,7 +73,7 @@ namespace libp2p::protocol::gossip { void onGraft(const PeerContextPtr &from, const TopicId &topic) override; void onPrune(const PeerContextPtr &from, const TopicId &topic, - uint64_t backoff_time) override; + std::optional backoff_time) override; void onTopicMessage(const PeerContextPtr &from, TopicMessage::Ptr msg) override; void onMessageEnd(const PeerContextPtr &from) override; @@ -115,6 +117,10 @@ namespace libp2p::protocol::gossip { /// Message cache w/expiration MessageCache msg_cache_; + std::shared_ptr score_; + + DuplicateCache duplicate_cache_; + /// Local subscriptions manager (this host subscribed to topics) std::shared_ptr local_subscriptions_; @@ -132,9 +138,6 @@ namespace libp2p::protocol::gossip { /// Network part of gossip component std::shared_ptr connectivity_; - /// Local {un}subscribe changes to be broadcasted to peers - std::map broadcast_on_heartbeat_; - /// Incremented msg sequence number uint64_t msg_seq_; diff --git a/src/protocol/gossip/impl/message_builder.cpp b/src/protocol/gossip/impl/message_builder.cpp index 3def3b54b..50bebe787 100644 --- a/src/protocol/gossip/impl/message_builder.cpp +++ b/src/protocol/gossip/impl/message_builder.cpp @@ -58,6 +58,13 @@ namespace libp2p::protocol::gossip { outcome::result MessageBuilder::serialize() { create_protobuf_structures(); + for (auto &[topic, subscribe] : subscriptions_) { + auto *subscription = pb_msg_->add_subscriptions(); + subscription->set_subscribe(subscribe); + subscription->set_topicid(topic); + } + subscriptions_.clear(); + for (auto &[topic, message_ids] : ihaves_) { auto *ih = control_pb_msg_->add_ihave(); ih->set_topicid(topic); @@ -109,11 +116,10 @@ namespace libp2p::protocol::gossip { } void MessageBuilder::addSubscription(bool subscribe, const TopicId &topic) { - create_protobuf_structures(); - - auto *dst = pb_msg_->add_subscriptions(); - dst->set_subscribe(subscribe); - dst->set_topicid(topic); + auto it = subscriptions_.emplace(topic, subscribe).first; + if (it->second != subscribe) { + subscriptions_.erase(it); + } empty_ = false; } @@ -137,10 +143,15 @@ namespace libp2p::protocol::gossip { empty_ = false; } - void MessageBuilder::addPrune(const TopicId &topic) { + void MessageBuilder::addPrune(const TopicId &topic, + std::optional backoff) { create_protobuf_structures(); - control_pb_msg_->add_prune()->set_topicid(topic); + auto *prune = control_pb_msg_->add_prune(); + prune->set_topicid(topic); + if (backoff.has_value()) { + prune->set_backoff(backoff->count()); + } control_not_empty_ = true; empty_ = false; } diff --git a/src/protocol/gossip/impl/message_builder.hpp b/src/protocol/gossip/impl/message_builder.hpp index e0e2585da..595dd5abb 100644 --- a/src/protocol/gossip/impl/message_builder.hpp +++ b/src/protocol/gossip/impl/message_builder.hpp @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include "common.hpp" @@ -54,7 +55,8 @@ namespace libp2p::protocol::gossip { void addGraft(const TopicId &topic); /// Adds prune request - void addPrune(const TopicId &topic); + void addPrune(const TopicId &topic, + std::optional backoff); /// Adds message to be forwarded void addMessage(const TopicMessage &msg, const MessageId &msg_id); @@ -73,6 +75,7 @@ namespace libp2p::protocol::gossip { std::unique_ptr control_pb_msg_; bool empty_; bool control_not_empty_; + std::unordered_map subscriptions_; /// Intermediate struct for building IHave messages std::map> ihaves_; diff --git a/src/protocol/gossip/impl/message_parser.cpp b/src/protocol/gossip/impl/message_parser.cpp index 15fc22924..f1ed61559 100644 --- a/src/protocol/gossip/impl/message_parser.cpp +++ b/src/protocol/gossip/impl/message_parser.cpp @@ -88,12 +88,10 @@ namespace libp2p::protocol::gossip { if (!pr.has_topicid()) { continue; } - uint64_t backoff_time = 60; + std::optional backoff_time; if (pr.has_backoff()) { - backoff_time = pr.backoff(); + backoff_time = std::chrono::seconds{pr.backoff()}; } - log()->debug( - "prune backoff={}, {} peers", backoff_time, pr.peers_size()); for (const auto &peer : pr.peers()) { // TODO(artem): meshsub 1.1.0 + signed peer records NYI diff --git a/src/protocol/gossip/impl/message_receiver.hpp b/src/protocol/gossip/impl/message_receiver.hpp index 22401a7eb..c19144c7e 100644 --- a/src/protocol/gossip/impl/message_receiver.hpp +++ b/src/protocol/gossip/impl/message_receiver.hpp @@ -37,7 +37,7 @@ namespace libp2p::protocol::gossip { /// backoff_time seconds virtual void onPrune(const PeerContextPtr &from, const TopicId &topic, - uint64_t backoff_time) = 0; + std::optional backoff_time) = 0; /// Message received virtual void onTopicMessage(const PeerContextPtr &from, diff --git a/src/protocol/gossip/impl/peer_context.cpp b/src/protocol/gossip/impl/peer_context.cpp index b3fa791c8..163633c57 100644 --- a/src/protocol/gossip/impl/peer_context.cpp +++ b/src/protocol/gossip/impl/peer_context.cpp @@ -22,6 +22,22 @@ namespace libp2p::protocol::gossip { str(makeStringRepr(peer_id)), message_builder(std::make_shared()) {} + bool PeerContext::isFloodsub() const { + return not peer_kind or peer_kind.value() == PeerKind::Floodsub; + } + + bool PeerContext::isGossipsub() const { + return peer_kind and peer_kind.value() >= PeerKind::Gossipsub; + } + + bool PeerContext::isGossipsubv1_1() const { + return peer_kind and peer_kind.value() >= PeerKind::Gossipsubv1_1; + } + + bool PeerContext::isGossipsubv1_2() const { + return peer_kind and peer_kind.value() >= PeerKind::Gossipsubv1_2; + } + bool operator<(const PeerContextPtr &ctx, const peer::PeerId &peer) { if (!ctx) { return false; diff --git a/src/protocol/gossip/impl/peer_context.hpp b/src/protocol/gossip/impl/peer_context.hpp index 56f33538c..a10dedb4b 100644 --- a/src/protocol/gossip/impl/peer_context.hpp +++ b/src/protocol/gossip/impl/peer_context.hpp @@ -7,6 +7,7 @@ #pragma once #include +#include #include "common.hpp" @@ -42,6 +43,8 @@ namespace libp2p::protocol::gossip { /// If true, then outbound connection is in progress bool is_connecting = false; + std::optional peer_kind; + ~PeerContext() = default; PeerContext(PeerContext &&) = delete; PeerContext(const PeerContext &) = delete; @@ -50,6 +53,11 @@ namespace libp2p::protocol::gossip { explicit PeerContext(peer::PeerId id); + bool isFloodsub() const; + bool isGossipsub() const; + bool isGossipsubv1_1() const; + bool isGossipsubv1_2() const; + LIBP2P_METRICS_INSTANCE_COUNT_IF_ENABLED( libp2p::protocol::gossip::PeerContext); }; diff --git a/src/protocol/gossip/impl/peer_set.cpp b/src/protocol/gossip/impl/peer_set.cpp index fa9d13410..57e29a2b6 100644 --- a/src/protocol/gossip/impl/peer_set.cpp +++ b/src/protocol/gossip/impl/peer_set.cpp @@ -24,7 +24,11 @@ namespace libp2p::protocol::gossip { } bool PeerSet::contains(const peer::PeerId &id) const { - return peers_.count(id) != 0; + return peers_.contains(id); + } + + bool PeerSet::contains(const PeerContextPtr &ctx) const { + return peers_.contains(ctx); } bool PeerSet::insert(PeerContextPtr ctx) { diff --git a/src/protocol/gossip/impl/peer_set.hpp b/src/protocol/gossip/impl/peer_set.hpp index c9e08c660..c326617aa 100644 --- a/src/protocol/gossip/impl/peer_set.hpp +++ b/src/protocol/gossip/impl/peer_set.hpp @@ -21,6 +21,7 @@ namespace libp2p::protocol::gossip { /// Returns if the set contains such a peer bool contains(const peer::PeerId &id) const; + bool contains(const PeerContextPtr &ctx) const; /// Inserts peer context into set, returns false if already inserted bool insert(PeerContextPtr ctx); @@ -56,6 +57,18 @@ namespace libp2p::protocol::gossip { /// Conditionally erases peers from the set void eraseIf(const FilterCallback &filter); + auto begin() const { + return peers_.begin(); + } + auto end() const { + return peers_.end(); + } + void extend(auto &&peers) { + for (auto &ctx : peers) { + insert(ctx); + } + } + private: std::set> peers_; }; diff --git a/src/protocol/gossip/impl/remote_subscriptions.cpp b/src/protocol/gossip/impl/remote_subscriptions.cpp index 4a2f5e7aa..2e97875a0 100644 --- a/src/protocol/gossip/impl/remote_subscriptions.cpp +++ b/src/protocol/gossip/impl/remote_subscriptions.cpp @@ -7,31 +7,45 @@ #include "remote_subscriptions.hpp" #include +#include +#include "choose_peers.hpp" #include "connectivity.hpp" #include "message_builder.hpp" namespace libp2p::protocol::gossip { - RemoteSubscriptions::RemoteSubscriptions(const Config &config, - Connectivity &connectivity, - basic::Scheduler &scheduler, - log::SubLogger &log) + RemoteSubscriptions::RemoteSubscriptions( + const Config &config, + Connectivity &connectivity, + std::shared_ptr score, + std::shared_ptr scheduler, + log::SubLogger &log) : config_(config), connectivity_(connectivity), - scheduler_(scheduler), + choose_peers_{std::make_shared()}, + explicit_peers_{std::make_shared()}, + score_{std::move(score)}, + scheduler_{std::move(scheduler)}, log_(log) {} void RemoteSubscriptions::onSelfSubscribed(bool subscribed, const TopicId &topic) { + if (subscribed == isSubscribed(topic)) { + return; + } auto res = getItem(topic, subscribed); if (!res) { log_.error("error in self subscribe to {}", topic); return; } TopicSubscriptions &subs = res.value(); - subs.onSelfSubscribed(subscribed); - if (subs.empty()) { + if (subscribed) { + subs.subscribe(); + } else { + subs.unsubscribe(); + } + if (not subs.isUsed()) { table_.erase(topic); } log_.debug("self {} {}", @@ -48,7 +62,7 @@ namespace libp2p::protocol::gossip { } log_.debug("peer {} subscribing to {}", peer->str, topic); - auto res = getItem(topic, true); + auto res = getItem(topic, false); if (!res) { // not error in this case, this is request from wire... log_.debug("entry doesnt exist for {}", topic); @@ -78,7 +92,7 @@ namespace libp2p::protocol::gossip { TopicSubscriptions &subs = res.value(); subs.onPeerUnsubscribed(peer); - if (subs.empty()) { + if (not subs.isUsed()) { table_.erase(topic); } } @@ -92,38 +106,44 @@ namespace libp2p::protocol::gossip { } } - bool RemoteSubscriptions::hasTopic(const TopicId &topic) const { - return table_.count(topic) != 0; + bool RemoteSubscriptions::isSubscribed(const TopicId &topic) const { + auto it = table_.find(topic); + return it != table_.end() and it->second.isSubscribed(); } void RemoteSubscriptions::onGraft(const PeerContextPtr &peer, const TopicId &topic) { + // implicit subscribe on graft + peer->subscribed_to.insert(topic); auto res = getItem(topic, false); if (!res) { // we don't have this topic anymore - peer->message_builder->addPrune(topic); - connectivity_.peerIsWritable(peer, true); + peer->message_builder->addPrune( + topic, + peer->isGossipsubv1_1() ? std::make_optional(config_.prune_backoff) + : std::nullopt); + connectivity_.peerIsWritable(peer); return; } res.value().onGraft(peer); } - void RemoteSubscriptions::onPrune(const PeerContextPtr &peer, - const TopicId &topic, - uint64_t backoff_time) { + void RemoteSubscriptions::onPrune( + const PeerContextPtr &peer, + const TopicId &topic, + std::optional backoff_time) { auto res = getItem(topic, false); if (!res) { return; } - res.value().onPrune(peer, - scheduler_.now() + std::chrono::seconds(backoff_time)); + res.value().onPrune(peer, backoff_time); } void RemoteSubscriptions::onNewMessage( const boost::optional &from, const TopicMessage::Ptr &msg, const MessageId &msg_id) { - auto now = scheduler_.now(); + auto now = scheduler_->now(); bool is_published_locally = !from.has_value(); auto res = getItem(msg->topic, is_published_locally); if (!res) { @@ -134,10 +154,10 @@ namespace libp2p::protocol::gossip { } void RemoteSubscriptions::onHeartbeat() { - auto now = scheduler_.now(); + auto now = scheduler_->now(); for (auto it = table_.begin(); it != table_.end();) { it->second.onHeartbeat(now); - if (it->second.empty()) { + if (not it->second.isUsed()) { // fanout interval expired - clean up log_.debug("deleted entry for topic {}", it->first); it = table_.erase(it); @@ -154,14 +174,16 @@ namespace libp2p::protocol::gossip { return it->second; } if (create_if_not_exist) { - auto [it, _] = table_.emplace( - topic, TopicSubscriptions(topic, config_, connectivity_, log_)); + auto [it, _] = table_.emplace(topic, + TopicSubscriptions(topic, + config_, + connectivity_, + scheduler_, + choose_peers_, + explicit_peers_, + score_, + log_)); TopicSubscriptions &item = it->second; - connectivity_.getConnectedPeers().selectIf( - [&item](const PeerContextPtr &ctx) { item.onPeerSubscribed(ctx); }, - [&topic](const PeerContextPtr &ctx) { - return ctx->subscribed_to.count(topic) != 0; - }); log_.debug("created entry for topic {}", topic); return item; } diff --git a/src/protocol/gossip/impl/remote_subscriptions.hpp b/src/protocol/gossip/impl/remote_subscriptions.hpp index 765d5bcdb..ceec19dad 100644 --- a/src/protocol/gossip/impl/remote_subscriptions.hpp +++ b/src/protocol/gossip/impl/remote_subscriptions.hpp @@ -20,7 +20,8 @@ namespace libp2p::protocol::gossip { /// GossipCore and lives only within its scope RemoteSubscriptions(const Config &config, Connectivity &connectivity, - basic::Scheduler &scheduler, + std::shared_ptr score, + std::shared_ptr scheduler, log::SubLogger &log); /// This host subscribes or unsubscribes @@ -38,7 +39,7 @@ namespace libp2p::protocol::gossip { void onPeerDisconnected(const PeerContextPtr &peer); /// Returns if topic exists in the table - bool hasTopic(const TopicId &topic) const; + bool isSubscribed(const TopicId &topic) const; /// Remote peer adds topic into its mesh void onGraft(const PeerContextPtr &peer, const TopicId &topic); @@ -46,7 +47,7 @@ namespace libp2p::protocol::gossip { /// Remote peer removes topic from its mesh void onPrune(const PeerContextPtr &peer, const TopicId &topic, - uint64_t backoff_time); + std::optional backoff_time); /// Forwards message to its topics. If 'from' is not set then the message is /// published locally @@ -64,10 +65,11 @@ namespace libp2p::protocol::gossip { const Config &config_; Connectivity &connectivity_; - basic::Scheduler &scheduler_; + std::shared_ptr choose_peers_; + std::shared_ptr explicit_peers_; + std::shared_ptr score_; + std::shared_ptr scheduler_; - // TODO(artem): bound table size (which may grow!) - // by removing items not subscribed to locally. LRU(???) std::unordered_map table_; log::SubLogger &log_; diff --git a/src/protocol/gossip/impl/stream.cpp b/src/protocol/gossip/impl/stream.cpp index 98baf3a74..d8f4eace5 100644 --- a/src/protocol/gossip/impl/stream.cpp +++ b/src/protocol/gossip/impl/stream.cpp @@ -23,7 +23,7 @@ namespace libp2p::protocol::gossip { const Config &config, basic::Scheduler &scheduler, const Feedback &feedback, - MessageReceiver &msg_receiver, + std::weak_ptr msg_receiver, std::shared_ptr stream, PeerContextPtr peer) : stream_id_(stream_id), @@ -31,7 +31,7 @@ namespace libp2p::protocol::gossip { scheduler_(scheduler), max_message_size_(config.max_message_size), feedback_(feedback), - msg_receiver_(msg_receiver), + msg_receiver_{std::move(msg_receiver)}, stream_(std::move(stream)), peer_(std::move(peer)), read_buffer_(std::make_shared>()) { @@ -119,7 +119,12 @@ namespace libp2p::protocol::gossip { return; } - parser.dispatch(peer_, msg_receiver_); + if (auto msg_receiver = msg_receiver_.lock()) { + parser.dispatch(peer_, *msg_receiver); + } else { + close(); + return; + } // reads again read(); diff --git a/src/protocol/gossip/impl/stream.hpp b/src/protocol/gossip/impl/stream.hpp index 12477a061..ff3a9790c 100644 --- a/src/protocol/gossip/impl/stream.hpp +++ b/src/protocol/gossip/impl/stream.hpp @@ -34,7 +34,7 @@ namespace libp2p::protocol::gossip { const Config &config, basic::Scheduler &scheduler, const Feedback &feedback, - MessageReceiver &msg_receiver, + std::weak_ptr msg_receiver, std::shared_ptr stream, PeerContextPtr peer); @@ -61,7 +61,7 @@ namespace libp2p::protocol::gossip { basic::Scheduler &scheduler_; const size_t max_message_size_; const Feedback &feedback_; - MessageReceiver &msg_receiver_; + std::weak_ptr msg_receiver_; std::shared_ptr stream_; PeerContextPtr peer_; diff --git a/src/protocol/gossip/impl/topic_subscriptions.cpp b/src/protocol/gossip/impl/topic_subscriptions.cpp index 37e8d74b0..2ae554583 100644 --- a/src/protocol/gossip/impl/topic_subscriptions.cpp +++ b/src/protocol/gossip/impl/topic_subscriptions.cpp @@ -8,7 +8,10 @@ #include #include +#include +#include +#include "choose_peers.hpp" #include "connectivity.hpp" #include "message_builder.hpp" @@ -29,21 +32,41 @@ namespace libp2p::protocol::gossip { } // namespace - TopicSubscriptions::TopicSubscriptions(TopicId topic, - const Config &config, - Connectivity &connectivity, - log::SubLogger &log) + TopicSubscriptions::TopicSubscriptions( + TopicId topic, + const Config &config, + Connectivity &connectivity, + std::shared_ptr scheduler, + std::shared_ptr choose_peers, + std::shared_ptr explicit_peers, + std::shared_ptr score, + log::SubLogger &log) : topic_(std::move(topic)), config_(config), connectivity_(connectivity), + scheduler_{std::move(scheduler)}, + choose_peers_{std::move(choose_peers)}, + explicit_peers_{std::move(explicit_peers)}, + score_{std::move(score)}, self_subscribed_(false), - fanout_period_ends_(0), - log_(log) {} + log_(log) { + connectivity_.getConnectedPeers().selectIf( + [&](const PeerContextPtr &ctx) { subscribed_peers_.insert(ctx); }, + [&](const PeerContextPtr &ctx) { + return ctx->subscribed_to.count(topic_) != 0; + }); + if (config_.history_gossip == 0) { + throw std::logic_error{"gossip config.history_gossip must not be zero"}; + } + history_gossip_.resize(config_.history_gossip); + } + + bool TopicSubscriptions::isUsed() const { + return self_subscribed_ or fanout_.has_value(); + } - bool TopicSubscriptions::empty() const { - return (not self_subscribed_) - && (fanout_period_ends_ == std::chrono::milliseconds::zero()) - && subscribed_peers_.empty() && mesh_peers_.empty(); + bool TopicSubscriptions::isSubscribed() const { + return self_subscribed_; } void TopicSubscriptions::onNewMessage( @@ -53,72 +76,99 @@ namespace libp2p::protocol::gossip { Time now) { bool is_published_locally = !from.has_value(); - if (is_published_locally) { - fanout_period_ends_ = now + config_.seen_cache_lifetime_msec; - log_.debug("setting fanout period for {}, {}->{}", - topic_, - now.count(), - fanout_period_ends_.count()); + if (not self_subscribed_ and not is_published_locally) { + return; } auto origin = peerFrom(*msg); - mesh_peers_.selectAll( - [this, &msg, &msg_id, &from, &origin](const PeerContextPtr &ctx) { - assert(ctx->message_builder); - - if (needToForward(ctx, from, origin)) { - ctx->message_builder->addMessage(*msg, msg_id); - - // forward immediately to those in mesh - connectivity_.peerIsWritable(ctx, true); - } - }); - - auto peers = subscribed_peers_.selectRandomPeers(config_.D_max * 2); - for (const auto &ctx : peers) { - assert(ctx->message_builder); - + auto add_peer = [&](const PeerContextPtr &ctx) { if (needToForward(ctx, from, origin)) { - ctx->message_builder->addIHave(topic_, msg_id); + ctx->message_builder->addMessage(*msg, msg_id); + connectivity_.peerIsWritable(ctx); + } + }; + + if (config_.flood_publish and is_published_locally) { + for (auto &ctx : subscribed_peers_) { + if (explicit_peers_->contains(ctx->peer_id) + or not score_->below(ctx->peer_id, + config_.score.publish_threshold)) { + add_peer(ctx); + } + } + } else { + for (auto &ctx : subscribed_peers_) { + if (ctx->isFloodsub()) { + add_peer(ctx); + } + } - // local messages announce themselves immediately - connectivity_.peerIsWritable(ctx, is_published_locally); + if (self_subscribed_) { + for (auto &ctx : mesh_peers_) { + add_peer(ctx); + } + } else { + if (not fanout_.has_value()) { + fanout_.emplace(); + } + if (fanout_->peers.empty()) { + fanout_->peers.extend(choose_peers_->choose( + subscribed_peers_, + [&](const PeerContextPtr &ctx) { + return not explicit_peers_->contains(ctx->peer_id) + and not score_->below(ctx->peer_id, + config_.score.publish_threshold); + }, + config_.D)); + } + fanout_->until = now + config_.fanout_ttl; + for (auto &ctx : fanout_->peers) { + add_peer(ctx); + } } } - seen_cache_.emplace_back(now + config_.seen_cache_lifetime_msec, msg_id); - - log_.debug("message forwarded, topic={}, m={}, s={}", - topic_, - mesh_peers_.size(), - subscribed_peers_.size()); + history_gossip_.back().emplace_back(msg_id); } void TopicSubscriptions::onHeartbeat(Time now) { - if (self_subscribed_ && !subscribed_peers_.empty()) { - // add/remove mesh members according to desired network density D - size_t sz = mesh_peers_.size(); + auto slack = config_.backoff_slack * config_.heartbeat_interval_msec; + for (auto it = dont_bother_until_.begin(); + it != dont_bother_until_.end();) { + if (it->second + slack > now) { + ++it; + } else { + it = dont_bother_until_.erase(it); + } + } - if (sz < config_.D_min) { - auto peers = subscribed_peers_.selectRandomPeers(config_.D_min - sz); - for (auto &p : peers) { - auto it = dont_bother_until_.find(p); - if (it != dont_bother_until_.end()) { - if (it->second < now) { - dont_bother_until_.erase(it); - } else { - continue; - } - } - - addToMesh(p); - subscribed_peers_.erase(p->peer_id); + if (self_subscribed_) { + // add/remove mesh members according to desired network density D + mesh_peers_.eraseIf([&](const PeerContextPtr &ctx) { + if (score_->below(ctx->peer_id, config_.score.zero)) { + sendPrune(ctx, false); + return true; + } + return false; + }); + if (mesh_peers_.size() < config_.D_min) { + for (auto &ctx : choose_peers_->choose( + subscribed_peers_, + [&](const PeerContextPtr &ctx) { + return not mesh_peers_.contains(ctx) + and not explicit_peers_->contains(ctx->peer_id) + and not isBackoffWithSlack(ctx->peer_id) + and not score_->below(ctx->peer_id, config_.score.zero); + }, + config_.D - mesh_peers_.size())) { + addToMesh(ctx); } - } else if (sz > config_.D_max) { - auto peers = mesh_peers_.selectRandomPeers(sz - config_.D_max); + } else if (mesh_peers_.size() > config_.D_max) { + auto peers = + mesh_peers_.selectRandomPeers(mesh_peers_.size() - config_.D_max); for (auto &p : peers) { - removeFromMesh(p); + sendPrune(p, false); mesh_peers_.erase(p->peer_id); } } @@ -126,109 +176,139 @@ namespace libp2p::protocol::gossip { // fanout ends some time after this host ends publishing to the topic, // to save space and traffic - if (fanout_period_ends_ != Time::zero() && fanout_period_ends_ < now) { - fanout_period_ends_ = Time::zero(); + if (fanout_.has_value() and fanout_->until < now) { + fanout_.reset(); log_.debug("fanout period reset for {}", topic_); } + if (fanout_.has_value()) { + fanout_->peers.eraseIf([&](const PeerContextPtr &ctx) { + return score_->below(ctx->peer_id, config_.score.publish_threshold); + }); + if (fanout_->peers.size() < config_.D) { + fanout_->peers.extend(choose_peers_->choose( + subscribed_peers_, + [&](const PeerContextPtr &ctx) { + return not fanout_->peers.contains(ctx) + and not explicit_peers_->contains(ctx->peer_id) + and not score_->below(ctx->peer_id, + config_.score.publish_threshold); + }, + config_.D - fanout_->peers.size())); + } + } + + emitGossip(); // shift msg ids cache - auto seen_cache_size = seen_cache_.size(); - bool changed = false; - - if (seen_cache_size > config_.seen_cache_limit) { - auto b = seen_cache_.begin(); - auto e = b + ssize_t(seen_cache_size - config_.seen_cache_limit); - seen_cache_.erase(b, e); - changed = true; - } else if (seen_cache_size != 0) { - auto it = std::find_if(seen_cache_.begin(), - seen_cache_.end(), - [now](const auto &p) { return p.first >= now; }); - if (it != seen_cache_.begin()) { - seen_cache_.erase(seen_cache_.begin(), it); - changed = true; + history_gossip_.pop_front(); + history_gossip_.emplace_back(); + } + + void TopicSubscriptions::subscribe() { + if (self_subscribed_) { + return; + } + self_subscribed_ = true; + if (fanout_.has_value()) { + for (auto &ctx : fanout_->peers) { + if (explicit_peers_->contains(ctx->peer_id)) { + continue; + } + if (isBackoffWithSlack(ctx->peer_id)) { + continue; + } + if (mesh_peers_.size() >= config_.D) { + break; + } + addToMesh(ctx); } + fanout_.reset(); } - - if (changed) { - log_.debug("seen cache size changed {}->{} for {}", - seen_cache_size, - seen_cache_.size(), - topic_); + if (mesh_peers_.size() < config_.D) { + for (auto &ctx : choose_peers_->choose( + subscribed_peers_, + [&](const PeerContextPtr &ctx) { + return not mesh_peers_.contains(ctx) + and not explicit_peers_->contains(ctx->peer_id) + and not isBackoffWithSlack(ctx->peer_id) + and not score_->below(ctx->peer_id, config_.score.zero); + }, + config_.D - mesh_peers_.size())) { + addToMesh(ctx); + } } } - void TopicSubscriptions::onSelfSubscribed(bool self_subscribed) { - self_subscribed_ = self_subscribed; - if (!self_subscribed_) { - // remove the mesh - log_.debug("removing mesh for {}", topic_); - mesh_peers_.selectAll( - [this](const PeerContextPtr &p) { removeFromMesh(p); }); - mesh_peers_.clear(); + void TopicSubscriptions::unsubscribe() { + if (not self_subscribed_) { + return; + } + self_subscribed_ = false; + for (auto &ctx : mesh_peers_) { + sendPrune(ctx, true); } + mesh_peers_.clear(); } - void TopicSubscriptions::onPeerSubscribed(const PeerContextPtr &p) { - assert(p->subscribed_to.count(topic_) != 0); + void TopicSubscriptions::onPeerSubscribed(const PeerContextPtr &ctx) { + assert(ctx->subscribed_to.count(topic_) != 0); - subscribed_peers_.insert(p); + subscribed_peers_.insert(ctx); - // announce the peer about messages available for the topic - for (const auto &[_, msg_id] : seen_cache_) { - p->message_builder->addIHave(topic_, msg_id); + if (ctx->isGossipsub() and mesh_peers_.size() < config_.D_min + and not explicit_peers_->contains(ctx->peer_id) + and not isBackoffWithSlack(ctx->peer_id) + and not score_->below(ctx->peer_id, config_.score.zero)) { + addToMesh(ctx); } - // will be sent on next heartbeat - connectivity_.peerIsWritable(p, false); } void TopicSubscriptions::onPeerUnsubscribed(const PeerContextPtr &p) { - auto res = subscribed_peers_.erase(p->peer_id); - if (!res) { - res = mesh_peers_.erase(p->peer_id); + subscribed_peers_.erase(p->peer_id); + if (fanout_) { + fanout_->peers.erase(p->peer_id); + if (fanout_->peers.empty()) { + fanout_.reset(); + } + } + if (mesh_peers_.erase(p->peer_id)) { + updateBackoff(p->peer_id, config_.prune_backoff); } - dont_bother_until_.erase(p); } - void TopicSubscriptions::onGraft(const PeerContextPtr &p) { - auto res = mesh_peers_.find(p->peer_id); + void TopicSubscriptions::onGraft(const PeerContextPtr &ctx) { + auto res = mesh_peers_.find(ctx->peer_id); if (res) { // already there return; } - if (!subscribed_peers_.contains(p->peer_id)) { - // subscribe first - p->subscribed_to.insert(topic_); - onPeerSubscribed(p); - } + // implicit subscribe on graft + subscribed_peers_.insert(ctx); bool mesh_is_full = (mesh_peers_.size() >= config_.D_max); - if (self_subscribed_ && !mesh_is_full) { - mesh_peers_.insert(p); - subscribed_peers_.erase(p->peer_id); + if (self_subscribed_ and not mesh_is_full + and not explicit_peers_->contains(ctx->peer_id) + and not isBackoff(ctx->peer_id, std::chrono::milliseconds{0})) { + mesh_peers_.insert(ctx); } else { // we don't have mesh for the topic - p->message_builder->addPrune(topic_); - connectivity_.peerIsWritable(p, true); + sendPrune(ctx, false); } } - void TopicSubscriptions::onPrune(const PeerContextPtr &p, - Time dont_bother_until) { + void TopicSubscriptions::onPrune( + const PeerContextPtr &p, std::optional backoff) { mesh_peers_.erase(p->peer_id); - if (p->subscribed_to.count(topic_) != 0) { - subscribed_peers_.insert(p); - dont_bother_until_.insert({p, dont_bother_until}); - } + updateBackoff(p->peer_id, backoff.value_or(config_.prune_backoff)); } void TopicSubscriptions::addToMesh(const PeerContextPtr &p) { assert(p->message_builder); p->message_builder->addGraft(topic_); - connectivity_.peerIsWritable(p, false); + connectivity_.peerIsWritable(p); mesh_peers_.insert(p); log_.debug("peer {} added to mesh (size={}) for topic {}", p->str, @@ -236,16 +316,79 @@ namespace libp2p::protocol::gossip { topic_); } - void TopicSubscriptions::removeFromMesh(const PeerContextPtr &p) { - assert(p->message_builder); + void TopicSubscriptions::sendPrune(const PeerContextPtr &ctx, + bool unsubscribe) { + assert(ctx->message_builder); - p->message_builder->addPrune(topic_); - connectivity_.peerIsWritable(p, false); - subscribed_peers_.insert(p); + auto backoff = unsubscribe ? config_.prune_backoff : config_.prune_backoff; + updateBackoff(ctx->peer_id, backoff); + ctx->message_builder->addPrune( + topic_, + ctx->isGossipsubv1_1() ? std::make_optional(backoff) : std::nullopt); + connectivity_.peerIsWritable(ctx); log_.debug("peer {} removed from mesh (size={}) for topic {}", - p->str, + ctx->str, mesh_peers_.size(), topic_); } + bool TopicSubscriptions::isBackoff(const PeerId &peer_id, + std::chrono::milliseconds slack) const { + auto it = dont_bother_until_.find(peer_id); + if (it == dont_bother_until_.end()) { + return false; + } + return it->second + slack > scheduler_->now(); + } + + bool TopicSubscriptions::isBackoffWithSlack(const PeerId &peer_id) const { + return isBackoff(peer_id, + config_.backoff_slack * config_.heartbeat_interval_msec); + } + + void TopicSubscriptions::updateBackoff(const PeerId &peer_id, + std::chrono::milliseconds duration) { + auto until = scheduler_->now() + duration; + auto [it, inserted] = dont_bother_until_.emplace(peer_id, until); + if (not inserted and it->second < until) { + it->second = until; + } + } + + void TopicSubscriptions::emitGossip() { + auto &mesh = fanout_.has_value() ? fanout_->peers : mesh_peers_; + auto peers = choose_peers_->choose( + subscribed_peers_, + [&](const PeerContextPtr &ctx) { + return not mesh.contains(ctx) + and not explicit_peers_->contains(ctx->peer_id) + and not score_->below(ctx->peer_id, + config_.score.gossip_threshold); + }, + config_.D); + size_t message_count = 0; + for (auto &x : history_gossip_) { + message_count += x.size(); + } + if (message_count == 0) { + return; + } + std::vector messages; + messages.reserve(message_count); + for (auto &x : history_gossip_) { + messages.insert(messages.end(), x.begin(), x.end()); + } + std::ranges::shuffle(messages, gossip_random_); + for (auto &ctx : peers) { + std::span messages_span{messages}; + if (messages.size() > config_.max_ihave_length) { + std::ranges::shuffle(messages, gossip_random_); + messages_span = messages_span.first(config_.max_ihave_length); + } + for (auto &message_id : messages_span) { + ctx->message_builder->addIHave(topic_, message_id); + } + connectivity_.peerIsWritable(ctx); + } + } } // namespace libp2p::protocol::gossip diff --git a/src/protocol/gossip/impl/topic_subscriptions.hpp b/src/protocol/gossip/impl/topic_subscriptions.hpp index 83b4b2ad1..7e1357e62 100644 --- a/src/protocol/gossip/impl/topic_subscriptions.hpp +++ b/src/protocol/gossip/impl/topic_subscriptions.hpp @@ -7,14 +7,17 @@ #pragma once #include +#include #include #include "peer_set.hpp" namespace libp2p::protocol::gossip { - + class ChoosePeers; class Connectivity; + class ExplicitPeers; + class Score; /// Per-topic subscriptions class TopicSubscriptions { @@ -24,11 +27,17 @@ namespace libp2p::protocol::gossip { TopicSubscriptions(TopicId topic, const Config &config, Connectivity &connectivity, + std::shared_ptr scheduler, + std::shared_ptr choose_peers, + std::shared_ptr explicit_peers, + std::shared_ptr score, log::SubLogger &log); - /// Returns true if no peers subscribed and not self-subscribed and + /// Returns true if not self-subscribed and /// no fanout period at the moment (empty item may be erased) - bool empty() const; + bool isUsed() const; + + bool isSubscribed() const; /// Forwards message to mesh members and announce to other subscribers void onNewMessage(const boost::optional &from, @@ -39,8 +48,8 @@ namespace libp2p::protocol::gossip { /// Periodic job needed to update meshes and shift "I have" caches void onHeartbeat(Time now); - /// Local host subscribes or unsubscribes, this affects mesh - void onSelfSubscribed(bool self_subscribed); + void subscribe(); + void unsubscribe(); /// Remote peer subscribes to topic void onPeerSubscribed(const PeerContextPtr &p); @@ -52,24 +61,41 @@ namespace libp2p::protocol::gossip { void onGraft(const PeerContextPtr &p); /// Remote peer kicks this host out of its mesh - void onPrune(const PeerContextPtr &p, Time dont_bother_until); + void onPrune(const PeerContextPtr &p, + std::optional backoff); private: + struct Fanout { + Time until; + PeerSet peers; + }; + /// Adds a peer to mesh void addToMesh(const PeerContextPtr &p); /// Removes a peer from mesh - void removeFromMesh(const PeerContextPtr &p); + void sendPrune(const PeerContextPtr &ctx, bool unsubscribe); + + void emitGossip(); + bool isBackoff(const PeerId &peer_id, + std::chrono::milliseconds slack) const; + bool isBackoffWithSlack(const PeerId &peer_id) const; + void updateBackoff(const PeerId &peer_id, + std::chrono::milliseconds duration); const TopicId topic_; const Config &config_; Connectivity &connectivity_; + std::shared_ptr scheduler_; + std::shared_ptr choose_peers_; + std::shared_ptr explicit_peers_; + std::shared_ptr score_; /// This host subscribed to this topic or not, this affects mesh behavior bool self_subscribed_; /// Fanout period allows for publishing from this host without subscribing - Time fanout_period_ends_; + std::optional fanout_; /// Peers subscribed to this topic, but not mesh members PeerSet subscribed_peers_; @@ -77,13 +103,15 @@ namespace libp2p::protocol::gossip { /// Mesh members to whom messages are forwarded in push manner PeerSet mesh_peers_; - /// "I have" notifications for new subscribers aka seen messages cache - std::deque> seen_cache_; + /// "I have" notifications for new subscribers + std::deque> history_gossip_; /// Prune backoff times per peer - std::unordered_map dont_bother_until_; + std::unordered_map dont_bother_until_; log::SubLogger &log_; + + std::default_random_engine gossip_random_; }; } // namespace libp2p::protocol::gossip diff --git a/src/protocol/gossip/protobuf/rpc.proto b/src/protocol/gossip/protobuf/rpc.proto index 6de40af24..2bfa62783 100644 --- a/src/protocol/gossip/protobuf/rpc.proto +++ b/src/protocol/gossip/protobuf/rpc.proto @@ -28,6 +28,7 @@ message ControlMessage { repeated ControlIWant iwant = 2; repeated ControlGraft graft = 3; repeated ControlPrune prune = 4; + repeated ControlIDontWant idontwant = 5; } message ControlIHave { @@ -53,3 +54,7 @@ message ControlPrune { repeated PeerInfo peers = 2; optional uint64 backoff = 3; } + +message ControlIDontWant { + repeated bytes message_ids = 1; +} diff --git a/test/libp2p/protocol/gossip/CMakeLists.txt b/test/libp2p/protocol/gossip/CMakeLists.txt index dcd6aba15..19b6baa09 100644 --- a/test/libp2p/protocol/gossip/CMakeLists.txt +++ b/test/libp2p/protocol/gossip/CMakeLists.txt @@ -20,3 +20,15 @@ target_link_libraries(gossip_local_subs_test p2p_gossip p2p_testutil_peer ) + +addtest(gossip_mock_test + gossip_mock_test.cpp + ) +target_link_libraries(gossip_mock_test + p2p_basic_scheduler + p2p_gossip + p2p_manual_scheduler_backend + p2p_message_read_writer + p2p_stream_pair + p2p_testutil_peer + ) diff --git a/test/libp2p/protocol/gossip/gossip_mock_test.cpp b/test/libp2p/protocol/gossip/gossip_mock_test.cpp new file mode 100644 index 000000000..6cbe00329 --- /dev/null +++ b/test/libp2p/protocol/gossip/gossip_mock_test.cpp @@ -0,0 +1,459 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mock/libp2p/crypto/crypto_provider.hpp" +#include "mock/libp2p/crypto/key_marshaller_mock.hpp" +#include "mock/libp2p/host/host_mock.hpp" +#include "mock/libp2p/peer/address_repository_mock.hpp" +#include "mock/libp2p/peer/identity_manager_mock.hpp" +#include "mock/libp2p/peer/peer_repository_mock.hpp" +#include "testutil/libp2p/peer.hpp" +#include "testutil/prepare_loggers.hpp" + +using libp2p::Bytes; +using libp2p::BytesIn; +using libp2p::HostMock; +using libp2p::PeerId; +using libp2p::PeerInfo; +using libp2p::ProtocolName; +using libp2p::ProtocolPredicate; +using libp2p::StreamAndProtocol; +using libp2p::StreamAndProtocolCb; +using libp2p::StreamAndProtocolOrErrorCb; +using libp2p::StreamProtocols; +using libp2p::basic::ManualSchedulerBackend; +using libp2p::basic::MessageReadWriterUvarint; +using libp2p::basic::SchedulerImpl; +using libp2p::connection::Stream; +using libp2p::crypto::CryptoProviderMock; +using libp2p::crypto::marshaller::KeyMarshallerMock; +using libp2p::peer::AddressRepositoryMock; +using libp2p::peer::IdentityManagerMock; +using libp2p::peer::PeerRepositoryMock; +using libp2p::protocol::gossip::Gossip; +using libp2p::protocol::gossip::PeerKind; +using libp2p::protocol::gossip::TopicId; +using testing::_; +using testing::Return; +using testing::ReturnRef; +using testutil::randomPeerId; + +using MessageIds = std::vector; + +inline Bytes encodeMessageId(uint8_t i) { + return Bytes{i}; +} +inline uint8_t decodeMessageId(std::string_view data) { + return data.at(0); +} + +struct MockPeer { + MockPeer(PeerId peer_id, ProtocolName version, std::shared_ptr writer) + : peer_id_{std::move(peer_id)}, + version_{std::move(version)}, + writer_{std::move(writer)}, + framing_{std::make_shared(writer_)} {} + + struct Received { + std::vector subscriptions; + MessageIds messages; + std::vector graft; + MessageIds ihave; + MessageIds iwant; + MessageIds idontwant; + }; + + void expect(Received expected) { + EXPECT_EQ(received_.subscriptions, expected.subscriptions); + EXPECT_EQ(received_.messages, expected.messages); + EXPECT_EQ(received_.graft, expected.graft); + EXPECT_EQ(received_.ihave, expected.ihave); + EXPECT_EQ(received_.iwant, expected.iwant); + received_ = {}; + } + + void write(const auto &f) { + pubsub::pb::RPC rpc; + f(rpc); + Bytes buffer; + buffer.resize(rpc.ByteSizeLong()); + rpc.SerializeToArray(buffer.data(), buffer.size()); + framing_->write( + buffer, [](outcome::result r) { EXPECT_TRUE(r.has_value()); }); + } + + PeerId peer_id_; + ProtocolName version_; + std::shared_ptr writer_; + std::shared_ptr framing_; + Received received_; +}; + +struct GossipMockTest : testing::Test { + PeerId gossip_peer_id_{randomPeerId()}; + TopicId topic1{"topic1"}; + std::unordered_map> peers_; + libp2p::protocol::gossip::Config config_; + std::optional host_handler_; + std::shared_ptr scheduler_backend_ = + std::make_shared(); + std::shared_ptr scheduler_ = std::make_shared( + scheduler_backend_, SchedulerImpl::Config{}); + std::shared_ptr peer_repo_ = + std::make_shared(); + std::shared_ptr address_repo_ = + std::make_shared(); + std::shared_ptr host_ = std::make_shared(); + std::shared_ptr idmgr_ = + std::make_shared(); + std::shared_ptr crypto_provider_ = + std::make_shared(); + std::shared_ptr key_marshaller_ = + std::make_shared(); + std::shared_ptr gossip_; + + static void SetUpTestCase() { + testutil::prepareLoggers(); + } + + void SetUp() override { + config_.D_min = 1; + config_.D = 1; + config_.flood_publish = false; + } + + void setup() { + EXPECT_CALL(*host_, getPeerRepository()) + .WillRepeatedly(ReturnRef(*peer_repo_)); + EXPECT_CALL(*peer_repo_, getAddressRepository()) + .WillRepeatedly(ReturnRef(*address_repo_)); + EXPECT_CALL(*address_repo_, updateAddresses(_, _)) + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*host_, getPeerInfo()) + .WillRepeatedly(Return(PeerInfo{gossip_peer_id_, {}})); + EXPECT_CALL(*host_, setProtocolHandler(_, _, _)) + .WillRepeatedly([&](StreamProtocols, + StreamAndProtocolCb cb, + ProtocolPredicate) { host_handler_ = cb; }); + EXPECT_CALL(*host_, newStream(_, _, _)) + .WillRepeatedly([&](const PeerInfo &info, + StreamProtocols, + StreamAndProtocolOrErrorCb cb) { + auto &peer_id = info.id; + auto peer = peers_.at(peer_id); + auto [stream1, stream2] = libp2p::connection::streamPair( + scheduler_, peer_id, gossip_peer_id_); + read(peer, std::make_shared(stream2)); + scheduler_->schedule([cb, stream1, peer] { + cb(StreamAndProtocol{stream1, peer->version_}); + }); + }); + gossip_ = libp2p::protocol::gossip::create( + scheduler_, host_, idmgr_, crypto_provider_, key_marshaller_, config_); + gossip_->setMessageIdFn( + [](BytesIn from, BytesIn seqno, Bytes data) { return data; }); + gossip_->start(); + } + + void TearDown() override { + for (auto &p : peers_) { + p.second->expect({}); + } + } + + auto connect(PeerKind peer_kind) { + auto version_it = + std::ranges::find_if(config_.protocol_versions, + [&](auto &p) { return p.second == peer_kind; }); + if (version_it == config_.protocol_versions.end()) { + throw std::logic_error{"PeerKind"}; + } + auto &version = version_it->first; + auto peer_id = randomPeerId(); + auto [stream1, stream2] = + libp2p::connection::streamPair(scheduler_, peer_id, gossip_peer_id_); + auto peer = std::make_shared(peer_id, version, stream1); + peers_.emplace(peer_id, peer); + scheduler_->schedule([this, stream2, version] { + host_handler_.value()({stream2, version}); + }); + scheduler_backend_->callDeferred(); + return peer; + } + + void read(std::shared_ptr peer, + std::shared_ptr framing) { + framing->read([this, peer, framing]( + outcome::result> frame_res) { + if (frame_res.has_error()) { + return; + } + auto &frame = frame_res.value(); + pubsub::pb::RPC rpc; + EXPECT_TRUE(rpc.ParseFromArray(frame->data(), frame->size())); + for (auto &sub : rpc.subscriptions()) { + EXPECT_EQ(sub.topicid(), topic1); + peer->received_.subscriptions.emplace_back(sub.subscribe()); + } + for (auto &pub : rpc.publish()) { + peer->received_.messages.emplace_back(decodeMessageId(pub.data())); + } + for (auto &graft : rpc.control().graft()) { + EXPECT_EQ(graft.topicid(), topic1); + peer->received_.graft.emplace_back(true); + } + EXPECT_EQ(rpc.control().prune().size(), 0); + for (auto &ihave : rpc.control().ihave()) { + EXPECT_EQ(ihave.topicid(), topic1); + for (auto &id : ihave.messageids()) { + peer->received_.ihave.emplace_back(decodeMessageId(id)); + } + } + for (auto &iwant : rpc.control().iwant()) { + for (auto &id : iwant.messageids()) { + peer->received_.iwant.emplace_back(decodeMessageId(id)); + } + } + for (auto &idontwant : rpc.control().idontwant()) { + for (auto &id : idontwant.message_ids()) { + peer->received_.idontwant.emplace_back(decodeMessageId(id)); + } + } + read(peer, framing); + }); + } + + auto subscribe() { + auto sub = gossip_->subscribe({topic1}, [](Gossip::SubscriptionData) {}); + scheduler_backend_->callDeferred(); + for (auto &p : peers_) { + p.second->expect({.subscriptions = {true}}); + } + return sub; + } + + void subscribe(MockPeer &peer, bool subscribe) { + peer.write([&](pubsub::pb::RPC &rpc) { + auto *sub = rpc.add_subscriptions(); + sub->set_topicid(topic1); + sub->set_subscribe(subscribe); + }); + scheduler_backend_->callDeferred(); + } + + void publish(uint8_t i) { + gossip_->publish(topic1, {i}); + scheduler_backend_->callDeferred(); + } + + void publish(MockPeer &peer, + uint8_t i, + std::optional author = std::nullopt) { + peer.write([&](pubsub::pb::RPC &rpc) { + auto *pub = rpc.add_publish(); + *pub->mutable_from() = + qtils::byte2str(author.value_or(peer.peer_id_).toVector()); + pub->set_seqno(""); + pub->set_topic(topic1); + *pub->mutable_data() = qtils::byte2str(encodeMessageId(i)); + }); + scheduler_backend_->callDeferred(); + } + + void ihave(MockPeer &peer, uint8_t i) { + peer.write([&](pubsub::pb::RPC &rpc) { + auto *ihave = rpc.mutable_control()->add_ihave(); + ihave->set_topicid(topic1); + *ihave->add_messageids() = qtils::byte2str(encodeMessageId(i)); + }); + scheduler_backend_->callDeferred(); + } + + void heartbeat() { + scheduler_backend_->shift(config_.heartbeat_interval_msec); + } +}; + +/** + * notify peers when subscribing and unsubscribing. + */ +TEST_F(GossipMockTest, SubscribeUnsubscribe) { + setup(); + auto peer1 = connect(PeerKind::Floodsub); + + auto sub = subscribe(); + + sub.cancel(); + scheduler_backend_->callDeferred(); + peer1->expect({.subscriptions = {false}}); +} + +/** + * publish to subscribed peers only. + * don't publish until peers subscribe. + * don't publish after peers unsubscribe. + */ +TEST_F(GossipMockTest, PublishToFloodsub) { + setup(); + auto peer1 = connect(PeerKind::Floodsub); + + publish(1); + peer1->expect({}); + + subscribe(*peer1, true); + publish(2); + peer1->expect({.messages = {2}}); + + subscribe(*peer1, false); + publish(3); + peer1->expect({}); +} + +/** + * forwards message to floodsub peers except: + * - peer who is not subscribed + * - peer who sent the message + * - message author + * don't forward same message more than once. + */ +TEST_F(GossipMockTest, ForwardToFloodsub) { + setup(); + auto peer1 = connect(PeerKind::Floodsub); + auto peer2 = connect(PeerKind::Floodsub); + auto peer3 = connect(PeerKind::Floodsub); + auto peer4 = connect(PeerKind::Floodsub); + + auto sub = subscribe(); + subscribe(*peer1, true); + subscribe(*peer2, true); + subscribe(*peer3, true); + publish(*peer2, 1, peer1->peer_id_); + peer3->expect({.messages = {1}}); + + publish(*peer2, 1, peer1->peer_id_); +} + +/** + * publish to fanout peers. + * must publish to same initially chosen fanout peers. + * don't forward messages to fanout peers. + */ +TEST_F(GossipMockTest, PublishToFanout) { + setup(); + auto peer1 = connect(PeerKind::Gossipsub); + auto peer2 = connect(PeerKind::Gossipsub); + uint8_t i = 1; + + subscribe(*peer1, true); + subscribe(*peer2, true); + publish(i); + if (peer1->received_.messages.empty()) { + std::swap(peer1, peer2); + } + peer1->expect({.messages = {i}}); + i++; + while (i < 30) { + publish(i); + peer1->expect({.messages = {i}}); + i++; + } + + publish(*peer2, i); +} + +/** + * notify peers when grafting. + * publish to grafted peers. + * forward to grafted peers. + */ +TEST_F(GossipMockTest, PublishToMesh) { + setup(); + auto peer1 = connect(PeerKind::Gossipsub); + auto peer2 = connect(PeerKind::Gossipsub); + + auto sub = subscribe(); + subscribe(*peer1, true); + subscribe(*peer2, true); + peer1->expect({.graft = {true}}); + + publish(1); + peer1->expect({.messages = {1}}); + + publish(*peer2, 2, randomPeerId()); + peer1->expect({.messages = {2}}); +} + +/** + * publish to all peers. + * don't forward to all peers. + */ +TEST_F(GossipMockTest, FloodPublish) { + config_.flood_publish = true; + setup(); + auto peer1 = connect(PeerKind::Gossipsub); + auto peer2 = connect(PeerKind::Gossipsub); + + auto sub = subscribe(); + subscribe(*peer1, true); + subscribe(*peer2, true); + publish(1); + publish(*peer2, 2, randomPeerId()); + peer1->expect({.graft = {true}, .messages = {1, 2}}); + peer2->expect({.messages = {1}}); +} + +/** + * gossip recent messages to random peers. + * don't gossip to mesh or fanout peers. + */ +TEST_F(GossipMockTest, Gossip) { + setup(); + auto peer1 = connect(PeerKind::Gossipsub); + auto peer2 = connect(PeerKind::Gossipsub); + auto peer3 = connect(PeerKind::Gossipsub); + + auto sub = subscribe(); + subscribe(*peer1, true); + subscribe(*peer2, true); + subscribe(*peer3, true); + publish(1); + peer1->expect({.graft = {true}, .messages = {1}}); + + heartbeat(); + if (peer2->received_.ihave.empty()) { + std::swap(peer2, peer3); + } + peer2->expect({.ihave = {1}}); +} + +/** + * send iwant after receiving ihave. + * don't send iwant until subscribed to topic. + * don't send iwant after receiving message. + */ +TEST_F(GossipMockTest, IhaveIwant) { + setup(); + auto peer1 = connect(PeerKind::Gossipsub); + + ihave(*peer1, 1); + peer1->expect({}); + + auto sub = subscribe(); + ihave(*peer1, 1); + peer1->expect({.iwant = {1}}); + + publish(*peer1, 1); + ihave(*peer1, 1); +} diff --git a/test/mock/libp2p/crypto/crypto_provider.hpp b/test/mock/libp2p/crypto/crypto_provider.hpp new file mode 100644 index 000000000..b04624bf2 --- /dev/null +++ b/test/mock/libp2p/crypto/crypto_provider.hpp @@ -0,0 +1,45 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include + +namespace libp2p::crypto { + class CryptoProviderMock : public CryptoProvider { + public: + MOCK_METHOD(outcome::result, + generateKeys, + (Key::Type, common::RSAKeyType), + (const, override)); + + MOCK_METHOD(outcome::result, + derivePublicKey, + (const PrivateKey &), + (const, override)); + + MOCK_METHOD(outcome::result, + sign, + (BytesIn, const PrivateKey &), + (const, override)); + + MOCK_METHOD(outcome::result, + verify, + (BytesIn, BytesIn, const PublicKey &), + (const, override)); + MOCK_METHOD(outcome::result, + generateEphemeralKeyPair, + (common::CurveType), + (const, override)); + + MOCK_METHOD((outcome::result>), + stretchKey, + (common::CipherType, common::HashType, const Buffer &), + (const, override)); + }; +} // namespace libp2p::crypto From 6def9d8a523488893285b751f23f57d3d71c39d7 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 17 Mar 2025 12:09:46 +0500 Subject: [PATCH 02/11] comment Signed-off-by: turuslan --- include/libp2p/connection/stream_pair.hpp | 4 ++ include/libp2p/protocol/gossip/gossip.hpp | 38 +++++++++++++++++++ .../libp2p/protocol/gossip/score_config.hpp | 5 +++ include/libp2p/protocol/gossip/time_cache.hpp | 2 + 4 files changed, 49 insertions(+) diff --git a/include/libp2p/connection/stream_pair.hpp b/include/libp2p/connection/stream_pair.hpp index 52af49e40..74104abb0 100644 --- a/include/libp2p/connection/stream_pair.hpp +++ b/include/libp2p/connection/stream_pair.hpp @@ -15,6 +15,10 @@ namespace libp2p::basic { namespace libp2p::connection { struct Stream; + /** + * Create pair of connected bidirectional read-writers + * implementing `Stream` interface. + */ std::pair, std::shared_ptr> streamPair( std::shared_ptr post, PeerId peer1, PeerId peer2); } // namespace libp2p::connection diff --git a/include/libp2p/protocol/gossip/gossip.hpp b/include/libp2p/protocol/gossip/gossip.hpp index a05ef8eb8..7248e91aa 100644 --- a/include/libp2p/protocol/gossip/gossip.hpp +++ b/include/libp2p/protocol/gossip/gossip.hpp @@ -87,22 +87,60 @@ namespace libp2p::protocol::gossip { /// Sign published messages bool sign_messages = false; + /// Number of heartbeats to keep in the `memcache` size_t history_length{5}; + /// Number of past heartbeats to gossip about (default is 3). size_t history_gossip{3}; + /// Time to live for fanout peers (default is 60 seconds). std::chrono::seconds fanout_ttl{60}; + /// Duplicates are prevented by storing message id's of known messages in an + /// LRU time cache. This settings sets the time period that messages are + /// stored in the cache. Duplicates can be received if duplicate messages + /// are sent at a time greater than this setting apart. The default is 1 + /// minute. std::chrono::seconds duplicate_cache_time{60}; + /// Controls the backoff time for pruned peers. This is how long + /// a peer must wait before attempting to graft into our mesh again after + /// being pruned. When pruning a peer, we send them our value of + /// `prune_backoff` so they know the minimum time to wait. Peers running + /// older versions may not send a backoff time, so if we receive a prune + /// message without one, we will wait at least `prune_backoff` before + /// attempting to re-graft. The default is one minute. std::chrono::seconds prune_backoff{60}; + /// Controls the backoff time when unsubscribing from a topic. + /// + /// This is how long to wait before resubscribing to the topic. A short + /// backoff period in case of an unsubscribe event allows reaching a healthy + /// mesh in a more timely manner. The default is 10 seconds. std::chrono::seconds unsubscribe_backoff{10}; + /// Number of heartbeat slots considered as slack for backoffs. This + /// guarantees that we wait at least backoff_slack heartbeats after a + /// backoff is over before we try to graft. This solves problems occurring + /// through high latencies. In particular if `backoff_slack * + /// heartbeat_interval` is longer than any latencies between processing + /// prunes on our side and processing prunes on the receiving side this + /// guarantees that we get not punished for too early grafting. The default + /// is 1. size_t backoff_slack = 1; + /// Whether to do flood publishing or not. If enabled newly created messages + /// will always be + /// sent to all peers that are subscribed to the topic and have a good + /// enough score. The default is true. bool flood_publish = true; + /// The maximum number of messages to include in an IHAVE message. + /// Also controls the maximum number of IHAVE ids we will accept and request + /// with IWANT from a peer within a heartbeat, to protect from IHAVE floods. + /// You should adjust this value from the default if your system is pushing + /// more than 5000 messages in GossipSubHistoryGossip heartbeats; with the + /// defaults this is 1666 messages/s. The default is 5000. size_t max_ihave_length = 5000; ScoreConfig score; diff --git a/include/libp2p/protocol/gossip/score_config.hpp b/include/libp2p/protocol/gossip/score_config.hpp index 258385e34..cb071fcb6 100644 --- a/include/libp2p/protocol/gossip/score_config.hpp +++ b/include/libp2p/protocol/gossip/score_config.hpp @@ -9,7 +9,12 @@ namespace libp2p::protocol::gossip { struct ScoreConfig { double zero = 0; + /// The score threshold below which gossip propagation is suppressed; + /// should be negative. double gossip_threshold = -10; + /// The score threshold below which we shouldn't publish when using flood + /// publishing (also applies to fanout peers); should be negative and <= + /// `gossip_threshold`. double publish_threshold = -50; }; } // namespace libp2p::protocol::gossip diff --git a/include/libp2p/protocol/gossip/time_cache.hpp b/include/libp2p/protocol/gossip/time_cache.hpp index 60b1d27d7..9ae701969 100644 --- a/include/libp2p/protocol/gossip/time_cache.hpp +++ b/include/libp2p/protocol/gossip/time_cache.hpp @@ -17,6 +17,8 @@ namespace libp2p::protocol::gossip::time_cache { using Clock = std::chrono::steady_clock; using Time = Clock::time_point; + /// This implements a time-based LRU cache for checking gossipsub message + /// duplicates. template class TimeCache { public: From 381afc86722f3190ff481d7264e8e80db9ac8ac0 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 17 Mar 2025 13:08:34 +0500 Subject: [PATCH 03/11] default init Signed-off-by: turuslan --- .../protocol/gossip/gossip_mock_test.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/libp2p/protocol/gossip/gossip_mock_test.cpp b/test/libp2p/protocol/gossip/gossip_mock_test.cpp index 6cbe00329..71f452fd0 100644 --- a/test/libp2p/protocol/gossip/gossip_mock_test.cpp +++ b/test/libp2p/protocol/gossip/gossip_mock_test.cpp @@ -59,6 +59,15 @@ inline uint8_t decodeMessageId(std::string_view data) { return data.at(0); } +struct Received { + std::vector subscriptions{}; + MessageIds messages{}; + std::vector graft{}; + MessageIds ihave{}; + MessageIds iwant{}; + MessageIds idontwant{}; +}; + struct MockPeer { MockPeer(PeerId peer_id, ProtocolName version, std::shared_ptr writer) : peer_id_{std::move(peer_id)}, @@ -66,15 +75,6 @@ struct MockPeer { writer_{std::move(writer)}, framing_{std::make_shared(writer_)} {} - struct Received { - std::vector subscriptions; - MessageIds messages; - std::vector graft; - MessageIds ihave; - MessageIds iwant; - MessageIds idontwant; - }; - void expect(Received expected) { EXPECT_EQ(received_.subscriptions, expected.subscriptions); EXPECT_EQ(received_.messages, expected.messages); From 1e600c52866c2f1ac4e17d7db41c547241af7af2 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 17 Mar 2025 14:49:19 +0500 Subject: [PATCH 04/11] designator order Signed-off-by: turuslan --- test/libp2p/protocol/gossip/gossip_mock_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/libp2p/protocol/gossip/gossip_mock_test.cpp b/test/libp2p/protocol/gossip/gossip_mock_test.cpp index 71f452fd0..98858fab9 100644 --- a/test/libp2p/protocol/gossip/gossip_mock_test.cpp +++ b/test/libp2p/protocol/gossip/gossip_mock_test.cpp @@ -410,7 +410,7 @@ TEST_F(GossipMockTest, FloodPublish) { subscribe(*peer2, true); publish(1); publish(*peer2, 2, randomPeerId()); - peer1->expect({.graft = {true}, .messages = {1, 2}}); + peer1->expect({.messages = {1, 2}, .graft = {true}}); peer2->expect({.messages = {1}}); } @@ -429,7 +429,7 @@ TEST_F(GossipMockTest, Gossip) { subscribe(*peer2, true); subscribe(*peer3, true); publish(1); - peer1->expect({.graft = {true}, .messages = {1}}); + peer1->expect({.messages = {1}, .graft = {true}}); heartbeat(); if (peer2->received_.ihave.empty()) { From 545a2fdaf9635e967a980a742f7ee1777fc455af Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 25 Mar 2025 15:30:52 +0500 Subject: [PATCH 05/11] idontwant Signed-off-by: turuslan --- cmake/Hunter/config.cmake | 18 --- cmake/Hunter/init.cmake | 4 +- include/libp2p/protocol/gossip/gossip.hpp | 20 ++++ include/libp2p/protocol/gossip/score.hpp | 1 + include/libp2p/protocol/gossip/time_cache.hpp | 104 ++++++++++++++++++ src/protocol/gossip/impl/gossip_core.cpp | 26 ++++- src/protocol/gossip/impl/gossip_core.hpp | 4 + src/protocol/gossip/impl/message_builder.cpp | 54 ++++++--- src/protocol/gossip/impl/message_builder.hpp | 8 ++ src/protocol/gossip/impl/message_parser.cpp | 10 ++ src/protocol/gossip/impl/message_receiver.hpp | 3 + src/protocol/gossip/impl/peer_context.hpp | 3 + .../gossip/impl/remote_subscriptions.cpp | 3 + .../gossip/impl/remote_subscriptions.hpp | 2 + .../gossip/impl/topic_subscriptions.cpp | 24 +++- .../gossip/impl/topic_subscriptions.hpp | 2 + .../protocol/gossip/gossip_mock_test.cpp | 55 +++++++++ 17 files changed, 305 insertions(+), 36 deletions(-) diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake index 60760abab..1ffd537ba 100644 --- a/cmake/Hunter/config.cmake +++ b/cmake/Hunter/config.cmake @@ -15,27 +15,9 @@ # CMAKE_ARGS "CMAKE_VARIABLE=value" # ) -hunter_config( - soralog - VERSION 0.2.5 - URL https://github.com/qdrvm/soralog/archive/refs/tags/v0.2.5.tar.gz - SHA1 1dafdb9e1921b4069f9e1dad0d0acfae24166bd2 - KEEP_PACKAGE_SOURCES -) - hunter_config( ZLIB VERSION 1.3.0-p0 URL https://github.com/cpp-pm/zlib/archive/refs/tags/v1.3.0-p0.tar.gz SHA1 311ca59e20cbbfe9d9e05196c12c6ae109093987 ) - -hunter_config( - qtils - VERSION 0.1.0 - URL https://github.com/qdrvm/qtils/archive/refs/tags/v0.1.0.tar.gz - SHA1 acc28902af7dc5d74ac33d486ad2261906716f5e - CMAKE_ARGS - FORMAT_ERROR_WITH_FULLTYPE=ON - KEEP_PACKAGE_SOURCES -) diff --git a/cmake/Hunter/init.cmake b/cmake/Hunter/init.cmake index 6f8fc7f35..3e6981fab 100644 --- a/cmake/Hunter/init.cmake +++ b/cmake/Hunter/init.cmake @@ -31,7 +31,7 @@ set( include(${CMAKE_CURRENT_LIST_DIR}/HunterGate.cmake) HunterGate( - URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm28.tar.gz - SHA1 a4f1b0f42464e07790b7f90b783a822d71be6c6d + URL https://github.com/qdrvm/hunter/archive/86e53c752977bf5a5efb6590c26941ed3fec8187.zip + SHA1 7121f2cbf052cc6cb0a526a89e9b4aca5bf3cd54 LOCAL ) diff --git a/include/libp2p/protocol/gossip/gossip.hpp b/include/libp2p/protocol/gossip/gossip.hpp index 7248e91aa..c49402ede 100644 --- a/include/libp2p/protocol/gossip/gossip.hpp +++ b/include/libp2p/protocol/gossip/gossip.hpp @@ -143,6 +143,26 @@ namespace libp2p::protocol::gossip { /// defaults this is 1666 messages/s. The default is 5000. size_t max_ihave_length = 5000; + /// Time to wait for a message requested through IWANT following an IHAVE + /// advertisement. If the message is not received within this window, a + /// broken promise is declared and the router may apply behavioural + /// penalties. The default is 3 seconds. + std::chrono::milliseconds iwant_followup_time = std::chrono::seconds{3}; + + /// The message size threshold for which IDONTWANT messages are sent. + /// Sending IDONTWANT messages for small messages can have a negative effect + /// to the overall traffic and CPU load. This acts as a lower bound cutoff + /// for the message size to which IDONTWANT won't be sent to peers. Only + /// works if the peers support Gossipsub1.2 (see + /// ) + /// default is 1kB + size_t idontwant_message_size_threshold = 1000; + + /// Send IDONTWANT messages after publishing message on gossip. This is an + /// optimisation to avoid bandwidth consumption by downloading the published + /// message over gossip. By default it is false. + bool idontwant_on_publish = false; + ScoreConfig score; }; diff --git a/include/libp2p/protocol/gossip/score.hpp b/include/libp2p/protocol/gossip/score.hpp index 8923b4db9..dca122efa 100644 --- a/include/libp2p/protocol/gossip/score.hpp +++ b/include/libp2p/protocol/gossip/score.hpp @@ -14,5 +14,6 @@ namespace libp2p::protocol::gossip { bool below(const PeerId &peer_id, double threshold) { return false; } + void addPenalty(const PeerId &peer_id, size_t count) {} }; } // namespace libp2p::protocol::gossip diff --git a/include/libp2p/protocol/gossip/time_cache.hpp b/include/libp2p/protocol/gossip/time_cache.hpp index 9ae701969..830eb3dc7 100644 --- a/include/libp2p/protocol/gossip/time_cache.hpp +++ b/include/libp2p/protocol/gossip/time_cache.hpp @@ -8,10 +8,16 @@ #include #include +#include +#include #include #include #include +namespace libp2p::protocol::gossip { + using MessageId = Bytes; +} // namespace libp2p::protocol::gossip + namespace libp2p::protocol::gossip::time_cache { using Ttl = std::chrono::milliseconds; using Clock = std::chrono::steady_clock; @@ -24,6 +30,10 @@ namespace libp2p::protocol::gossip::time_cache { public: TimeCache(Ttl ttl) : ttl_{ttl} {} + size_t size() const { + return map_.size(); + } + bool contains(const K &key) const { return map_.contains(key); } @@ -45,6 +55,14 @@ namespace libp2p::protocol::gossip::time_cache { return it->second; } + void popFront() { + if (expirations_.empty()) { + throw std::logic_error{"TimeCache::popFront empty"}; + } + map_.erase(expirations_.front().second); + expirations_.pop_front(); + } + private: using Map = std::unordered_map; @@ -75,9 +93,95 @@ namespace libp2p::protocol::gossip::time_cache { private: TimeCache cache_; }; + + template + class IDontWantCache { + static constexpr size_t kCapacity = 10000; + static constexpr Ttl kTtl = std::chrono::seconds{3}; + + public: + void clearExpired(Time now = Clock::now()) { + cache_.clearExpired(now); + } + + bool contains(const K &key) const { + return cache_.contains(key); + } + + void insert(const K &key) { + if (cache_.contains(key)) { + return; + } + if (cache_.size() >= kCapacity) { + cache_.popFront(); + } + cache_.getOrDefault(key); + } + + private: + TimeCache cache_{kTtl}; + }; + + class GossipPromises { + public: + GossipPromises(Ttl ttl) : ttl_{ttl} {} + + bool contains(const MessageId &message_id) const { + return map_.contains(message_id); + } + + void add(const MessageId &message_id, + const PeerId &peer_id, + Time now = Clock::now()) { + map_[message_id].emplace(peer_id, now + ttl_); + } + + void remove(const MessageId &message_id) { + map_.erase(message_id); + } + + void peers(const MessageId &message_id, const auto &f) { + auto it = map_.find(message_id); + if (it != map_.end()) { + for (auto &p : it->second) { + f(p.first); + } + } + } + + auto clearExpired(Time now = Clock::now()) { + std::unordered_map result; + for (auto it1 = map_.begin(); it1 != map_.end();) { + auto &map2 = it1->second; + for (auto it2 = map2.begin(); it2 != map2.end();) { + if (it2->second < now) { + ++result[it2->first]; + it2 = map2.erase(it2); + } else { + ++it2; + } + } + if (map2.empty()) { + it1 = map_.erase(it1); + } else { + ++it1; + } + } + return result; + } + + private: + Ttl ttl_; + std::unordered_map, + qtils::BytesStdHash> + map_; + }; } // namespace libp2p::protocol::gossip::time_cache namespace libp2p::protocol::gossip { using time_cache::DuplicateCache; + using time_cache::GossipPromises; + using time_cache::IDontWantCache; using time_cache::TimeCache; } // namespace libp2p::protocol::gossip diff --git a/src/protocol/gossip/impl/gossip_core.cpp b/src/protocol/gossip/impl/gossip_core.cpp index 8be418a72..70d54b48a 100644 --- a/src/protocol/gossip/impl/gossip_core.cpp +++ b/src/protocol/gossip/impl/gossip_core.cpp @@ -60,6 +60,7 @@ namespace libp2p::protocol::gossip { ), score_{std::make_shared()}, duplicate_cache_{config.duplicate_cache_time}, + gossip_promises_{std::make_shared(config.iwant_followup_time)}, local_subscriptions_(std::make_shared( [this](bool subscribe, const TopicId &topic) { onLocalSubscriptionChanged(subscribe, topic); @@ -112,7 +113,7 @@ namespace libp2p::protocol::gossip { } remote_subscriptions_ = std::make_shared( - config_, *connectivity_, score_, scheduler_, log_); + config_, *connectivity_, score_, gossip_promises_, scheduler_, log_); started_ = true; @@ -240,9 +241,11 @@ namespace libp2p::protocol::gossip { log_.debug("peer {} has msg for topic {}", from->str, topic); if (remote_subscriptions_->isSubscribed(topic) - and not duplicate_cache_.contains(msg_id)) { + and not duplicate_cache_.contains(msg_id) + and not gossip_promises_->contains(msg_id)) { log_.debug("requesting msg id {:x}", msg_id); + gossip_promises_->add(msg_id, from->peer_id); from->message_builder->addIWant(msg_id); connectivity_->peerIsWritable(from); } @@ -256,6 +259,9 @@ namespace libp2p::protocol::gossip { if (score_->below(from->peer_id, config_.score.gossip_threshold)) { return; } + if (from->idontwant.contains(msg_id)) { + return; + } log_.debug("peer {} wants message {:x}", from->str, msg_id); @@ -337,6 +343,18 @@ namespace libp2p::protocol::gossip { local_subscriptions_->forwardMessage(msg); remote_subscriptions_->onNewMessage(from, msg, msg_id); + + gossip_promises_->remove(msg_id); + } + + void GossipCore::onIDontWant(const PeerContextPtr &from, + const std::vector &message_ids) { + if (not from->isGossipsubv1_2()) { + return; + } + for (auto &message_id : message_ids) { + from->idontwant.insert(message_id); + } } void GossipCore::onMessageEnd(const PeerContextPtr &from) { @@ -348,6 +366,10 @@ namespace libp2p::protocol::gossip { void GossipCore::onHeartbeat() { assert(started_); + for (auto &[peer_id, count] : gossip_promises_->clearExpired()) { + score_->addPenalty(peer_id, count); + } + // shift cache msg_cache_.shift(); diff --git a/src/protocol/gossip/impl/gossip_core.hpp b/src/protocol/gossip/impl/gossip_core.hpp index bc133e09a..140278699 100644 --- a/src/protocol/gossip/impl/gossip_core.hpp +++ b/src/protocol/gossip/impl/gossip_core.hpp @@ -76,6 +76,8 @@ namespace libp2p::protocol::gossip { std::optional backoff_time) override; void onTopicMessage(const PeerContextPtr &from, TopicMessage::Ptr msg) override; + void onIDontWant(const PeerContextPtr &from, + const std::vector &message_ids) override; void onMessageEnd(const PeerContextPtr &from) override; /// Periodic heartbeat timer fn @@ -121,6 +123,8 @@ namespace libp2p::protocol::gossip { DuplicateCache duplicate_cache_; + std::shared_ptr gossip_promises_; + /// Local subscriptions manager (this host subscribed to topics) std::shared_ptr local_subscriptions_; diff --git a/src/protocol/gossip/impl/message_builder.cpp b/src/protocol/gossip/impl/message_builder.cpp index 50bebe787..88d002182 100644 --- a/src/protocol/gossip/impl/message_builder.cpp +++ b/src/protocol/gossip/impl/message_builder.cpp @@ -7,6 +7,7 @@ #include "message_builder.hpp" #include +#include #include @@ -29,6 +30,8 @@ namespace libp2p::protocol::gossip { control_pb_msg_->Clear(); empty_ = true; control_not_empty_ = false; + subscriptions_.clear(); + idontwant_.clear(); ihaves_.clear(); iwant_.clear(); messages_added_.clear(); @@ -39,6 +42,8 @@ namespace libp2p::protocol::gossip { control_pb_msg_.reset(); empty_ = true; control_not_empty_ = false; + std::exchange(subscriptions_, {}); + std::exchange(idontwant_, {}); decltype(ihaves_){}.swap(ihaves_); decltype(iwant_){}.swap(iwant_); decltype(messages_added_){}.swap(messages_added_); @@ -63,7 +68,13 @@ namespace libp2p::protocol::gossip { subscription->set_subscribe(subscribe); subscription->set_topicid(topic); } - subscriptions_.clear(); + + if (not idontwant_.empty()) { + auto *idontwant = control_pb_msg_->add_idontwant(); + for (auto &msg_id : idontwant_) { + *idontwant->add_message_ids() = qtils::byte2str(msg_id); + } + } for (auto &[topic, message_ids] : ihaves_) { auto *ih = control_pb_msg_->add_ihave(); @@ -123,6 +134,12 @@ namespace libp2p::protocol::gossip { empty_ = false; } + void MessageBuilder::addIDontWant(const MessageId &msg_id) { + idontwant_.emplace(msg_id); + control_not_empty_ = true; + empty_ = false; + } + void MessageBuilder::addIHave(const TopicId &topic, const MessageId &msg_id) { ihaves_[topic].push_back(msg_id); control_not_empty_ = true; @@ -166,18 +183,7 @@ namespace libp2p::protocol::gossip { } messages_added_.insert(msg_id); - auto *dst = pb_msg_->add_publish(); - dst->set_from(msg.from.data(), msg.from.size()); - dst->set_data(msg.data.data(), msg.data.size()); - dst->set_seqno(msg.seq_no.data(), msg.seq_no.size()); - dst->set_topic(msg.topic); - if (msg.signature) { - dst->set_signature(msg.signature.value().data(), - msg.signature.value().size()); - } - if (msg.key) { - dst->set_key(msg.key.value().data(), msg.key.value().size()); - } + toPb(*pb_msg_->add_publish(), msg); empty_ = false; } @@ -199,4 +205,26 @@ namespace libp2p::protocol::gossip { } return signable; } + + void MessageBuilder::toPb(pubsub::pb::Message &pb_message, + const TopicMessage &message) { + pb_message.set_from(message.from.data(), message.from.size()); + pb_message.set_data(message.data.data(), message.data.size()); + pb_message.set_seqno(message.seq_no.data(), message.seq_no.size()); + pb_message.set_topic(message.topic); + if (message.signature) { + pb_message.set_signature(message.signature.value().data(), + message.signature.value().size()); + } + if (message.key) { + pb_message.set_key(message.key.value().data(), + message.key.value().size()); + } + } + + size_t MessageBuilder::pbSize(const TopicMessage &message) { + static thread_local pubsub::pb::Message pb_message; + toPb(pb_message, message); + return pb_message.ByteSizeLong(); + } } // namespace libp2p::protocol::gossip diff --git a/src/protocol/gossip/impl/message_builder.hpp b/src/protocol/gossip/impl/message_builder.hpp index 595dd5abb..df529daa8 100644 --- a/src/protocol/gossip/impl/message_builder.hpp +++ b/src/protocol/gossip/impl/message_builder.hpp @@ -14,6 +14,7 @@ namespace pubsub::pb { // protobuf entities forward declaration + class Message; class RPC; class ControlMessage; } // namespace pubsub::pb @@ -45,6 +46,8 @@ namespace libp2p::protocol::gossip { /// Adds subscription notification void addSubscription(bool subscribe, const TopicId &topic); + void addIDontWant(const MessageId &msg_id); + /// Adds "I have" notification void addIHave(const TopicId &topic, const MessageId &msg_id); @@ -63,6 +66,10 @@ namespace libp2p::protocol::gossip { static outcome::result signableMessage(const TopicMessage &msg); + static void toPb(pubsub::pb::Message &pb_message, + const TopicMessage &message); + static size_t pbSize(const TopicMessage &message); + private: /// Creates protobuf structures if needed void create_protobuf_structures(); @@ -76,6 +83,7 @@ namespace libp2p::protocol::gossip { bool empty_; bool control_not_empty_; std::unordered_map subscriptions_; + std::unordered_set idontwant_; /// Intermediate struct for building IHave messages std::map> ihaves_; diff --git a/src/protocol/gossip/impl/message_parser.cpp b/src/protocol/gossip/impl/message_parser.cpp index f1ed61559..9a845247b 100644 --- a/src/protocol/gossip/impl/message_parser.cpp +++ b/src/protocol/gossip/impl/message_parser.cpp @@ -7,6 +7,7 @@ #include "message_parser.hpp" #include +#include #include "message_receiver.hpp" @@ -52,6 +53,15 @@ namespace libp2p::protocol::gossip { if (pb_msg_->has_control()) { const auto &c = pb_msg_->control(); + std::vector idontwant_message_ids; + for (const auto &idontwant : c.idontwant()) { + for (auto &msg_id : idontwant.message_ids()) { + idontwant_message_ids.emplace_back( + qtils::asVec(qtils::str2byte(msg_id))); + } + } + receiver.onIDontWant(from, idontwant_message_ids); + for (const auto &h : c.ihave()) { if (!h.has_topicid() || h.messageids_size() == 0) { continue; diff --git a/src/protocol/gossip/impl/message_receiver.hpp b/src/protocol/gossip/impl/message_receiver.hpp index c19144c7e..153c3111c 100644 --- a/src/protocol/gossip/impl/message_receiver.hpp +++ b/src/protocol/gossip/impl/message_receiver.hpp @@ -43,6 +43,9 @@ namespace libp2p::protocol::gossip { virtual void onTopicMessage(const PeerContextPtr &from, TopicMessage::Ptr msg) = 0; + virtual void onIDontWant(const PeerContextPtr &from, + const std::vector &message_ids) = 0; + /// Current wire protocol message dispatch ended virtual void onMessageEnd(const PeerContextPtr &from) = 0; }; diff --git a/src/protocol/gossip/impl/peer_context.hpp b/src/protocol/gossip/impl/peer_context.hpp index a10dedb4b..d434f22ed 100644 --- a/src/protocol/gossip/impl/peer_context.hpp +++ b/src/protocol/gossip/impl/peer_context.hpp @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include "common.hpp" @@ -45,6 +46,8 @@ namespace libp2p::protocol::gossip { std::optional peer_kind; + IDontWantCache idontwant; + ~PeerContext() = default; PeerContext(PeerContext &&) = delete; PeerContext(const PeerContext &) = delete; diff --git a/src/protocol/gossip/impl/remote_subscriptions.cpp b/src/protocol/gossip/impl/remote_subscriptions.cpp index 2e97875a0..c448f5e89 100644 --- a/src/protocol/gossip/impl/remote_subscriptions.cpp +++ b/src/protocol/gossip/impl/remote_subscriptions.cpp @@ -19,6 +19,7 @@ namespace libp2p::protocol::gossip { const Config &config, Connectivity &connectivity, std::shared_ptr score, + std::shared_ptr gossip_promises, std::shared_ptr scheduler, log::SubLogger &log) : config_(config), @@ -26,6 +27,7 @@ namespace libp2p::protocol::gossip { choose_peers_{std::make_shared()}, explicit_peers_{std::make_shared()}, score_{std::move(score)}, + gossip_promises_{std::move(gossip_promises)}, scheduler_{std::move(scheduler)}, log_(log) {} @@ -182,6 +184,7 @@ namespace libp2p::protocol::gossip { choose_peers_, explicit_peers_, score_, + gossip_promises_, log_)); TopicSubscriptions &item = it->second; log_.debug("created entry for topic {}", topic); diff --git a/src/protocol/gossip/impl/remote_subscriptions.hpp b/src/protocol/gossip/impl/remote_subscriptions.hpp index ceec19dad..408874e23 100644 --- a/src/protocol/gossip/impl/remote_subscriptions.hpp +++ b/src/protocol/gossip/impl/remote_subscriptions.hpp @@ -21,6 +21,7 @@ namespace libp2p::protocol::gossip { RemoteSubscriptions(const Config &config, Connectivity &connectivity, std::shared_ptr score, + std::shared_ptr gossip_promises, std::shared_ptr scheduler, log::SubLogger &log); @@ -68,6 +69,7 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers_; std::shared_ptr explicit_peers_; std::shared_ptr score_; + std::shared_ptr gossip_promises_; std::shared_ptr scheduler_; std::unordered_map table_; diff --git a/src/protocol/gossip/impl/topic_subscriptions.cpp b/src/protocol/gossip/impl/topic_subscriptions.cpp index 2ae554583..e8fa874b6 100644 --- a/src/protocol/gossip/impl/topic_subscriptions.cpp +++ b/src/protocol/gossip/impl/topic_subscriptions.cpp @@ -40,6 +40,7 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers, std::shared_ptr explicit_peers, std::shared_ptr score, + std::shared_ptr gossip_promises, log::SubLogger &log) : topic_(std::move(topic)), config_(config), @@ -48,6 +49,7 @@ namespace libp2p::protocol::gossip { choose_peers_{std::move(choose_peers)}, explicit_peers_{std::move(explicit_peers)}, score_{std::move(score)}, + gossip_promises_{std::move(gossip_promises)}, self_subscribed_(false), log_(log) { connectivity_.getConnectedPeers().selectIf( @@ -83,7 +85,8 @@ namespace libp2p::protocol::gossip { auto origin = peerFrom(*msg); auto add_peer = [&](const PeerContextPtr &ctx) { - if (needToForward(ctx, from, origin)) { + if (needToForward(ctx, from, origin) + and not ctx->idontwant.contains(msg_id)) { ctx->message_builder->addMessage(*msg, msg_id); connectivity_.peerIsWritable(ctx); } @@ -130,6 +133,25 @@ namespace libp2p::protocol::gossip { } history_gossip_.back().emplace_back(msg_id); + + if ((not is_published_locally or config_.idontwant_on_publish) + and MessageBuilder::pbSize(*msg) + > config_.idontwant_message_size_threshold) { + auto add_idontwant_peer = [&](const PeerContextPtr &ctx) { + if (ctx->isGossipsubv1_2()) { + ctx->message_builder->addIDontWant(msg_id); + connectivity_.peerIsWritable(ctx); + } + }; + for (auto &ctx : mesh_peers_) { + add_idontwant_peer(ctx); + } + gossip_promises_->peers(msg_id, [&](const PeerId &peer_id) { + if (auto ctx = subscribed_peers_.find(peer_id)) { + add_idontwant_peer(*ctx); + } + }); + } } void TopicSubscriptions::onHeartbeat(Time now) { diff --git a/src/protocol/gossip/impl/topic_subscriptions.hpp b/src/protocol/gossip/impl/topic_subscriptions.hpp index 7e1357e62..ae8b09c7e 100644 --- a/src/protocol/gossip/impl/topic_subscriptions.hpp +++ b/src/protocol/gossip/impl/topic_subscriptions.hpp @@ -31,6 +31,7 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers, std::shared_ptr explicit_peers, std::shared_ptr score, + std::shared_ptr gossip_promises, log::SubLogger &log); /// Returns true if not self-subscribed and @@ -90,6 +91,7 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers_; std::shared_ptr explicit_peers_; std::shared_ptr score_; + std::shared_ptr gossip_promises_; /// This host subscribed to this topic or not, this affects mesh behavior bool self_subscribed_; diff --git a/test/libp2p/protocol/gossip/gossip_mock_test.cpp b/test/libp2p/protocol/gossip/gossip_mock_test.cpp index 98858fab9..82fcef658 100644 --- a/test/libp2p/protocol/gossip/gossip_mock_test.cpp +++ b/test/libp2p/protocol/gossip/gossip_mock_test.cpp @@ -81,6 +81,7 @@ struct MockPeer { EXPECT_EQ(received_.graft, expected.graft); EXPECT_EQ(received_.ihave, expected.ihave); EXPECT_EQ(received_.iwant, expected.iwant); + EXPECT_EQ(received_.idontwant, expected.idontwant); received_ = {}; } @@ -281,6 +282,22 @@ struct GossipMockTest : testing::Test { scheduler_backend_->callDeferred(); } + void iwant(MockPeer &peer, uint8_t i) { + peer.write([&](pubsub::pb::RPC &rpc) { + auto *iwant = rpc.mutable_control()->add_iwant(); + *iwant->add_messageids() = qtils::byte2str(encodeMessageId(i)); + }); + scheduler_backend_->callDeferred(); + } + + void idontwant(MockPeer &peer, uint8_t i) { + peer.write([&](pubsub::pb::RPC &rpc) { + auto *idontwant = rpc.mutable_control()->add_idontwant(); + *idontwant->add_message_ids() = qtils::byte2str(encodeMessageId(i)); + }); + scheduler_backend_->callDeferred(); + } + void heartbeat() { scheduler_backend_->shift(config_.heartbeat_interval_msec); } @@ -456,4 +473,42 @@ TEST_F(GossipMockTest, IhaveIwant) { publish(*peer1, 1); ihave(*peer1, 1); + + iwant(*peer1, 1); + peer1->expect({.messages = {1}}); +} + +/** + * send idontwant after receiving message. + * send idontwant to mesh peers. + * send idontwant to cancel pending iwant requests. + * don't reply to iwant from peer after receiving idontwant. + * don't forward message to peer after receiving idontwant. + */ +TEST_F(GossipMockTest, Idontwant) { + config_.idontwant_message_size_threshold = 1; + setup(); + auto peer1 = connect(PeerKind::Gossipsubv1_2); + auto peer2 = connect(PeerKind::Gossipsubv1_2); + + auto sub = subscribe(); + subscribe(*peer1, true); + subscribe(*peer2, true); + publish(*peer2, 1); + peer1->expect({.messages = {1}, .graft = {1}, .idontwant = {1}}); + + ihave(*peer2, 2); + peer2->expect({.iwant = {2}}); + publish(*peer1, 2); + peer1->expect({.idontwant = {2}}); + peer2->expect({.idontwant = {2}}); + + idontwant(*peer2, 2); + iwant(*peer2, 2); + peer2->expect({}); + + idontwant(*peer2, 3); + publish(*peer1, 3); + peer1->expect({.idontwant = {3}}); + peer2->expect({}); } From fd1d1ae2feb761c6b15c6a33868d86f399ba168d Mon Sep 17 00:00:00 2001 From: turuslan Date: Wed, 26 Mar 2025 12:15:54 +0500 Subject: [PATCH 06/11] adaptive gossip Signed-off-by: turuslan --- include/libp2p/protocol/gossip/gossip.hpp | 7 +++++++ src/protocol/gossip/impl/topic_subscriptions.cpp | 4 +++- test/libp2p/protocol/gossip/gossip_mock_test.cpp | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/libp2p/protocol/gossip/gossip.hpp b/include/libp2p/protocol/gossip/gossip.hpp index c49402ede..d67631ca6 100644 --- a/include/libp2p/protocol/gossip/gossip.hpp +++ b/include/libp2p/protocol/gossip/gossip.hpp @@ -45,9 +45,16 @@ namespace libp2p::protocol::gossip { struct Config { /// Network density factors for gossip meshes size_t D_min = 5; + size_t D_lazy = 6; size_t D = 6; size_t D_max = 10; + /// Affects how many peers we will emit gossip to at each heartbeat. + /// + /// We will send gossip to `gossip_factor * (total number of non-mesh + /// peers)`, or `gossip_lazy`, whichever is greater. The default is 0.25. + double gossip_factor = 0.25; + /// Ideal number of connected peers to support the network size_t ideal_connections_num = 100; diff --git a/src/protocol/gossip/impl/topic_subscriptions.cpp b/src/protocol/gossip/impl/topic_subscriptions.cpp index e8fa874b6..f4a29b4c4 100644 --- a/src/protocol/gossip/impl/topic_subscriptions.cpp +++ b/src/protocol/gossip/impl/topic_subscriptions.cpp @@ -387,7 +387,9 @@ namespace libp2p::protocol::gossip { and not score_->below(ctx->peer_id, config_.score.gossip_threshold); }, - config_.D); + [&](size_t n) { + return std::max(config_.D_lazy, n * config_.gossip_factor); + }); size_t message_count = 0; for (auto &x : history_gossip_) { message_count += x.size(); diff --git a/test/libp2p/protocol/gossip/gossip_mock_test.cpp b/test/libp2p/protocol/gossip/gossip_mock_test.cpp index 82fcef658..ad14ac776 100644 --- a/test/libp2p/protocol/gossip/gossip_mock_test.cpp +++ b/test/libp2p/protocol/gossip/gossip_mock_test.cpp @@ -132,6 +132,7 @@ struct GossipMockTest : testing::Test { void SetUp() override { config_.D_min = 1; config_.D = 1; + config_.D_lazy = 1; config_.flood_publish = false; } From e5a6e4c1b06bbba9720079d3f78a696bc79cbf93 Mon Sep 17 00:00:00 2001 From: turuslan Date: Thu, 27 Mar 2025 16:43:01 +0500 Subject: [PATCH 07/11] score stub Signed-off-by: turuslan --- include/libp2p/protocol/gossip/gossip.hpp | 5 ++ include/libp2p/protocol/gossip/score.hpp | 15 ++++ .../libp2p/protocol/gossip/score_config.hpp | 29 +++++++ src/protocol/gossip/impl/gossip_core.cpp | 85 +++++++++++++++++-- src/protocol/gossip/impl/gossip_core.hpp | 6 +- .../gossip/impl/remote_subscriptions.cpp | 3 - .../gossip/impl/remote_subscriptions.hpp | 2 - .../gossip/impl/topic_subscriptions.cpp | 82 +++++++++++------- .../gossip/impl/topic_subscriptions.hpp | 2 - .../protocol/gossip/gossip_mock_test.cpp | 2 + 10 files changed, 185 insertions(+), 46 deletions(-) diff --git a/include/libp2p/protocol/gossip/gossip.hpp b/include/libp2p/protocol/gossip/gossip.hpp index d67631ca6..4a0adb6d8 100644 --- a/include/libp2p/protocol/gossip/gossip.hpp +++ b/include/libp2p/protocol/gossip/gossip.hpp @@ -142,6 +142,11 @@ namespace libp2p::protocol::gossip { /// enough score. The default is true. bool flood_publish = true; + /// If a GRAFT comes before `graft_flood_threshold` has elapsed since the + /// last PRUNE, then there is an extra score penalty applied to the peer + /// through P7. + std::chrono::milliseconds graft_flood_threshold = std::chrono::seconds{10}; + /// The maximum number of messages to include in an IHAVE message. /// Also controls the maximum number of IHAVE ids we will accept and request /// with IWANT from a peer within a heartbeat, to protect from IHAVE floods. diff --git a/include/libp2p/protocol/gossip/score.hpp b/include/libp2p/protocol/gossip/score.hpp index dca122efa..30204659b 100644 --- a/include/libp2p/protocol/gossip/score.hpp +++ b/include/libp2p/protocol/gossip/score.hpp @@ -9,11 +9,26 @@ #include namespace libp2p::protocol::gossip { + using TopicId = std::string; + using MessageId = Bytes; + class Score { public: bool below(const PeerId &peer_id, double threshold) { return false; } void addPenalty(const PeerId &peer_id, size_t count) {} + void graft(const PeerId &peer_id, const TopicId &topic) {} + void prune(const PeerId &peer_id, const TopicId &topic) {} + void duplicateMessage(const PeerId &peer_id, + const MessageId &msg_id, + const TopicId &topic) {} + void validateMessage(const PeerId &peer_id, + const MessageId &msg_id, + const TopicId &topic) {} + void connect(const PeerId &peer_id) {} + void disconnect(const PeerId &peer_id) {} + + void onDecay() {} }; } // namespace libp2p::protocol::gossip diff --git a/include/libp2p/protocol/gossip/score_config.hpp b/include/libp2p/protocol/gossip/score_config.hpp index cb071fcb6..bf407c385 100644 --- a/include/libp2p/protocol/gossip/score_config.hpp +++ b/include/libp2p/protocol/gossip/score_config.hpp @@ -6,6 +6,8 @@ #pragma once +#include + namespace libp2p::protocol::gossip { struct ScoreConfig { double zero = 0; @@ -16,5 +18,32 @@ namespace libp2p::protocol::gossip { /// publishing (also applies to fanout peers); should be negative and <= /// `gossip_threshold`. double publish_threshold = -50; + /// The score threshold below which message processing is suppressed + /// altogether, implementing an effective graylist according to peer score; + /// should be negative and + /// <= `publish_threshold`. + double graylist_threshold = -80; + /// The median mesh score threshold before triggering opportunistic + /// grafting; this should have a small positive value. + double opportunistic_graft_threshold = 20; + + /// The decay interval for parameter counters. + std::chrono::milliseconds decay_interval = std::chrono::seconds{1}; + + bool valid() const { + if (gossip_threshold > 0) { + return false; + } + if (publish_threshold > 0 or publish_threshold > gossip_threshold) { + return false; + } + if (graylist_threshold > 0 or graylist_threshold > publish_threshold) { + return false; + } + if (opportunistic_graft_threshold < 0) { + return false; + } + return true; + } }; } // namespace libp2p::protocol::gossip diff --git a/src/protocol/gossip/impl/gossip_core.cpp b/src/protocol/gossip/impl/gossip_core.cpp index 70d54b48a..71c8119b7 100644 --- a/src/protocol/gossip/impl/gossip_core.cpp +++ b/src/protocol/gossip/impl/gossip_core.cpp @@ -60,7 +60,7 @@ namespace libp2p::protocol::gossip { ), score_{std::make_shared()}, duplicate_cache_{config.duplicate_cache_time}, - gossip_promises_{std::make_shared(config.iwant_followup_time)}, + gossip_promises_{config.iwant_followup_time}, local_subscriptions_(std::make_shared( [this](bool subscribe, const TopicId &topic) { onLocalSubscriptionChanged(subscribe, topic); @@ -113,7 +113,7 @@ namespace libp2p::protocol::gossip { } remote_subscriptions_ = std::make_shared( - config_, *connectivity_, score_, gossip_promises_, scheduler_, log_); + config_, *connectivity_, score_, scheduler_, log_); started_ = true; @@ -122,8 +122,36 @@ namespace libp2p::protocol::gossip { } setTimerHeartbeat(); + setTimerScore(); connectivity_->start(); + new_connection_sub_ = + host_->getBus() + .getChannel() + .subscribe( + [weak_self{weak_from_this()}]( + std::weak_ptr weak_conn) { + if (auto self = weak_self.lock()) { + if (auto conn = weak_conn.lock()) { + auto peer_id = conn->remotePeer().value(); + if (self->host_->getNetwork() + .getConnectionManager() + .getConnectionsToPeer(peer_id) + .size() + == 1) { + self->score_->connect(peer_id); + } + } + } + }); + peer_disconnected_sub_ = + host_->getBus() + .getChannel() + .subscribe([weak_self{weak_from_this()}](const PeerId &peer_id) { + if (auto self = weak_self.lock()) { + self->score_->disconnect(peer_id); + } + }); } void GossipCore::stop() { @@ -242,10 +270,10 @@ namespace libp2p::protocol::gossip { if (remote_subscriptions_->isSubscribed(topic) and not duplicate_cache_.contains(msg_id) - and not gossip_promises_->contains(msg_id)) { + and not gossip_promises_.contains(msg_id)) { log_.debug("requesting msg id {:x}", msg_id); - gossip_promises_->add(msg_id, from->peer_id); + gossip_promises_.add(msg_id, from->peer_id); from->message_builder->addIWant(msg_id); connectivity_->peerIsWritable(from); } @@ -280,6 +308,9 @@ namespace libp2p::protocol::gossip { if (not from->isGossipsub()) { return; } + if (score_->below(from->peer_id, config_.score.graylist_threshold)) { + return; + } log_.debug("graft from peer {} for topic {}", from->str, topic); @@ -294,6 +325,9 @@ namespace libp2p::protocol::gossip { if (not from->isGossipsub()) { return; } + if (score_->below(from->peer_id, config_.score.graylist_threshold)) { + return; + } log_.debug("prune from peer {} for topic {}", from->str, topic); @@ -304,6 +338,10 @@ namespace libp2p::protocol::gossip { TopicMessage::Ptr msg) { assert(started_); + if (score_->below(from->peer_id, config_.score.graylist_threshold)) { + return; + } + // do we need this message? auto subscribed = remote_subscriptions_->isSubscribed(msg->topic); if (!subscribed) { @@ -312,7 +350,22 @@ namespace libp2p::protocol::gossip { } MessageId msg_id = create_message_id_(msg->from, msg->seq_no, msg->data); + + if (MessageBuilder::pbSize(*msg) + > config_.idontwant_message_size_threshold) { + gossip_promises_.peers(msg_id, [&](const PeerId &peer_id) { + if (auto ctx_opt = connectivity_->getConnectedPeers().find(peer_id)) { + auto &ctx = *ctx_opt; + if (ctx->isGossipsubv1_2()) { + ctx->message_builder->addIDontWant(msg_id); + connectivity_->peerIsWritable(ctx); + } + } + }); + } + if (not duplicate_cache_.insert(msg_id)) { + score_->duplicateMessage(from->peer_id, msg_id, msg->topic); return; } @@ -334,6 +387,10 @@ namespace libp2p::protocol::gossip { return; } + score_->validateMessage(from->peer_id, msg_id, msg->topic); + + gossip_promises_.remove(msg_id); + if (!msg_cache_.insert(msg, msg_id)) { log_.error("message cache error"); return; @@ -343,8 +400,6 @@ namespace libp2p::protocol::gossip { local_subscriptions_->forwardMessage(msg); remote_subscriptions_->onNewMessage(from, msg, msg_id); - - gossip_promises_->remove(msg_id); } void GossipCore::onIDontWant(const PeerContextPtr &from, @@ -352,6 +407,9 @@ namespace libp2p::protocol::gossip { if (not from->isGossipsubv1_2()) { return; } + if (score_->below(from->peer_id, config_.score.graylist_threshold)) { + return; + } for (auto &message_id : message_ids) { from->idontwant.insert(message_id); } @@ -366,7 +424,7 @@ namespace libp2p::protocol::gossip { void GossipCore::onHeartbeat() { assert(started_); - for (auto &[peer_id, count] : gossip_promises_->clearExpired()) { + for (auto &[peer_id, count] : gossip_promises_.clearExpired()) { score_->addPenalty(peer_id, count); } @@ -423,4 +481,17 @@ namespace libp2p::protocol::gossip { }, config_.heartbeat_interval_msec); } + + void GossipCore::setTimerScore() { + scheduler_->schedule( + [weak_self{weak_from_this()}] { + auto self = weak_self.lock(); + if (not self) { + return; + } + self->score_->onDecay(); + self->setTimerScore(); + }, + config_.score.decay_interval); + } } // namespace libp2p::protocol::gossip diff --git a/src/protocol/gossip/impl/gossip_core.hpp b/src/protocol/gossip/impl/gossip_core.hpp index 140278699..665845258 100644 --- a/src/protocol/gossip/impl/gossip_core.hpp +++ b/src/protocol/gossip/impl/gossip_core.hpp @@ -90,6 +90,7 @@ namespace libp2p::protocol::gossip { void onPeerConnection(bool connected, const PeerContextPtr &ctx); void setTimerHeartbeat(); + void setTimerScore(); /// Configuration parameters const Config config_; @@ -123,7 +124,7 @@ namespace libp2p::protocol::gossip { DuplicateCache duplicate_cache_; - std::shared_ptr gossip_promises_; + GossipPromises gossip_promises_; /// Local subscriptions manager (this host subscribed to topics) std::shared_ptr local_subscriptions_; @@ -153,6 +154,9 @@ namespace libp2p::protocol::gossip { /// Logger log::SubLogger log_; + + event::Handle new_connection_sub_; + event::Handle peer_disconnected_sub_; }; } // namespace libp2p::protocol::gossip diff --git a/src/protocol/gossip/impl/remote_subscriptions.cpp b/src/protocol/gossip/impl/remote_subscriptions.cpp index c448f5e89..2e97875a0 100644 --- a/src/protocol/gossip/impl/remote_subscriptions.cpp +++ b/src/protocol/gossip/impl/remote_subscriptions.cpp @@ -19,7 +19,6 @@ namespace libp2p::protocol::gossip { const Config &config, Connectivity &connectivity, std::shared_ptr score, - std::shared_ptr gossip_promises, std::shared_ptr scheduler, log::SubLogger &log) : config_(config), @@ -27,7 +26,6 @@ namespace libp2p::protocol::gossip { choose_peers_{std::make_shared()}, explicit_peers_{std::make_shared()}, score_{std::move(score)}, - gossip_promises_{std::move(gossip_promises)}, scheduler_{std::move(scheduler)}, log_(log) {} @@ -184,7 +182,6 @@ namespace libp2p::protocol::gossip { choose_peers_, explicit_peers_, score_, - gossip_promises_, log_)); TopicSubscriptions &item = it->second; log_.debug("created entry for topic {}", topic); diff --git a/src/protocol/gossip/impl/remote_subscriptions.hpp b/src/protocol/gossip/impl/remote_subscriptions.hpp index 408874e23..ceec19dad 100644 --- a/src/protocol/gossip/impl/remote_subscriptions.hpp +++ b/src/protocol/gossip/impl/remote_subscriptions.hpp @@ -21,7 +21,6 @@ namespace libp2p::protocol::gossip { RemoteSubscriptions(const Config &config, Connectivity &connectivity, std::shared_ptr score, - std::shared_ptr gossip_promises, std::shared_ptr scheduler, log::SubLogger &log); @@ -69,7 +68,6 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers_; std::shared_ptr explicit_peers_; std::shared_ptr score_; - std::shared_ptr gossip_promises_; std::shared_ptr scheduler_; std::unordered_map table_; diff --git a/src/protocol/gossip/impl/topic_subscriptions.cpp b/src/protocol/gossip/impl/topic_subscriptions.cpp index f4a29b4c4..36eba15cc 100644 --- a/src/protocol/gossip/impl/topic_subscriptions.cpp +++ b/src/protocol/gossip/impl/topic_subscriptions.cpp @@ -40,7 +40,6 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers, std::shared_ptr explicit_peers, std::shared_ptr score, - std::shared_ptr gossip_promises, log::SubLogger &log) : topic_(std::move(topic)), config_(config), @@ -49,7 +48,6 @@ namespace libp2p::protocol::gossip { choose_peers_{std::move(choose_peers)}, explicit_peers_{std::move(explicit_peers)}, score_{std::move(score)}, - gossip_promises_{std::move(gossip_promises)}, self_subscribed_(false), log_(log) { connectivity_.getConnectedPeers().selectIf( @@ -102,7 +100,9 @@ namespace libp2p::protocol::gossip { } } else { for (auto &ctx : subscribed_peers_) { - if (ctx->isFloodsub()) { + if (ctx->isFloodsub() + and not score_->below(ctx->peer_id, + config_.score.publish_threshold)) { add_peer(ctx); } } @@ -137,20 +137,12 @@ namespace libp2p::protocol::gossip { if ((not is_published_locally or config_.idontwant_on_publish) and MessageBuilder::pbSize(*msg) > config_.idontwant_message_size_threshold) { - auto add_idontwant_peer = [&](const PeerContextPtr &ctx) { + for (auto &ctx : mesh_peers_) { if (ctx->isGossipsubv1_2()) { ctx->message_builder->addIDontWant(msg_id); connectivity_.peerIsWritable(ctx); } - }; - for (auto &ctx : mesh_peers_) { - add_idontwant_peer(ctx); } - gossip_promises_->peers(msg_id, [&](const PeerId &peer_id) { - if (auto ctx = subscribed_peers_.find(peer_id)) { - add_idontwant_peer(*ctx); - } - }); } } @@ -236,6 +228,9 @@ namespace libp2p::protocol::gossip { if (explicit_peers_->contains(ctx->peer_id)) { continue; } + if (score_->below(ctx->peer_id, config_.score.publish_threshold)) { + continue; + } if (isBackoffWithSlack(ctx->peer_id)) { continue; } @@ -278,6 +273,7 @@ namespace libp2p::protocol::gossip { subscribed_peers_.insert(ctx); if (ctx->isGossipsub() and mesh_peers_.size() < config_.D_min + and not mesh_peers_.contains(ctx) and not explicit_peers_->contains(ctx->peer_id) and not isBackoffWithSlack(ctx->peer_id) and not score_->below(ctx->peer_id, config_.score.zero)) { @@ -285,16 +281,17 @@ namespace libp2p::protocol::gossip { } } - void TopicSubscriptions::onPeerUnsubscribed(const PeerContextPtr &p) { - subscribed_peers_.erase(p->peer_id); + void TopicSubscriptions::onPeerUnsubscribed(const PeerContextPtr &ctx) { + subscribed_peers_.erase(ctx->peer_id); if (fanout_) { - fanout_->peers.erase(p->peer_id); + fanout_->peers.erase(ctx->peer_id); if (fanout_->peers.empty()) { fanout_.reset(); } } - if (mesh_peers_.erase(p->peer_id)) { - updateBackoff(p->peer_id, config_.prune_backoff); + if (mesh_peers_.erase(ctx->peer_id)) { + score_->prune(ctx->peer_id, topic_); + updateBackoff(ctx->peer_id, config_.prune_backoff); } } @@ -310,30 +307,52 @@ namespace libp2p::protocol::gossip { bool mesh_is_full = (mesh_peers_.size() >= config_.D_max); - if (self_subscribed_ and not mesh_is_full - and not explicit_peers_->contains(ctx->peer_id) - and not isBackoff(ctx->peer_id, std::chrono::milliseconds{0})) { + bool prune = [&] { + if (self_subscribed_) { + return true; + } + if (explicit_peers_->contains(ctx->peer_id)) { + return true; + } + if (isBackoff(ctx->peer_id, std::chrono::milliseconds{0})) { + score_->addPenalty(ctx->peer_id, 1); + if (isBackoff(ctx->peer_id, + config_.graft_flood_threshold - config_.prune_backoff)) { + score_->addPenalty(ctx->peer_id, 1); + } + } + if (score_->below(ctx->peer_id, config_.score.zero)) { + return true; + } + if (mesh_is_full) { + return true; + } + score_->graft(ctx->peer_id, topic_); mesh_peers_.insert(ctx); - } else { - // we don't have mesh for the topic + return false; + }(); + if (prune) { sendPrune(ctx, false); } } void TopicSubscriptions::onPrune( - const PeerContextPtr &p, std::optional backoff) { - mesh_peers_.erase(p->peer_id); - updateBackoff(p->peer_id, backoff.value_or(config_.prune_backoff)); + const PeerContextPtr &ctx, std::optional backoff) { + if (mesh_peers_.erase(ctx->peer_id)) { + score_->prune(ctx->peer_id, topic_); + } + updateBackoff(ctx->peer_id, backoff.value_or(config_.prune_backoff)); } - void TopicSubscriptions::addToMesh(const PeerContextPtr &p) { - assert(p->message_builder); + void TopicSubscriptions::addToMesh(const PeerContextPtr &ctx) { + assert(ctx->message_builder); - p->message_builder->addGraft(topic_); - connectivity_.peerIsWritable(p); - mesh_peers_.insert(p); + ctx->message_builder->addGraft(topic_); + connectivity_.peerIsWritable(ctx); + score_->graft(ctx->peer_id, topic_); + mesh_peers_.insert(ctx); log_.debug("peer {} added to mesh (size={}) for topic {}", - p->str, + ctx->str, mesh_peers_.size(), topic_); } @@ -347,6 +366,7 @@ namespace libp2p::protocol::gossip { ctx->message_builder->addPrune( topic_, ctx->isGossipsubv1_1() ? std::make_optional(backoff) : std::nullopt); + score_->prune(ctx->peer_id, topic_); connectivity_.peerIsWritable(ctx); log_.debug("peer {} removed from mesh (size={}) for topic {}", ctx->str, diff --git a/src/protocol/gossip/impl/topic_subscriptions.hpp b/src/protocol/gossip/impl/topic_subscriptions.hpp index ae8b09c7e..7e1357e62 100644 --- a/src/protocol/gossip/impl/topic_subscriptions.hpp +++ b/src/protocol/gossip/impl/topic_subscriptions.hpp @@ -31,7 +31,6 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers, std::shared_ptr explicit_peers, std::shared_ptr score, - std::shared_ptr gossip_promises, log::SubLogger &log); /// Returns true if not self-subscribed and @@ -91,7 +90,6 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers_; std::shared_ptr explicit_peers_; std::shared_ptr score_; - std::shared_ptr gossip_promises_; /// This host subscribed to this topic or not, this affects mesh behavior bool self_subscribed_; diff --git a/test/libp2p/protocol/gossip/gossip_mock_test.cpp b/test/libp2p/protocol/gossip/gossip_mock_test.cpp index ad14ac776..d6255e5c5 100644 --- a/test/libp2p/protocol/gossip/gossip_mock_test.cpp +++ b/test/libp2p/protocol/gossip/gossip_mock_test.cpp @@ -112,6 +112,7 @@ struct GossipMockTest : testing::Test { std::make_shared(); std::shared_ptr scheduler_ = std::make_shared( scheduler_backend_, SchedulerImpl::Config{}); + libp2p::event::Bus bus_; std::shared_ptr peer_repo_ = std::make_shared(); std::shared_ptr address_repo_ = @@ -137,6 +138,7 @@ struct GossipMockTest : testing::Test { } void setup() { + EXPECT_CALL(*host_, getBus()).WillRepeatedly(ReturnRef(bus_)); EXPECT_CALL(*host_, getPeerRepository()) .WillRepeatedly(ReturnRef(*peer_repo_)); EXPECT_CALL(*peer_repo_, getAddressRepository()) From 7e683c88e201882b01540ffb39cd67f61ccd3af3 Mon Sep 17 00:00:00 2001 From: turuslan Date: Thu, 27 Mar 2025 17:38:31 +0500 Subject: [PATCH 08/11] outbound Signed-off-by: turuslan --- include/libp2p/protocol/gossip/gossip.hpp | 13 ++++ include/libp2p/protocol/gossip/score.hpp | 3 + src/protocol/gossip/impl/choose_peers.hpp | 4 ++ src/protocol/gossip/impl/common.hpp | 3 + src/protocol/gossip/impl/gossip_core.cpp | 7 ++- src/protocol/gossip/impl/gossip_core.hpp | 2 + .../gossip/impl/remote_subscriptions.cpp | 3 + .../gossip/impl/remote_subscriptions.hpp | 2 + .../gossip/impl/topic_subscriptions.cpp | 60 ++++++++++++++++--- .../gossip/impl/topic_subscriptions.hpp | 2 + 10 files changed, 90 insertions(+), 9 deletions(-) diff --git a/include/libp2p/protocol/gossip/gossip.hpp b/include/libp2p/protocol/gossip/gossip.hpp index 4a0adb6d8..60e9e8ccd 100644 --- a/include/libp2p/protocol/gossip/gossip.hpp +++ b/include/libp2p/protocol/gossip/gossip.hpp @@ -49,6 +49,14 @@ namespace libp2p::protocol::gossip { size_t D = 6; size_t D_max = 10; + /// Affects how peers are selected when pruning a mesh due to over + /// subscription. + /// + /// At least `retain_scores` of the retained peers will be high-scoring, + /// while the remainder + /// are chosen randomly (D_score in the spec, default is 4). + size_t retain_scores = 4; + /// Affects how many peers we will emit gossip to at each heartbeat. /// /// We will send gossip to `gossip_factor * (total number of non-mesh @@ -147,6 +155,11 @@ namespace libp2p::protocol::gossip { /// through P7. std::chrono::milliseconds graft_flood_threshold = std::chrono::seconds{10}; + /// Minimum number of outbound peers in the mesh network before adding more + /// (D_out in the spec). This value must be smaller or equal than `mesh_n / + /// 2` and smaller than `mesh_n_low`. The default is 2. + size_t mesh_outbound_min = 2; + /// The maximum number of messages to include in an IHAVE message. /// Also controls the maximum number of IHAVE ids we will accept and request /// with IWANT from a peer within a heartbeat, to protect from IHAVE floods. diff --git a/include/libp2p/protocol/gossip/score.hpp b/include/libp2p/protocol/gossip/score.hpp index 30204659b..05cc51353 100644 --- a/include/libp2p/protocol/gossip/score.hpp +++ b/include/libp2p/protocol/gossip/score.hpp @@ -17,6 +17,9 @@ namespace libp2p::protocol::gossip { bool below(const PeerId &peer_id, double threshold) { return false; } + double get(const PeerId &peer_id) { + return 0; + } void addPenalty(const PeerId &peer_id, size_t count) {} void graft(const PeerId &peer_id, const TopicId &topic) {} void prune(const PeerId &peer_id, const TopicId &topic) {} diff --git a/src/protocol/gossip/impl/choose_peers.hpp b/src/protocol/gossip/impl/choose_peers.hpp index 2a0ee8451..906fb0205 100644 --- a/src/protocol/gossip/impl/choose_peers.hpp +++ b/src/protocol/gossip/impl/choose_peers.hpp @@ -40,6 +40,10 @@ namespace libp2p::protocol::gossip { }); } + void shuffle(auto &&r) { + std::ranges::shuffle(std::forward(r), random_); + } + private: std::default_random_engine random_; }; diff --git a/src/protocol/gossip/impl/common.hpp b/src/protocol/gossip/impl/common.hpp index 0097aebfa..79bffa2d2 100644 --- a/src/protocol/gossip/impl/common.hpp +++ b/src/protocol/gossip/impl/common.hpp @@ -8,6 +8,7 @@ #include #include +#include #include @@ -47,6 +48,8 @@ namespace libp2p::protocol::gossip { /// Remote peer and its context using PeerContextPtr = std::shared_ptr; + using OutboundPeers = std::unordered_set; + /// Message being published struct TopicMessage { using Ptr = std::shared_ptr; diff --git a/src/protocol/gossip/impl/gossip_core.cpp b/src/protocol/gossip/impl/gossip_core.cpp index 71c8119b7..05b27a784 100644 --- a/src/protocol/gossip/impl/gossip_core.cpp +++ b/src/protocol/gossip/impl/gossip_core.cpp @@ -59,6 +59,7 @@ namespace libp2p::protocol::gossip { [sch = scheduler_] { return sch->now(); } ), score_{std::make_shared()}, + outbound_peers_{std::make_shared()}, duplicate_cache_{config.duplicate_cache_time}, gossip_promises_{config.iwant_followup_time}, local_subscriptions_(std::make_shared( @@ -113,7 +114,7 @@ namespace libp2p::protocol::gossip { } remote_subscriptions_ = std::make_shared( - config_, *connectivity_, score_, scheduler_, log_); + config_, *connectivity_, score_, outbound_peers_, scheduler_, log_); started_ = true; @@ -140,6 +141,9 @@ namespace libp2p::protocol::gossip { .size() == 1) { self->score_->connect(peer_id); + if (conn->isInitiator()) { + self->outbound_peers_->insert(peer_id); + } } } } @@ -150,6 +154,7 @@ namespace libp2p::protocol::gossip { .subscribe([weak_self{weak_from_this()}](const PeerId &peer_id) { if (auto self = weak_self.lock()) { self->score_->disconnect(peer_id); + self->outbound_peers_->erase(peer_id); } }); } diff --git a/src/protocol/gossip/impl/gossip_core.hpp b/src/protocol/gossip/impl/gossip_core.hpp index 665845258..2c95fb94e 100644 --- a/src/protocol/gossip/impl/gossip_core.hpp +++ b/src/protocol/gossip/impl/gossip_core.hpp @@ -122,6 +122,8 @@ namespace libp2p::protocol::gossip { std::shared_ptr score_; + std::shared_ptr outbound_peers_; + DuplicateCache duplicate_cache_; GossipPromises gossip_promises_; diff --git a/src/protocol/gossip/impl/remote_subscriptions.cpp b/src/protocol/gossip/impl/remote_subscriptions.cpp index 2e97875a0..432c39cbf 100644 --- a/src/protocol/gossip/impl/remote_subscriptions.cpp +++ b/src/protocol/gossip/impl/remote_subscriptions.cpp @@ -19,6 +19,7 @@ namespace libp2p::protocol::gossip { const Config &config, Connectivity &connectivity, std::shared_ptr score, + std::shared_ptr outbound_peers, std::shared_ptr scheduler, log::SubLogger &log) : config_(config), @@ -26,6 +27,7 @@ namespace libp2p::protocol::gossip { choose_peers_{std::make_shared()}, explicit_peers_{std::make_shared()}, score_{std::move(score)}, + outbound_peers_{std::move(outbound_peers)}, scheduler_{std::move(scheduler)}, log_(log) {} @@ -182,6 +184,7 @@ namespace libp2p::protocol::gossip { choose_peers_, explicit_peers_, score_, + outbound_peers_, log_)); TopicSubscriptions &item = it->second; log_.debug("created entry for topic {}", topic); diff --git a/src/protocol/gossip/impl/remote_subscriptions.hpp b/src/protocol/gossip/impl/remote_subscriptions.hpp index ceec19dad..8eec4fc6f 100644 --- a/src/protocol/gossip/impl/remote_subscriptions.hpp +++ b/src/protocol/gossip/impl/remote_subscriptions.hpp @@ -21,6 +21,7 @@ namespace libp2p::protocol::gossip { RemoteSubscriptions(const Config &config, Connectivity &connectivity, std::shared_ptr score, + std::shared_ptr outbound_peers, std::shared_ptr scheduler, log::SubLogger &log); @@ -68,6 +69,7 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers_; std::shared_ptr explicit_peers_; std::shared_ptr score_; + std::shared_ptr outbound_peers_; std::shared_ptr scheduler_; std::unordered_map table_; diff --git a/src/protocol/gossip/impl/topic_subscriptions.cpp b/src/protocol/gossip/impl/topic_subscriptions.cpp index 36eba15cc..bd654062a 100644 --- a/src/protocol/gossip/impl/topic_subscriptions.cpp +++ b/src/protocol/gossip/impl/topic_subscriptions.cpp @@ -40,6 +40,7 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers, std::shared_ptr explicit_peers, std::shared_ptr score, + std::shared_ptr outbound_peers, log::SubLogger &log) : topic_(std::move(topic)), config_(config), @@ -48,6 +49,7 @@ namespace libp2p::protocol::gossip { choose_peers_{std::move(choose_peers)}, explicit_peers_{std::move(explicit_peers)}, score_{std::move(score)}, + outbound_peers_{std::move(outbound_peers)}, self_subscribed_(false), log_(log) { connectivity_.getConnectedPeers().selectIf( @@ -166,6 +168,10 @@ namespace libp2p::protocol::gossip { } return false; }); + auto outbound = static_cast( + std::ranges::count_if(mesh_peers_, [&](const PeerContextPtr &ctx) { + return outbound_peers_->contains(ctx->peer_id); + })); if (mesh_peers_.size() < config_.D_min) { for (auto &ctx : choose_peers_->choose( subscribed_peers_, @@ -176,14 +182,53 @@ namespace libp2p::protocol::gossip { and not score_->below(ctx->peer_id, config_.score.zero); }, config_.D - mesh_peers_.size())) { + if (outbound_peers_->contains(ctx->peer_id)) { + ++outbound; + } addToMesh(ctx); } } else if (mesh_peers_.size() > config_.D_max) { - auto peers = - mesh_peers_.selectRandomPeers(mesh_peers_.size() - config_.D_max); - for (auto &p : peers) { - sendPrune(p, false); - mesh_peers_.erase(p->peer_id); + std::vector shuffled; + for (auto &ctx : mesh_peers_) { + shuffled.emplace_back(ctx->peer_id); + } + choose_peers_->shuffle(shuffled); + std::ranges::sort(shuffled, [&](const PeerId &l, const PeerId &r) { + return score_->get(r) < score_->get(l); + }); + if (shuffled.size() > config_.retain_scores) { + choose_peers_->shuffle( + std::span{shuffled}.subspan(config_.retain_scores)); + } + for (auto &peer_id : shuffled) { + if (mesh_peers_.size() <= config_.D_max) { + break; + } + if (outbound_peers_->contains(peer_id)) { + if (outbound <= config_.mesh_outbound_min) { + continue; + } + --outbound; + } + auto ctx = mesh_peers_.find(peer_id).value(); + sendPrune(ctx, false); + mesh_peers_.erase(ctx->peer_id); + } + } + if (mesh_peers_.size() >= config_.D_min) { + if (outbound < config_.mesh_outbound_min) { + for (auto &ctx : choose_peers_->choose( + subscribed_peers_, + [&](const PeerContextPtr &ctx) { + return not mesh_peers_.contains(ctx) + and not explicit_peers_->contains(ctx->peer_id) + and not isBackoffWithSlack(ctx->peer_id) + and not score_->below(ctx->peer_id, config_.score.zero) + and outbound_peers_->contains(ctx->peer_id); + }, + config_.mesh_outbound_min - outbound)) { + addToMesh(ctx); + } } } } @@ -305,8 +350,6 @@ namespace libp2p::protocol::gossip { // implicit subscribe on graft subscribed_peers_.insert(ctx); - bool mesh_is_full = (mesh_peers_.size() >= config_.D_max); - bool prune = [&] { if (self_subscribed_) { return true; @@ -324,7 +367,8 @@ namespace libp2p::protocol::gossip { if (score_->below(ctx->peer_id, config_.score.zero)) { return true; } - if (mesh_is_full) { + if (mesh_peers_.size() >= config_.D_max + and not outbound_peers_->contains(ctx->peer_id)) { return true; } score_->graft(ctx->peer_id, topic_); diff --git a/src/protocol/gossip/impl/topic_subscriptions.hpp b/src/protocol/gossip/impl/topic_subscriptions.hpp index 7e1357e62..4bcc38a59 100644 --- a/src/protocol/gossip/impl/topic_subscriptions.hpp +++ b/src/protocol/gossip/impl/topic_subscriptions.hpp @@ -31,6 +31,7 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers, std::shared_ptr explicit_peers, std::shared_ptr score, + std::shared_ptr outbound_peers, log::SubLogger &log); /// Returns true if not self-subscribed and @@ -90,6 +91,7 @@ namespace libp2p::protocol::gossip { std::shared_ptr choose_peers_; std::shared_ptr explicit_peers_; std::shared_ptr score_; + std::shared_ptr outbound_peers_; /// This host subscribed to this topic or not, this affects mesh behavior bool self_subscribed_; From 498942d983a0ede84e3aeb90a5bc17529da9d21f Mon Sep 17 00:00:00 2001 From: turuslan Date: Fri, 28 Mar 2025 13:03:52 +0500 Subject: [PATCH 09/11] score Signed-off-by: turuslan --- include/libp2p/protocol/gossip/score.hpp | 325 +++++++++++++++++- .../libp2p/protocol/gossip/score_config.hpp | 38 +- src/protocol/gossip/impl/gossip_core.cpp | 2 +- src/protocol/gossip/impl/gossip_core.hpp | 6 +- .../gossip/impl/topic_subscriptions.cpp | 2 +- .../gossip/impl/topic_subscriptions.hpp | 6 +- 6 files changed, 363 insertions(+), 16 deletions(-) diff --git a/include/libp2p/protocol/gossip/score.hpp b/include/libp2p/protocol/gossip/score.hpp index 05cc51353..3838c4c8f 100644 --- a/include/libp2p/protocol/gossip/score.hpp +++ b/include/libp2p/protocol/gossip/score.hpp @@ -6,32 +6,335 @@ #pragma once +#include #include +#include +#include +#include +#include +#include namespace libp2p::protocol::gossip { using TopicId = std::string; using MessageId = Bytes; +} // namespace libp2p::protocol::gossip + +namespace libp2p::protocol::gossip::score { + using Duration = std::chrono::milliseconds; + using Clock = std::chrono::steady_clock; + using Time = Clock::time_point; + + constexpr std::chrono::seconds kTimeCacheDuration{120}; + + struct DeliveryStatusUnknown {}; + struct DeliveryStatusValid { + Time time; + }; + struct DeliveryStatusInvalid {}; + struct DeliveryStatusIgnored {}; + using DeliveryStatus = std::variant; + + struct DeliveryRecord { + DeliveryStatus status; + Time first_seen; + std::unordered_set peers; + }; + + struct MeshActive { + Time graft_time; + Duration mesh_time; + }; + + struct TopicStats { + std::optional mesh_active; + double first_message_deliveries = 0; + bool mesh_message_deliveries_active = false; + double mesh_message_deliveries = 0; + double mesh_failure_penalty = 0; + double invalid_message_deliveries = 0; + }; + + struct PeerStats { + std::optional