net: CAddress & CAddrMan: (un)serialize as ADDRv2

vasild · dongcarl · vasild · commit 896a5afebf88 · 2020-09-18T14:05:19.000+02:00
Change the serialization of `CAddrMan` to serialize its addresses
in ADDRv2/BIP155 format by default. Introduce a new `CAddrMan` format
version (3).

Add support for ADDRv2 format in `CAddress` (un)serialization.

Co-authored-by: Carl Dong &lt;contact@carldong.me&gt;
diff --git a/src/addrman.h b/src/addrman.h
@@ -264,6 +264,17 @@ friend class CAddrManTest;
     void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs);
 
 public:
+    //! Serialization versions.
+    enum class Format : uint8_t {
+        V0 = 0, //!< historic format, before commit e6b343d88
+        V1 = 1, //!< for pre-asmap files
+        V2 = 2, //!< for files including asmap version
+        V3 = 3, //!< same as V2 plus addresses are in BIP155 format
+    };
+
+    //! Serialization / unserialization format.
+    Format m_format = Format::V3;
+
     // Compressed IP->ASN mapping, loaded from a file when a node starts.
     // Should be always empty if no file was provided.
     // This mapping is then used for bucketing nodes in Addrman.
@@ -285,8 +296,8 @@ friend class CAddrManTest;
 
 
     /**
-     * serialized format:
-     * * version byte (1 for pre-asmap files, 2 for files including asmap version)
+     * Serialized format.
+     * * version byte (@see `Format`)
      * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility)
      * * nNew
      * * nTried
@@ -318,8 +329,15 @@ friend class CAddrManTest;
     {
         LOCK(cs);
 
-        unsigned char nVersion = 2;
-        s << nVersion;
+        const auto stream_version_orig = s.GetVersion();
+
+        if (m_format >= Format::V3) {
+            // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress
+            // serialize methods produce an address in v2 format.
+            s.SetVersion(stream_version_orig | ADDRV2_FORMAT);
+        }
+
+        s << (uint8_t)m_format;
         s << ((unsigned char)32);
         s << nKey;
         s << nNew;
@@ -368,6 +386,8 @@ friend class CAddrManTest;
             asmap_version = SerializeHash(m_asmap);
         }
         s << asmap_version;
+
+        s.SetVersion(stream_version_orig);
     }
 
     template<typename Stream>
@@ -376,8 +396,17 @@ friend class CAddrManTest;
         LOCK(cs);
 
         Clear();
-        unsigned char nVersion;
-        s >> nVersion;
+
+        const auto stream_version_orig = s.GetVersion();
+
+        uint8_t format_tmp;
+        s >> format_tmp;
+        m_format = (Format)format_tmp;
+        if (m_format >= Format::V3) {
+            // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress
+            // unserialize methods know that an address in v2 format is coming.
+            s.SetVersion(stream_version_orig | ADDRV2_FORMAT);
+        }
         unsigned char nKeySize;
         s >> nKeySize;
         if (nKeySize != 32) throw std::ios_base::failure("Incorrect keysize in addrman deserialization");
@@ -386,15 +415,17 @@ friend class CAddrManTest;
         s >> nTried;
         int nUBuckets = 0;
         s >> nUBuckets;
-        if (nVersion != 0) {
+        if (m_format > Format::V0) {
             nUBuckets ^= (1 << 30);
         }
 
         if (nNew > ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE) {
+            s.SetVersion(stream_version_orig);
             throw std::ios_base::failure("Corrupt CAddrMan serialization, nNew exceeds limit.");
         }
 
         if (nTried > ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE) {
+            s.SetVersion(stream_version_orig);
             throw std::ios_base::failure("Corrupt CAddrMan serialization, nTried exceeds limit.");
         }
 
@@ -449,21 +480,21 @@ friend class CAddrManTest;
             supplied_asmap_version = SerializeHash(m_asmap);
         }
         uint256 serialized_asmap_version;
-        if (nVersion > 1) {
+        if (m_format > Format::V1) {
             s >> serialized_asmap_version;
         }
 
         for (int n = 0; n < nNew; n++) {
             CAddrInfo &info = mapInfo[n];
             int bucket = entryToBucket[n];
             int nUBucketPos = info.GetBucketPosition(nKey, true, bucket);
-            if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 &&
+            if (m_format >= Format::V2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 &&
                 info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) {
                 // Bucketing has not changed, using existing bucket positions for the new table
                 vvNew[bucket][nUBucketPos] = n;
                 info.nRefCount++;
             } else {
-                // In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap),
+                // In case the new table data cannot be used (m_format unknown, bucket count wrong or new asmap),
                 // try to give them a reference based on their primary source address.
                 LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n");
                 bucket = info.GetNewBucket(nKey, m_asmap);
@@ -491,6 +522,8 @@ friend class CAddrManTest;
         }
 
         Check();
+
+        s.SetVersion(stream_version_orig);
     }
 
     void Clear()
diff --git a/src/hash.h b/src/hash.h
@@ -102,13 +102,14 @@ class CHashWriter
     CSHA256 ctx;
 
     const int nType;
-    const int nVersion;
+    int nVersion;
 public:
 
     CHashWriter(int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) {}
 
     int GetType() const { return nType; }
     int GetVersion() const { return nVersion; }
+    void SetVersion(int n) { nVersion = n; }
 
     void write(const char *pch, size_t size) {
         ctx.Write((const unsigned char*)pch, size);
diff --git a/src/protocol.h b/src/protocol.h
@@ -351,7 +351,8 @@ class CAddress : public CService
 
 public:
     CAddress() : CService{} {};
-    explicit CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {};
+    CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {};
+    CAddress(CService ipIn, ServiceFlags nServicesIn, uint32_t nTimeIn) : CService{ipIn}, nTime{nTimeIn}, nServices{nServicesIn} {};
 
     SERIALIZE_METHODS(CAddress, obj)
     {
@@ -370,7 +371,14 @@ class CAddress : public CService
             // nTime.
             READWRITE(obj.nTime);
         }
-        READWRITE(Using<CustomUintFormatter<8>>(obj.nServices));
+        if (nVersion & ADDRV2_FORMAT) {
+            uint64_t services_tmp;
+            SER_WRITE(obj, services_tmp = obj.nServices);
+            READWRITE(COMPACTSIZE(services_tmp));
+            SER_READ(obj, obj.nServices = static_cast<ServiceFlags>(services_tmp));
+        } else {
+            READWRITE(Using<CustomUintFormatter<8>>(obj.nServices));
+        }
         READWRITEAS(CService, obj);
     }
 
diff --git a/src/streams.h b/src/streams.h
@@ -602,7 +602,7 @@ class CAutoFile
 {
 private:
     const int nType;
-    const int nVersion;
+    int nVersion;
 
     FILE* file;
 
@@ -650,6 +650,7 @@ class CAutoFile
     //
     int GetType() const          { return nType; }
     int GetVersion() const       { return nVersion; }
+    void SetVersion(int n)       { nVersion = n; }
 
     void read(char* pch, size_t nSize)
     {
diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp
@@ -80,7 +80,7 @@ static CDataStream AddrmanToStream(CAddrManSerializationMock& _addrman)
     ssPeersIn << _addrman;
     std::string str = ssPeersIn.str();
     std::vector<unsigned char> vchData(str.begin(), str.end());
-    return CDataStream(vchData, SER_DISK, CLIENT_VERSION);
+    return CDataStream(vchData, ssPeersIn.GetType(), ssPeersIn.GetVersion());
 }
 
 BOOST_FIXTURE_TEST_SUITE(net_tests, BasicTestingSetup)
@@ -142,6 +142,36 @@ BOOST_AUTO_TEST_CASE(caddrdb_read)
     BOOST_CHECK(addrman2.size() == 3);
 }
 
+BOOST_AUTO_TEST_CASE(caddrdb_read_v1_ser)
+{
+    // Construct an AddrMan that serializes addresses in pre-BIP155 format.
+    CAddrManUncorrupted addrmanUncorrupted;
+    addrmanUncorrupted.m_format = CAddrMan::Format::V2;
+    addrmanUncorrupted.MakeDeterministic();
+
+    CService addr1, addr2, addr3;
+    BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false));
+    BOOST_CHECK(Lookup("250.7.2.2", addr2, 9999, false));
+    BOOST_CHECK(Lookup("250.7.3.3", addr3, 9999, false));
+
+    // Add three addresses to new table.
+    CService source;
+    BOOST_CHECK(Lookup("252.5.1.1", source, 8333, false));
+    BOOST_CHECK(addrmanUncorrupted.Add(CAddress(addr1, NODE_NONE), source));
+    BOOST_CHECK(addrmanUncorrupted.Add(CAddress(addr2, NODE_NONE), source));
+    BOOST_CHECK(addrmanUncorrupted.Add(CAddress(addr3, NODE_NONE), source));
+
+    // Test that the de-serialization does not throw an exception.
+    CDataStream ssPeers1 = AddrmanToStream(addrmanUncorrupted);
+    CAddrMan addrman1;
+
+    BOOST_CHECK(addrman1.size() == 0);
+    unsigned char message_start[4];
+    ssPeers1 >> message_start;
+    ssPeers1 >> addrman1;
+    BOOST_CHECK_EQUAL(addrman1.m_format, CAddrMan::Format::V2);
+    BOOST_CHECK_EQUAL(addrman1.size(), 3U);
+}
 
 BOOST_AUTO_TEST_CASE(caddrdb_read_corrupted)
 {
diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp
@@ -4,9 +4,13 @@
 
 #include <net_permissions.h>
 #include <netbase.h>
+#include <protocol.h>
+#include <serialize.h>
+#include <streams.h>
 #include <test/util/setup_common.h>
 #include <util/strencodings.h>
 #include <util/translation.h>
+#include <version.h>
 
 #include <string>
 
@@ -443,4 +447,105 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters)
     BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret));
 }
 
+// Since CNetAddr (un)ser is tested separately in net_tests.cpp here we only
+// try a few edge cases for port, service flags and time.
+
+static const std::vector<CAddress> fixture_addresses({
+    CAddress(
+        CService(CNetAddr(in6addr_loopback), 0 /* port */),
+        NODE_NONE,
+        0x4966bc61U /* Fri Jan  9 02:54:25 UTC 2009 */
+    ),
+    CAddress(
+        CService(CNetAddr(in6addr_loopback), 0x00f1 /* port */),
+        NODE_NETWORK,
+        0x83766279U /* Tue Nov 22 11:22:33 UTC 2039 */
+    ),
+    CAddress(
+        CService(CNetAddr(in6addr_loopback), 0xf1f2 /* port */),
+        static_cast<ServiceFlags>(NODE_WITNESS | NODE_COMPACT_FILTERS | NODE_NETWORK_LIMITED),
+        0xffffffffU /* Sun Feb  7 06:28:15 UTC 2106 */
+    )
+});
+
+// fixture_addresses should equal to this when serialized in V1 format.
+// When this is unserialized from V1 format it should equal to fixture_addresses.
+static constexpr const char* stream_addrv1_hex =
+    "03" // number of entries
+
+    "61bc6649"                         // time, Fri Jan  9 02:54:25 UTC 2009
+    "0000000000000000"                 // service flags, NODE_NONE
+    "00000000000000000000000000000001" // address, fixed 16 bytes (IPv4 embedded in IPv6)
+    "0000"                             // port
+
+    "79627683"                         // time, Tue Nov 22 11:22:33 UTC 2039
+    "0100000000000000"                 // service flags, NODE_NETWORK
+    "00000000000000000000000000000001" // address, fixed 16 bytes (IPv6)
+    "00f1"                             // port
+
+    "ffffffff"                         // time, Sun Feb  7 06:28:15 UTC 2106
+    "4804000000000000"                 // service flags, NODE_WITNESS | NODE_COMPACT_FILTERS | NODE_NETWORK_LIMITED
+    "00000000000000000000000000000001" // address, fixed 16 bytes (IPv6)
+    "f1f2";                            // port
+
+// fixture_addresses should equal to this when serialized in V2 format.
+// When this is unserialized from V2 format it should equal to fixture_addresses.
+static constexpr const char* stream_addrv2_hex =
+    "03" // number of entries
+
+    "61bc6649"                         // time, Fri Jan  9 02:54:25 UTC 2009
+    "00"                               // service flags, COMPACTSIZE(NODE_NONE)
+    "02"                               // network id, IPv6
+    "10"                               // address length, COMPACTSIZE(16)
+    "00000000000000000000000000000001" // address
+    "0000"                             // port
+
+    "79627683"                         // time, Tue Nov 22 11:22:33 UTC 2039
+    "01"                               // service flags, COMPACTSIZE(NODE_NETWORK)
+    "02"                               // network id, IPv6
+    "10"                               // address length, COMPACTSIZE(16)
+    "00000000000000000000000000000001" // address
+    "00f1"                             // port
+
+    "ffffffff"                         // time, Sun Feb  7 06:28:15 UTC 2106
+    "fd4804"                           // service flags, COMPACTSIZE(NODE_WITNESS | NODE_COMPACT_FILTERS | NODE_NETWORK_LIMITED)
+    "02"                               // network id, IPv6
+    "10"                               // address length, COMPACTSIZE(16)
+    "00000000000000000000000000000001" // address
+    "f1f2";                            // port
+
+BOOST_AUTO_TEST_CASE(caddress_serialize_v1)
+{
+    CDataStream s(SER_NETWORK, PROTOCOL_VERSION);
+
+    s << fixture_addresses;
+    BOOST_CHECK_EQUAL(HexStr(s), stream_addrv1_hex);
+}
+
+BOOST_AUTO_TEST_CASE(caddress_unserialize_v1)
+{
+    CDataStream s(ParseHex(stream_addrv1_hex), SER_NETWORK, PROTOCOL_VERSION);
+    std::vector<CAddress> addresses_unserialized;
+
+    s >> addresses_unserialized;
+    BOOST_CHECK(fixture_addresses == addresses_unserialized);
+}
+
+BOOST_AUTO_TEST_CASE(caddress_serialize_v2)
+{
+    CDataStream s(SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT);
+
+    s << fixture_addresses;
+    BOOST_CHECK_EQUAL(HexStr(s), stream_addrv2_hex);
+}
+
+BOOST_AUTO_TEST_CASE(caddress_unserialize_v2)
+{
+    CDataStream s(ParseHex(stream_addrv2_hex), SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT);
+    std::vector<CAddress> addresses_unserialized;
+
+    s >> addresses_unserialized;
+    BOOST_CHECK(fixture_addresses == addresses_unserialized);
+}
+
 BOOST_AUTO_TEST_SUITE_END()

Original file line number	Diff line number	Diff line change
`@@ -602,7 +602,7 @@ class CAutoFile`
`602`	`602`	`{`
`603`	`603`	`private:`
`604`	`604`	`const int nType;`
`605`		`- const int nVersion;`
	`605`	`+ int nVersion;`
`606`	`606`
`607`	`607`	`FILE* file;`
`608`	`608`
`@@ -650,6 +650,7 @@ class CAutoFile`
`650`	`650`	`//`
`651`	`651`	`int GetType() const { return nType; }`
`652`	`652`	`int GetVersion() const { return nVersion; }`
	`653`	`+ void SetVersion(int n) { nVersion = n; }`
`653`	`654`
`654`	`655`	`void read(char* pch, size_t nSize)`
`655`	`656`	`{`