From b8cca0dd52a4264e3b7a19f60bd94873c6328c60 Mon Sep 17 00:00:00 2001 From: jbride Date: Tue, 15 Oct 2024 19:12:31 -0600 Subject: [PATCH 01/13] PoC implementation of https://github.com/SurmountSystems/gabriel/issues/6 using zmqpubrawblock --- Cargo.lock | 514 +++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 + README.md | 61 ++++-- src/block.rs | 18 +- src/main.rs | 107 +++++++++-- 5 files changed, 663 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1589e6..cf7477a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,30 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -78,12 +102,51 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "async-trait" +version = "0.1.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "asynchronous-codec" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a860072022177f903e59730004fb5dc13db9275b79bb2aef7ba8ce831956c233" +dependencies = [ + "bytes", + "futures-sink", + "futures-util", + "memchr", + "pin-project-lite", +] + [[package]] name = "autocfg" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + [[package]] name = "base58ck" version = "0.1.0" @@ -154,6 +217,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + [[package]] name = "block-buffer" version = "0.10.4" @@ -175,6 +244,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" + [[package]] name = "cc" version = "1.1.18" @@ -297,6 +372,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -313,6 +397,19 @@ dependencies = [ "typenum", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "digest" version = "0.10.7" @@ -335,6 +432,68 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "gabriel" version = "0.2.0" @@ -352,6 +511,8 @@ dependencies = [ "serde", "serde_json", "sha2", + "tokio", + "zeromq", ] [[package]] @@ -364,12 +525,41 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "hex" version = "0.4.3" @@ -469,6 +659,16 @@ version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.22" @@ -487,6 +687,27 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi", + "libc", + "wasi", + "windows-sys", +] + [[package]] name = "nom" version = "7.1.3" @@ -512,18 +733,71 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.36.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "portable-atomic" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -542,6 +816,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "rayon" version = "1.10.0" @@ -562,12 +866,62 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + [[package]] name = "ryu" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "secp256k1" version = "0.29.1" @@ -637,6 +991,40 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "strsim" version = "0.11.1" @@ -654,6 +1042,69 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio" +version = "1.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "typenum" version = "1.17.0" @@ -678,12 +1129,27 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +dependencies = [ + "getrandom", +] + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.93" @@ -820,3 +1286,51 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeromq" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a4528179201f6eecf211961a7d3276faa61554c82651ecc66387f68fc3004bd" +dependencies = [ + "async-trait", + "asynchronous-codec", + "bytes", + "crossbeam-queue", + "dashmap", + "futures-channel", + "futures-io", + "futures-task", + "futures-util", + "log", + "num-traits", + "once_cell", + "parking_lot", + "rand", + "regex", + "thiserror", + "tokio", + "tokio-util", + "uuid", +] diff --git a/Cargo.toml b/Cargo.toml index 6c106f2..cb8880c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,3 +20,5 @@ serde = { version = "1.0.201", features = ["derive"] } serde_json = "1.0" sha2 = "0.10" log = "0.4.22" +zeromq = "0.4.1" +tokio = "1.40.0" diff --git a/README.md b/README.md index b3486c5..14fdcb2 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,29 @@ -# gabriel - +gabriel + +- [1. Introduction](#1-introduction) +- [2. Setup](#2-setup) + - [2.1. Pre-reqs](#21-pre-reqs) + - [2.1.1. Hardware](#211-hardware) + - [2.1.2. Software](#212-software) + - [2.1.2.1. Rust](#2121-rust) + - [2.1.2.2. bitcoind](#2122-bitcoind) + - [2.2. Clone code](#22-clone-code) + - [2.3. Build](#23-build) + - [2.4. Execute tests](#24-execute-tests) +- [3. Run Gabriel](#3-run-gabriel) + - [3.1. analyze single block data file](#31-analyze-single-block-data-file) + - [3.2. analyze all block data files](#32-analyze-all-block-data-files) + - [3.3. consume and analyze new raw blocks](#33-consume-and-analyze-new-raw-blocks) + - [3.3.1. Test](#331-test) +- [4. Debug in VSCode:](#4-debug-in-vscode) + + +## 1. Introduction Measures how many unspent public key addresses there are, and how many coins are in them over time. Early Satoshi-era coins that are just sitting with exposed public keys. If we see lots of coins move... That's a potential sign that quantum computers have silently broken bitcoin. -## Execution +## 2. Setup -### Pre-reqs +### 2.1. Pre-reqs ``` $ bitcoind \ @@ -12,26 +31,24 @@ $ bitcoind \ -daemon=0 ``` -#### Hardware +#### 2.1.1. Hardware -#### Software -##### Rust +#### 2.1.2. Software +##### 2.1.2.1. Rust The best way to install Rust is to use [rustup](https://rustup.rs). -##### bitcoind +##### 2.1.2.2. bitcoind If on bitcoind v28.0, ensure the following flag is set prior to initial block download: `-blocksxor=0` -#### Environment Variables - -### Clone code +### 2.2. Clone code ``` $ git clone https://github.com/SurmountSystems/gabriel.git $ git checkout HB/gabriel-v2 ``` -### Build +### 2.3. Build * execute: @@ -42,15 +59,15 @@ $ git checkout HB/gabriel-v2 $ ./target/debug/gabriel -### Execute tests +### 2.4. Execute tests ``` $ cargo test ``` -### Run Gabriel +## 3. Run Gabriel -* execute indexer on a specific bitcoin block data file : +### 3.1. analyze single block data file $ export BITCOIND_DATA_DIR=/path/to/bitcoind/data/dir $ export BITCOIND_BLOCK_DATA_FILE=xxx.dat @@ -60,14 +77,24 @@ $ cargo test -o /tmp/$BITCOIND_BLOCK_DATA_FILE.csv -* execute indexer across all bitcoin block data files : +### 3.2. analyze all block data files $ export BITCOIND_DATA_DIR=/path/to/bitcoind/data/dir $ ./target/debug/gabriel index \ --input $BITCOIND_DATA_DIR/blocks \ --output /tmp/gabriel-testnet4.csv -#### Debug in VSCode: +### 3.3. consume and analyze new raw blocks + + $ ./target/debug/gabriel block-async-eval \ + --zmqpubrawblock-socket-url tcp://127.0.0.1:29001 \ + --output /tmp/async_blocks.txt + +#### 3.3.1. Test + +TO-DO: generate a test P2PK address and send block rewards + +## 4. Debug in VSCode: Add and edit the following to $PROJECT_HOME/.vscode/launch.json: diff --git a/src/block.rs b/src/block.rs index 988f9fb..33f6dab 100644 --- a/src/block.rs +++ b/src/block.rs @@ -422,12 +422,19 @@ fn parse_block_with_magic(input: &[u8]) -> IResult<&[u8], BitcoinBlock> { } /// Parse the entire blkxxxx.dat file, returning a list of blocks and any remaining input -fn parse_blk_file(input: &[u8]) -> IResult<&[u8], Vec> { +fn parse_blk_file(input: &[u8], use_magic: bool) -> IResult<&[u8], Vec> { let mut blocks = Vec::new(); let mut remaining_input = input; while !remaining_input.is_empty() { - match parse_block_with_magic(remaining_input) { + let mut _block_result: IResult<&[u8], BitcoinBlock>; + if use_magic { + _block_result = parse_block_with_magic(remaining_input); + }else { + _block_result = parse_block(remaining_input); + } + + match _block_result { Ok((remaining, block)) => { blocks.push(block); remaining_input = remaining; @@ -456,16 +463,17 @@ fn is_p2pk(script: &[u8]) -> bool { } /// Process a single block from the input data -fn process_block( +pub fn process_block( input: &[u8], pb: &ProgressBar, result_map: &ResultMap, tx_map: &TxMap, header_map: &HeaderMap, + use_magic: bool ) -> usize { let mut blocks_processed = 0; - match parse_blk_file(input) { + match parse_blk_file(input, use_magic) { Ok((_, blocks)) => { for block in blocks { let block_hash = compute_block_hash(&block.header); @@ -556,7 +564,7 @@ pub fn process_block_file( file.read_to_end(&mut buffer) .expect("Failed to read block file"); // Process the blk file containing multiple blocks - process_block(&buffer, pb, result_map, tx_map, header_map) + process_block(&buffer, pb, result_map, tx_map, header_map, true) } /// Iterate through the blocks directory and process each blkxxxxx.dat file in parallel diff --git a/src/main.rs b/src/main.rs index 3ed14c1..906ec50 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,14 @@ use std::{ - fs::OpenOptions, - io::{Seek, Write}, + fs::{OpenOptions, File}, + io::{Read, Seek, Write}, path::PathBuf, }; use anyhow::{Ok, Result}; -use block::{process_block_file, process_blocks_in_parallel, Record}; -use clap::{Parser, Subcommand}; // Updated import +use block::{process_block, process_block_file, process_blocks_in_parallel, Record}; +use clap::{Parser, Subcommand}; +use nom::AsBytes; +use zeromq::{Socket, SocketRecv}; mod block; mod tx; @@ -26,6 +28,7 @@ struct Cli { #[derive(Subcommand, Debug)] enum Commands { BlockFileEval(BlockFileEvalArgs), + BlockAsyncEval(BlockAsyncEvalArgs), Index(IndexArgs), Graph(GraphArgs), } @@ -41,6 +44,17 @@ struct BlockFileEvalArgs { output: PathBuf, } +#[derive(Parser, Debug)] +struct BlockAsyncEvalArgs { + /// zmqpubrawblock socket URL + #[arg(short, long)] + zmqpubrawblock_socket_url: String, + + /// CSV output file path + #[arg(short, long)] + output: PathBuf, +} + #[derive(Parser, Debug)] struct IndexArgs { /// Bitcoin directory path @@ -57,17 +71,38 @@ struct GraphArgs { // Add arguments for the graph command if needed } -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { let cli = Cli::parse(); match &cli.command { Commands::BlockFileEval(args) => run_block_file_eval(args), Commands::Index(args) => run_index(args), Commands::Graph(args) => run_graph(args), + Commands::BlockAsyncEval(args) => run_async_block_eval_listener(args).await } } +fn append_to_output(mut file: &File, result_map: &ResultMap) -> Result<()> { + let result_map_read = result_map.read().unwrap(); + for (_key, record) in result_map_read.iter() { + // write a record to the file + let mut p2pk_addresses = &record.p2pk_addresses_added; + let binding = p2pk_addresses - &record.p2pk_addresses_spent; + p2pk_addresses = &binding; + let mut p2pk_coins = record.p2pk_sats_added.to_owned() as f64 / 100_000_000.0; + p2pk_coins -= record.p2pk_sats_spent.to_owned() as f64 / 100_000_000.0; + let date = &record.date; + let output_line = format!("0,{date},{p2pk_addresses},{p2pk_coins}"); + writeln!(file, "{}", output_line)?; + } + Ok(()) + +} + fn run_block_file_eval(args: &BlockFileEvalArgs) -> Result<()> { + + // Maps previous block hash to next merkle root let header_map: HeaderMap = Default::default(); @@ -107,22 +142,60 @@ fn run_block_file_eval(args: &BlockFileEvalArgs) -> Result<()> { file.set_len(0)?; // Truncate the file file.write_all(HEADER.as_bytes())?; - let result_map_read = result_map.read().unwrap(); - for (_key, record) in result_map_read.iter() { - // write a record to the file - let mut p2pk_addresses = &record.p2pk_addresses_added; - let binding = p2pk_addresses - &record.p2pk_addresses_spent; - p2pk_addresses = &binding; - let mut p2pk_coins = record.p2pk_sats_added.to_owned() as f64 / 100_000_000.0; - p2pk_coins -= record.p2pk_sats_spent.to_owned() as f64 / 100_000_000.0; - let date = &record.date; - let output_line = format!("0,{date},{p2pk_addresses},{p2pk_coins}"); - writeln!(file, "{}", output_line)?; - } + append_to_output(&file, &result_map); Ok(()) } +async fn run_async_block_eval_listener(args: &BlockAsyncEvalArgs) -> Result<()> { + + println!( + "zmqpubrawblock_socket_url: {} ; output file = {}", + &args.zmqpubrawblock_socket_url, + &args.output.display() + ); + + // Maps previous block hash to next merkle root + let header_map: HeaderMap = Default::default(); + // Maps txid to tx value + let tx_map: TxMap = Default::default(); + + // Maps header hash to result Record + let result_map: ResultMap = Default::default(); + let pb = ProgressBar::new(1); + + let mut socket = zeromq::SubSocket::new(); + socket + .connect(&args.zmqpubrawblock_socket_url) + .await + .expect(&format!("Failed to connect: {}", &args.zmqpubrawblock_socket_url)); + + socket.subscribe("").await?; + + // prep output file + let mut file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&args.output)?; + + loop { + let zmq_message = socket.recv().await?; + + let second_element = zmq_message.get(1); + match second_element { + Some(block_bytes) => { + let u8_byte_array = block_bytes.as_bytes(); + let tx_count = process_block(u8_byte_array, &pb, &result_map, &tx_map, &header_map, false); + println!("received block! byte length: {}; tx_count: {}", u8_byte_array.len(), tx_count); + let _ = append_to_output(&file, &result_map); + } + None => panic!("second element from zeromq raw block is non-existent!") + } + } +} + fn run_index(args: &IndexArgs) -> Result<()> { // Maps previous block hash to next merkle root let header_map: HeaderMap = Default::default(); From 0db1643136dfad2cb85a71263fb330845fcc5c20 Mon Sep 17 00:00:00 2001 From: jbride Date: Thu, 17 Oct 2024 20:42:55 -0600 Subject: [PATCH 02/13] First draft of documenting how to create a trnx with a P2PK output (useful for testing) --- README.md | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++-- src/main.rs | 19 ++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 14fdcb2..46fe949 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ gabriel - [3.1. analyze single block data file](#31-analyze-single-block-data-file) - [3.2. analyze all block data files](#32-analyze-all-block-data-files) - [3.3. consume and analyze new raw blocks](#33-consume-and-analyze-new-raw-blocks) - - [3.3.1. Test](#331-test) + - [3.3.1. Fund a P2PK address on reg-test](#331-fund-a-p2pk-address-on-reg-test) + - [3.3.2. Generate block:](#332-generate-block) + - [3.3.3. Test](#333-test) - [4. Debug in VSCode:](#4-debug-in-vscode) @@ -90,7 +92,71 @@ $ cargo test --zmqpubrawblock-socket-url tcp://127.0.0.1:29001 \ --output /tmp/async_blocks.txt -#### 3.3.1. Test +#### 3.3.1. Fund a P2PK address on reg-test + +NOTE: FOr this exercise, start w/ a fresh reg-test environment (with no blocks yet generated) + +1. On reg-test, create a new address and corresponding public key: + + $ TARGET_ADDR=$( b-reg getnewaddress ) \ + && echo $TARGET_ADDR \ + && TARGET_PUB_KEY=$( b-reg getaddressinfo $TARGET_ADDR | jq -r .pubkey ) \ + && echo $TARGET_PUB_KEY + +2. Create an initial raw trnx: + + $ INITIAL_RAW_TRNX=$( b-reg createrawtransaction "[]" "[{\"$TARGET_ADDR\":49.99971800}]" 0 true ) \ + && echo $INITIAL_RAW_TRNX + +3. Fund initial trnx: + + $ FUNDED_RAW_TRNX=$( b-reg fundrawtransaction $INITIAL_RAW_TRNX '{"subtractFeeFromOutputs":[0],"fee_rate":200}' \ + | jq -r .hex ) \ + && echo $FUNDED_RAW_TRNX + +4. View decoded trnx: + + $ b-reg decoderawtransaction $FUNDED_RAW_TRNX + +5. Generate ScriptPubKey from your public key: + + $ P2PK_SCRIPT_PUB_KEY=$( ./target/debug/gabriel \ + generate-script-pub-key-from-pub-key --pub-key=$TARGET_PUB_KEY ) \ + && b-reg decodescript $P2PK_SCRIPT_PUB_KEY + +6. TO-DO: TOTAL HACK : Swap output of funded trnx with P2PK: + + // https://learnmeabitcoin.com/technical/transaction/output/#scriptpubkey-size + // https://bitcointalk.org/index.php?topic=5465605.msg62794648#msg62794648 + + 020000000138597989d9eb741c551d3c5949ef47330dbfbad99e85ee1e4aad4a5bf752a5a80100000000fdffffff012531000000000000 + + 160014203e1c96fc3083329aaa12e4deafdcd621ffc856 //this part should be replaced + 00000000 + + P2PK_RAW_TRNX=020000000116206f68ec8b12b3c1d4b13e045cb3750191d490f7932e814ba29bf1d38177de0000000000fdffffff01109c052a010000002321033fac86cc916b4750c434641e86f08c50a43f3f83d0f1869ec51403833f57ae43ac00000000 + +7. View funded trnx w/ P2PK output: + + $ b-reg decoderawtransaction $P2PK_RAW_TRNX + +8. Sign trnx: + + $ SIGNED_P2PK_RAW_TRNX=$( b-reg signrawtransactionwithwallet $P2PK_RAW_TRNX \ + | jq -r .hex ) \ + && echo $SIGNED_P2PK_RAW_TRNX + +9. Send trnx: + + $ b-reg sendrawtransaction $SIGNED_P2PK_RAW_TRNX + +#### 3.3.2. Generate block: + + $ b-reg -generate 1 + + + +#### 3.3.3. Test TO-DO: generate a test P2PK address and send block rewards @@ -114,3 +180,4 @@ Add and edit the following to $PROJECT_HOME/.vscode/launch.json: ] } ````` + diff --git a/src/main.rs b/src/main.rs index 906ec50..3e4a042 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,7 @@ use std::{ }; use anyhow::{Ok, Result}; +use bitcoin::{PublicKey, ScriptBuf}; use block::{process_block, process_block_file, process_blocks_in_parallel, Record}; use clap::{Parser, Subcommand}; use nom::AsBytes; @@ -31,8 +32,17 @@ enum Commands { BlockAsyncEval(BlockAsyncEvalArgs), Index(IndexArgs), Graph(GraphArgs), + GenerateScriptPubKeyFromPubKey(GenerateScriptPubKeyFromPubKeyArgs), } +#[derive(Parser, Debug)] +struct GenerateScriptPubKeyFromPubKeyArgs { + #[arg(short, long)] + pub_key: String +} + + + #[derive(Parser, Debug)] struct BlockFileEvalArgs { /// Bitcoin directory path @@ -79,10 +89,19 @@ async fn main() -> Result<()> { Commands::BlockFileEval(args) => run_block_file_eval(args), Commands::Index(args) => run_index(args), Commands::Graph(args) => run_graph(args), + Commands::GenerateScriptPubKeyFromPubKey(args) => generate_script_pub_key_from_pub_key(args), Commands::BlockAsyncEval(args) => run_async_block_eval_listener(args).await } } +fn generate_script_pub_key_from_pub_key(args: &GenerateScriptPubKeyFromPubKeyArgs) -> Result<()> { + let pub_key_string = &args.pub_key; + let pubkey = pub_key_string.parse::().unwrap(); + let p2pk = ScriptBuf::new_p2pk(&pubkey); + println!("{}", p2pk.to_hex_string()); + Ok(()) +} + fn append_to_output(mut file: &File, result_map: &ResultMap) -> Result<()> { let result_map_read = result_map.read().unwrap(); for (_key, record) in result_map_read.iter() { From ee77f8adce31df184c2e7c55887ee4f5220a19de Mon Sep 17 00:00:00 2001 From: jbride Date: Mon, 21 Oct 2024 08:21:00 -0600 Subject: [PATCH 03/13] p2pk_trnx: now invoking bitcoind so as to automate testing --- Cargo.lock | 143 ++++++++++++++++------- Cargo.toml | 1 + README.md | 87 ++++++-------- src/main.rs | 52 ++++++--- src/p2pktrnx.rs | 303 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 473 insertions(+), 113 deletions(-) create mode 100644 src/p2pktrnx.rs diff --git a/Cargo.lock b/Cargo.lock index cf7477a..d7c818e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,9 +92,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.88" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e1496f8fb1fbf272686b8d37f523dab3e4a7443300055e74cdaa449f3114356" +checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" [[package]] name = "arrayvec" @@ -128,9 +128,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "backtrace" @@ -157,6 +157,12 @@ dependencies = [ "bitcoin_hashes", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bech32" version = "0.11.0" @@ -165,9 +171,9 @@ checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" [[package]] name = "bitcoin" -version = "0.32.2" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea507acc1cd80fc084ace38544bbcf7ced7c2aa65b653b102de0ce718df668f6" +checksum = "0032b0e8ead7074cda7fc4f034409607e3f03a6f71d66ade8a307f79b4d99e73" dependencies = [ "base58ck", "bech32", @@ -217,6 +223,30 @@ dependencies = [ "serde", ] +[[package]] +name = "bitcoincore-rpc" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee" +dependencies = [ + "bitcoincore-rpc-json", + "jsonrpc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bitcoincore-rpc-json" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a" +dependencies = [ + "bitcoin", + "serde", + "serde_json", +] + [[package]] name = "bitflags" version = "2.6.0" @@ -252,9 +282,9 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.18" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" dependencies = [ "shlex", ] @@ -281,9 +311,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -291,9 +321,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -303,9 +333,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", @@ -500,6 +530,7 @@ version = "0.2.0" dependencies = [ "anyhow", "bitcoin", + "bitcoincore-rpc", "byteorder", "chrono", "clap", @@ -583,9 +614,9 @@ checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -640,13 +671,25 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonrpc" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf" +dependencies = [ + "base64", + "minreq", + "serde", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -655,9 +698,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "lock_api" @@ -696,6 +739,17 @@ dependencies = [ "adler2", ] +[[package]] +name = "minreq" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0" +dependencies = [ + "log", + "serde", + "serde_json", +] + [[package]] name = "mio" version = "1.0.2" @@ -744,9 +798,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "parking_lot" @@ -785,9 +839,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "ppv-lite86" @@ -800,9 +854,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9" dependencies = [ "unicode-ident", ] @@ -929,6 +983,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" dependencies = [ "bitcoin_hashes", + "rand", "secp256k1-sys", "serde", ] @@ -964,9 +1019,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -1033,9 +1088,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.77" +version = "2.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "e6e185e337f816bc8da115b8afcb3324006ccc82eeaddf35113888d3bd8e44ac" dependencies = [ "proc-macro2", "quote", @@ -1119,9 +1174,9 @@ checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "utf8parse" @@ -1131,9 +1186,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", ] @@ -1152,9 +1207,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -1163,9 +1218,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", @@ -1178,9 +1233,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1188,9 +1243,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", @@ -1201,9 +1256,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "windows-core" diff --git a/Cargo.toml b/Cargo.toml index cb8880c..3923c05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,3 +22,4 @@ sha2 = "0.10" log = "0.4.22" zeromq = "0.4.1" tokio = "1.40.0" +bitcoincore-rpc = "0.19.0" diff --git a/README.md b/README.md index 46fe949..bd0c6b4 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ gabriel - [3.1. analyze single block data file](#31-analyze-single-block-data-file) - [3.2. analyze all block data files](#32-analyze-all-block-data-files) - [3.3. consume and analyze new raw blocks](#33-consume-and-analyze-new-raw-blocks) - - [3.3.1. Fund a P2PK address on reg-test](#331-fund-a-p2pk-address-on-reg-test) + - [3.3.1. Fund a trnx w/ a P2PK output on reg-test](#331-fund-a-trnx-w-a-p2pk-output-on-reg-test) - [3.3.2. Generate block:](#332-generate-block) - [3.3.3. Test](#333-test) - [4. Debug in VSCode:](#4-debug-in-vscode) @@ -26,7 +26,6 @@ Measures how many unspent public key addresses there are, and how many coins are ## 2. Setup ### 2.1. Pre-reqs - ``` $ bitcoind \ -conf=$GITEA_HOME/blockchain/bitcoin/admin/bitcoind/bitcoin.conf \ @@ -43,6 +42,29 @@ The best way to install Rust is to use [rustup](https://rustup.rs). If on bitcoind v28.0, ensure the following flag is set prior to initial block download: `-blocksxor=0` +1. Start Bitcoin Core in Regtest mode, for example: + + + $ bitcoind \ + -regtest \ + -server -daemon \ + -fallbackfee=0.0002 \ + -rpcuser=admin -rpcpassword=pass -rpcallowip=127.0.0.1/0 -rpcbind=127.0.0.1 \ + -blockfilterindex=1 -peerblockfilters=1 \ + -blocksxor=0 + +2. Define a shell alias to `bitcoin-cli`, for example: + + $ `alias b-reg=bitcoin-cli -rpcuser=admin -rpcpassword=pass -rpcport=18443` + +3. Create (or load) a default wallet, for example: + + $ `b-reg createwallet ` + +4. Mine some blocks, for example: + + $ `b-reg generatetoaddress 110 $(b-reg getnewaddress)` + ### 2.2. Clone code ``` @@ -92,61 +114,20 @@ $ cargo test --zmqpubrawblock-socket-url tcp://127.0.0.1:29001 \ --output /tmp/async_blocks.txt -#### 3.3.1. Fund a P2PK address on reg-test +#### 3.3.1. Fund a trnx w/ a P2PK output on reg-test -NOTE: FOr this exercise, start w/ a fresh reg-test environment (with no blocks yet generated) +1. Get extended private key: -1. On reg-test, create a new address and corresponding public key: - - $ TARGET_ADDR=$( b-reg getnewaddress ) \ - && echo $TARGET_ADDR \ - && TARGET_PUB_KEY=$( b-reg getaddressinfo $TARGET_ADDR | jq -r .pubkey ) \ - && echo $TARGET_PUB_KEY - -2. Create an initial raw trnx: - - $ INITIAL_RAW_TRNX=$( b-reg createrawtransaction "[]" "[{\"$TARGET_ADDR\":49.99971800}]" 0 true ) \ - && echo $INITIAL_RAW_TRNX + $ export W_NAME=lightning && export WPASS=lightning + $ b-reg -rpcwallet=$W_NAME walletpassphrase $WPASS 120 + $ XPRV=$( b-reg gethdkeys '{"active_only":true, "private":true}' | jq -r .[].xprv ) && echo $XPRV -3. Fund initial trnx: +2. Create a trnx w/ P2PK output: - $ FUNDED_RAW_TRNX=$( b-reg fundrawtransaction $INITIAL_RAW_TRNX '{"subtractFeeFromOutputs":[0],"fee_rate":200}' \ - | jq -r .hex ) \ - && echo $FUNDED_RAW_TRNX - -4. View decoded trnx: - - $ b-reg decoderawtransaction $FUNDED_RAW_TRNX - -5. Generate ScriptPubKey from your public key: - - $ P2PK_SCRIPT_PUB_KEY=$( ./target/debug/gabriel \ - generate-script-pub-key-from-pub-key --pub-key=$TARGET_PUB_KEY ) \ - && b-reg decodescript $P2PK_SCRIPT_PUB_KEY - -6. TO-DO: TOTAL HACK : Swap output of funded trnx with P2PK: - - // https://learnmeabitcoin.com/technical/transaction/output/#scriptpubkey-size - // https://bitcointalk.org/index.php?topic=5465605.msg62794648#msg62794648 - - 020000000138597989d9eb741c551d3c5949ef47330dbfbad99e85ee1e4aad4a5bf752a5a80100000000fdffffff012531000000000000 - - 160014203e1c96fc3083329aaa12e4deafdcd621ffc856 //this part should be replaced - 00000000 - - P2PK_RAW_TRNX=020000000116206f68ec8b12b3c1d4b13e045cb3750191d490f7932e814ba29bf1d38177de0000000000fdffffff01109c052a010000002321033fac86cc916b4750c434641e86f08c50a43f3f83d0f1869ec51403833f57ae43ac00000000 - -7. View funded trnx w/ P2PK output: - - $ b-reg decoderawtransaction $P2PK_RAW_TRNX - -8. Sign trnx: - - $ SIGNED_P2PK_RAW_TRNX=$( b-reg signrawtransactionwithwallet $P2PK_RAW_TRNX \ - | jq -r .hex ) \ - && echo $SIGNED_P2PK_RAW_TRNX + $ export RUST_BACKTRACE=1 + $ SIGNED_P2PK_RAW_TRNX=$( ./target/debug/gabriel generate-p2pk-trnx ) -9. Send trnx: +3. Send trnx: $ b-reg sendrawtransaction $SIGNED_P2PK_RAW_TRNX @@ -171,7 +152,7 @@ Add and edit the following to $PROJECT_HOME/.vscode/launch.json: { "type": "lldb", "request": "launch", - "name": "Debug gabriel local: 'block-file-eval'", + "name": "gabriel local: 'block-file-eval'", "args": ["block-file-eval", "-b=/u04/bitcoin/datadir/blocks/blk00000.dat", "-o=/tmp/blk00000.dat.csv"], "cwd": "${workspaceFolder}", "program": "./target/debug/gabriel", diff --git a/src/main.rs b/src/main.rs index 3e4a042..c07ae4a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,21 +1,23 @@ use std::{ - fs::{OpenOptions, File}, + fs::{File, OpenOptions}, io::{Read, Seek, Write}, - path::PathBuf, + path::PathBuf, str::FromStr, }; use anyhow::{Ok, Result}; -use bitcoin::{PublicKey, ScriptBuf}; +use bitcoin::{Amount, PublicKey, ScriptBuf}; use block::{process_block, process_block_file, process_blocks_in_parallel, Record}; use clap::{Parser, Subcommand}; use nom::AsBytes; use zeromq::{Socket, SocketRecv}; +mod p2pktrnx; mod block; mod tx; use block::{HeaderMap, ResultMap, TxMap}; use indicatif::ProgressBar; +use p2pktrnx::BitcoindRpcInfo; const HEADER: &str = "Height,Date,Total P2PK addresses,Total P2PK coins\n"; @@ -32,17 +34,24 @@ enum Commands { BlockAsyncEval(BlockAsyncEvalArgs), Index(IndexArgs), Graph(GraphArgs), - GenerateScriptPubKeyFromPubKey(GenerateScriptPubKeyFromPubKeyArgs), + GenerateP2PKTrnx(GenerateP2PKTrnxArgs), } #[derive(Parser, Debug)] -struct GenerateScriptPubKeyFromPubKeyArgs { - #[arg(short, long)] - pub_key: String +struct GenerateP2PKTrnxArgs { + + #[arg(long, default_value="http://127.0.0.1:18443")] + rpc_url: String, + #[arg(long, default_value="regtest")] + rpc_user_id: String, + #[arg(long, default_value="regtest")] + rpc_password: String, + #[arg(short, long, default_value="1.0 BTC")] + output_amount_btc: String, + #[arg(short, long, default_value="changeme")] + extended_master_private_key: String } - - #[derive(Parser, Debug)] struct BlockFileEvalArgs { /// Bitcoin directory path @@ -85,21 +94,31 @@ struct GraphArgs { async fn main() -> Result<()> { let cli = Cli::parse(); + match &cli.command { Commands::BlockFileEval(args) => run_block_file_eval(args), Commands::Index(args) => run_index(args), Commands::Graph(args) => run_graph(args), - Commands::GenerateScriptPubKeyFromPubKey(args) => generate_script_pub_key_from_pub_key(args), + Commands::GenerateP2PKTrnx(args) => generate_p2pk_trnx(args), Commands::BlockAsyncEval(args) => run_async_block_eval_listener(args).await } } -fn generate_script_pub_key_from_pub_key(args: &GenerateScriptPubKeyFromPubKeyArgs) -> Result<()> { - let pub_key_string = &args.pub_key; - let pubkey = pub_key_string.parse::().unwrap(); - let p2pk = ScriptBuf::new_p2pk(&pubkey); - println!("{}", p2pk.to_hex_string()); - Ok(()) +fn generate_p2pk_trnx(args: &GenerateP2PKTrnxArgs) -> Result<()> { + + let rpc_info = BitcoindRpcInfo{ + rpc_url: args.rpc_url.clone(), + rpc_user_id: args.rpc_user_id.clone(), + rpc_password: args.rpc_password.clone(), + }; + + let to_amount = Amount::from_str(&args.output_amount_btc)?; + let e_master_key = "tprv8ZgxMBicQKsPdvbo9jfEM6484s6KHpfX27HwiU6YriRAwxDYWRnNCEooQqveWQ4mBkicD2SFthXXUL4pB3vV9cpRfYidKU1LE1LHLXQDECQ"; + p2pktrnx::generate_p2pk_trnx( + e_master_key, + to_amount, + rpc_info + ) } fn append_to_output(mut file: &File, result_map: &ResultMap) -> Result<()> { @@ -198,6 +217,7 @@ async fn run_async_block_eval_listener(args: &BlockAsyncEvalArgs) -> Result<()> .create(true) .truncate(false) .open(&args.output)?; + file.seek(std::io::SeekFrom::End(0))?; loop { let zmq_message = socket.recv().await?; diff --git a/src/p2pktrnx.rs b/src/p2pktrnx.rs new file mode 100644 index 0000000..4ad5633 --- /dev/null +++ b/src/p2pktrnx.rs @@ -0,0 +1,303 @@ +//! Implements an example PSBT workflow. +//! +//! The workflow we simulate is that of a setup using a watch-only online wallet (contains only +//! public keys) and a cold-storage signing wallet (contains the private keys). +//! + +//! + +use std::collections::BTreeMap; +use std::fmt; +use std::str::FromStr; + +use anyhow::{Ok, Result, anyhow}; + +use bitcoin::bip32::{self, ChildNumber, DerivationPath, Fingerprint, IntoDerivationPath, Xpriv, Xpub}; +use bitcoin::consensus::encode; +use bitcoin::key::rand; +use bitcoin::locktime::absolute; +use bitcoin::psbt::{self, Input, Psbt, PsbtSighashType}; +use bitcoin::secp256k1::{Secp256k1, Signing, Verification}; +use bitcoin::{ + key, transaction, Address, Amount, CompressedPublicKey, Network, OutPoint, PrivateKey, PublicKey, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Witness +}; + +extern crate bitcoincore_rpc; +use bitcoincore_rpc::json::{self, GetAddressInfoResult, ListUnspentResultEntry}; +use bitcoincore_rpc::{Auth, Client, RpcApi}; + +const INPUT_UTXO_VOUT: u32 = 0; + +#[derive(Debug)] +pub struct BitcoindRpcInfo { + pub rpc_url: String, + pub rpc_user_id: String, + pub rpc_password: String +} + +pub fn generate_p2pk_trnx( + extended_master_private_key: &str, + output_amount: Amount, + rpc_info: BitcoindRpcInfo, + ) -> Result<()> { + + let secp = Secp256k1::new(); + let mut rng = rand::thread_rng(); + let (_, secp256k1_pubkey) = secp.generate_keypair(&mut rng); + let p2pk_pubkey = PublicKey::new(secp256k1_pubkey); + + let output_amount_btc = output_amount.to_btc(); + let results: (ListUnspentResultEntry, GetAddressInfoResult, Address, Amount) = get_bitcoind_info(output_amount_btc, rpc_info)?; + let unspent_trnx = results.0; + let input_utxo_address = results.1; + let change_addr = results.2; + let network_relay_fee = results.3; + + let input_utxo_derivation_path = input_utxo_address.hd_key_path.unwrap(); + let input_utxo_txid = unspent_trnx.txid.to_string(); + let input_utxo_script_pubkey = unspent_trnx.script_pub_key; + let input_utxo_value = unspent_trnx.amount; + + let (offline, fingerprint, account_0_xpub, input_xpub) = + ColdStorage::new(&secp, extended_master_private_key, &input_utxo_derivation_path)?; + + let online = WatchOnly::new(account_0_xpub, input_xpub, fingerprint); + + let created = online.create_psbt(&input_utxo_txid, &input_utxo_value, &output_amount, p2pk_pubkey,change_addr, network_relay_fee)?; + + let updated = online.update_psbt(created, input_utxo_script_pubkey, &input_utxo_derivation_path, &input_utxo_value)?; + + let signed = offline.sign_psbt(&secp, updated)?; + + let finalized = online.finalize_psbt(signed)?; + + // You can use `bt sendrawtransaction` to broadcast the extracted transaction. + let tx = finalized.extract_tx_unchecked_fee_rate(); + + let tx_hex = encode::serialize_hex(&tx); + //println!("You should now be able to broadcast the following transaction: \n\n{}", hex); + println!("{}", tx_hex); + + Ok(()) +} + +fn get_bitcoind_info(output_amount_btc: f64, rpc_info: BitcoindRpcInfo) -> Result<(ListUnspentResultEntry, GetAddressInfoResult, Address, Amount)> { + + let rpc = Client::new(&rpc_info.rpc_url, + Auth::UserPass(rpc_info.rpc_user_id.to_string(), + rpc_info.rpc_password.to_string())).unwrap(); + + let network_relay_fee = rpc.get_network_info()?.relay_fee; + let output_trnx_total = network_relay_fee.to_btc() + output_amount_btc; + + let mut unspent_option: Option = None; + let unspent_vec = rpc.list_unspent(Some(0), None, None, None, None).unwrap(); + for unspent_candidate in unspent_vec { + //println!("unspent_candidate txid={}, amount={}", unspent_candidate.txid, unspent_candidate.amount.to_btc()); + if unspent_candidate.amount.to_btc() > output_trnx_total { + unspent_option = Some(unspent_candidate); + break; + } + } + if unspent_option == None { + return Err(anyhow!("No unspent trnxs have sufficient funds: {}", output_trnx_total)); + } + + let unspent_trnx = unspent_option.unwrap(); + let input_utxo_address = unspent_trnx.address.clone().unwrap().assume_checked(); + + let input_utxo_address_info = rpc.get_address_info(&input_utxo_address)?; + + let change_addr = rpc.get_raw_change_address(Some(json::AddressType::Bech32)).unwrap().assume_checked(); + + Ok((unspent_trnx, input_utxo_address_info, change_addr, network_relay_fee)) +} + +// We cache the pubkeys for convenience because it requires a scep context to convert the private key. +/// An example of an offline signer i.e., a cold-storage device. +struct ColdStorage { + /// The master extended private key. + master_xpriv: Xpriv, + /// The master extended public key. + master_xpub: Xpub, +} + +/// The data exported from an offline wallet to enable creation of a watch-only online wallet. +/// (wallet, fingerprint, account_0_xpub, input_utxo_xpub) +type ExportData = (ColdStorage, Fingerprint, Xpub, Xpub); + +impl ColdStorage { + + /// Constructs a new `ColdStorage` signer. + /// + /// # Returns + /// The newly created signer along with the data needed to configure a watch-only wallet. + fn new(secp: &Secp256k1, xpriv: &str, input_utxo_derivation_path: &DerivationPath) -> Result { + let master_xpriv = Xpriv::from_str(xpriv)?; + let master_xpub = Xpub::from_priv(secp, &master_xpriv); + + // Hardened children require secret data to derive. + let account_0_xpriv = master_xpriv.derive_priv(secp, &input_utxo_derivation_path)?; + let account_0_xpub = Xpub::from_priv(secp, &account_0_xpriv); + + let input_xpriv = master_xpriv.derive_priv(secp, &input_utxo_derivation_path)?; + let input_xpub = Xpub::from_priv(secp, &input_xpriv); + + let wallet = ColdStorage { master_xpriv, master_xpub }; + let fingerprint = wallet.master_fingerprint(); + + Ok((wallet, fingerprint, account_0_xpub, input_xpub)) + } + + /// Returns the fingerprint for the master extended public key. + fn master_fingerprint(&self) -> Fingerprint { self.master_xpub.fingerprint() } + + /// Signs `psbt` with this signer. + fn sign_psbt( + &self, + secp: &Secp256k1, + mut psbt: Psbt, + ) -> Result { + match psbt.sign(&self.master_xpriv, secp) { + std::result::Result::Ok(keys) => assert_eq!(keys.len(), 1), + Err((_, e)) => { + let e = e.get(&0).expect("at least one error"); + return Err(e.clone().into()); + } + }; + Ok(psbt) + } +} + +/// An example of an watch-only online wallet. +struct WatchOnly { + /// The xpub for account 0 derived from derivation path "m/84h/0h/0h". + account_0_xpub: Xpub, + /// The xpub derived from `INPUT_UTXO_DERIVATION_PATH`. + input_xpub: Xpub, + /// The master extended pubkey fingerprint. + master_fingerprint: Fingerprint, +} + +impl WatchOnly { + /// Constructs a new watch-only wallet. + /// + /// A watch-only wallet would typically be online and connected to the Bitcoin network. We + /// 'import' into the wallet the `account_0_xpub` and `master_fingerprint`. + /// + /// The reason for importing the `input_xpub` is so one can use bitcoind to grab a valid input + /// to verify the workflow presented in this file. + fn new(account_0_xpub: Xpub, input_xpub: Xpub, master_fingerprint: Fingerprint) -> Self { + WatchOnly { account_0_xpub, input_xpub, master_fingerprint } + } + + /// Creates the PSBT, in BIP174 parlance this is the 'Creater'. + fn create_psbt( + &self, + input_utxo_txid: &str, + input_utxo_value: &Amount, + output_amount_btc: &Amount, + p2pk_pubkey: PublicKey, + change_address: Address, + network_relay_fee: Amount) -> Result { + + let output_change_total = input_utxo_value.to_btc() - network_relay_fee.to_btc() - output_amount_btc.to_btc(); + let change_amount_rounded = (output_change_total * 1000000.0).round() / 1000000.0; +/* println!("input_utxo_value={}, network_relay_fee={}, output_amount_btc={}, change_amount_rounded={}", + input_utxo_value.to_btc(), + network_relay_fee.to_btc(), + output_amount_btc.to_btc(), + change_amount_rounded + ); */ + let change_amount = Amount::from_float_in(change_amount_rounded, bitcoin::Denomination::Bitcoin)?; + //let change_amount: Amount = Amount::from_str("46.99999 BTC")?; // 1000 sat transaction fee. + + let p2pk_pubkey_scriptbuf = ScriptBuf::new_p2pk(&p2pk_pubkey); + + let tx = Transaction { + version: transaction::Version::TWO, + lock_time: absolute::LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint { txid: input_utxo_txid.parse()?, vout: INPUT_UTXO_VOUT }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, // Disable LockTime and RBF. + witness: Witness::default(), + }], + output: vec![ + TxOut { value: *output_amount_btc, script_pubkey: p2pk_pubkey_scriptbuf }, + TxOut { value: change_amount, script_pubkey: change_address.script_pubkey() } + ], + }; + + let psbt = Psbt::from_unsigned_tx(tx)?; + + Ok(psbt) + } + + /// Updates the PSBT, in BIP174 parlance this is the 'Updater'. + fn update_psbt(&self, + mut psbt: Psbt, + input_utxo_script_pubkey: ScriptBuf, + input_utxo_derivation_path: &DerivationPath, + input_utxo_value: &Amount) -> Result { + let t_out = TxOut { value: *input_utxo_value, script_pubkey: input_utxo_script_pubkey}; + let mut input = Input { witness_utxo: Some(t_out), ..Default::default() }; + + let pk = self.input_xpub.to_pub(); + let wpkh = pk.wpubkey_hash(); + + let redeem_script = ScriptBuf::new_p2wpkh(&wpkh); + input.redeem_script = Some(redeem_script); + + let fingerprint = self.master_fingerprint; + let mut map = BTreeMap::new(); + map.insert(pk.0, (fingerprint, input_utxo_derivation_path.clone())); + input.bip32_derivation = map; + + let ty = PsbtSighashType::from_str("SIGHASH_ALL")?; + input.sighash_type = Some(ty); + + psbt.inputs = vec![input]; + + Ok(psbt) + } + + /// Finalizes the PSBT, in BIP174 parlance this is the 'Finalizer'. + /// This is just an example. For a production-ready PSBT Finalizer, use [rust-miniscript](https://docs.rs/miniscript/latest/miniscript/psbt/trait.PsbtExt.html#tymethod.finalize) + fn finalize_psbt(&self, mut psbt: Psbt) -> Result { + if psbt.inputs.is_empty() { + return Err(psbt::SignError::MissingInputUtxo.into()); + } + + let sigs: Vec<_> = psbt.inputs[0].partial_sigs.values().collect(); + let mut script_witness: Witness = Witness::new(); + script_witness.push(&sigs[0].to_vec()); + script_witness.push(self.input_xpub.to_pub().to_bytes()); + psbt.inputs[0].final_script_witness = Some(script_witness); + + // Clear all the data fields as per the spec. + psbt.inputs[0].partial_sigs = BTreeMap::new(); + psbt.inputs[0].sighash_type = None; + psbt.inputs[0].redeem_script = None; + psbt.inputs[0].witness_script = None; + psbt.inputs[0].bip32_derivation = BTreeMap::new(); + + Ok(psbt) + } + +} + +fn input_derivation_path(input_utxo_derivation_path: &str) -> Result { + let path = input_utxo_derivation_path.into_derivation_path()?; + Ok(path) +} + +struct Error(Box); + +impl From for Error { + fn from(e: T) -> Self { Error(Box::new(e)) } +} + +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&self.0, f) } +} From abad91d6104dd8c9eea30e45aac9cde62952676f Mon Sep 17 00:00:00 2001 From: jbride Date: Tue, 22 Oct 2024 15:19:56 -0600 Subject: [PATCH 04/13] s/trnx/tx/g --- README.md | 109 ++++++++++++++++++++------------- src/main.rs | 18 +++--- src/{p2pktrnx.rs => p2pktx.rs} | 24 ++++---- 3 files changed, 90 insertions(+), 61 deletions(-) rename src/{p2pktrnx.rs => p2pktx.rs} (93%) diff --git a/README.md b/README.md index bd0c6b4..d6bec8c 100644 --- a/README.md +++ b/README.md @@ -7,16 +7,16 @@ gabriel - [2.1.2. Software](#212-software) - [2.1.2.1. Rust](#2121-rust) - [2.1.2.2. bitcoind](#2122-bitcoind) - - [2.2. Clone code](#22-clone-code) + - [2.2. Clone](#22-clone) - [2.3. Build](#23-build) - [2.4. Execute tests](#24-execute-tests) - [3. Run Gabriel](#3-run-gabriel) - - [3.1. analyze single block data file](#31-analyze-single-block-data-file) - [3.2. analyze all block data files](#32-analyze-all-block-data-files) - - [3.3. consume and analyze new raw blocks](#33-consume-and-analyze-new-raw-blocks) - - [3.3.1. Fund a trnx w/ a P2PK output on reg-test](#331-fund-a-trnx-w-a-p2pk-output-on-reg-test) + - [3.1. analyze single block data file](#31-analyze-single-block-data-file) + - [Optional: debug via VSCode:](#optional--debug-via-vscode) + - [3.3. consume and analyze new raw block events](#33-consume-and-analyze-new-raw-block-events) + - [3.3.1. Fund a tx w/ a P2PK output on reg-test](#331-fund-a-tx-w-a-p2pk-output-on-reg-test) - [3.3.2. Generate block:](#332-generate-block) - - [3.3.3. Test](#333-test) - [4. Debug in VSCode:](#4-debug-in-vscode) @@ -65,7 +65,7 @@ If on bitcoind v28.0, ensure the following flag is set prior to initial block do $ `b-reg generatetoaddress 110 $(b-reg getnewaddress)` -### 2.2. Clone code +### 2.2. Clone ``` $ git clone https://github.com/SurmountSystems/gabriel.git @@ -91,8 +91,20 @@ $ cargo test ## 3. Run Gabriel +### 3.2. analyze all block data files + +Execute the following if analyzing the entire (previously downloaded) Bitcoin blockchain: + + $ export BITCOIND_DATA_DIR=/path/to/bitcoind/data/dir + $ ./target/debug/gabriel index \ + --input $BITCOIND_DATA_DIR/blocks \ + --output /tmp/gabriel-testnet4.csv + ### 3.1. analyze single block data file +Alternatively, you can have (likely for testing purposes) Gabriel analyze a single Bitcoin Core block data file. +Execute as follows: + $ export BITCOIND_DATA_DIR=/path/to/bitcoind/data/dir $ export BITCOIND_BLOCK_DATA_FILE=xxx.dat @@ -100,65 +112,80 @@ $ cargo test -b $BITCOIND_DATA_DIR/blocks/$BITCOIND_BLOCK_DATA_FILE \ -o /tmp/$BITCOIND_BLOCK_DATA_FILE.csv +#### Optional: debug via VSCode: -### 3.2. analyze all block data files - - $ export BITCOIND_DATA_DIR=/path/to/bitcoind/data/dir - $ ./target/debug/gabriel index \ - --input $BITCOIND_DATA_DIR/blocks \ - --output /tmp/gabriel-testnet4.csv +Modify the following as appropriate and add to your vscode `launch.json`: + + { + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "gabriel local: 'block-file-eval'", + "args": ["block-file-eval", "-b=/tmp/.dat", "-o=/tmp/.dat.csv"], + "cwd": "${workspaceFolder}", + "program": "./target/debug/gabriel", + "sourceLanguages": ["rust"] + } + ] + } -### 3.3. consume and analyze new raw blocks +### 3.3. consume and analyze new raw block events $ ./target/debug/gabriel block-async-eval \ --zmqpubrawblock-socket-url tcp://127.0.0.1:29001 \ --output /tmp/async_blocks.txt -#### 3.3.1. Fund a trnx w/ a P2PK output on reg-test +#### 3.3.1. Fund a tx w/ a P2PK output on reg-test -1. Get extended private key: +If interested in testing Gabriel's ability to consume and process a block with a P2PK utxo, you can use the following in a new terminal: - $ export W_NAME=lightning && export WPASS=lightning - $ b-reg -rpcwallet=$W_NAME walletpassphrase $WPASS 120 - $ XPRV=$( b-reg gethdkeys '{"active_only":true, "private":true}' | jq -r .[].xprv ) && echo $XPRV +1. Get extended private key from bitcoind:\ + + NOTE: for the following command, you'll already need to have unlocked your wallet via the bitcoin cli. + + $ XPRV=$( b-reg gethdkeys '{"active_only":true, "private":true}' \ + | jq -r .[].xprv ) && echo $XPRV + +2. Create a tx w/ P2PK output: + + $ SIGNED_P2PK_RAW_TX=$( ./target/debug/gabriel \ + generate-p2pk-tx \ + -e $XPRV ) \ + && echo $SIGNED_P2PK_RAW_TX -2. Create a trnx w/ P2PK output: +3. View decoded tx: - $ export RUST_BACKTRACE=1 - $ SIGNED_P2PK_RAW_TRNX=$( ./target/debug/gabriel generate-p2pk-trnx ) + $ b-reg decoderawtransaction $SIGNED_P2PK_RAW_TX -3. Send trnx: +4. Send tx: - $ b-reg sendrawtransaction $SIGNED_P2PK_RAW_TRNX + $ b-reg sendrawtransaction $SIGNED_P2PK_RAW_TX #### 3.3.2. Generate block: $ b-reg -generate 1 + NOTE: You should now see a new record in Gabriel's output file indicating the new P2PK utxo. -#### 3.3.3. Test - -TO-DO: generate a test P2PK address and send block rewards - ## 4. Debug in VSCode: Add and edit the following to $PROJECT_HOME/.vscode/launch.json: -````` -{ - "version": "0.2.0", - "configurations": [ { - "type": "lldb", - "request": "launch", - "name": "gabriel local: 'block-file-eval'", - "args": ["block-file-eval", "-b=/u04/bitcoin/datadir/blocks/blk00000.dat", "-o=/tmp/blk00000.dat.csv"], - "cwd": "${workspaceFolder}", - "program": "./target/debug/gabriel", - "sourceLanguages": ["rust"] + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "gabriel local: 'generate-p2pk-trnx'", + "args": ["generate-p2pk-trnx", "-e=$XPRV-CHANGEME"], + "cwd": "${workspaceFolder}", + "program": "./target/debug/gabriel", + "sourceLanguages": ["rust"] + } + ] } - ] -} -````` diff --git a/src/main.rs b/src/main.rs index c07ae4a..4b8def2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,13 +11,13 @@ use clap::{Parser, Subcommand}; use nom::AsBytes; use zeromq::{Socket, SocketRecv}; -mod p2pktrnx; +mod p2pktx; mod block; mod tx; use block::{HeaderMap, ResultMap, TxMap}; use indicatif::ProgressBar; -use p2pktrnx::BitcoindRpcInfo; +use p2pktx::BitcoindRpcInfo; const HEADER: &str = "Height,Date,Total P2PK addresses,Total P2PK coins\n"; @@ -34,11 +34,11 @@ enum Commands { BlockAsyncEval(BlockAsyncEvalArgs), Index(IndexArgs), Graph(GraphArgs), - GenerateP2PKTrnx(GenerateP2PKTrnxArgs), + GenerateP2PKTx(GenerateP2PKTxArgs), } #[derive(Parser, Debug)] -struct GenerateP2PKTrnxArgs { +struct GenerateP2PKTxArgs { #[arg(long, default_value="http://127.0.0.1:18443")] rpc_url: String, @@ -48,7 +48,7 @@ struct GenerateP2PKTrnxArgs { rpc_password: String, #[arg(short, long, default_value="1.0 BTC")] output_amount_btc: String, - #[arg(short, long, default_value="changeme")] + #[arg(short, long)] extended_master_private_key: String } @@ -99,12 +99,12 @@ async fn main() -> Result<()> { Commands::BlockFileEval(args) => run_block_file_eval(args), Commands::Index(args) => run_index(args), Commands::Graph(args) => run_graph(args), - Commands::GenerateP2PKTrnx(args) => generate_p2pk_trnx(args), + Commands::GenerateP2PKTx(args) => generate_p2pk_tx(args), Commands::BlockAsyncEval(args) => run_async_block_eval_listener(args).await } } -fn generate_p2pk_trnx(args: &GenerateP2PKTrnxArgs) -> Result<()> { +fn generate_p2pk_tx(args: &GenerateP2PKTxArgs) -> Result<()> { let rpc_info = BitcoindRpcInfo{ rpc_url: args.rpc_url.clone(), @@ -113,8 +113,8 @@ fn generate_p2pk_trnx(args: &GenerateP2PKTrnxArgs) -> Result<()> { }; let to_amount = Amount::from_str(&args.output_amount_btc)?; - let e_master_key = "tprv8ZgxMBicQKsPdvbo9jfEM6484s6KHpfX27HwiU6YriRAwxDYWRnNCEooQqveWQ4mBkicD2SFthXXUL4pB3vV9cpRfYidKU1LE1LHLXQDECQ"; - p2pktrnx::generate_p2pk_trnx( + let e_master_key = &args.extended_master_private_key; + p2pktx::generate_p2pk_tx( e_master_key, to_amount, rpc_info diff --git a/src/p2pktrnx.rs b/src/p2pktx.rs similarity index 93% rename from src/p2pktrnx.rs rename to src/p2pktx.rs index 4ad5633..ab80bcf 100644 --- a/src/p2pktrnx.rs +++ b/src/p2pktx.rs @@ -35,7 +35,7 @@ pub struct BitcoindRpcInfo { pub rpc_password: String } -pub fn generate_p2pk_trnx( +pub fn generate_p2pk_tx( extended_master_private_key: &str, output_amount: Amount, rpc_info: BitcoindRpcInfo, @@ -48,15 +48,17 @@ pub fn generate_p2pk_trnx( let output_amount_btc = output_amount.to_btc(); let results: (ListUnspentResultEntry, GetAddressInfoResult, Address, Amount) = get_bitcoind_info(output_amount_btc, rpc_info)?; - let unspent_trnx = results.0; + let unspent_tx = results.0; let input_utxo_address = results.1; let change_addr = results.2; let network_relay_fee = results.3; let input_utxo_derivation_path = input_utxo_address.hd_key_path.unwrap(); - let input_utxo_txid = unspent_trnx.txid.to_string(); - let input_utxo_script_pubkey = unspent_trnx.script_pub_key; - let input_utxo_value = unspent_trnx.amount; + let input_utxo_xkey_identifier = input_utxo_address.hd_seed_id.unwrap(); + + let input_utxo_txid = unspent_tx.txid.to_string(); + let input_utxo_script_pubkey = unspent_tx.script_pub_key; + let input_utxo_value = unspent_tx.amount; let (offline, fingerprint, account_0_xpub, input_xpub) = ColdStorage::new(&secp, extended_master_private_key, &input_utxo_derivation_path)?; @@ -88,29 +90,29 @@ fn get_bitcoind_info(output_amount_btc: f64, rpc_info: BitcoindRpcInfo) -> Resul rpc_info.rpc_password.to_string())).unwrap(); let network_relay_fee = rpc.get_network_info()?.relay_fee; - let output_trnx_total = network_relay_fee.to_btc() + output_amount_btc; + let output_tx_total = network_relay_fee.to_btc() + output_amount_btc; let mut unspent_option: Option = None; let unspent_vec = rpc.list_unspent(Some(0), None, None, None, None).unwrap(); for unspent_candidate in unspent_vec { //println!("unspent_candidate txid={}, amount={}", unspent_candidate.txid, unspent_candidate.amount.to_btc()); - if unspent_candidate.amount.to_btc() > output_trnx_total { + if unspent_candidate.amount.to_btc() > output_tx_total { unspent_option = Some(unspent_candidate); break; } } if unspent_option == None { - return Err(anyhow!("No unspent trnxs have sufficient funds: {}", output_trnx_total)); + return Err(anyhow!("No unspent txs have sufficient funds: {}", output_tx_total)); } - let unspent_trnx = unspent_option.unwrap(); - let input_utxo_address = unspent_trnx.address.clone().unwrap().assume_checked(); + let unspent_tx = unspent_option.unwrap(); + let input_utxo_address = unspent_tx.address.clone().unwrap().assume_checked(); let input_utxo_address_info = rpc.get_address_info(&input_utxo_address)?; let change_addr = rpc.get_raw_change_address(Some(json::AddressType::Bech32)).unwrap().assume_checked(); - Ok((unspent_trnx, input_utxo_address_info, change_addr, network_relay_fee)) + Ok((unspent_tx, input_utxo_address_info, change_addr, network_relay_fee)) } // We cache the pubkeys for convenience because it requires a scep context to convert the private key. From 4e0adcb2894929314a8c24791f0db80ade97fa48 Mon Sep 17 00:00:00 2001 From: jbride Date: Tue, 22 Oct 2024 20:26:04 -0600 Subject: [PATCH 05/13] Now using bitcoin RPC cookie (as opposed to userId/password) --- README.md | 3 ++ src/main.rs | 15 +------- src/p2pktx.rs | 96 +++++++++++++++++++++++++++++---------------------- 3 files changed, 59 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index d6bec8c..a349c46 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,9 @@ If interested in testing Gabriel's ability to consume and process a block with a 2. Create a tx w/ P2PK output: + $ export URL=http://127.0.0.1:18443 \ + && export COOKIE=/path/to/bitcoind/datadir/regtest/.cookie + $ SIGNED_P2PK_RAW_TX=$( ./target/debug/gabriel \ generate-p2pk-tx \ -e $XPRV ) \ diff --git a/src/main.rs b/src/main.rs index 4b8def2..b236ccc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,12 +40,6 @@ enum Commands { #[derive(Parser, Debug)] struct GenerateP2PKTxArgs { - #[arg(long, default_value="http://127.0.0.1:18443")] - rpc_url: String, - #[arg(long, default_value="regtest")] - rpc_user_id: String, - #[arg(long, default_value="regtest")] - rpc_password: String, #[arg(short, long, default_value="1.0 BTC")] output_amount_btc: String, #[arg(short, long)] @@ -106,18 +100,11 @@ async fn main() -> Result<()> { fn generate_p2pk_tx(args: &GenerateP2PKTxArgs) -> Result<()> { - let rpc_info = BitcoindRpcInfo{ - rpc_url: args.rpc_url.clone(), - rpc_user_id: args.rpc_user_id.clone(), - rpc_password: args.rpc_password.clone(), - }; - let to_amount = Amount::from_str(&args.output_amount_btc)?; let e_master_key = &args.extended_master_private_key; p2pktx::generate_p2pk_tx( e_master_key, - to_amount, - rpc_info + to_amount ) } diff --git a/src/p2pktx.rs b/src/p2pktx.rs index ab80bcf..85e4902 100644 --- a/src/p2pktx.rs +++ b/src/p2pktx.rs @@ -7,10 +7,10 @@ //! use std::collections::BTreeMap; -use std::fmt; +use std::{env, fmt}; use std::str::FromStr; -use anyhow::{Ok, Result, anyhow}; +use anyhow::{Result, anyhow}; use bitcoin::bip32::{self, ChildNumber, DerivationPath, Fingerprint, IntoDerivationPath, Xpriv, Xpub}; use bitcoin::consensus::encode; @@ -30,24 +30,70 @@ const INPUT_UTXO_VOUT: u32 = 0; #[derive(Debug)] pub struct BitcoindRpcInfo { - pub rpc_url: String, - pub rpc_user_id: String, - pub rpc_password: String + rpc_client: Client +} + +impl BitcoindRpcInfo { + + pub fn new() -> Result { + let url = env::var("URL")?; + let cookie = env::var("COOKIE"); + let auth = match cookie { + Ok(cookiefile) => Auth::CookieFile(cookiefile.into()), + Err(_) => { + let user = env::var("USER")?; + let pass = env::var("PASS")?; + + Auth::UserPass(user, pass) + } + }; + let rpc_client = Client::new(&url, auth)?; + Ok(BitcoindRpcInfo{rpc_client}) + } + + pub fn get_bitcoind_info(&self, output_amount_btc: f64) -> Result<(ListUnspentResultEntry, GetAddressInfoResult, Address, Amount)> { + + let network_relay_fee = self.rpc_client.get_network_info()?.relay_fee; + let output_tx_total = network_relay_fee.to_btc() + output_amount_btc; + + let mut unspent_option: Option = None; + let unspent_vec = self.rpc_client.list_unspent(Some(0), None, None, None, None).unwrap(); + for unspent_candidate in unspent_vec { + //println!("unspent_candidate txid={}, amount={}", unspent_candidate.txid, unspent_candidate.amount.to_btc()); + if unspent_candidate.amount.to_btc() > output_tx_total { + unspent_option = Some(unspent_candidate); + break; + } + } + if unspent_option == None { + return Err(anyhow!("No unspent txs have sufficient funds: {}", output_tx_total)); + } + + let unspent_tx = unspent_option.unwrap(); + let input_utxo_address = unspent_tx.address.clone().unwrap().assume_checked(); + + let input_utxo_address_info = self.rpc_client.get_address_info(&input_utxo_address)?; + + let change_addr = self.rpc_client.get_raw_change_address(Some(json::AddressType::Bech32)).unwrap().assume_checked(); + + Ok((unspent_tx, input_utxo_address_info, change_addr, network_relay_fee)) + } + } pub fn generate_p2pk_tx( extended_master_private_key: &str, - output_amount: Amount, - rpc_info: BitcoindRpcInfo, + output_amount: Amount ) -> Result<()> { let secp = Secp256k1::new(); let mut rng = rand::thread_rng(); let (_, secp256k1_pubkey) = secp.generate_keypair(&mut rng); let p2pk_pubkey = PublicKey::new(secp256k1_pubkey); - let output_amount_btc = output_amount.to_btc(); - let results: (ListUnspentResultEntry, GetAddressInfoResult, Address, Amount) = get_bitcoind_info(output_amount_btc, rpc_info)?; + + let bitcoind_info = BitcoindRpcInfo::new()?; + let results: (ListUnspentResultEntry, GetAddressInfoResult, Address, Amount) = bitcoind_info.get_bitcoind_info(output_amount_btc)?; let unspent_tx = results.0; let input_utxo_address = results.1; let change_addr = results.2; @@ -83,38 +129,6 @@ pub fn generate_p2pk_tx( Ok(()) } -fn get_bitcoind_info(output_amount_btc: f64, rpc_info: BitcoindRpcInfo) -> Result<(ListUnspentResultEntry, GetAddressInfoResult, Address, Amount)> { - - let rpc = Client::new(&rpc_info.rpc_url, - Auth::UserPass(rpc_info.rpc_user_id.to_string(), - rpc_info.rpc_password.to_string())).unwrap(); - - let network_relay_fee = rpc.get_network_info()?.relay_fee; - let output_tx_total = network_relay_fee.to_btc() + output_amount_btc; - - let mut unspent_option: Option = None; - let unspent_vec = rpc.list_unspent(Some(0), None, None, None, None).unwrap(); - for unspent_candidate in unspent_vec { - //println!("unspent_candidate txid={}, amount={}", unspent_candidate.txid, unspent_candidate.amount.to_btc()); - if unspent_candidate.amount.to_btc() > output_tx_total { - unspent_option = Some(unspent_candidate); - break; - } - } - if unspent_option == None { - return Err(anyhow!("No unspent txs have sufficient funds: {}", output_tx_total)); - } - - let unspent_tx = unspent_option.unwrap(); - let input_utxo_address = unspent_tx.address.clone().unwrap().assume_checked(); - - let input_utxo_address_info = rpc.get_address_info(&input_utxo_address)?; - - let change_addr = rpc.get_raw_change_address(Some(json::AddressType::Bech32)).unwrap().assume_checked(); - - Ok((unspent_tx, input_utxo_address_info, change_addr, network_relay_fee)) -} - // We cache the pubkeys for convenience because it requires a scep context to convert the private key. /// An example of an offline signer i.e., a cold-storage device. struct ColdStorage { From 4696b085598daea735a84cb6ec2c1a804d4d4dba Mon Sep 17 00:00:00 2001 From: jbride Date: Wed, 23 Oct 2024 06:25:20 -0600 Subject: [PATCH 06/13] generate_p2pk_tx: network fee now subtracted from p2pk utxo --- src/p2pktx.rs | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/p2pktx.rs b/src/p2pktx.rs index 85e4902..9422522 100644 --- a/src/p2pktx.rs +++ b/src/p2pktx.rs @@ -57,9 +57,14 @@ impl BitcoindRpcInfo { let output_tx_total = network_relay_fee.to_btc() + output_amount_btc; let mut unspent_option: Option = None; - let unspent_vec = self.rpc_client.list_unspent(Some(0), None, None, None, None).unwrap(); + let unspent_vec = self.rpc_client.list_unspent(Some(3), None, None, None, None).unwrap(); for unspent_candidate in unspent_vec { - //println!("unspent_candidate txid={}, amount={}", unspent_candidate.txid, unspent_candidate.amount.to_btc()); + println!("unspent_candidate txid={}, vout={}, tx_amount={}, output_tx_total={}", + unspent_candidate.txid, + unspent_candidate.vout, + unspent_candidate.amount.to_btc(), + output_tx_total + ); if unspent_candidate.amount.to_btc() > output_tx_total { unspent_option = Some(unspent_candidate); break; @@ -217,16 +222,21 @@ impl WatchOnly { change_address: Address, network_relay_fee: Amount) -> Result { - let output_change_total = input_utxo_value.to_btc() - network_relay_fee.to_btc() - output_amount_btc.to_btc(); - let change_amount_rounded = (output_change_total * 1000000.0).round() / 1000000.0; + // network fee subtracted from payment total. Similar to specifying "subtractFeeFromOutputs" argument with "fundrawtransaction" CLI + let payment_float = output_amount_btc.to_btc() - network_relay_fee.to_btc(); + let payment_total = Amount::from_float_in(payment_float, bitcoin::Denomination::Bitcoin)?; + + + let change_float = input_utxo_value.to_btc() - output_amount_btc.to_btc(); + let change_rounded = (change_float * 1000000.0).round() / 1000000.0; /* println!("input_utxo_value={}, network_relay_fee={}, output_amount_btc={}, change_amount_rounded={}", input_utxo_value.to_btc(), network_relay_fee.to_btc(), output_amount_btc.to_btc(), change_amount_rounded ); */ - let change_amount = Amount::from_float_in(change_amount_rounded, bitcoin::Denomination::Bitcoin)?; - //let change_amount: Amount = Amount::from_str("46.99999 BTC")?; // 1000 sat transaction fee. + let change_total = Amount::from_float_in(change_rounded, bitcoin::Denomination::Bitcoin)?; + let p2pk_pubkey_scriptbuf = ScriptBuf::new_p2pk(&p2pk_pubkey); @@ -240,8 +250,8 @@ impl WatchOnly { witness: Witness::default(), }], output: vec![ - TxOut { value: *output_amount_btc, script_pubkey: p2pk_pubkey_scriptbuf }, - TxOut { value: change_amount, script_pubkey: change_address.script_pubkey() } + TxOut { value: payment_total, script_pubkey: p2pk_pubkey_scriptbuf }, + TxOut { value: change_total, script_pubkey: change_address.script_pubkey() } ], }; From d1dbcc8c588622a1c4fcdaceec50cbce489610c2 Mon Sep 17 00:00:00 2001 From: jbride Date: Wed, 23 Oct 2024 14:37:38 -0600 Subject: [PATCH 07/13] Documentation regarding use of ZeroMQ unix domain sockets. (As alternative to TCP sockets). --- README.md | 78 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index a349c46..8581d2d 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,13 @@ gabriel - [2.3. Build](#23-build) - [2.4. Execute tests](#24-execute-tests) - [3. Run Gabriel](#3-run-gabriel) - - [3.2. analyze all block data files](#32-analyze-all-block-data-files) - - [3.1. analyze single block data file](#31-analyze-single-block-data-file) - - [Optional: debug via VSCode:](#optional--debug-via-vscode) + - [3.1. analyze all block data files](#31-analyze-all-block-data-files) + - [3.2. analyze single block data file](#32-analyze-single-block-data-file) + - [3.2.1. Optional: debug via VSCode:](#321-optional--debug-via-vscode) - [3.3. consume and analyze new raw block events](#33-consume-and-analyze-new-raw-block-events) - [3.3.1. Fund a tx w/ a P2PK output on reg-test](#331-fund-a-tx-w-a-p2pk-output-on-reg-test) - [3.3.2. Generate block:](#332-generate-block) -- [4. Debug in VSCode:](#4-debug-in-vscode) + - [3.3.3. Optional: dDebug in VSCode:](#333-optional-ddebug-in-vscode) ## 1. Introduction @@ -26,32 +26,42 @@ Measures how many unspent public key addresses there are, and how many coins are ## 2. Setup ### 2.1. Pre-reqs -``` -$ bitcoind \ - -conf=$GITEA_HOME/blockchain/bitcoin/admin/bitcoind/bitcoin.conf \ - -daemon=0 -``` #### 2.1.1. Hardware +Gabriel requires a fully synced Bitcoin Core daemon to be running. +Your hardware requirements will vary depending on the bitcoin network(ie: main, testnet4, regtest, etc) you choose. + +If running in _regtest_ (ie: for dev / test purposes) then use of a modern laptop will be plenty sufficient. + #### 2.1.2. Software ##### 2.1.2.1. Rust The best way to install Rust is to use [rustup](https://rustup.rs). ##### 2.1.2.2. bitcoind +Gabriel requires a fully synced Bitcoin Core daemon to be running. +For testing and development purposes, running Bitcoin Core on _regtest_ is sufficient. + If on bitcoind v28.0, ensure the following flag is set prior to initial block download: `-blocksxor=0` -1. Start Bitcoin Core in Regtest mode, for example: +1. Start Bitcoin Core: + The following example starts Bitcoin Core in _regtest_ mode. - $ bitcoind \ - -regtest \ - -server -daemon \ - -fallbackfee=0.0002 \ - -rpcuser=admin -rpcpassword=pass -rpcallowip=127.0.0.1/0 -rpcbind=127.0.0.1 \ - -blockfilterindex=1 -peerblockfilters=1 \ - -blocksxor=0 + $ bitcoind \ + -regtest \ + -server -daemon \ + -fallbackfee=0.0002 \ + -rpcuser=admin -rpcpassword=pass \ + -rpcallowip=127.0.0.1/0 -rpcbind=127.0.0.1 \ + -blockfilterindex=1 -peerblockfilters=1 \ + -zmqpubrawblock=unix:/tmp/zmqpubrawblock.unix \ + -blocksxor=0 + + NOTE: Gabriel includes functionality that consumes block events from Bitcoin Core via its _zmqpubrawblock_ ZeroMQ interface. + The example above specifies a Unix domain socket. + Alternatively, you could choose to specify a tcp socket and port similar to the following: `-zmqpubrawblock=tcp://127.0.0.1:29001` 2. Define a shell alias to `bitcoin-cli`, for example: @@ -67,6 +77,8 @@ If on bitcoind v28.0, ensure the following flag is set prior to initial block do ### 2.2. Clone +You'll need the Gabriel source code: + ``` $ git clone https://github.com/SurmountSystems/gabriel.git $ git checkout HB/gabriel-v2 @@ -78,7 +90,7 @@ $ git checkout HB/gabriel-v2 $ cargo build -* view gabriel command line options: +* view Gabriel's command line options: $ ./target/debug/gabriel @@ -91,7 +103,9 @@ $ cargo test ## 3. Run Gabriel -### 3.2. analyze all block data files +### 3.1. analyze all block data files + +Gabriel can be used to identify P2PK utxos across all transactions. Execute the following if analyzing the entire (previously downloaded) Bitcoin blockchain: @@ -100,9 +114,10 @@ Execute the following if analyzing the entire (previously downloaded) Bitcoin bl --input $BITCOIND_DATA_DIR/blocks \ --output /tmp/gabriel-testnet4.csv -### 3.1. analyze single block data file +### 3.2. analyze single block data file Alternatively, you can have (likely for testing purposes) Gabriel analyze a single Bitcoin Core block data file. + Execute as follows: $ export BITCOIND_DATA_DIR=/path/to/bitcoind/data/dir @@ -112,7 +127,7 @@ Execute as follows: -b $BITCOIND_DATA_DIR/blocks/$BITCOIND_BLOCK_DATA_FILE \ -o /tmp/$BITCOIND_BLOCK_DATA_FILE.csv -#### Optional: debug via VSCode: +#### 3.2.1. Optional: debug via VSCode: Modify the following as appropriate and add to your vscode `launch.json`: @@ -133,9 +148,22 @@ Modify the following as appropriate and add to your vscode `launch.json`: ### 3.3. consume and analyze new raw block events - $ ./target/debug/gabriel block-async-eval \ - --zmqpubrawblock-socket-url tcp://127.0.0.1:29001 \ - --output /tmp/async_blocks.txt +After identifying P2PK utxos from an Initial Block Download (IBD), Gabriel can run to wait for and consume new block events as generated by your Bitcoin Core node. + +Execute as follows: +``` +$ ./target/debug/gabriel block-async-eval \ + --zmqpubrawblock-socket-url ipc:/tmp/zmqpubrawblock.unix \ + --output /tmp/async_blocks.txt +``` + +NOTE: The following example configures Gabriel to consume block events using the same ZeroMQ Unix domain socket that Bitcoin Core was previously configured to produce to. +If your Bitcoin Core daemon is configured to use TCP for its ZeroMQ interfaces, then you will want Gabriel to use a TCP consumer as well: + +``` +--zmqpubrawblock-socket-url=tcp://127.0.0.1:29001 +``` + #### 3.3.1. Fund a tx w/ a P2PK output on reg-test @@ -173,7 +201,7 @@ If interested in testing Gabriel's ability to consume and process a block with a NOTE: You should now see a new record in Gabriel's output file indicating the new P2PK utxo. -## 4. Debug in VSCode: +#### 3.3.3. Optional: dDebug in VSCode: Add and edit the following to $PROJECT_HOME/.vscode/launch.json: From 74287fc6fe6bed1deeec5975d3b52e9c02437b84 Mon Sep 17 00:00:00 2001 From: jbride Date: Wed, 23 Oct 2024 15:40:53 -0600 Subject: [PATCH 08/13] cargo fmt --- src/block.rs | 6 +- src/main.rs | 41 ++++++----- src/p2pktx.rs | 194 ++++++++++++++++++++++++++++++++++---------------- 3 files changed, 155 insertions(+), 86 deletions(-) diff --git a/src/block.rs b/src/block.rs index 33f6dab..a61b29a 100644 --- a/src/block.rs +++ b/src/block.rs @@ -430,10 +430,10 @@ fn parse_blk_file(input: &[u8], use_magic: bool) -> IResult<&[u8], Vec; if use_magic { _block_result = parse_block_with_magic(remaining_input); - }else { + } else { _block_result = parse_block(remaining_input); } - + match _block_result { Ok((remaining, block)) => { blocks.push(block); @@ -469,7 +469,7 @@ pub fn process_block( result_map: &ResultMap, tx_map: &TxMap, header_map: &HeaderMap, - use_magic: bool + use_magic: bool, ) -> usize { let mut blocks_processed = 0; diff --git a/src/main.rs b/src/main.rs index b236ccc..c0f39c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,8 @@ use std::{ fs::{File, OpenOptions}, io::{Read, Seek, Write}, - path::PathBuf, str::FromStr, + path::PathBuf, + str::FromStr, }; use anyhow::{Ok, Result}; @@ -11,8 +12,8 @@ use clap::{Parser, Subcommand}; use nom::AsBytes; use zeromq::{Socket, SocketRecv}; -mod p2pktx; mod block; +mod p2pktx; mod tx; use block::{HeaderMap, ResultMap, TxMap}; @@ -39,11 +40,10 @@ enum Commands { #[derive(Parser, Debug)] struct GenerateP2PKTxArgs { - - #[arg(short, long, default_value="1.0 BTC")] + #[arg(short, long, default_value = "1.0 BTC")] output_amount_btc: String, #[arg(short, long)] - extended_master_private_key: String + extended_master_private_key: String, } #[derive(Parser, Debug)] @@ -88,24 +88,19 @@ struct GraphArgs { async fn main() -> Result<()> { let cli = Cli::parse(); - match &cli.command { Commands::BlockFileEval(args) => run_block_file_eval(args), Commands::Index(args) => run_index(args), Commands::Graph(args) => run_graph(args), Commands::GenerateP2PKTx(args) => generate_p2pk_tx(args), - Commands::BlockAsyncEval(args) => run_async_block_eval_listener(args).await + Commands::BlockAsyncEval(args) => run_async_block_eval_listener(args).await, } } fn generate_p2pk_tx(args: &GenerateP2PKTxArgs) -> Result<()> { - let to_amount = Amount::from_str(&args.output_amount_btc)?; let e_master_key = &args.extended_master_private_key; - p2pktx::generate_p2pk_tx( - e_master_key, - to_amount - ) + p2pktx::generate_p2pk_tx(e_master_key, to_amount) } fn append_to_output(mut file: &File, result_map: &ResultMap) -> Result<()> { @@ -122,12 +117,9 @@ fn append_to_output(mut file: &File, result_map: &ResultMap) -> Result<()> { writeln!(file, "{}", output_line)?; } Ok(()) - } fn run_block_file_eval(args: &BlockFileEvalArgs) -> Result<()> { - - // Maps previous block hash to next merkle root let header_map: HeaderMap = Default::default(); @@ -173,7 +165,6 @@ fn run_block_file_eval(args: &BlockFileEvalArgs) -> Result<()> { } async fn run_async_block_eval_listener(args: &BlockAsyncEvalArgs) -> Result<()> { - println!( "zmqpubrawblock_socket_url: {} ; output file = {}", &args.zmqpubrawblock_socket_url, @@ -193,7 +184,10 @@ async fn run_async_block_eval_listener(args: &BlockAsyncEvalArgs) -> Result<()> socket .connect(&args.zmqpubrawblock_socket_url) .await - .expect(&format!("Failed to connect: {}", &args.zmqpubrawblock_socket_url)); + .expect(&format!( + "Failed to connect: {}", + &args.zmqpubrawblock_socket_url + )); socket.subscribe("").await?; @@ -208,16 +202,21 @@ async fn run_async_block_eval_listener(args: &BlockAsyncEvalArgs) -> Result<()> loop { let zmq_message = socket.recv().await?; - + let second_element = zmq_message.get(1); match second_element { Some(block_bytes) => { let u8_byte_array = block_bytes.as_bytes(); - let tx_count = process_block(u8_byte_array, &pb, &result_map, &tx_map, &header_map, false); - println!("received block! byte length: {}; tx_count: {}", u8_byte_array.len(), tx_count); + let tx_count = + process_block(u8_byte_array, &pb, &result_map, &tx_map, &header_map, false); + println!( + "received block! byte length: {}; tx_count: {}", + u8_byte_array.len(), + tx_count + ); let _ = append_to_output(&file, &result_map); } - None => panic!("second element from zeromq raw block is non-existent!") + None => panic!("second element from zeromq raw block is non-existent!"), } } } diff --git a/src/p2pktx.rs b/src/p2pktx.rs index 9422522..782edc9 100644 --- a/src/p2pktx.rs +++ b/src/p2pktx.rs @@ -7,19 +7,22 @@ //! use std::collections::BTreeMap; -use std::{env, fmt}; use std::str::FromStr; +use std::{env, fmt}; -use anyhow::{Result, anyhow}; +use anyhow::{anyhow, Result}; -use bitcoin::bip32::{self, ChildNumber, DerivationPath, Fingerprint, IntoDerivationPath, Xpriv, Xpub}; +use bitcoin::bip32::{ + self, ChildNumber, DerivationPath, Fingerprint, IntoDerivationPath, Xpriv, Xpub, +}; use bitcoin::consensus::encode; use bitcoin::key::rand; use bitcoin::locktime::absolute; use bitcoin::psbt::{self, Input, Psbt, PsbtSighashType}; use bitcoin::secp256k1::{Secp256k1, Signing, Verification}; use bitcoin::{ - key, transaction, Address, Amount, CompressedPublicKey, Network, OutPoint, PrivateKey, PublicKey, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Witness + key, transaction, Address, Amount, CompressedPublicKey, Network, OutPoint, PrivateKey, + PublicKey, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Witness, }; extern crate bitcoincore_rpc; @@ -30,11 +33,10 @@ const INPUT_UTXO_VOUT: u32 = 0; #[derive(Debug)] pub struct BitcoindRpcInfo { - rpc_client: Client + rpc_client: Client, } impl BitcoindRpcInfo { - pub fn new() -> Result { let url = env::var("URL")?; let cookie = env::var("COOKIE"); @@ -48,49 +50,67 @@ impl BitcoindRpcInfo { } }; let rpc_client = Client::new(&url, auth)?; - Ok(BitcoindRpcInfo{rpc_client}) + Ok(BitcoindRpcInfo { rpc_client }) } - pub fn get_bitcoind_info(&self, output_amount_btc: f64) -> Result<(ListUnspentResultEntry, GetAddressInfoResult, Address, Amount)> { - + pub fn get_bitcoind_info( + &self, + output_amount_btc: f64, + ) -> Result<( + ListUnspentResultEntry, + GetAddressInfoResult, + Address, + Amount, + )> { let network_relay_fee = self.rpc_client.get_network_info()?.relay_fee; let output_tx_total = network_relay_fee.to_btc() + output_amount_btc; - + let mut unspent_option: Option = None; - let unspent_vec = self.rpc_client.list_unspent(Some(3), None, None, None, None).unwrap(); + let unspent_vec = self + .rpc_client + .list_unspent(Some(3), None, None, None, None) + .unwrap(); for unspent_candidate in unspent_vec { - println!("unspent_candidate txid={}, vout={}, tx_amount={}, output_tx_total={}", + println!( + "unspent_candidate txid={}, vout={}, tx_amount={}, output_tx_total={}", unspent_candidate.txid, unspent_candidate.vout, unspent_candidate.amount.to_btc(), output_tx_total ); - if unspent_candidate.amount.to_btc() > output_tx_total { + if unspent_candidate.amount.to_btc() > output_tx_total { unspent_option = Some(unspent_candidate); break; } } if unspent_option == None { - return Err(anyhow!("No unspent txs have sufficient funds: {}", output_tx_total)); + return Err(anyhow!( + "No unspent txs have sufficient funds: {}", + output_tx_total + )); } - + let unspent_tx = unspent_option.unwrap(); let input_utxo_address = unspent_tx.address.clone().unwrap().assume_checked(); - + let input_utxo_address_info = self.rpc_client.get_address_info(&input_utxo_address)?; - - let change_addr = self.rpc_client.get_raw_change_address(Some(json::AddressType::Bech32)).unwrap().assume_checked(); - - Ok((unspent_tx, input_utxo_address_info, change_addr, network_relay_fee)) - } + let change_addr = self + .rpc_client + .get_raw_change_address(Some(json::AddressType::Bech32)) + .unwrap() + .assume_checked(); + + Ok(( + unspent_tx, + input_utxo_address_info, + change_addr, + network_relay_fee, + )) + } } -pub fn generate_p2pk_tx( - extended_master_private_key: &str, - output_amount: Amount - ) -> Result<()> { - +pub fn generate_p2pk_tx(extended_master_private_key: &str, output_amount: Amount) -> Result<()> { let secp = Secp256k1::new(); let mut rng = rand::thread_rng(); let (_, secp256k1_pubkey) = secp.generate_keypair(&mut rng); @@ -98,7 +118,12 @@ pub fn generate_p2pk_tx( let output_amount_btc = output_amount.to_btc(); let bitcoind_info = BitcoindRpcInfo::new()?; - let results: (ListUnspentResultEntry, GetAddressInfoResult, Address, Amount) = bitcoind_info.get_bitcoind_info(output_amount_btc)?; + let results: ( + ListUnspentResultEntry, + GetAddressInfoResult, + Address, + Amount, + ) = bitcoind_info.get_bitcoind_info(output_amount_btc)?; let unspent_tx = results.0; let input_utxo_address = results.1; let change_addr = results.2; @@ -106,19 +131,34 @@ pub fn generate_p2pk_tx( let input_utxo_derivation_path = input_utxo_address.hd_key_path.unwrap(); let input_utxo_xkey_identifier = input_utxo_address.hd_seed_id.unwrap(); - + let input_utxo_txid = unspent_tx.txid.to_string(); let input_utxo_script_pubkey = unspent_tx.script_pub_key; let input_utxo_value = unspent_tx.amount; - let (offline, fingerprint, account_0_xpub, input_xpub) = - ColdStorage::new(&secp, extended_master_private_key, &input_utxo_derivation_path)?; + let (offline, fingerprint, account_0_xpub, input_xpub) = ColdStorage::new( + &secp, + extended_master_private_key, + &input_utxo_derivation_path, + )?; let online = WatchOnly::new(account_0_xpub, input_xpub, fingerprint); - let created = online.create_psbt(&input_utxo_txid, &input_utxo_value, &output_amount, p2pk_pubkey,change_addr, network_relay_fee)?; - - let updated = online.update_psbt(created, input_utxo_script_pubkey, &input_utxo_derivation_path, &input_utxo_value)?; + let created = online.create_psbt( + &input_utxo_txid, + &input_utxo_value, + &output_amount, + p2pk_pubkey, + change_addr, + network_relay_fee, + )?; + + let updated = online.update_psbt( + created, + input_utxo_script_pubkey, + &input_utxo_derivation_path, + &input_utxo_value, + )?; let signed = offline.sign_psbt(&secp, updated)?; @@ -148,12 +188,15 @@ struct ColdStorage { type ExportData = (ColdStorage, Fingerprint, Xpub, Xpub); impl ColdStorage { - /// Constructs a new `ColdStorage` signer. /// /// # Returns /// The newly created signer along with the data needed to configure a watch-only wallet. - fn new(secp: &Secp256k1, xpriv: &str, input_utxo_derivation_path: &DerivationPath) -> Result { + fn new( + secp: &Secp256k1, + xpriv: &str, + input_utxo_derivation_path: &DerivationPath, + ) -> Result { let master_xpriv = Xpriv::from_str(xpriv)?; let master_xpub = Xpub::from_priv(secp, &master_xpriv); @@ -164,14 +207,19 @@ impl ColdStorage { let input_xpriv = master_xpriv.derive_priv(secp, &input_utxo_derivation_path)?; let input_xpub = Xpub::from_priv(secp, &input_xpriv); - let wallet = ColdStorage { master_xpriv, master_xpub }; + let wallet = ColdStorage { + master_xpriv, + master_xpub, + }; let fingerprint = wallet.master_fingerprint(); Ok((wallet, fingerprint, account_0_xpub, input_xpub)) } /// Returns the fingerprint for the master extended public key. - fn master_fingerprint(&self) -> Fingerprint { self.master_xpub.fingerprint() } + fn master_fingerprint(&self) -> Fingerprint { + self.master_xpub.fingerprint() + } /// Signs `psbt` with this signer. fn sign_psbt( @@ -209,27 +257,30 @@ impl WatchOnly { /// The reason for importing the `input_xpub` is so one can use bitcoind to grab a valid input /// to verify the workflow presented in this file. fn new(account_0_xpub: Xpub, input_xpub: Xpub, master_fingerprint: Fingerprint) -> Self { - WatchOnly { account_0_xpub, input_xpub, master_fingerprint } + WatchOnly { + account_0_xpub, + input_xpub, + master_fingerprint, + } } /// Creates the PSBT, in BIP174 parlance this is the 'Creater'. fn create_psbt( - &self, - input_utxo_txid: &str, - input_utxo_value: &Amount, - output_amount_btc: &Amount, - p2pk_pubkey: PublicKey, - change_address: Address, - network_relay_fee: Amount) -> Result { - + &self, + input_utxo_txid: &str, + input_utxo_value: &Amount, + output_amount_btc: &Amount, + p2pk_pubkey: PublicKey, + change_address: Address, + network_relay_fee: Amount, + ) -> Result { // network fee subtracted from payment total. Similar to specifying "subtractFeeFromOutputs" argument with "fundrawtransaction" CLI let payment_float = output_amount_btc.to_btc() - network_relay_fee.to_btc(); let payment_total = Amount::from_float_in(payment_float, bitcoin::Denomination::Bitcoin)?; - let change_float = input_utxo_value.to_btc() - output_amount_btc.to_btc(); let change_rounded = (change_float * 1000000.0).round() / 1000000.0; -/* println!("input_utxo_value={}, network_relay_fee={}, output_amount_btc={}, change_amount_rounded={}", + /* println!("input_utxo_value={}, network_relay_fee={}, output_amount_btc={}, change_amount_rounded={}", input_utxo_value.to_btc(), network_relay_fee.to_btc(), output_amount_btc.to_btc(), @@ -237,21 +288,29 @@ impl WatchOnly { ); */ let change_total = Amount::from_float_in(change_rounded, bitcoin::Denomination::Bitcoin)?; - let p2pk_pubkey_scriptbuf = ScriptBuf::new_p2pk(&p2pk_pubkey); let tx = Transaction { version: transaction::Version::TWO, lock_time: absolute::LockTime::ZERO, input: vec![TxIn { - previous_output: OutPoint { txid: input_utxo_txid.parse()?, vout: INPUT_UTXO_VOUT }, + previous_output: OutPoint { + txid: input_utxo_txid.parse()?, + vout: INPUT_UTXO_VOUT, + }, script_sig: ScriptBuf::new(), sequence: Sequence::MAX, // Disable LockTime and RBF. witness: Witness::default(), }], output: vec![ - TxOut { value: payment_total, script_pubkey: p2pk_pubkey_scriptbuf }, - TxOut { value: change_total, script_pubkey: change_address.script_pubkey() } + TxOut { + value: payment_total, + script_pubkey: p2pk_pubkey_scriptbuf, + }, + TxOut { + value: change_total, + script_pubkey: change_address.script_pubkey(), + }, ], }; @@ -261,13 +320,21 @@ impl WatchOnly { } /// Updates the PSBT, in BIP174 parlance this is the 'Updater'. - fn update_psbt(&self, - mut psbt: Psbt, - input_utxo_script_pubkey: ScriptBuf, - input_utxo_derivation_path: &DerivationPath, - input_utxo_value: &Amount) -> Result { - let t_out = TxOut { value: *input_utxo_value, script_pubkey: input_utxo_script_pubkey}; - let mut input = Input { witness_utxo: Some(t_out), ..Default::default() }; + fn update_psbt( + &self, + mut psbt: Psbt, + input_utxo_script_pubkey: ScriptBuf, + input_utxo_derivation_path: &DerivationPath, + input_utxo_value: &Amount, + ) -> Result { + let t_out = TxOut { + value: *input_utxo_value, + script_pubkey: input_utxo_script_pubkey, + }; + let mut input = Input { + witness_utxo: Some(t_out), + ..Default::default() + }; let pk = self.input_xpub.to_pub(); let wpkh = pk.wpubkey_hash(); @@ -310,7 +377,6 @@ impl WatchOnly { Ok(psbt) } - } fn input_derivation_path(input_utxo_derivation_path: &str) -> Result { @@ -321,9 +387,13 @@ fn input_derivation_path(input_utxo_derivation_path: &str) -> Result); impl From for Error { - fn from(e: T) -> Self { Error(Box::new(e)) } + fn from(e: T) -> Self { + Error(Box::new(e)) + } } impl fmt::Debug for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&self.0, f) } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } } From ffb7e27c7aac7850cba841c560f718ee5b34984f Mon Sep 17 00:00:00 2001 From: jbride Date: Mon, 28 Oct 2024 07:10:05 -0600 Subject: [PATCH 09/13] 1st draft using sqlite --- Cargo.lock | 74 ++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 4 +++ src/block.rs | 2 +- src/persistence.rs | 39 ++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 src/persistence.rs diff --git a/Cargo.lock b/Cargo.lock index d7c818e..8c54b02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,18 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -462,6 +474,18 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "futures-channel" version = "0.3.31" @@ -539,6 +563,7 @@ dependencies = [ "log", "nom", "rayon", + "rusqlite", "serde", "serde_json", "sha2", @@ -578,6 +603,18 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown", +] [[package]] name = "heck" @@ -702,6 +739,17 @@ version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "lock_api" version = "0.4.12" @@ -837,6 +885,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + [[package]] name = "portable-atomic" version = "1.9.0" @@ -958,6 +1012,20 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -1193,6 +1261,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 3923c05..97f9076 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,3 +23,7 @@ log = "0.4.22" zeromq = "0.4.1" tokio = "1.40.0" bitcoincore-rpc = "0.19.0" + +# features= bundled +# This causes rusqlite to compile its own private libsqlite3 and link it with your Rust code, instead of using /usr/lib/x86_64-linux-gnu/libsqlite3.so +rusqlite = { version = "0.32.1", features = ["bundled"] } diff --git a/src/block.rs b/src/block.rs index a61b29a..6a7ace1 100644 --- a/src/block.rs +++ b/src/block.rs @@ -49,7 +49,7 @@ pub type HeaderMap = Arc>>; pub type TxMap = Arc>>; pub type ResultMap = Arc>>; -/// Parses a Bitcoin block header +/// Parses a Bitcoin block header as per: https://learnmeabitcoin.com/technical/block/#header fn parse_block_header(input: &[u8]) -> IResult<&[u8], BlockHeader> { let (input, version) = le_u32(input)?; let (input, previous_block_hash) = take(32usize)(input)?; diff --git a/src/persistence.rs b/src/persistence.rs new file mode 100644 index 0000000..4adbcae --- /dev/null +++ b/src/persistence.rs @@ -0,0 +1,39 @@ +use std::env; + +use rusqlite::{Connection, Result}; + +use crate::block::{HeaderMap, ResultMap}; + +#[derive(Debug)] +pub struct SQLitePersistence { + sql_conn: Connection, + sqlite_absolute_path: String +} + +impl SQLitePersistence { + pub fn new() -> anyhow::Result<(Self)> { + let sqlite_absolute_path = env::var("SQLITE_ABSOLUTE_PATH")?; + let sql_conn = Connection::open(&sqlite_absolute_path)?; + + let db_exec_results = sql_conn.execute( + "create table if not exists p2pk_utxo_block_aggregates ( + block_height integer primary key, + date integer not null, + p2pk_utxo_count integer not null, + p2pk_utxo_value real not null + )", + [], + )?; + println!("p2pk_utxo_block_aggregates: table now exists at: {}", sqlite_absolute_path); + + Ok(SQLitePersistence{sql_conn, sqlite_absolute_path}) + + } + + pub fn persist_block_aggregates(header_map: &HeaderMap,result_map: &ResultMap) -> anyhow::Result<()> { + + let result_map_read = result_map.read().unwrap(); + + Ok(()) + } +} \ No newline at end of file From df3d7964ed8d7ffd196f9136f1370ceeec8b8693 Mon Sep 17 00:00:00 2001 From: jbride Date: Mon, 28 Oct 2024 15:29:21 -0600 Subject: [PATCH 10/13] zmqpubrawblock: now determining block height using bitcoind rpc --- README.md | 4 +- src/bitcoind_rpc.rs | 98 ++++++++++++++++++++++++++++++++++++++++++++ src/block.rs | 4 ++ src/main.rs | 84 ++++++++++++++++++++++++++++---------- src/p2pktx.rs | 99 ++++----------------------------------------- 5 files changed, 174 insertions(+), 115 deletions(-) create mode 100644 src/bitcoind_rpc.rs diff --git a/README.md b/README.md index 8581d2d..52e7309 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ After identifying P2PK utxos from an Initial Block Download (IBD), Gabriel can r Execute as follows: ``` $ ./target/debug/gabriel block-async-eval \ - --zmqpubrawblock-socket-url ipc:/tmp/zmqpubrawblock.unix \ + --zmqpubrawblock-socket-url ipc:///tmp/zmqpubrawblock.unix \ --output /tmp/async_blocks.txt ``` @@ -174,7 +174,7 @@ If interested in testing Gabriel's ability to consume and process a block with a NOTE: for the following command, you'll already need to have unlocked your wallet via the bitcoin cli. $ XPRV=$( b-reg gethdkeys '{"active_only":true, "private":true}' \ - | jq -r .[].xprv ) && echo $XPRV + | jq -r .[].xprv ) && echo $XPRV 2. Create a tx w/ P2PK output: diff --git a/src/bitcoind_rpc.rs b/src/bitcoind_rpc.rs new file mode 100644 index 0000000..60483b4 --- /dev/null +++ b/src/bitcoind_rpc.rs @@ -0,0 +1,98 @@ +use std::env; + +use anyhow::{anyhow, Result}; +use bitcoin::{Address, Amount, BlockHash}; + +use bitcoin::hashes::sha256d::Hash; +use bitcoincore_rpc::json::{self, GetAddressInfoResult, ListUnspentResultEntry}; +use bitcoincore_rpc::{Auth, Client, RpcApi}; + +#[derive(Debug)] +pub struct BitcoindRpcInfo { + rpc_client: Client, +} + +impl BitcoindRpcInfo { + pub fn new() -> Result { + let url = env::var("URL")?; + let cookie = env::var("COOKIE"); + let auth = match cookie { + Ok(cookiefile) => Auth::CookieFile(cookiefile.into()), + Err(_) => { + let user = env::var("USER")?; + let pass = env::var("PASS")?; + + Auth::UserPass(user, pass) + } + }; + let rpc_client = Client::new(&url, auth)?; + Ok(BitcoindRpcInfo { rpc_client }) + } + + pub fn get_bitcoind_info_for_test_p2pk( + &self, + output_amount_btc: f64, + ) -> Result<( + ListUnspentResultEntry, + GetAddressInfoResult, + Address, + Amount, + )> { + // 1) get default bitcoind relay_fee + let network_relay_fee = self.rpc_client.get_network_info()?.relay_fee; + let output_tx_total = network_relay_fee.to_btc() + output_amount_btc; + + // 2) identify first utxo managed by bitcoind wallet with a value > desired p2pk output + let mut unspent_option: Option = None; + let unspent_vec = self + .rpc_client + .list_unspent(Some(3), None, None, None, None) + .unwrap(); + for unspent_candidate in unspent_vec { + println!( + "unspent_candidate txid={}, vout={}, tx_amount={}, output_tx_total={}", + unspent_candidate.txid, + unspent_candidate.vout, + unspent_candidate.amount.to_btc(), + output_tx_total + ); + if unspent_candidate.amount.to_btc() > output_tx_total { + unspent_option = Some(unspent_candidate); + break; + } + } + if unspent_option == None { + return Err(anyhow!( + "No unspent txs have sufficient funds: {}", + output_tx_total + )); + } + + let unspent_tx = unspent_option.unwrap(); + let input_utxo_address = unspent_tx.address.clone().unwrap().assume_checked(); + + // 3) Get info about the input utxo address + let input_utxo_address_info = self.rpc_client.get_address_info(&input_utxo_address)?; + + // 4) + let change_addr = self + .rpc_client + .get_raw_change_address(Some(json::AddressType::Bech32)) + .unwrap() + .assume_checked(); + + Ok(( + unspent_tx, + input_utxo_address_info, + change_addr, + network_relay_fee, + )) + } + + pub fn get_block_height(&self, sha256d_hash: &Hash) -> Result { + let hash = BlockHash::from_raw_hash(*sha256d_hash); + let block_header = self.rpc_client.get_block_header_info(&hash)?; + Ok(block_header.height) + + } +} diff --git a/src/block.rs b/src/block.rs index a61b29a..4c474fc 100644 --- a/src/block.rs +++ b/src/block.rs @@ -45,8 +45,12 @@ pub struct Record { pub p2pk_sats_spent: u64, } +// pub type HeaderMap = Arc>>; + pub type TxMap = Arc>>; + +// pub type ResultMap = Arc>>; /// Parses a Bitcoin block header diff --git a/src/main.rs b/src/main.rs index c0f39c3..3d7961c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,26 +1,24 @@ use std::{ - fs::{File, OpenOptions}, - io::{Read, Seek, Write}, - path::PathBuf, - str::FromStr, + fs::{File, OpenOptions}, io::{Seek, Write}, path::PathBuf, str::FromStr }; use anyhow::{Ok, Result}; -use bitcoin::{Amount, PublicKey, ScriptBuf}; +use bitcoin::{hashes::sha256d::Hash, Amount}; +use bitcoind_rpc::BitcoindRpcInfo; use block::{process_block, process_block_file, process_blocks_in_parallel, Record}; use clap::{Parser, Subcommand}; use nom::AsBytes; use zeromq::{Socket, SocketRecv}; +mod bitcoind_rpc; mod block; mod p2pktx; mod tx; use block::{HeaderMap, ResultMap, TxMap}; use indicatif::ProgressBar; -use p2pktx::BitcoindRpcInfo; -const HEADER: &str = "Height,Date,Total P2PK addresses,Total P2PK coins\n"; +const HEADER: &str = "Height,Block Hash,Date,Total P2PK addresses,Total P2PK coins\n"; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -103,19 +101,46 @@ fn generate_p2pk_tx(args: &GenerateP2PKTxArgs) -> Result<()> { p2pktx::generate_p2pk_tx(e_master_key, to_amount) } -fn append_to_output(mut file: &File, result_map: &ResultMap) -> Result<()> { - let result_map_read = result_map.read().unwrap(); - for (_key, record) in result_map_read.iter() { - // write a record to the file - let mut p2pk_addresses = &record.p2pk_addresses_added; - let binding = p2pk_addresses - &record.p2pk_addresses_spent; - p2pk_addresses = &binding; - let mut p2pk_coins = record.p2pk_sats_added.to_owned() as f64 / 100_000_000.0; - p2pk_coins -= record.p2pk_sats_spent.to_owned() as f64 / 100_000_000.0; - let date = &record.date; - let output_line = format!("0,{date},{p2pk_addresses},{p2pk_coins}"); - writeln!(file, "{}", output_line)?; +fn append_single_block_result_to_output( + mut file: &File, + bitcoind_info: &BitcoindRpcInfo, + h_map_entry: (&[u8; 32], &[u8; 32]) , + record: &Record, +) -> Result<()> { + // Get previous and current block hashes + let mut raw_previous_block_hash = h_map_entry.0.clone(); + raw_previous_block_hash.reverse(); + let previous_block_hash = hex::encode(raw_previous_block_hash); + + let mut raw_current_block_hash = h_map_entry.1.clone(); + let sha256d_hash = Hash::from_bytes_ref(&raw_current_block_hash); + + let mut block_height = 0; + + match bitcoind_info.get_block_height(sha256d_hash){ + std::result::Result::Ok(x) => { block_height = x}, + Err(e) => println!("block not found: exception={}", e), } + raw_current_block_hash.reverse(); + let current_block_hash_header = hex::encode(raw_current_block_hash); + + println!( + "previous_block_hash={} , current_block_hash={}, block_height={}", + previous_block_hash, current_block_hash_header, block_height + ); + + // Determine total p2pk addresses and value + let mut total_p2pk_addresses = record.p2pk_addresses_added.to_owned(); + total_p2pk_addresses -= record.p2pk_addresses_spent.to_owned(); + let mut total_p2pk_value = record.p2pk_sats_added.to_owned() as f64 / 100_000_000.0; + total_p2pk_value -= record.p2pk_sats_spent.to_owned() as f64 / 100_000_000.0; + let date = &record.date; + + let output_line = format!( + "{},{},{},{},{}", + block_height,current_block_hash_header, date, total_p2pk_addresses, total_p2pk_value + ); + writeln!(file, "{}", output_line)?; Ok(()) } @@ -159,8 +184,15 @@ fn run_block_file_eval(args: &BlockFileEvalArgs) -> Result<()> { file.set_len(0)?; // Truncate the file file.write_all(HEADER.as_bytes())?; - append_to_output(&file, &result_map); - + let bitcoind_info = BitcoindRpcInfo::new()?; + let h_binding = header_map.read().unwrap(); + let mut header_map_iter = h_binding.iter(); + while let Some(h_map_entry) = header_map_iter.next(){ + let r_binding = result_map.read().unwrap(); + let record = r_binding.get(h_map_entry.1); + + append_single_block_result_to_output(&file, &bitcoind_info, h_map_entry, record.unwrap())?; + } Ok(()) } @@ -200,6 +232,8 @@ async fn run_async_block_eval_listener(args: &BlockAsyncEvalArgs) -> Result<()> .open(&args.output)?; file.seek(std::io::SeekFrom::End(0))?; + let bitcoind_info = BitcoindRpcInfo::new()?; + loop { let zmq_message = socket.recv().await?; @@ -214,7 +248,13 @@ async fn run_async_block_eval_listener(args: &BlockAsyncEvalArgs) -> Result<()> u8_byte_array.len(), tx_count ); - let _ = append_to_output(&file, &result_map); + + let h_binding = header_map.read().unwrap(); + let h_map_entry = h_binding.first_key_value().unwrap(); + let r_binding = result_map.read().unwrap(); + let record_entry = r_binding.first_key_value().unwrap(); + let record = record_entry.1; + let _ = append_single_block_result_to_output(&file, &bitcoind_info, h_map_entry, record); } None => panic!("second element from zeromq raw block is non-existent!"), } diff --git a/src/p2pktx.rs b/src/p2pktx.rs index 782edc9..fa30ce2 100644 --- a/src/p2pktx.rs +++ b/src/p2pktx.rs @@ -8,12 +8,12 @@ use std::collections::BTreeMap; use std::str::FromStr; -use std::{env, fmt}; +use std::fmt; -use anyhow::{anyhow, Result}; +use anyhow::Result; use bitcoin::bip32::{ - self, ChildNumber, DerivationPath, Fingerprint, IntoDerivationPath, Xpriv, Xpub, + DerivationPath, Fingerprint, IntoDerivationPath, Xpriv, Xpub, }; use bitcoin::consensus::encode; use bitcoin::key::rand; @@ -21,94 +21,16 @@ use bitcoin::locktime::absolute; use bitcoin::psbt::{self, Input, Psbt, PsbtSighashType}; use bitcoin::secp256k1::{Secp256k1, Signing, Verification}; use bitcoin::{ - key, transaction, Address, Amount, CompressedPublicKey, Network, OutPoint, PrivateKey, + transaction, Address, Amount, OutPoint, PublicKey, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Witness, }; extern crate bitcoincore_rpc; -use bitcoincore_rpc::json::{self, GetAddressInfoResult, ListUnspentResultEntry}; -use bitcoincore_rpc::{Auth, Client, RpcApi}; +use bitcoincore_rpc::json::{GetAddressInfoResult, ListUnspentResultEntry}; -const INPUT_UTXO_VOUT: u32 = 0; - -#[derive(Debug)] -pub struct BitcoindRpcInfo { - rpc_client: Client, -} - -impl BitcoindRpcInfo { - pub fn new() -> Result { - let url = env::var("URL")?; - let cookie = env::var("COOKIE"); - let auth = match cookie { - Ok(cookiefile) => Auth::CookieFile(cookiefile.into()), - Err(_) => { - let user = env::var("USER")?; - let pass = env::var("PASS")?; - - Auth::UserPass(user, pass) - } - }; - let rpc_client = Client::new(&url, auth)?; - Ok(BitcoindRpcInfo { rpc_client }) - } - - pub fn get_bitcoind_info( - &self, - output_amount_btc: f64, - ) -> Result<( - ListUnspentResultEntry, - GetAddressInfoResult, - Address, - Amount, - )> { - let network_relay_fee = self.rpc_client.get_network_info()?.relay_fee; - let output_tx_total = network_relay_fee.to_btc() + output_amount_btc; - - let mut unspent_option: Option = None; - let unspent_vec = self - .rpc_client - .list_unspent(Some(3), None, None, None, None) - .unwrap(); - for unspent_candidate in unspent_vec { - println!( - "unspent_candidate txid={}, vout={}, tx_amount={}, output_tx_total={}", - unspent_candidate.txid, - unspent_candidate.vout, - unspent_candidate.amount.to_btc(), - output_tx_total - ); - if unspent_candidate.amount.to_btc() > output_tx_total { - unspent_option = Some(unspent_candidate); - break; - } - } - if unspent_option == None { - return Err(anyhow!( - "No unspent txs have sufficient funds: {}", - output_tx_total - )); - } - - let unspent_tx = unspent_option.unwrap(); - let input_utxo_address = unspent_tx.address.clone().unwrap().assume_checked(); - - let input_utxo_address_info = self.rpc_client.get_address_info(&input_utxo_address)?; +use crate::bitcoind_rpc::BitcoindRpcInfo; - let change_addr = self - .rpc_client - .get_raw_change_address(Some(json::AddressType::Bech32)) - .unwrap() - .assume_checked(); - - Ok(( - unspent_tx, - input_utxo_address_info, - change_addr, - network_relay_fee, - )) - } -} +const INPUT_UTXO_VOUT: u32 = 0; pub fn generate_p2pk_tx(extended_master_private_key: &str, output_amount: Amount) -> Result<()> { let secp = Secp256k1::new(); @@ -123,7 +45,7 @@ pub fn generate_p2pk_tx(extended_master_private_key: &str, output_amount: Amount GetAddressInfoResult, Address, Amount, - ) = bitcoind_info.get_bitcoind_info(output_amount_btc)?; + ) = bitcoind_info.get_bitcoind_info_for_test_p2pk(output_amount_btc)?; let unspent_tx = results.0; let input_utxo_address = results.1; let change_addr = results.2; @@ -379,11 +301,6 @@ impl WatchOnly { } } -fn input_derivation_path(input_utxo_derivation_path: &str) -> Result { - let path = input_utxo_derivation_path.into_derivation_path()?; - Ok(path) -} - struct Error(Box); impl From for Error { From 2f8c0e7b3559385a5f753751acfb368a24aa903b Mon Sep 17 00:00:00 2001 From: jbride Date: Tue, 29 Oct 2024 09:52:25 -0600 Subject: [PATCH 11/13] sqlite: now inserting block aggregate records into db --- README.md | 38 +++++++++++- src/bitcoind_rpc.rs | 1 - src/block.rs | 10 +++ src/main.rs | 144 +++++++++++++++++++++++++++++--------------- src/p2pktx.rs | 10 ++- src/persistence.rs | 45 ++++++++------ 6 files changed, 174 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 52e7309..e438517 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ gabriel - [2.1.2. Software](#212-software) - [2.1.2.1. Rust](#2121-rust) - [2.1.2.2. bitcoind](#2122-bitcoind) + - [2.1.2.3. SQLite client](#2123-sqlite-client) - [2.2. Clone](#22-clone) - [2.3. Build](#23-build) - [2.4. Execute tests](#24-execute-tests) @@ -18,6 +19,7 @@ gabriel - [3.3.1. Fund a tx w/ a P2PK output on reg-test](#331-fund-a-tx-w-a-p2pk-output-on-reg-test) - [3.3.2. Generate block:](#332-generate-block) - [3.3.3. Optional: dDebug in VSCode:](#333-optional-ddebug-in-vscode) +- [4. Inspect P2PK Analysis data](#4-inspect-p2pk-analysis-data) ## 1. Introduction @@ -75,6 +77,12 @@ If on bitcoind v28.0, ensure the following flag is set prior to initial block do $ `b-reg generatetoaddress 110 $(b-reg getnewaddress)` + +##### 2.1.2.3. SQLite client + +Gabriel persists P2PK utxo analysis to a local SQLite database. +If you would like to view that data, you'll want to download and install the [SQLite client](https://sqlite.org/download.html) for your operating system. + ### 2.2. Clone You'll need the Gabriel source code: @@ -122,6 +130,7 @@ Execute as follows: $ export BITCOIND_DATA_DIR=/path/to/bitcoind/data/dir $ export BITCOIND_BLOCK_DATA_FILE=xxx.dat + $ export SQLITE_ABSOLUTE_PATH=/tmp/gabriel_p2pk.db $ ./target/debug/gabriel block-file-eval \ -b $BITCOIND_DATA_DIR/blocks/$BITCOIND_BLOCK_DATA_FILE \ @@ -148,10 +157,12 @@ Modify the following as appropriate and add to your vscode `launch.json`: ### 3.3. consume and analyze new raw block events -After identifying P2PK utxos from an Initial Block Download (IBD), Gabriel can run to wait for and consume new block events as generated by your Bitcoin Core node. +After identifying P2PK utxos from an Initial Block Download (IBD), Gabriel can asynchronously consume new block events as generated by your Bitcoin Core node. Execute as follows: ``` +$ export SQLITE_ABSOLUTE_PATH=/tmp/gabriel_p2pk.db + $ ./target/debug/gabriel block-async-eval \ --zmqpubrawblock-socket-url ipc:///tmp/zmqpubrawblock.unix \ --output /tmp/async_blocks.txt @@ -220,3 +231,28 @@ Add and edit the following to $PROJECT_HOME/.vscode/launch.json: ] } +## 4. Inspect P2PK Analysis data +Gabriel will persist analysis of P2PK utxos in a SQLite database. + +The path of the SQLite database is the value of the SQLITE_ABSOLUTE_PATH environment variable. + +At the command line, you can inspect the data in SQLite database similar to the following: + +``` +$ sqlite3 $SQLITE_ABSOLUTE_PATH + +# list tables; +sqlite> .tables + +# view the schema of the p2pk_utxo_block_aggregates table: +sqlite> .schema p2pk_utxo_block_aggregates + +# identify number of records in p2pk_utxo_block_aggregates table +sqlite> select count(block_height) from p2pk_utxo_block_aggregates; + +# delete all records +sqlite> delete from p2pk_utxo_block_aggregates; + +# quit sqlite command line: press d + +``` diff --git a/src/bitcoind_rpc.rs b/src/bitcoind_rpc.rs index 60483b4..2d7252c 100644 --- a/src/bitcoind_rpc.rs +++ b/src/bitcoind_rpc.rs @@ -93,6 +93,5 @@ impl BitcoindRpcInfo { let hash = BlockHash::from_raw_hash(*sha256d_hash); let block_header = self.rpc_client.get_block_header_info(&hash)?; Ok(block_header.height) - } } diff --git a/src/block.rs b/src/block.rs index 9b7b5d0..789211f 100644 --- a/src/block.rs +++ b/src/block.rs @@ -45,6 +45,16 @@ pub struct Record { pub p2pk_sats_spent: u64, } + +#[derive(Debug)] +pub struct BlockAggregateOutput { + pub date: String, + pub block_height: usize, + pub block_hash_big_endian: String, + pub total_p2pk_addresses: u32, + pub total_p2pk_value: f64, +} + // pub type HeaderMap = Arc>>; diff --git a/src/main.rs b/src/main.rs index 3d7961c..bb017a3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,18 +1,22 @@ use std::{ - fs::{File, OpenOptions}, io::{Seek, Write}, path::PathBuf, str::FromStr + env, fs::{File, OpenOptions}, io::{Seek, Write}, path::PathBuf, str::FromStr }; use anyhow::{Ok, Result}; use bitcoin::{hashes::sha256d::Hash, Amount}; use bitcoind_rpc::BitcoindRpcInfo; -use block::{process_block, process_block_file, process_blocks_in_parallel, Record}; +use block::{ + process_block, process_block_file, process_blocks_in_parallel, BlockAggregateOutput, Record, +}; use clap::{Parser, Subcommand}; use nom::AsBytes; +use persistence::SQLitePersistence; use zeromq::{Socket, SocketRecv}; mod bitcoind_rpc; mod block; mod p2pktx; +mod persistence; mod tx; use block::{HeaderMap, ResultMap, TxMap}; @@ -86,6 +90,7 @@ struct GraphArgs { async fn main() -> Result<()> { let cli = Cli::parse(); + SQLitePersistence::new()?; match &cli.command { Commands::BlockFileEval(args) => run_block_file_eval(args), Commands::Index(args) => run_index(args), @@ -101,49 +106,6 @@ fn generate_p2pk_tx(args: &GenerateP2PKTxArgs) -> Result<()> { p2pktx::generate_p2pk_tx(e_master_key, to_amount) } -fn append_single_block_result_to_output( - mut file: &File, - bitcoind_info: &BitcoindRpcInfo, - h_map_entry: (&[u8; 32], &[u8; 32]) , - record: &Record, -) -> Result<()> { - // Get previous and current block hashes - let mut raw_previous_block_hash = h_map_entry.0.clone(); - raw_previous_block_hash.reverse(); - let previous_block_hash = hex::encode(raw_previous_block_hash); - - let mut raw_current_block_hash = h_map_entry.1.clone(); - let sha256d_hash = Hash::from_bytes_ref(&raw_current_block_hash); - - let mut block_height = 0; - - match bitcoind_info.get_block_height(sha256d_hash){ - std::result::Result::Ok(x) => { block_height = x}, - Err(e) => println!("block not found: exception={}", e), - } - raw_current_block_hash.reverse(); - let current_block_hash_header = hex::encode(raw_current_block_hash); - - println!( - "previous_block_hash={} , current_block_hash={}, block_height={}", - previous_block_hash, current_block_hash_header, block_height - ); - - // Determine total p2pk addresses and value - let mut total_p2pk_addresses = record.p2pk_addresses_added.to_owned(); - total_p2pk_addresses -= record.p2pk_addresses_spent.to_owned(); - let mut total_p2pk_value = record.p2pk_sats_added.to_owned() as f64 / 100_000_000.0; - total_p2pk_value -= record.p2pk_sats_spent.to_owned() as f64 / 100_000_000.0; - let date = &record.date; - - let output_line = format!( - "{},{},{},{},{}", - block_height,current_block_hash_header, date, total_p2pk_addresses, total_p2pk_value - ); - writeln!(file, "{}", output_line)?; - Ok(()) -} - fn run_block_file_eval(args: &BlockFileEvalArgs) -> Result<()> { // Maps previous block hash to next merkle root let header_map: HeaderMap = Default::default(); @@ -185,13 +147,22 @@ fn run_block_file_eval(args: &BlockFileEvalArgs) -> Result<()> { file.write_all(HEADER.as_bytes())?; let bitcoind_info = BitcoindRpcInfo::new()?; + let sqlite_persistence = persistence::SQLitePersistence::new()?; + let h_binding = header_map.read().unwrap(); let mut header_map_iter = h_binding.iter(); - while let Some(h_map_entry) = header_map_iter.next(){ + while let Some(h_map_entry) = header_map_iter.next() { let r_binding = result_map.read().unwrap(); let record = r_binding.get(h_map_entry.1); - - append_single_block_result_to_output(&file, &bitcoind_info, h_map_entry, record.unwrap())?; + + let block_aggregate = get_block_aggregate_output(&bitcoind_info, h_map_entry, record.unwrap())?; + append_single_block_result_to_file(&file, &block_aggregate)?; + match sqlite_persistence.persist_block_aggregates(&block_aggregate){ + std::result::Result::Ok(_) => {}, + Err(e) => { + eprintln!("Error persisting {}, error={}", block_aggregate.block_hash_big_endian, e); + }, + } } Ok(()) } @@ -233,6 +204,7 @@ async fn run_async_block_eval_listener(args: &BlockAsyncEvalArgs) -> Result<()> file.seek(std::io::SeekFrom::End(0))?; let bitcoind_info = BitcoindRpcInfo::new()?; + let sqlite_persistence = persistence::SQLitePersistence::new()?; loop { let zmq_message = socket.recv().await?; @@ -254,7 +226,14 @@ async fn run_async_block_eval_listener(args: &BlockAsyncEvalArgs) -> Result<()> let r_binding = result_map.read().unwrap(); let record_entry = r_binding.first_key_value().unwrap(); let record = record_entry.1; - let _ = append_single_block_result_to_output(&file, &bitcoind_info, h_map_entry, record); + let block_aggregate = get_block_aggregate_output(&bitcoind_info, h_map_entry, record)?; + append_single_block_result_to_file(&file, &block_aggregate)?; + match sqlite_persistence.persist_block_aggregates(&block_aggregate){ + std::result::Result::Ok(_) => {}, + Err(e) => { + eprintln!("Error persisting {}, error={}", block_aggregate.block_hash_big_endian, e); + }, + }; } None => panic!("second element from zeromq raw block is non-existent!"), } @@ -330,3 +309,70 @@ fn run_graph(_args: &GraphArgs) -> Result<()> { println!("Graph functionality not yet implemented"); Ok(()) } + +fn append_single_block_result_to_file( + mut file: &File, + block_aggregate: &BlockAggregateOutput +) -> Result<()> { + let write_output_env = env::var("WRITE_OUTPUT_TO_FILE"); + match write_output_env { + std::result::Result::Ok(x) => { + if x.parse().unwrap() { + let output_line = format!( + "{},{},{},{},{}", + block_aggregate.block_height, + block_aggregate.block_hash_big_endian, + block_aggregate.date, + block_aggregate.total_p2pk_addresses, + block_aggregate.total_p2pk_value + ); + writeln!(file, "{}", output_line)?; + } + }, + Err(_) => {}, + } + Ok(()) +} + +fn get_block_aggregate_output( + bitcoind_info: &BitcoindRpcInfo, + h_map_entry: (&[u8; 32], &[u8; 32]), + record: &Record, +) -> Result { + // Get previous and current block hashes + let mut raw_previous_block_hash = h_map_entry.0.clone(); + raw_previous_block_hash.reverse(); + let previous_block_hash = hex::encode(raw_previous_block_hash); + + let mut raw_current_block_hash = h_map_entry.1.clone(); + let sha256d_hash = Hash::from_bytes_ref(&raw_current_block_hash); + + let mut block_height = 0; + + match bitcoind_info.get_block_height(sha256d_hash) { + std::result::Result::Ok(x) => block_height = x, + Err(e) => println!("block not found: exception={}", e), + } + raw_current_block_hash.reverse(); + let current_block_hash_header = hex::encode(raw_current_block_hash); + + println!( + "previous_block_hash={} , current_block_hash={}, block_height={}", + previous_block_hash, current_block_hash_header, block_height + ); + + // Determine total p2pk addresses and value + let mut total_p2pk_addresses = record.p2pk_addresses_added.to_owned(); + total_p2pk_addresses -= record.p2pk_addresses_spent.to_owned(); + let mut total_p2pk_value = record.p2pk_sats_added.to_owned() as f64 / 100_000_000.0; + total_p2pk_value -= record.p2pk_sats_spent.to_owned() as f64 / 100_000_000.0; + let date = &record.date; + + Ok(BlockAggregateOutput { + date: date.clone(), + block_height, + block_hash_big_endian: current_block_hash_header, + total_p2pk_addresses, + total_p2pk_value, + }) +} diff --git a/src/p2pktx.rs b/src/p2pktx.rs index fa30ce2..ba2acf3 100644 --- a/src/p2pktx.rs +++ b/src/p2pktx.rs @@ -7,22 +7,20 @@ //! use std::collections::BTreeMap; -use std::str::FromStr; use std::fmt; +use std::str::FromStr; use anyhow::Result; -use bitcoin::bip32::{ - DerivationPath, Fingerprint, IntoDerivationPath, Xpriv, Xpub, -}; +use bitcoin::bip32::{DerivationPath, Fingerprint, Xpriv, Xpub}; use bitcoin::consensus::encode; use bitcoin::key::rand; use bitcoin::locktime::absolute; use bitcoin::psbt::{self, Input, Psbt, PsbtSighashType}; use bitcoin::secp256k1::{Secp256k1, Signing, Verification}; use bitcoin::{ - transaction, Address, Amount, OutPoint, - PublicKey, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Witness, + transaction, Address, Amount, OutPoint, PublicKey, ScriptBuf, Sequence, Transaction, TxIn, + TxOut, Witness, }; extern crate bitcoincore_rpc; diff --git a/src/persistence.rs b/src/persistence.rs index 4adbcae..ad186e5 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -1,13 +1,12 @@ use std::env; -use rusqlite::{Connection, Result}; +use rusqlite::Connection; -use crate::block::{HeaderMap, ResultMap}; +use crate::block::BlockAggregateOutput; #[derive(Debug)] pub struct SQLitePersistence { - sql_conn: Connection, - sqlite_absolute_path: String + sql_conn: Connection } impl SQLitePersistence { @@ -15,25 +14,37 @@ impl SQLitePersistence { let sqlite_absolute_path = env::var("SQLITE_ABSOLUTE_PATH")?; let sql_conn = Connection::open(&sqlite_absolute_path)?; - let db_exec_results = sql_conn.execute( + sql_conn.execute( "create table if not exists p2pk_utxo_block_aggregates ( - block_height integer primary key, - date integer not null, + block_height integer not null, + block_hash text primary key, + date text not null, p2pk_utxo_count integer not null, p2pk_utxo_value real not null )", [], )?; - println!("p2pk_utxo_block_aggregates: table now exists at: {}", sqlite_absolute_path); - - Ok(SQLitePersistence{sql_conn, sqlite_absolute_path}) - + println!( + "p2pk_utxo_block_aggregates: table now exists at: {}", + sqlite_absolute_path + ); + + Ok(SQLitePersistence { + sql_conn + }) } - pub fn persist_block_aggregates(header_map: &HeaderMap,result_map: &ResultMap) -> anyhow::Result<()> { - - let result_map_read = result_map.read().unwrap(); - - Ok(()) + pub fn persist_block_aggregates(&self, block_aggregate: &BlockAggregateOutput) -> anyhow::Result<(usize)> { + + let sql = "INSERT INTO p2pk_utxo_block_aggregates VALUES(?1,?2,?3,?4,?5)"; + let db_exec_results = self.sql_conn.execute(sql, [ + block_aggregate.block_height.to_string(), + block_aggregate.block_hash_big_endian.clone(), + block_aggregate.date.clone(), + block_aggregate.total_p2pk_addresses.to_string(), + block_aggregate.total_p2pk_value.to_string() + ])?; + + Ok(db_exec_results) } -} \ No newline at end of file +} From 18969fbfda95feb60a0b4de73869987aeb703ed7 Mon Sep 17 00:00:00 2001 From: jbride Date: Tue, 29 Oct 2024 20:43:41 -0600 Subject: [PATCH 12/13] sqlite persistence: now using connection pool --- Cargo.lock | 34 ++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/persistence.rs | 15 ++++++++++----- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8c54b02..e1982b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -562,6 +562,8 @@ dependencies = [ "indicatif", "log", "nom", + "r2d2", + "r2d2_sqlite", "rayon", "rusqlite", "serde", @@ -924,6 +926,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r2d2" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "r2d2_sqlite" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb14dba8247a6a15b7fdbc7d389e2e6f03ee9f184f87117706d509c092dfe846" +dependencies = [ + "r2d2", + "rusqlite", + "uuid", +] + [[package]] name = "rand" version = "0.8.5" @@ -1038,6 +1062,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1259,6 +1292,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", + "rand", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 97f9076..aa0ff2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,3 +27,5 @@ bitcoincore-rpc = "0.19.0" # features= bundled # This causes rusqlite to compile its own private libsqlite3 and link it with your Rust code, instead of using /usr/lib/x86_64-linux-gnu/libsqlite3.so rusqlite = { version = "0.32.1", features = ["bundled"] } +r2d2_sqlite = "0.25.0" +r2d2 = "0.8.10" diff --git a/src/persistence.rs b/src/persistence.rs index ad186e5..96ff117 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -1,19 +1,22 @@ use std::env; -use rusqlite::Connection; +use r2d2::Pool; +use r2d2_sqlite::SqliteConnectionManager; use crate::block::BlockAggregateOutput; #[derive(Debug)] pub struct SQLitePersistence { - sql_conn: Connection + pool: Pool } impl SQLitePersistence { pub fn new() -> anyhow::Result<(Self)> { let sqlite_absolute_path = env::var("SQLITE_ABSOLUTE_PATH")?; - let sql_conn = Connection::open(&sqlite_absolute_path)?; + let manager = SqliteConnectionManager::file(&sqlite_absolute_path); + let pool = r2d2::Pool::builder().max_size(15).build(manager).unwrap(); + let sql_conn = pool.get().unwrap(); sql_conn.execute( "create table if not exists p2pk_utxo_block_aggregates ( block_height integer not null, @@ -30,14 +33,16 @@ impl SQLitePersistence { ); Ok(SQLitePersistence { - sql_conn + pool }) } pub fn persist_block_aggregates(&self, block_aggregate: &BlockAggregateOutput) -> anyhow::Result<(usize)> { let sql = "INSERT INTO p2pk_utxo_block_aggregates VALUES(?1,?2,?3,?4,?5)"; - let db_exec_results = self.sql_conn.execute(sql, [ + + let sql_conn = self.pool.get().unwrap(); + let db_exec_results = sql_conn.execute(sql, [ block_aggregate.block_height.to_string(), block_aggregate.block_hash_big_endian.clone(), block_aggregate.date.clone(), From 6cf328c53bbbd1569acb9369cd41d963431dfcdf Mon Sep 17 00:00:00 2001 From: jbride Date: Wed, 30 Oct 2024 03:41:14 -0600 Subject: [PATCH 13/13] Suggested code changes from cursor --- src/bitcoind_rpc.rs | 90 ++++++++++++++++++--------------------------- src/block.rs | 2 +- src/main.rs | 31 ++++++++++------ src/persistence.rs | 7 ++-- 4 files changed, 61 insertions(+), 69 deletions(-) diff --git a/src/bitcoind_rpc.rs b/src/bitcoind_rpc.rs index 2d7252c..a3d89cf 100644 --- a/src/bitcoind_rpc.rs +++ b/src/bitcoind_rpc.rs @@ -7,6 +7,7 @@ use bitcoin::hashes::sha256d::Hash; use bitcoincore_rpc::json::{self, GetAddressInfoResult, ListUnspentResultEntry}; use bitcoincore_rpc::{Auth, Client, RpcApi}; +/// Represents a connection to a Bitcoin Core RPC interface #[derive(Debug)] pub struct BitcoindRpcInfo { rpc_client: Client, @@ -14,79 +15,60 @@ pub struct BitcoindRpcInfo { impl BitcoindRpcInfo { pub fn new() -> Result { - let url = env::var("URL")?; - let cookie = env::var("COOKIE"); - let auth = match cookie { + let url = env::var("URL").map_err(|e| anyhow!("Missing URL environment variable: {}", e))?; + + let auth = match env::var("COOKIE") { Ok(cookiefile) => Auth::CookieFile(cookiefile.into()), Err(_) => { - let user = env::var("USER")?; - let pass = env::var("PASS")?; - + let user = env::var("USER") + .map_err(|e| anyhow!("Missing USER environment variable: {}", e))?; + let pass = env::var("PASS") + .map_err(|e| anyhow!("Missing PASS environment variable: {}", e))?; Auth::UserPass(user, pass) } }; - let rpc_client = Client::new(&url, auth)?; - Ok(BitcoindRpcInfo { rpc_client }) + + Client::new(&url, auth) + .map(|rpc_client| BitcoindRpcInfo { rpc_client }) + .map_err(|e| anyhow!("Failed to create RPC client: {}", e)) } pub fn get_bitcoind_info_for_test_p2pk( &self, output_amount_btc: f64, - ) -> Result<( - ListUnspentResultEntry, - GetAddressInfoResult, - Address, - Amount, - )> { - // 1) get default bitcoind relay_fee - let network_relay_fee = self.rpc_client.get_network_info()?.relay_fee; + ) -> Result<(ListUnspentResultEntry, GetAddressInfoResult, Address, Amount)> { + // Get network relay fee + let network_relay_fee = self.rpc_client.get_network_info() + .map_err(|e| anyhow!("Failed to get network info: {}", e))? + .relay_fee; let output_tx_total = network_relay_fee.to_btc() + output_amount_btc; - // 2) identify first utxo managed by bitcoind wallet with a value > desired p2pk output - let mut unspent_option: Option = None; - let unspent_vec = self - .rpc_client + // Find suitable UTXO + let unspent_vec = self.rpc_client .list_unspent(Some(3), None, None, None, None) - .unwrap(); - for unspent_candidate in unspent_vec { - println!( - "unspent_candidate txid={}, vout={}, tx_amount={}, output_tx_total={}", - unspent_candidate.txid, - unspent_candidate.vout, - unspent_candidate.amount.to_btc(), - output_tx_total - ); - if unspent_candidate.amount.to_btc() > output_tx_total { - unspent_option = Some(unspent_candidate); - break; - } - } - if unspent_option == None { - return Err(anyhow!( - "No unspent txs have sufficient funds: {}", - output_tx_total - )); - } + .map_err(|e| anyhow!("Failed to list unspent transactions: {}", e))?; - let unspent_tx = unspent_option.unwrap(); - let input_utxo_address = unspent_tx.address.clone().unwrap().assume_checked(); + let unspent_tx = unspent_vec + .into_iter() + .find(|utxo| utxo.amount.to_btc() > output_tx_total) + .ok_or_else(|| anyhow!("No unspent txs have sufficient funds: {}", output_tx_total))?; - // 3) Get info about the input utxo address - let input_utxo_address_info = self.rpc_client.get_address_info(&input_utxo_address)?; + // Get input UTXO address info + let input_utxo_address = unspent_tx.address.clone() + .ok_or_else(|| anyhow!("UTXO has no address"))? + .assume_checked(); + + let input_utxo_address_info = self.rpc_client + .get_address_info(&input_utxo_address) + .map_err(|e| anyhow!("Failed to get address info: {}", e))?; - // 4) - let change_addr = self - .rpc_client + // Get change address + let change_addr = self.rpc_client .get_raw_change_address(Some(json::AddressType::Bech32)) - .unwrap() + .map_err(|e| anyhow!("Failed to get change address: {}", e))? .assume_checked(); - Ok(( - unspent_tx, - input_utxo_address_info, - change_addr, - network_relay_fee, - )) + Ok((unspent_tx, input_utxo_address_info, change_addr, network_relay_fee)) } pub fn get_block_height(&self, sha256d_hash: &Hash) -> Result { diff --git a/src/block.rs b/src/block.rs index 789211f..2d06ab1 100644 --- a/src/block.rs +++ b/src/block.rs @@ -36,7 +36,7 @@ pub struct BitcoinBlock { pub header: BlockHeader, pub transactions: Vec, } - +#[derive(Clone)] pub struct Record { pub date: String, pub p2pk_addresses_added: u32, diff --git a/src/main.rs b/src/main.rs index bb017a3..210da70 100644 --- a/src/main.rs +++ b/src/main.rs @@ -152,10 +152,13 @@ fn run_block_file_eval(args: &BlockFileEvalArgs) -> Result<()> { let h_binding = header_map.read().unwrap(); let mut header_map_iter = h_binding.iter(); while let Some(h_map_entry) = header_map_iter.next() { - let r_binding = result_map.read().unwrap(); - let record = r_binding.get(h_map_entry.1); - - let block_aggregate = get_block_aggregate_output(&bitcoind_info, h_map_entry, record.unwrap())?; + let record = { + let r_binding = result_map.read().unwrap(); + r_binding.get(h_map_entry.1).cloned() + }; + + let h_map_entry = (*h_map_entry.0, *h_map_entry.1); + let block_aggregate = get_block_aggregate_output(&bitcoind_info, &h_map_entry, &record.unwrap())?; append_single_block_result_to_file(&file, &block_aggregate)?; match sqlite_persistence.persist_block_aggregates(&block_aggregate){ std::result::Result::Ok(_) => {}, @@ -221,12 +224,18 @@ async fn run_async_block_eval_listener(args: &BlockAsyncEvalArgs) -> Result<()> tx_count ); - let h_binding = header_map.read().unwrap(); - let h_map_entry = h_binding.first_key_value().unwrap(); - let r_binding = result_map.read().unwrap(); - let record_entry = r_binding.first_key_value().unwrap(); - let record = record_entry.1; - let block_aggregate = get_block_aggregate_output(&bitcoind_info, h_map_entry, record)?; + let h_map_entry = { + let h_binding = header_map.read().unwrap(); + let (key, value) = h_binding.first_key_value().unwrap(); + (*key, *value) + }; + let record = { + let r_binding = result_map.read().unwrap(); + r_binding.first_key_value().unwrap().1.clone() + }; + + + let block_aggregate = get_block_aggregate_output(&bitcoind_info, &h_map_entry, &record)?; append_single_block_result_to_file(&file, &block_aggregate)?; match sqlite_persistence.persist_block_aggregates(&block_aggregate){ std::result::Result::Ok(_) => {}, @@ -336,7 +345,7 @@ fn append_single_block_result_to_file( fn get_block_aggregate_output( bitcoind_info: &BitcoindRpcInfo, - h_map_entry: (&[u8; 32], &[u8; 32]), + h_map_entry: &([u8; 32], [u8; 32]), record: &Record, ) -> Result { // Get previous and current block hashes diff --git a/src/persistence.rs b/src/persistence.rs index 96ff117..8cb81f8 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -12,11 +12,12 @@ pub struct SQLitePersistence { impl SQLitePersistence { pub fn new() -> anyhow::Result<(Self)> { - let sqlite_absolute_path = env::var("SQLITE_ABSOLUTE_PATH")?; + let sqlite_absolute_path = env::var("SQLITE_ABSOLUTE_PATH") + .map_err(|e| anyhow::anyhow!("Missing SQLITE_ABSOLUTE_PATH environment variable: {}", e))?; let manager = SqliteConnectionManager::file(&sqlite_absolute_path); - let pool = r2d2::Pool::builder().max_size(15).build(manager).unwrap(); + let pool = r2d2::Pool::builder().max_size(15).build(manager)?; - let sql_conn = pool.get().unwrap(); + let sql_conn = pool.get()?; sql_conn.execute( "create table if not exists p2pk_utxo_block_aggregates ( block_height integer not null,