diff --git a/Cargo.lock b/Cargo.lock index c0db0194..287d5462 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + [[package]] name = "bumpalo" version = "3.11.0" @@ -914,6 +920,22 @@ dependencies = [ "keccak", ] +[[package]] +name = "sol-balance" +version = "0.1.0" +dependencies = [ + "anyhow", + "bs58", + "ethabi", + "hex-literal", + "num-bigint", + "prost 0.11.3", + "substreams 0.3.2", + "substreams-common", + "substreams-helper", + "substreams-solana 0.2.0", +] + [[package]] name = "solana-sample" version = "0.1.0" @@ -925,7 +947,7 @@ dependencies = [ "prost-build 0.10.4", "prost-types 0.10.1", "substreams 0.0.11", - "substreams-solana", + "substreams-solana 0.1.0", ] [[package]] @@ -1155,6 +1177,17 @@ dependencies = [ "substreams 0.0.12", ] +[[package]] +name = "substreams-solana" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e5ad4a64b8af93729ca06debc69b2c72eaec3ee44872dd593e16dfcc56dae7" +dependencies = [ + "prost 0.11.3", + "prost-build 0.11.1", + "prost-types 0.11.1", +] + [[package]] name = "substreams-uniswap-v2" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5a1bd6a3..23899b1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "substreams-helper", "uniswap-v2", "solana-sample", + "sol-balance", "ens-names", ] exclude = ["messari-cli"] @@ -18,7 +19,7 @@ exclude = ["messari-cli"] [workspace.dependencies] substreams = "0.3.2" substreams-ethereum = "0.6.2" -substreams-solana = "0.1.0" +substreams-solana = "0.2.0" [build] target = "wasm32-unknown-unknown" diff --git a/Makefile b/Makefile index 150c34c2..3af031e4 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ build-all: $(MAKE) -C network build $(MAKE) -C solana-sample build $(MAKE) -C eth-balance build + $(MAKE) -C sol-balance build $(MAKE) -C ens-names build .PHONY: run-all @@ -19,6 +20,7 @@ run-all: $(MAKE) -C erc20-market-cap run $(MAKE) -C erc721 run $(MAKE) -C network run + $(MAKE) -C sol-balance run $(MAKE) -C ens-names run .PHONY: test diff --git a/README.md b/README.md index 8704de8e..0501edd1 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ | ENS Look Up | 🔨 | ENS records for lookup and reverse lookup | | Uniswap v2 | 🔨 | Substreams for Uniswap v2 | | Compound v2 | 🔨 | Substreams for Compound v2 | +| SOL Balance | 🛠 | SOL balance for every Solana address | ## Workflow diff --git a/common/proto/sol_token.proto b/common/proto/sol_token.proto new file mode 100644 index 00000000..fba92695 --- /dev/null +++ b/common/proto/sol_token.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package messari.sol_token.v1; + +message Transfers { + repeated Transfer transfers = 1; +} + +message Transfer { + string signature = 1; + string from = 2; + string to = 3; + TokenAccount token = 4; + string mint = 5; + uint64 native_amount = 6; + uint64 amount = 7; + BalanceChanges balance_changes = 8; +} + +message BalanceChanges { + repeated TokenBalance items = 1; +} + +// balance change +message TokenBalance { + TokenAccount token = 1; + string transaction_id = 2; + uint64 block_height = 3; + string address = 4; // account address of the balance change + string pre_balance = 5; // BigInt, in token's native amount + string post_balance = 6; // BigInt, in token's native amount +} + +message Tokens { + repeated TokenAccount tokens = 1; +} + +message TokenAccount { + string address = 1; + string name = 2; + string symbol = 3; + uint64 decimals = 4; + string freeze_authority = 5; + string mint_authority = 6; + string tx_created = 7; + uint64 block_created = 8; +} diff --git a/erc20-holdings/README.md b/erc20-holdings/README.md new file mode 100644 index 00000000..e69de29b diff --git a/erc20-holdings/src/pb.rs b/erc20-holdings/src/pb.rs index bf5cf178..6d3d4de7 100644 --- a/erc20-holdings/src/pb.rs +++ b/erc20-holdings/src/pb.rs @@ -1,39 +1,9 @@ #[rustfmt::skip] -#[path = "../target/pb/messari.chainlink.v1.rs"] -pub(in crate::pb) mod chainlink_v1; +#[path = "../target/pb/messari.sol_token.v1.rs"] +pub(in crate::pb) mod sol_token_v1; -pub mod chainlink { +pub mod sol_token { pub mod v1 { - pub use super::super::chainlink_v1::*; - } -} - -#[rustfmt::skip] -#[path = "../target/pb/messari.common.v1.rs"] -pub(in crate::pb) mod common_v1; - -pub mod common { - pub mod v1 { - pub use super::super::common_v1::*; - } -} - -#[rustfmt::skip] -#[path = "../target/pb/messari.erc20.v1.rs"] -pub(in crate::pb) mod erc20_v1; - -pub mod erc20 { - pub mod v1 { - pub use super::super::erc20_v1::*; - } -} - -#[rustfmt::skip] -#[path = "../target/pb/messari.erc20_price.v1.rs"] -pub(in crate::pb) mod erc20_price_v1; - -pub mod erc20_price { - pub mod v1 { - pub use super::super::erc20_price_v1::*; + pub use super::super::sol_token_v1::*; } } diff --git a/erc20-market-cap/src/pb.rs b/erc20-market-cap/src/pb.rs index 9a8fb3eb..eb61bfbe 100644 --- a/erc20-market-cap/src/pb.rs +++ b/erc20-market-cap/src/pb.rs @@ -37,3 +37,13 @@ pub mod erc20_price { pub use super::super::erc20_price_v1::*; } } + +#[rustfmt::skip] +#[path = "../target/pb/messari.uniswap.v1.rs"] +pub(in crate::pb) mod uniswap_v1; + +pub mod uniswap { + pub mod v1 { + pub use super::super::uniswap_v1::*; + } +} diff --git a/sol-balance/Cargo.toml b/sol-balance/Cargo.toml new file mode 100644 index 00000000..e9382f9d --- /dev/null +++ b/sol-balance/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "sol-balance" +version = "0.1.0" +description = "Messari's substream for solana SOL balances" +edition = "2021" +repository = "https://github.com/messari/substreams" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +bs58 = "0.4.0" +prost = "0.11.2" +ethabi = "17.2.0" +num-bigint = "0.4" +hex-literal = "0.3.4" +substreams = { workspace = true } +substreams-solana = { workspace = true } +substreams-helper = { path = "../substreams-helper" } + +[build-dependencies] +anyhow = "1" +substreams-common = { path = "../common" } \ No newline at end of file diff --git a/sol-balance/Makefile b/sol-balance/Makefile new file mode 100644 index 00000000..241a52ba --- /dev/null +++ b/sol-balance/Makefile @@ -0,0 +1,7 @@ +.PHONY: build +build: + cargo build --target wasm32-unknown-unknown --release + +.PHONY: run +run: + substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml map_balances -s 174257750 -t +1 diff --git a/sol-balance/README.md b/sol-balance/README.md new file mode 100644 index 00000000..9ceb7534 --- /dev/null +++ b/sol-balance/README.md @@ -0,0 +1,10 @@ +# SOL Balance Substream + +This substream is designed to store the SOL balance of every account. + +### Notes + +- We are not able fully map all the accounts to SOL balance changes. + - There is a missing field [`address_table_lookup`](https://github.com/streamingfast/firehose-solana/blob/develop/proto/sf/solana/type/v1/type.proto#L38) that stores the rest of the addresses. + - See the issue filed [here](https://github.com/streamingfast/substreams/issues/144) to track the status. + - This field is available, but there is decoding needed on the substream part in order to actually get that data. Next steps are understanding the Solana data structure better and seeing firesol output to get a better understanding of where this data lives. diff --git a/sol-balance/build.rs b/sol-balance/build.rs new file mode 100644 index 00000000..fb321d31 --- /dev/null +++ b/sol-balance/build.rs @@ -0,0 +1,10 @@ +use anyhow::{Ok, Result}; +use substreams_common::codegen; + +fn main() -> Result<(), anyhow::Error> { + println!("cargo:rerun-if-changed=proto"); + println!("cargo:rerun-if-changed=abi"); + codegen::generate(None)?; + + Ok(()) +} diff --git a/sol-balance/rust-toolchain.toml b/sol-balance/rust-toolchain.toml new file mode 100644 index 00000000..0a0b90ae --- /dev/null +++ b/sol-balance/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.64.0" +components = [ "rustfmt" ] +targets = [ "wasm32-unknown-unknown" ] diff --git a/sol-balance/src/abi.rs b/sol-balance/src/abi.rs new file mode 100644 index 00000000..d55d99b3 --- /dev/null +++ b/sol-balance/src/abi.rs @@ -0,0 +1 @@ +// DO NOT EDIT - the file is generated by build script diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs new file mode 100644 index 00000000..f40387ea --- /dev/null +++ b/sol-balance/src/lib.rs @@ -0,0 +1,41 @@ +use bs58; +use substreams_helper::pb::sol_token::v1 as proto; +use substreams_helper::token::get_sol_token; +use substreams_solana::pb::sol as solana; + +#[substreams::handlers::map] +fn map_balances( + block: solana::v1::Block, +) -> Result { + let mut balances = vec![]; + for tx in block.transactions { + if let Some(meta) = tx.meta { + if let Some(_) = meta.err { + continue; + } + if let Some(transaction) = tx.transaction { + if let Some(msg) = transaction.message { + for i in 0..meta.post_balances.len() { + let account_id: String; + if i < msg.account_keys.len() { + account_id = bs58::encode(&msg.account_keys[i]).into_string(); + } else { + account_id = "TODO".to_string(); + // TODO: use msg.address_table_lookup table, but it is not avail + } + balances.push(proto::TokenBalance { + token: get_sol_token(), + transaction_id: bs58::encode(&transaction.signatures[0]).into_string(), + block_height: block.block_height.as_ref().unwrap().block_height, + address: account_id, + pre_balance: meta.pre_balances[i].to_string(), + post_balance: meta.post_balances[i].to_string(), + }); + } + } + } + } + } + + Ok(proto::BalanceChanges { items: balances }) +} diff --git a/sol-balance/src/pb.rs b/sol-balance/src/pb.rs new file mode 100644 index 00000000..42aed8b4 --- /dev/null +++ b/sol-balance/src/pb.rs @@ -0,0 +1,19 @@ +#[rustfmt::skip] +#[path = "../target/pb/messari.sol_token.v1.rs"] +pub(in crate::pb) mod sol_token_v1; + +pub mod sol_token { + pub mod v1 { + pub use super::super::sol_token_v1::*; + } +} + +#[rustfmt::skip] +#[path = "../target/pb/messari.solana.type.rs"] +pub(in crate::pb) mod solana_type; + +pub mod solana { + pub mod type { + pub use super::super::solana_type::*; + } +} diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml new file mode 100644 index 00000000..038c966f --- /dev/null +++ b/sol-balance/substreams.yaml @@ -0,0 +1,27 @@ +specVersion: v0.1.0 +package: + name: sol_balance + version: v0.1.0 + +imports: + sol: https://github.com/streamingfast/firehose-solana/releases/download/v0.1.0/solana-v0.1.0.spkg + +protobuf: + files: + - sol_token.proto + importPaths: + - ../common/proto + +binaries: + default: + type: wasm/rust-v1 + file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm" + +modules: + # get the SOL balance change of each account in each transaction in a block + - name: map_balances + kind: map + inputs: + - source: sf.solana.type.v1.Block + output: + type: proto:messari.sol.token.BalanceChanges diff --git a/substreams-helper/build.rs b/substreams-helper/build.rs index 0f1a2864..05e44fdb 100644 --- a/substreams-helper/build.rs +++ b/substreams-helper/build.rs @@ -2,6 +2,7 @@ use anyhow::{Ok, Result}; use substreams_common::codegen; fn main() -> Result<(), anyhow::Error> { + codegen::generate(None); codegen::generate_abi(None)?; Ok(()) } diff --git a/substreams-helper/src/lib.rs b/substreams-helper/src/lib.rs index b6cf1767..ad0b4954 100644 --- a/substreams-helper/src/lib.rs +++ b/substreams-helper/src/lib.rs @@ -2,6 +2,8 @@ pub mod abi; pub mod erc20; pub mod keyer; pub mod math; +pub mod pb; pub mod price; +pub mod token; pub mod types; pub mod utils; diff --git a/substreams-helper/src/pb.rs b/substreams-helper/src/pb.rs new file mode 100644 index 00000000..6d3d4de7 --- /dev/null +++ b/substreams-helper/src/pb.rs @@ -0,0 +1,9 @@ +#[rustfmt::skip] +#[path = "../target/pb/messari.sol_token.v1.rs"] +pub(in crate::pb) mod sol_token_v1; + +pub mod sol_token { + pub mod v1 { + pub use super::super::sol_token_v1::*; + } +} diff --git a/substreams-helper/src/token.rs b/substreams-helper/src/token.rs new file mode 100644 index 00000000..7292295f --- /dev/null +++ b/substreams-helper/src/token.rs @@ -0,0 +1,16 @@ +use crate::pb::sol_token::v1::TokenAccount; + +pub fn get_sol_token() -> Option { + let sol_token = TokenAccount { + address: "So11111111111111111111111111111111111111111".to_string(), + name: "Solana".to_string(), + symbol: "SOL".to_string(), + decimals: 9_u64, + freeze_authority: "".to_string(), + mint_authority: "".to_string(), + tx_created: "".to_string(), + block_created: 0_u64, + }; + + Some(sol_token) +} diff --git a/substreams-helper/substreams.yaml b/substreams-helper/substreams.yaml new file mode 100644 index 00000000..ef91ee98 --- /dev/null +++ b/substreams-helper/substreams.yaml @@ -0,0 +1,19 @@ +specVersion: v0.1.0 +package: + name: substreams_helper + version: v0.1.0 + +imports: + eth: https://github.com/streamingfast/sf-ethereum/releases/download/v0.10.2/ethereum-v0.10.4.spkg + +protobuf: + files: + - sol_token.proto + - + importPaths: + - ../common/proto + +binaries: + default: + type: wasm/rust-v1 + file: "../target/wasm32-unknown-unknown/release/substreams_helper.wasm" \ No newline at end of file