proxy: add blocks support (#3338)

* add blocks support

* add rpc handlers

* reviews

* format

* catch only error exceptions

* remove unused imports

* review

* add basics tests

* fix

Author: chirag-parmar

nimbus_verified_proxy/header_store.nim

@@ -116,6 +116,12 @@ func finalized*(self: HeaderStore): Opt[Header] =
 func finalizedHash*(self: HeaderStore): Opt[Hash32] =
   self.finalizedHash
 
+func contains*(self: HeaderStore, hash: Hash32): bool =
+  self.headers.contains(hash)
+
+func contains*(self: HeaderStore, number: base.BlockNumber): bool =
+  self.hashes.contains(number)
+
 proc updateFinalized*(
     self: HeaderStore, header: ForkedLightClientHeader
 ): Result[bool, string] =

nimbus_verified_proxy/nimbus_verified_proxy.nim

@@ -84,7 +84,9 @@ proc run*(
     # header cache contains headers downloaded from p2p
     headerStore = HeaderStore.new(config.cacheLen)
 
-  let verifiedProxy = VerifiedRpcProxy.init(rpcProxy, headerStore, chainId)
+  # TODO: add config object to verified proxy for future config options
+  let verifiedProxy =
+    VerifiedRpcProxy.init(rpcProxy, headerStore, chainId, config.maxBlockWalk)
 
   # add handlers that verify RPC calls /rpc/rpc_eth_api.nim
   verifiedProxy.installEthApiHandlers()

nimbus_verified_proxy/nimbus_verified_proxy_conf.nim

@@ -141,6 +141,14 @@ type VerifiedProxyConf* = object # Config
     name: "max-peers"
   .}: int
 
+  maxBlockWalk* {.
+    hidden,
+    desc: "Maximum number of blocks that will be queried to serve a request",
+    defaultValue: 1000,
+    defaultValueDesc: "1000",
+    name: "debug-max-walk"
+  .}: uint64
+
   hardMaxPeers* {.
     desc: "The maximum number of peers to connect to. Defaults to maxPeers * 1.5",
     name: "hard-max-peers"

nimbus_verified_proxy/rpc/blocks.nim

@@ -9,59 +9,244 @@
 
 import
   std/strutils,
-  stint,
-  chronos,
   results,
+  chronicles,
+  web3/[eth_api_types, eth_api],
+  json_rpc/[rpcproxy, rpcserver, rpcclient],
   eth/common/eth_types_rlp,
-  web3/eth_api_types,
+  eth/rlp,
+  eth/trie/[ordered_trie, trie_defs],
+  ../../execution_chain/beacon/web3_eth_conv,
+  ../types,
   ../header_store,
-  ../types
+  ./transactions
 
-type
-  QuantityTagKind = enum
-    LatestBlock
-    BlockNumber
-
-  QuantityTag = object
-    case kind: QuantityTagKind
-    of LatestBlock:
-      discard
-    of BlockNumber:
-      blockNumber: Quantity
-
-func parseQuantityTag(blockTag: BlockTag): Result[QuantityTag, string] =
+proc resolveBlockTag*(
+    vp: VerifiedRpcProxy, blockTag: BlockTag
+): Result[base.BlockNumber, string] =
   if blockTag.kind == bidAlias:
-    let tag = blockTag.alias.toLowerAscii
+    let tag = blockTag.alias.toLowerAscii()
     case tag
     of "latest":
-      return ok(QuantityTag(kind: LatestBlock))
+      let hLatest = vp.headerStore.latest.valueOr:
+        return err("Couldn't get the latest block number from header store")
+      ok(hLatest.number)
     else:
-      return err("Unsupported blockTag: " & tag)
+      err("No support for block tag " & $blockTag)
   else:
-    let quantity = blockTag.number
-    return ok(QuantityTag(kind: BlockNumber, blockNumber: quantity))
-
-template checkPreconditions(proxy: VerifiedRpcProxy) =
-  if proxy.headerStore.isEmpty():
-    raise newException(ValueError, "Syncing")
-
-proc getHeaderByTag(
-    proxy: VerifiedRpcProxy, quantityTag: BlockTag
-): results.Opt[Header] {.raises: [ValueError].} =
-  checkPreconditions(proxy)
-
-  let tag = parseQuantityTag(quantityTag).valueOr:
-    raise newException(ValueError, error)
-
-  case tag.kind
-  of LatestBlock:
-    # this will always return some block, as we always checkPreconditions
-    proxy.headerStore.latest
-  of BlockNumber:
-    proxy.headerStore.get(base.BlockNumber(distinctBase(tag.blockNumber)))
-
-proc getHeaderByTagOrThrow*(
-    proxy: VerifiedRpcProxy, quantityTag: BlockTag
-): Header {.raises: [ValueError].} =
-  getHeaderByTag(proxy, quantityTag).valueOr:
-    raise newException(ValueError, "No block stored for given tag " & $quantityTag)
+    ok(base.BlockNumber(distinctBase(blockTag.number)))
+
+func convHeader(blk: eth_api_types.BlockObject): Header =
+  let nonce = blk.nonce.valueOr:
+    default(Bytes8)
+
+  return Header(
+    parentHash: blk.parentHash,
+    ommersHash: blk.sha3Uncles,
+    coinbase: blk.miner,
+    stateRoot: blk.stateRoot,
+    transactionsRoot: blk.transactionsRoot,
+    receiptsRoot: blk.receiptsRoot,
+    logsBloom: blk.logsBloom,
+    difficulty: blk.difficulty,
+    number: base.BlockNumber(distinctBase(blk.number)),
+    gasLimit: GasInt(blk.gasLimit.uint64),
+    gasUsed: GasInt(blk.gasUsed.uint64),
+    timestamp: ethTime(blk.timestamp),
+    extraData: seq[byte](blk.extraData),
+    mixHash: Bytes32(distinctBase(blk.mixHash)),
+    nonce: nonce,
+    baseFeePerGas: blk.baseFeePerGas,
+    withdrawalsRoot: blk.withdrawalsRoot,
+    blobGasUsed: blk.blobGasUsed.u64,
+    excessBlobGas: blk.excessBlobGas.u64,
+    parentBeaconBlockRoot: blk.parentBeaconBlockRoot,
+    requestsHash: blk.requestsHash,
+  )
+
+proc walkBlocks(
+    vp: VerifiedRpcProxy,
+    sourceNum: base.BlockNumber,
+    targetNum: base.BlockNumber,
+    sourceHash: Hash32,
+    targetHash: Hash32,
+): Future[Result[void, string]] {.async: (raises: []).} =
+  var nextHash = sourceHash
+  info "Starting block walk to verify requested block", blockHash = targetHash
+
+  let numBlocks = sourceNum - targetNum
+  if numBlocks > vp.maxBlockWalk:
+    return err(
+      "Cannot query more than " & $vp.maxBlockWalk &
+        " to verify the chain for the requested block"
+    )
+
+  for i in 0 ..< numBlocks:
+    let nextHeader =
+      if vp.headerStore.contains(nextHash):
+        vp.headerStore.get(nextHash).get()
+      else:
+        let blk =
+          try:
+            await vp.rpcClient.eth_getBlockByHash(nextHash, false)
+          except CatchableError as e:
+            return err(
+              "Couldn't get block " & $nextHash & " during the chain traversal: " & e.msg
+            )
+
+        trace "getting next block",
+          hash = nextHash,
+          number = blk.number,
+          remaining = distinctBase(blk.number) - targetNum
+
+        let header = convHeader(blk)
+
+        if header.computeBlockHash != nextHash:
+          return err("Encountered an invalid block header while walking the chain")
+
+        header
+
+    if nextHeader.parentHash == targetHash:
+      return ok()
+
+    nextHash = nextHeader.parentHash
+
+  err("the requested block is not part of the canonical chain")
+
+proc verifyHeader(
+    vp: VerifiedRpcProxy, header: Header, hash: Hash32
+): Future[Result[void, string]] {.async.} =
+  # verify calculated hash with the requested hash
+  if header.computeBlockHash != hash:
+    return err("hashed block header doesn't match with blk.hash(downloaded)")
+
+  if not vp.headerStore.contains(hash):
+    let latestHeader = vp.headerStore.latest.valueOr:
+      return err("Couldn't get the latest header, syncing in progress")
+
+    # walk blocks backwards(time) from source to target
+    ?(
+      await vp.walkBlocks(
+        latestHeader.number, header.number, latestHeader.parentHash, hash
+      )
+    )
+
+  ok()
+
+proc verifyBlock(
+    vp: VerifiedRpcProxy, blk: BlockObject, fullTransactions: bool
+): Future[Result[void, string]] {.async.} =
+  let header = convHeader(blk)
+
+  ?(await vp.verifyHeader(header, blk.hash))
+
+  # verify transactions
+  if fullTransactions:
+    ?verifyTransactions(header.transactionsRoot, blk.transactions)
+
+  # verify withdrawals
+  if blk.withdrawalsRoot.isSome():
+    if blk.withdrawalsRoot.get() != orderedTrieRoot(blk.withdrawals.get(@[])):
+      return err("Withdrawals within the block do not yield the same withdrawals root")
+  else:
+    if blk.withdrawals.isSome():
+      return err("Block contains withdrawals but no withdrawalsRoot")
+
+  ok()
+
+proc getBlock*(
+    vp: VerifiedRpcProxy, blockHash: Hash32, fullTransactions: bool
+): Future[Result[BlockObject, string]] {.async.} =
+  # get the target block
+  let blk =
+    try:
+      await vp.rpcClient.eth_getBlockByHash(blockHash, fullTransactions)
+    except CatchableError as e:
+      return err(e.msg)
+
+  # verify requested hash with the downloaded hash
+  if blockHash != blk.hash:
+    return err("the downloaded block hash doesn't match with the requested hash")
+
+  # verify the block
+  ?(await vp.verifyBlock(blk, fullTransactions))
+
+  ok(blk)
+
+proc getBlock*(
+    vp: VerifiedRpcProxy, blockTag: BlockTag, fullTransactions: bool
+): Future[Result[BlockObject, string]] {.async.} =
+  let n = vp.resolveBlockTag(blockTag).valueOr:
+    return err(error)
+
+  # get the target block
+  let blk =
+    try:
+      await vp.rpcClient.eth_getBlockByNumber(blockTag, fullTransactions)
+    except CatchableError as e:
+      return err(e.msg)
+
+  if n != distinctBase(blk.number):
+    return
+      err("the downloaded block number doesn't match with the requested block number")
+
+  # verify the block
+  ?(await vp.verifyBlock(blk, fullTransactions))
+
+  ok(blk)
+
+proc getHeader*(
+    vp: VerifiedRpcProxy, blockHash: Hash32
+): Future[Result[Header, string]] {.async.} =
+  let cachedHeader = vp.headerStore.get(blockHash)
+
+  if cachedHeader.isNone():
+    debug "did not find the header in the cache", blockHash = blockHash
+  else:
+    return ok(cachedHeader.get())
+
+  # get the target block
+  let blk =
+    try:
+      await vp.rpcClient.eth_getBlockByHash(blockHash, false)
+    except CatchableError as e:
+      return err(e.msg)
+
+  let header = convHeader(blk)
+
+  if blockHash != blk.hash:
+    return err("the blk.hash(downloaded) doesn't match with the provided hash")
+
+  ?(await vp.verifyHeader(header, blockHash))
+
+  ok(header)
+
+proc getHeader*(
+    vp: VerifiedRpcProxy, blockTag: BlockTag
+): Future[Result[Header, string]] {.async.} =
+  let
+    n = vp.resolveBlockTag(blockTag).valueOr:
+      return err(error)
+    cachedHeader = vp.headerStore.get(n)
+
+  if cachedHeader.isNone():
+    debug "did not find the header in the cache", blockTag = blockTag
+  else:
+    return ok(cachedHeader.get())
+
+  # get the target block
+  let blk =
+    try:
+      await vp.rpcClient.eth_getBlockByNumber(blockTag, false)
+    except CatchableError as e:
+      return err(e.msg)
+
+  let header = convHeader(blk)
+
+  if n != header.number:
+    return
+      err("the downloaded block number doesn't match with the requested block number")
+
+  ?(await vp.verifyHeader(header, blk.hash))
+
+  ok(header)