nimbus initial commit

.github/workflows/ci.yml

@@ -18,6 +18,7 @@ on:
       - '**/*.yml'
       - 'hive_integration/**'
       - 'portal/**'
+      - 'nimbus/**'
       - '.github/workflows/portal*.yml'
       - 'nimbus_verified_proxy/**'
       - '.github/workflows/nimbus_verified_proxy.yml'

.github/workflows/kurtosis.yml

@@ -20,6 +20,7 @@ on:
       - '**/*.md'
       - 'hive_integration/**'
       - 'portal/**'
+      - 'nimbus/**'
       - '.github/workflows/portal*.yml'
       - 'nimbus_verified_proxy/**'
       - '.github/workflows/nimbus_verified_proxy.yml'
@@ -32,6 +33,7 @@ on:
       - '**/*.md'
       - 'hive_integration/**'
       - 'portal/**'
+      - 'nimbus/**'
       - '.github/workflows/portal*.yml'
       - 'nimbus_verified_proxy/**'
       - '.github/workflows/nimbus_verified_proxy.yml'

.gitmodules

@@ -161,7 +161,7 @@
 [submodule "vendor/nimbus-eth2"]
 	path = vendor/nimbus-eth2
 	url = https://github.com/status-im/nimbus-eth2.git
-	branch = unstable
+	branch = dev/pedro/pubBN
 [submodule "vendor/nim-taskpools"]
 	path = vendor/nim-taskpools
 	url = https://github.com/status-im/nim-taskpools.git

Makefile

@@ -215,9 +215,6 @@ nimbus_execution_client: | build deps rocksdb
 	echo -e $(BUILD_MSG) "build/nimbus_execution_client" && \
 		$(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/nimbus_execution_client "execution_chain/nimbus_execution_client.nim"
 
-nimbus: nimbus_execution_client
-	echo "The nimbus target is deprecated and will soon change meaning, use 'nimbus_execution_client' instead"
-
 # symlink
 nimbus.nims:
 	ln -s nimbus.nimble $@
@@ -378,13 +375,26 @@ txparse: | build deps
 
 # usual cleaning
 clean: | clean-common
-	rm -rf build/{nimbus,nimbus_execution_client,nimbus_portal_client,fluffy,portal_bridge,libverifproxy,nimbus_verified_proxy,$(TOOLS_CSV),$(PORTAL_TOOLS_CSV),all_tests,test_kvstore_rocksdb,test_rpc,all_portal_tests,all_history_network_custom_chain_tests,test_portal_testnet,utp_test_app,utp_test,*.dSYM}
+	rm -rf build/{nimbus_client,nimbus_execution_client,nimbus_portal_client,fluffy,portal_bridge,libverifproxy,nimbus_verified_proxy}
+	rm -rf build/{$(TOOLS_CSV),$(PORTAL_TOOLS_CSV)}
+	rm -rf build/{all_tests_nimbus,all_tests,test_kvstore_rocksdb,test_rpc,all_portal_tests,all_history_network_custom_chain_tests,test_portal_testnet,utp_test_app,utp_test}
+	rm -rf build/*.dSYM
 	rm -rf tools/t8n/{t8n,t8n_test}
 	rm -rf tools/evmstate/{evmstate,evmstate_test}
 ifneq ($(USE_LIBBACKTRACE), 0)
 	+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
 endif
 
+# Nimbus
+NIM_PARAMS := -d:release --parallelBuild:1 -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,lodestar,prysm,teku,grandine $(NIM_PARAMS)
+nimbus: | build deps rocksdb
+	echo -e $(BUILD_MSG) "build/nimbus_client" && \
+	$(ENV_SCRIPT) nim c $(NIM_PARAMS) --threads:on -d:disable_libbacktrace -d:libp2p_pki_schemes=secp256k1 -o:build//nimbus_client "nimbus/nimbus.nim"
+
+all_tests_nimbus: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:testing --threads:on -d:chronicles_log_level=ERROR -o:build/$@ "nimbus/tests/[email protected]"
+
 # Note about building Nimbus as a library:
 #
 # There were `wrappers`, `wrappers-static`, `libnimbus.so` and `libnimbus.a`

config.nims

@@ -132,6 +132,7 @@ if not defined(windows):
 --mm:refc
 
 switch("define", "withoutPCRE")
+switch("import", "testutils/moduletests")
 
 when not defined(disable_libbacktrace):
   --define:nimStackTraceOverride

execution_chain/nimbus_execution_client.nim

@@ -192,7 +192,7 @@ proc preventLoadingDataDirForTheWrongNetwork(db: CoreDbRef; conf: NimbusConf) =
       expected=calculatedId
     quit(QuitFailure)
 
-proc run(nimbus: NimbusNode, conf: NimbusConf) =
+proc run*(nimbus: NimbusNode, conf: NimbusConf) =
   info "Launching execution client",
       version = FullVersionStr,
       conf

nimbus.nimble

@@ -40,6 +40,7 @@ when declared(namedBin):
     "execution_chain/nimbus_execution_client": "nimbus_execution_client",
     "portal/client/nimbus_portal_client": "nimbus_portal_client",
     "nimbus_verified_proxy/nimbus_verified_proxy": "nimbus_verified_proxy",
+    "nimbus/nimbus_client": "nimbus_client",
   }.toTable()
 
 import std/[os, strutils]
@@ -135,3 +136,11 @@ task build_fuzzers, "Build fuzzer test cases":
   for file in walkDirRec("tests/networking/fuzzing/"):
     if file.endsWith("nim"):
       exec "nim c -c -d:release " & file
+
+## Nimbus tasks
+
+task nimbus, "Build Nimbus":
+  buildBinary "nimbus", "nimbus/", "-d:chronicles_log_level=TRACE"
+
+task nimbus_test, "Run Nimbus tests":
+  test "nimbus/tests/", "all_tests_nimbus", "-d:chronicles_log_level=ERROR -d:testing"

nimbus/README.md

@@ -0,0 +1,43 @@
+# Nimbus
+
+<!-- [![Github Actions CI](tbd) -->
+[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
+[![Discord: Nimbus](https://img.shields.io/badge/discord-nimbus-orange.svg)](https://discord.gg/XRxWahP)
+[![Status: #nimbus-general](https://img.shields.io/badge/status-nimbus--general-orange.svg)](https://join.status.im/nimbus-general)
+
+
+# description
+tbd
+For in-depth configuration and functionality of Nimbus execution and consensus layer refer to:
+- [Nimbus-eth1 - Execution layer client](https://github.com/status-im/nimbus-eth1) Documentation
+- [Nimbus-eth2 - Consensus layer client](https://github.com/status-im/nimbus-eth2) Documentation
+
+tbc
+# dependencies
+tbd
+# how to
+## configuration
+    todo
+## commands
+    todo
+## compile
+tbd
+    - mac os, windows, and linux
+
+    ]$ make nimbus
+## collaborate
+- Use [Status Nim style guide](https://status-im.github.io/nim-style-guide/) to maintain code consistency.
+- Format your code using the [Nim Pretty Printer (nph)](https://github.com/arnetheduck/nph) to ensure consistency across the codebase. Run it as part of your pull request process.
+## License
+
+Licensed and distributed under either of
+
+* MIT license: [LICENSE-MIT](LICENSE-MIT) or https://opensource.org/licenses/MIT
+
+or
+
+* Apache License, Version 2.0: [LICENSE-APACHEv2](LICENSE-APACHEv2) or https://www.apache.org/licenses/LICENSE-2.0
+
+at your option. These files may not be copied, modified, or distributed except according to those terms.

nimbus/common/utils.nim

@@ -0,0 +1,83 @@
+# Nimbus
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import results, chronicles, stew/shims/macros, confutils
+
+logScope:
+  topics = "utils"
+
+## Macro that collects names and abbreviations per layer from configuration
+macro extractFieldNames*(configType: typed): untyped =
+  var names: seq[string] = newSeq[string]()
+  let recDef = configType.getImpl()
+
+  for field in recordFields(recDef):
+    let
+      name = field.readPragma("name")
+      abbr = field.readPragma("abbr")
+
+    if name.kind == nnkNilLit:
+      continue
+
+    names.add($name)
+
+    if abbr.kind != nnkNilLit:
+      names.add($abbr)
+
+  result = quote:
+    `names . mapIt ( newLit ( it ))`
+
+## Write a string into a raw memory buffer (prefixed with length)
+proc writeConfigString*(offset: var uint, elem: string) =
+  if offset <= 0:
+    fatal "memory offset can't be zero"
+    quit(QuitFailure)
+
+  let optLen = uint(elem.len)
+  copyMem(cast[pointer](offset), addr optLen, sizeof(uint))
+  offset += uint(sizeof(uint))
+
+  if optLen > 0:
+    copyMem(cast[pointer](offset), unsafeAddr elem[0], elem.len)
+    offset += uint(elem.len)
+
+## Read a string from a raw memory buffer (expects length prefix)
+proc readConfigString*(offset: var uint): string =
+  var strLen: uint
+  copyMem(addr strLen, cast[pointer](offset), sizeof(uint))
+  offset += uint(sizeof(uint))
+
+  var strData = ""
+  if strLen > 0:
+    strData = newString(strLen)
+    copyMem(addr strData[0], cast[pointer](offset), uint(strLen))
+    offset += uint(strLen)
+
+  strData
+
+## Parse configuration options from a memory block.
+## Format: (table size:uint) | [ (key size:uint)(key:string) (val size:uint)(val:string) ]*
+proc deserializeConfigArgs*(p: pointer): Result[seq[string], string] =
+  var
+    readOffset = cast[uint](p)
+    optionsList = newSeq[string]()
+    totalSize: uint = 0
+
+  copyMem(addr totalSize, cast[pointer](readOffset), sizeof(uint))
+  readOffset += uint(sizeof(uint))
+
+  while readOffset < cast[uint](p) + totalSize:
+    let
+      optName = readConfigString(readOffset)
+      arg = readConfigString(readOffset)
+      option = optName & arg
+
+    optionsList.add(option)
+
+  ok optionsList

nimbus/conf.nim

@@ -0,0 +1,53 @@
+# Nimbus
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import
+  std/[atomics, tables],
+  chronicles
+
+
+## log
+logScope:
+  topics = "Service manager"
+
+## Exceptions
+type NimbusServiceError* = object of CatchableError
+
+## Constants
+const
+  cNimbusServiceTimeoutMs* = 3000
+  cThreadTimeAck* = 10
+
+# configuration read by threads
+var isConfigRead*: Atomic[bool]
+isConfigRead.store(false)
+
+## Nimbus service arguments
+type
+  NimbusConfigTable* = Table[string, string]
+
+  ConfigKind* = enum
+    Execution
+    Consensus
+
+  LayerConfig* = object
+    case kind*: ConfigKind
+    of Consensus:
+      consensusOptions*: NimbusConfigTable
+    of Execution:
+      executionOptions*: NimbusConfigTable
+
+  NimbusService* = ref object
+    name*: string
+    layerConfig*: LayerConfig
+    serviceHandler*: Thread[ptr Channel[pointer]]
+    serviceFunc*: proc(ch: ptr Channel[pointer]) {.thread.}
+
+  Nimbus* = ref object
+    serviceList*: seq[NimbusService]

nimbus/consensus/consensus_layer.nim

@@ -0,0 +1,100 @@
+# Nimbus
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms .
+
+{.push raises: [].}
+
+import
+  std/[atomics, os],
+  chronicles,
+  results,
+  confutils,
+  ../conf,
+  ../common/utils,
+  beacon_chain/validators/keystore_management,
+  beacon_chain/[beacon_node_status, nimbus_binary_common]
+
+from beacon_chain/nimbus_beacon_node import handleStartUpCmd
+from beacon_chain/conf import
+  BeaconNodeConf, SlashingDbKind, BNStartUpCmd, defaultDataDir
+
+logScope:
+  topics = "Consensus layer"
+
+## Request to shutdown Consensus layer
+proc shutdownConsensus*() =
+  bnStatus = BeaconNodeStatus.Stopping
+
+## Creates required beacon node configuration and possibility of additional sources
+proc makeConfig(cmdCommandList: seq[string], ConfType: type): Result[ConfType, string] =
+  {.push warning[ProveInit]: off.}
+  let config =
+    try:
+      ConfType.load(
+        cmdLine = cmdCommandList,
+        secondarySources = proc(
+            config: ConfType, sources: auto
+        ) {.raises: [ConfigurationError], gcsafe.} =
+          if config.configFile.isSome:
+            sources.addConfigFile(Toml, config.configFile.get)
+        ,
+      )
+    except CatchableError as exc:
+      # We need to log to stderr here, because logging hasn't been configured yet
+      var msg = "Failure while loading the configuration:\p" & exc.msg & "\p"
+      if (exc[] of ConfigurationError) and not (isNil(exc.parent)) and
+          (exc.parent[] of TomlFieldReadingError):
+        let fieldName = ((ref TomlFieldReadingError)(exc.parent)).field
+        if fieldName in
+            [
+              "el", "web3-url", "bootstrap-node", "direct-peer",
+              "validator-monitor-pubkey",
+            ]:
+          msg &=
+            "Since the '" & fieldName & "' option is allowed to " &
+            "have more than one value, please make sure to supply " &
+            "a properly formatted TOML array\p"
+      return err(msg)
+  {.pop.}
+  ok(config)
+
+## starts beacon node
+proc startBeaconNode(paramsList: seq[string]) {.raises: [CatchableError].} =
+  var config = makeConfig(paramsList, BeaconNodeConf).valueOr:
+    error "Error starting consensus", err = error
+    quit QuitFailure
+
+  # required for db
+  if not (checkAndCreateDataDir(string(config.dataDir))):
+    quit QuitFailure
+
+  setupLogging(config.logLevel, config.logStdout, config.logFile)
+
+  handleStartUpCmd(config)
+
+## Consensus Layer handler
+proc consensusLayerHandler*(channel: ptr Channel[pointer]) =
+  var p: pointer
+  try:
+    p = channel[].recv()
+  except Exception as e:
+    fatal " service unable to receive configuration", err = e.msg
+    quit(QuitFailure)
+
+  let configList = deserializeConfigArgs(p).valueOr:
+    fatal "unable to parse service data", message = error
+    quit(QuitFailure)
+
+  #signal main thread that data is read
+  isConfigRead.store(true)
+
+  try:
+    {.gcsafe.}:
+      startBeaconNode(configList)
+  except CatchableError as e:
+    fatal "error", message = e.msg
+
+  warn "\tExiting consensus layer"