From c73d8c660edabdb83962b76f2467812a11d93a7a Mon Sep 17 00:00:00 2001 From: hweawer Date: Mon, 14 Apr 2025 12:37:09 +0200 Subject: [PATCH 01/13] feat: Run oracle on a specified refslot --- src/main.py | 117 +++++++++++++++++------- src/metrics/healthcheck_server.py | 2 +- src/modules/checks/checks_module.py | 27 ++---- src/modules/submodules/consensus.py | 3 +- src/modules/submodules/oracle_module.py | 20 ++-- 5 files changed, 106 insertions(+), 63 deletions(-) diff --git a/src/main.py b/src/main.py index 4369ae208..485eb7166 100644 --- a/src/main.py +++ b/src/main.py @@ -1,3 +1,4 @@ +import argparse import sys from typing import Iterator, cast @@ -10,11 +11,12 @@ from src.metrics.logging import logging from src.metrics.prometheus.basic import ENV_VARIABLES_INFO, BUILD_INFO from src.modules.accounting.accounting import Accounting -from src.modules.checks.checks_module import ChecksModule +from src.modules.checks.checks_module import execute_checks from src.modules.csm.csm import CSOracle from src.modules.ejector.ejector import Ejector from src.providers.ipfs import GW3, IPFSProvider, MultiIPFSProvider, Pinata, PublicIPFS -from src.types import OracleModule +from src.types import OracleModule, BlockRoot, SlotNumber +from src.utils.blockstamp import build_blockstamp from src.utils.build import get_build_info from src.utils.exception import IncompatibleException from src.web3py.contract_tweak import tweak_w3_contracts @@ -33,25 +35,7 @@ logger = logging.getLogger(__name__) -def main(module_name: OracleModule): - build_info = get_build_info() - logger.info({ - 'msg': 'Oracle startup.', - 'variables': { - **build_info, - 'module': module_name, - **variables.PUBLIC_ENV_VARS, - }, - }) - ENV_VARIABLES_INFO.info(variables.PUBLIC_ENV_VARS) - BUILD_INFO.info(build_info) - - logger.info({'msg': f'Start healthcheck server for Docker container on port {variables.HEALTHCHECK_SERVER_PORT}'}) - start_pulse_server() - - logger.info({'msg': f'Start http server with prometheus metrics on port {variables.PROMETHEUS_PORT}'}) - start_http_server(variables.PROMETHEUS_PORT) - +def _construct_web3() -> Web3: logger.info({'msg': 'Initialize multi web3 provider.'}) web3 = Web3(FallbackProviderModule( variables.EXECUTION_CLIENT_URI, @@ -92,9 +76,10 @@ def main(module_name: OracleModule): logger.info({'msg': 'Add metrics middleware for ETH1 requests.'}) add_requests_metric_middleware(web3) + return web3 - logger.info({'msg': 'Sanity checks.'}) +def _construct_module(web3: Web3, module_name: OracleModule) -> Accounting | Ejector | CSOracle: instance: Accounting | Ejector | CSOracle if module_name == OracleModule.ACCOUNTING: logger.info({'msg': 'Initialize Accounting module.'}) @@ -108,18 +93,47 @@ def main(module_name: OracleModule): else: raise ValueError(f'Unexpected arg: {module_name=}.') + logger.info({'msg': 'Sanity checks.'}) instance.check_contract_configs() + return instance + + +def main(module_name: OracleModule): + build_info = get_build_info() + logger.info({ + 'msg': 'Oracle startup.', + 'variables': { + **build_info, + 'module': module_name, + **variables.PUBLIC_ENV_VARS, + }, + }) + ENV_VARIABLES_INFO.info(variables.PUBLIC_ENV_VARS) + BUILD_INFO.info(build_info) + + logger.info({'msg': f'Start healthcheck server for Docker container on port {variables.HEALTHCHECK_SERVER_PORT}'}) + start_pulse_server() + logger.info({'msg': f'Start http server with prometheus metrics on port {variables.PROMETHEUS_PORT}'}) + start_http_server(variables.PROMETHEUS_PORT) + web3 = _construct_web3() + instance: Accounting | Ejector | CSOracle = _construct_module(web3, module_name) if variables.DAEMON: instance.run_as_daemon() else: instance.cycle_handler() -def check(): - logger.info({'msg': 'Check oracle is ready to work in the current environment.'}) +def run_on_refslot(module_name: OracleModule, slot: int): + w3 = _construct_web3() + instance: Accounting | Ejector | CSOracle = _construct_module(w3, module_name) + instance.check_contract_configs() + + block_root = BlockRoot(w3.cc.get_block_root(SlotNumber(slot + 3 * 32)).root) + block_details = w3.cc.get_block_details(block_root) + bs = build_blockstamp(block_details) - return ChecksModule().execute_module() + instance.refresh_contracts_and_run_cycle(bs) def check_providers_chain_ids(web3: Web3, cc: ConsensusClientModule, kac: KeysAPIClientModule): @@ -155,19 +169,56 @@ def ipfs_providers() -> Iterator[IPFSProvider]: yield PublicIPFS(timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS) +def parse_args(): + """ + Parse command-line arguments using argparse. + The 'module' argument is restricted to valid OracleModule values. + """ + valid_modules = [str(item) for item in OracleModule] + + parser = argparse.ArgumentParser(description="Run the Oracle module process.") + subparsers = parser.add_subparsers(dest="module", required=True, help=f"Module to run. One of: {valid_modules}") + check_parser = subparsers.add_parser("check", help="Run the check module.") + check_parser.add_argument( + "--name", + "-n", + type=str, + default=None, + help="Module name to check for a refslot execution." + ) + check_parser.add_argument( + "--refslot", + "-r", + type=str, + default=None, + help="Refslot parameter for the check module. If it is set it will run oracle on a specific refslot." + ) + for mod in OracleModule: + if mod == OracleModule.CSM: + continue + subparsers.add_parser(mod.value(), help=f"Run the {mod.value()} module.") + + return parser.parse_args() + + if __name__ == '__main__': - module_name_arg = sys.argv[-1] - if module_name_arg not in OracleModule: - msg = f'Last arg should be one of {[str(item) for item in OracleModule]}, received {module_name_arg}.' + args = parse_args() + if args.module not in OracleModule: + msg = f'Last arg should be one of {[str(item) for item in OracleModule]}, received {args.module}.' logger.error({'msg': msg}) raise ValueError(msg) - module = OracleModule(module_name_arg) + module = OracleModule(args.module) if module is OracleModule.CHECK: - errors = variables.check_uri_required_variables() - variables.raise_from_errors(errors) - - sys.exit(check()) + if args.refslot is None and args.name is None: + errors = variables.check_uri_required_variables() + variables.raise_from_errors(errors) + sys.exit(execute_checks()) + else: + errors = variables.check_all_required_variables(module) + variables.raise_from_errors(errors) + run_on_refslot(args.name, args.refslot) + sys.exit(0) errors = variables.check_all_required_variables(module) variables.raise_from_errors(errors) diff --git a/src/metrics/healthcheck_server.py b/src/metrics/healthcheck_server.py index 83c5efe40..ea166b73d 100644 --- a/src/metrics/healthcheck_server.py +++ b/src/metrics/healthcheck_server.py @@ -9,7 +9,6 @@ from src import variables from src.variables import MAX_CYCLE_LIFETIME_IN_SECONDS - _last_pulse = datetime.now() logger = logging.getLogger(__name__) @@ -50,6 +49,7 @@ def start_pulse_server(): If bot didn't call pulse for a while (5 minutes but should be changed individually) Docker healthcheck fails to do request """ + logger.info({'msg': f'Start healthcheck server for Docker container on port {variables.HEALTHCHECK_SERVER_PORT}'}) server = HTTPServer(('localhost', variables.HEALTHCHECK_SERVER_PORT), RequestHandlerClass=PulseRequestHandler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() diff --git a/src/modules/checks/checks_module.py b/src/modules/checks/checks_module.py index 59dd9ee09..8f29053ae 100644 --- a/src/modules/checks/checks_module.py +++ b/src/modules/checks/checks_module.py @@ -1,22 +1,13 @@ +import logging + import pytest +logger = logging.getLogger(__name__) -class ChecksModule: - """ - Module that executes all tests to figure out that environment is ready for Oracle. - Checks: - - Consensus Layer node - - Execution Layer node - - Keys API service - if LIDO_LOCATOR address provided - - Checks configs in Accounting module and Ejector module - if CSM_MODULE_ADDRESS provided - - Checks configs in CSM oracle module - - Checks with special blockstamp value (6300 slots in the past) - """ - def execute_module(self): - return pytest.main([ - 'src/modules/checks/suites', - '-c', 'src/modules/checks/pytest.ini', - ]) +def execute_checks(): + logger.info({'msg': 'Check oracle is ready to work in the current environment.'}) + return pytest.main([ + 'src/modules/checks/suites', + '-c', 'src/modules/checks/pytest.ini', + ]) diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index a71dc71a9..8cb802592 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -390,8 +390,7 @@ def _encode_data_hash(self, report_data: tuple) -> HexBytes: # Transform str abi to tuple, because ReportData is struct encoded = encode([f'({report_str_abi})'], [report_data]) - report_hash = self.w3.keccak(encoded) - return report_hash + return self.w3.keccak(encoded) def _send_report_hash(self, blockstamp: ReferenceBlockStamp, report_hash: bytes, consensus_version: int): consensus_contract = self._get_consensus_contract(blockstamp) diff --git a/src/modules/submodules/oracle_module.py b/src/modules/submodules/oracle_module.py index d2bd42241..51dd434fc 100644 --- a/src/modules/submodules/oracle_module.py +++ b/src/modules/submodules/oracle_module.py @@ -8,22 +8,21 @@ from requests.exceptions import ConnectionError as RequestsConnectionError from timeout_decorator import timeout, TimeoutError as DecoratorTimeoutError from web3.exceptions import Web3Exception +from web3_multi_provider import NoActiveProviderError +from src import variables from src.metrics.healthcheck_server import pulse from src.metrics.prometheus.basic import ORACLE_BLOCK_NUMBER, ORACLE_SLOT_NUMBER from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch from src.providers.http_provider import NotOkResponse from src.providers.ipfs import IPFSError from src.providers.keys.client import KeysOutdatedException -from src.utils.cache import clear_global_cache -from src.web3py.extensions.lido_validators import CountOfKeysDiffersException +from src.types import SlotNumber, BlockStamp, BlockRoot from src.utils.blockstamp import build_blockstamp +from src.utils.cache import clear_global_cache from src.utils.slot import NoSlotsAvailable, SlotNotFinalized, InconsistentData +from src.web3py.extensions.lido_validators import CountOfKeysDiffersException from src.web3py.types import Web3 -from web3_multi_provider import NoActiveProviderError - -from src import variables -from src.types import SlotNumber, BlockStamp, BlockRoot logger = logging.getLogger(__name__) @@ -60,6 +59,10 @@ def cycle_handler(self): self._cycle() self._sleep_cycle() + def refresh_contracts_and_run_cycle(self, blockstamp: BlockStamp): + self.refresh_contracts_if_address_change() + self.run_cycle(blockstamp) + @timeout(variables.MAX_CYCLE_LIFETIME_IN_SECONDS) def _cycle(self): """ @@ -78,8 +81,8 @@ def _cycle(self): }) return - self.refresh_contracts_if_address_change() - self.run_cycle(blockstamp) + self.refresh_contracts_and_run_cycle(blockstamp) + pulse() except IsNotMemberException as error: logger.error({'msg': 'Provided account is not part of Oracle`s committee.'}) raise error @@ -130,7 +133,6 @@ def _receive_last_finalized_slot(self) -> BlockStamp: def run_cycle(self, blockstamp: BlockStamp): logger.info({'msg': 'Execute module.', 'value': blockstamp}) result = self.execute_module(blockstamp) - pulse() if result is ModuleExecuteDelay.NEXT_FINALIZED_EPOCH: self._slot_threshold = blockstamp.slot_number From 3d0e8bf250cefc2d4c5cd284778ac2ee03eaf198 Mon Sep 17 00:00:00 2001 From: hweawer Date: Mon, 14 Apr 2025 19:16:20 +0200 Subject: [PATCH 02/13] feat: Run Oracle on refsol --- src/main.py | 56 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/src/main.py b/src/main.py index 485eb7166..b0ac6b4f8 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,10 @@ import argparse +import os import sys from typing import Iterator, cast +import requests +from eth_typing import HexStr from packaging.version import Version from prometheus_client import start_http_server @@ -124,16 +127,48 @@ def main(module_name: OracleModule): instance.cycle_handler() -def run_on_refslot(module_name: OracleModule, slot: int): +def get_transactions(contract_address, selector: str, limit: int = 10): + etherscan_key = os.getenv('ETHERSCAN_API_KEY') + if etherscan_key is None: + raise ValueError('ETHERSCAN_API_KEY is not set') + url = f"https://api.etherscan.io/api?module=account&action=txlist&address={contract_address}&sort=desc&apikey={etherscan_key}" + + response = requests.get(url, timeout=30) + data = response.json() + + if data["status"] != "1": + logger.error("Error fetching transactions: %s", data["message"]) + return [] + + transactions = data["result"] + successful_transactions = [tx for tx in transactions if tx.get("txreceipt_status") == "1"] + return [tx for tx in successful_transactions if tx["input"].startswith(selector)][:limit] + + +def run_on_refslot(module_name: OracleModule): w3 = _construct_web3() instance: Accounting | Ejector | CSOracle = _construct_module(w3, module_name) instance.check_contract_configs() + submit_report_fn = instance.report_contract.get_function_by_name("submitReportData") + selector = HexStr(submit_report_fn.selector) - block_root = BlockRoot(w3.cc.get_block_root(SlotNumber(slot + 3 * 32)).root) - block_details = w3.cc.get_block_details(block_root) - bs = build_blockstamp(block_details) + txs = get_transactions(instance.report_contract.address, selector) + if not txs: + print("No submitReportData transactions found!") + sys.exit(0) - instance.refresh_contracts_and_run_cycle(bs) + print(f"✅ Found {len(txs)} submitReportData calls") + + for tx in txs: + tx_hash = tx["hash"] + _, data = instance.report_contract.decode_function_input(tx["input"]) + refslot = int(data['data']['refslot']) + print(f"🔹 Tx: {tx_hash} → X: {refslot}") + print(data) + block_root = BlockRoot(w3.cc.get_block_root(SlotNumber(refslot + 3 * 32)).root) + block_details = w3.cc.get_block_details(block_root) + bs = build_blockstamp(block_details) + instance.refresh_contracts_and_run_cycle(bs) def check_providers_chain_ids(web3: Web3, cc: ConsensusClientModule, kac: KeysAPIClientModule): @@ -186,13 +221,6 @@ def parse_args(): default=None, help="Module name to check for a refslot execution." ) - check_parser.add_argument( - "--refslot", - "-r", - type=str, - default=None, - help="Refslot parameter for the check module. If it is set it will run oracle on a specific refslot." - ) for mod in OracleModule: if mod == OracleModule.CSM: continue @@ -210,14 +238,14 @@ def parse_args(): module = OracleModule(args.module) if module is OracleModule.CHECK: - if args.refslot is None and args.name is None: + if args.name is None: errors = variables.check_uri_required_variables() variables.raise_from_errors(errors) sys.exit(execute_checks()) else: errors = variables.check_all_required_variables(module) variables.raise_from_errors(errors) - run_on_refslot(args.name, args.refslot) + run_on_refslot(args.name) sys.exit(0) errors = variables.check_all_required_variables(module) From 27815fcf1f9203c7268c4be60827d284eedcf9c3 Mon Sep 17 00:00:00 2001 From: hweawer Date: Tue, 15 Apr 2025 17:33:54 +0200 Subject: [PATCH 03/13] feat: Run refslot --- src/constants.py | 2 +- src/{types.py => custom_types.py} | 0 src/main.py | 31 ++++++++--------- src/modules/accounting/accounting.py | 5 +-- .../accounting/third_phase/extra_data.py | 2 +- src/modules/accounting/types.py | 2 +- src/modules/checks/suites/conftest.py | 5 ++- src/modules/csm/checkpoint.py | 2 +- src/modules/csm/csm.py | 19 ++++++----- src/modules/csm/log.py | 2 +- src/modules/csm/state.py | 2 +- src/modules/csm/types.py | 2 +- src/modules/ejector/data_encode.py | 3 +- src/modules/ejector/ejector.py | 6 ++-- src/modules/ejector/sweep.py | 2 +- src/modules/ejector/types.py | 2 +- src/modules/submodules/consensus.py | 33 +++++++++++-------- src/modules/submodules/oracle_module.py | 2 +- src/modules/submodules/types.py | 3 +- src/providers/consensus/client.py | 4 +-- src/providers/consensus/types.py | 2 +- .../execution/contracts/base_oracle.py | 5 ++- .../execution/contracts/hash_consensus.py | 4 +-- src/providers/execution/contracts/lido.py | 3 +- src/providers/keys/client.py | 2 +- src/providers/keys/types.py | 2 +- src/services/bunker.py | 2 +- .../bunker_cases/abnormal_cl_rebase.py | 2 +- .../bunker_cases/midterm_slashing_penalty.py | 2 +- src/services/exit_order_iterator.py | 2 +- src/services/prediction.py | 3 +- src/services/safe_border.py | 2 +- src/services/validator_state.py | 2 +- src/services/withdrawal.py | 8 ++--- src/utils/blockstamp.py | 2 +- src/utils/dataclass.py | 2 +- src/utils/events.py | 2 +- src/utils/slot.py | 2 +- src/utils/units.py | 2 +- src/utils/validator_state.py | 2 +- src/utils/web3converter.py | 2 +- src/variables.py | 2 +- src/web3py/extensions/contracts.py | 4 +-- src/web3py/extensions/csm.py | 2 +- src/web3py/extensions/lido_validators.py | 2 +- tests/conftest.py | 2 +- tests/factory/blockstamp.py | 2 +- tests/factory/no_registry.py | 2 +- tests/fork/conftest.py | 2 +- tests/modules/accounting/bunker/conftest.py | 4 +-- .../modules/accounting/bunker/test_bunker.py | 2 +- .../bunker/test_bunker_abnormal_cl_rebase.py | 2 +- .../bunker/test_bunker_midterm_penalty.py | 2 +- .../accounting/test_accounting_module.py | 2 +- .../test_safe_border_integration.py | 2 +- .../accounting/test_validator_state.py | 2 +- tests/modules/csm/test_checkpoint.py | 2 +- tests/modules/csm/test_csm_module.py | 2 +- tests/modules/csm/test_log.py | 3 +- tests/modules/csm/test_state.py | 2 +- tests/modules/csm/test_tree.py | 2 +- tests/modules/ejector/test_data_encode.py | 3 +- tests/modules/ejector/test_ejector.py | 2 +- tests/modules/ejector/test_prediction.py | 2 +- tests/modules/ejector/test_sweep.py | 2 +- .../test_validator_exit_order_iterator.py | 2 +- .../modules/submodules/consensus/conftest.py | 2 +- .../submodules/consensus/test_consensus.py | 2 +- .../modules/submodules/test_oracle_module.py | 6 ++-- .../consensus/test_consensus_client.py | 2 +- .../providers_clients/test_keys_api_client.py | 2 +- tests/utils/test_slot.py | 2 +- tests/utils/test_validator_state_utils.py | 2 +- tests/utils/test_web3_converter.py | 2 +- 74 files changed, 129 insertions(+), 131 deletions(-) rename src/{types.py => custom_types.py} (100%) diff --git a/src/constants.py b/src/constants.py index 2ba467222..f19ff51c3 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,6 +1,6 @@ from packaging.version import Version -from src.types import Gwei, SlotNumber +from src.custom_types import Gwei, SlotNumber # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#misc GENESIS_SLOT = SlotNumber(0) diff --git a/src/types.py b/src/custom_types.py similarity index 100% rename from src/types.py rename to src/custom_types.py diff --git a/src/main.py b/src/main.py index b0ac6b4f8..561db9f10 100644 --- a/src/main.py +++ b/src/main.py @@ -4,12 +4,12 @@ from typing import Iterator, cast import requests -from eth_typing import HexStr from packaging.version import Version from prometheus_client import start_http_server from src import constants from src import variables +from src.custom_types import OracleModule, BlockRoot, SlotNumber from src.metrics.healthcheck_server import start_pulse_server from src.metrics.logging import logging from src.metrics.prometheus.basic import ENV_VARIABLES_INFO, BUILD_INFO @@ -18,7 +18,6 @@ from src.modules.csm.csm import CSOracle from src.modules.ejector.ejector import Ejector from src.providers.ipfs import GW3, IPFSProvider, MultiIPFSProvider, Pinata, PublicIPFS -from src.types import OracleModule, BlockRoot, SlotNumber from src.utils.blockstamp import build_blockstamp from src.utils.build import get_build_info from src.utils.exception import IncompatibleException @@ -82,17 +81,17 @@ def _construct_web3() -> Web3: return web3 -def _construct_module(web3: Web3, module_name: OracleModule) -> Accounting | Ejector | CSOracle: +def _construct_module(web3: Web3, module_name: OracleModule, run_past: bool = False) -> Accounting | Ejector | CSOracle: instance: Accounting | Ejector | CSOracle if module_name == OracleModule.ACCOUNTING: logger.info({'msg': 'Initialize Accounting module.'}) - instance = Accounting(web3) + instance = Accounting(web3, run_past) elif module_name == OracleModule.EJECTOR: logger.info({'msg': 'Initialize Ejector module.'}) - instance = Ejector(web3) + instance = Ejector(web3, run_past) elif module_name == OracleModule.CSM: logger.info({'msg': 'Initialize CSM performance oracle module.'}) - instance = CSOracle(web3) + instance = CSOracle(web3, run_past) else: raise ValueError(f'Unexpected arg: {module_name=}.') @@ -147,28 +146,30 @@ def get_transactions(contract_address, selector: str, limit: int = 10): def run_on_refslot(module_name: OracleModule): w3 = _construct_web3() - instance: Accounting | Ejector | CSOracle = _construct_module(w3, module_name) + instance: Accounting | Ejector | CSOracle = _construct_module(w3, module_name, True) instance.check_contract_configs() submit_report_fn = instance.report_contract.get_function_by_name("submitReportData") - selector = HexStr(submit_report_fn.selector) + selector = '0x' + w3.keccak(text=submit_report_fn.abi_element_identifier)[:4].hex() + logger.info("Manual Selector: %s", selector) txs = get_transactions(instance.report_contract.address, selector) if not txs: - print("No submitReportData transactions found!") + logger.info("No submitReportData transactions found!") sys.exit(0) - print(f"✅ Found {len(txs)} submitReportData calls") + logger.info("Found %d submitReportData calls", len(txs)) for tx in txs: tx_hash = tx["hash"] _, data = instance.report_contract.decode_function_input(tx["input"]) - refslot = int(data['data']['refslot']) - print(f"🔹 Tx: {tx_hash} → X: {refslot}") - print(data) + refslot = int(data['data']['refSlot']) + print(f"Tx: {tx_hash} → X: {refslot}") + logger.info(data) block_root = BlockRoot(w3.cc.get_block_root(SlotNumber(refslot + 3 * 32)).root) block_details = w3.cc.get_block_details(block_root) bs = build_blockstamp(block_details) instance.refresh_contracts_and_run_cycle(bs) + instance = _construct_module(w3, module_name, True) def check_providers_chain_ids(web3: Web3, cc: ConsensusClientModule, kac: KeysAPIClientModule): @@ -222,9 +223,9 @@ def parse_args(): help="Module name to check for a refslot execution." ) for mod in OracleModule: - if mod == OracleModule.CSM: + if mod == OracleModule.CHECK: continue - subparsers.add_parser(mod.value(), help=f"Run the {mod.value()} module.") + subparsers.add_parser(mod.value, help=f"Run the {mod.value} module.") return parser.parse_args() diff --git a/src/modules/accounting/accounting.py b/src/modules/accounting/accounting.py index 5d1231ad8..8c51e43f4 100644 --- a/src/modules/accounting/accounting.py +++ b/src/modules/accounting/accounting.py @@ -8,6 +8,7 @@ from src import variables from src.constants import SHARE_RATE_PRECISION_E27 +from src.custom_types import BlockStamp, Gwei, ReferenceBlockStamp, StakingModuleId, NodeOperatorGlobalIndex, FinalizationBatches from src.metrics.prometheus.accounting import ( ACCOUNTING_IS_BUNKER, ACCOUNTING_CL_BALANCE_GWEI, @@ -36,7 +37,6 @@ from src.services.bunker import BunkerService from src.services.validator_state import LidoValidatorStateService from src.services.withdrawal import Withdrawal -from src.types import BlockStamp, Gwei, ReferenceBlockStamp, StakingModuleId, NodeOperatorGlobalIndex, FinalizationBatches from src.utils.cache import global_lru_cache as lru_cache from src.utils.units import gwei_to_wei from src.variables import ALLOW_REPORTING_IN_BUNKER_MODE @@ -59,8 +59,9 @@ class Accounting(BaseModule, ConsensusModule): """ COMPATIBLE_ONCHAIN_VERSIONS = [(2, 2), (2, 3)] - def __init__(self, w3: Web3): + def __init__(self, w3: Web3, run_past: bool = False): self.report_contract: AccountingOracleContract = w3.lido_contracts.accounting_oracle + self.run_past = run_past super().__init__(w3) self.lido_validator_state_service = LidoValidatorStateService(self.w3) diff --git a/src/modules/accounting/third_phase/extra_data.py b/src/modules/accounting/third_phase/extra_data.py index 8861af1a8..a6393ad5e 100644 --- a/src/modules/accounting/third_phase/extra_data.py +++ b/src/modules/accounting/third_phase/extra_data.py @@ -2,9 +2,9 @@ from itertools import groupby, batched from typing import Sequence +from src.custom_types import NodeOperatorGlobalIndex from src.modules.accounting.third_phase.types import ExtraData, ItemType, ExtraDataLengths, FormatList from src.modules.submodules.types import ZERO_HASH -from src.types import NodeOperatorGlobalIndex from src.web3py.types import Web3 diff --git a/src/modules/accounting/types.py b/src/modules/accounting/types.py index 9843c61ae..d4e3a29b8 100644 --- a/src/modules/accounting/types.py +++ b/src/modules/accounting/types.py @@ -5,7 +5,7 @@ from hexbytes import HexBytes from web3.types import Wei -from src.types import ( +from src.custom_types import ( SlotNumber, Gwei, StakingModuleId, diff --git a/src/modules/checks/suites/conftest.py b/src/modules/checks/suites/conftest.py index 06cf97977..d06e23342 100644 --- a/src/modules/checks/suites/conftest.py +++ b/src/modules/checks/suites/conftest.py @@ -4,9 +4,9 @@ from xdist.dsession import TerminalDistReporter # type: ignore[import] from src import variables -from src.types import EpochNumber, SlotNumber, BlockRoot -from src.utils.blockstamp import build_blockstamp +from src.custom_types import EpochNumber, SlotNumber, BlockRoot from src.utils.api import opsgenie_api +from src.utils.blockstamp import build_blockstamp from src.utils.slot import get_reference_blockstamp from src.web3py.contract_tweak import tweak_w3_contracts from src.web3py.extensions import ( @@ -20,7 +20,6 @@ ) from src.web3py.types import Web3 - TITLE_PROPERTY_NAME = "test_title" _config = None diff --git a/src/modules/csm/checkpoint.py b/src/modules/csm/checkpoint.py index 0efc326c6..6001834f7 100644 --- a/src/modules/csm/checkpoint.py +++ b/src/modules/csm/checkpoint.py @@ -7,11 +7,11 @@ from src import variables from src.constants import SLOTS_PER_HISTORICAL_ROOT +from src.custom_types import BlockRoot, BlockStamp, CommitteeIndex, EpochNumber, SlotNumber, ValidatorIndex from src.metrics.prometheus.csm import CSM_MIN_UNPROCESSED_EPOCH, CSM_UNPROCESSED_EPOCHS_COUNT from src.modules.csm.state import State from src.providers.consensus.client import ConsensusClient from src.providers.consensus.types import BlockAttestation, BlockAttestationEIP7549 -from src.types import BlockRoot, BlockStamp, CommitteeIndex, EpochNumber, SlotNumber, ValidatorIndex from src.utils.range import sequence from src.utils.timeit import timeit from src.utils.types import hex_str_to_bytes diff --git a/src/modules/csm/csm.py b/src/modules/csm/csm.py index e003408d5..bc57a19ba 100644 --- a/src/modules/csm/csm.py +++ b/src/modules/csm/csm.py @@ -5,6 +5,14 @@ from hexbytes import HexBytes from src.constants import TOTAL_BASIS_POINTS, UINT64_MAX +from src.custom_types import ( + BlockStamp, + EpochNumber, + ReferenceBlockStamp, + SlotNumber, + StakingModuleAddress, + StakingModuleId, +) from src.metrics.prometheus.business import CONTRACT_ON_PAUSE from src.metrics.prometheus.csm import ( CSM_CURRENT_FRAME_RANGE_L_EPOCH, @@ -22,14 +30,6 @@ from src.providers.execution.contracts.cs_fee_oracle import CSFeeOracleContract from src.providers.execution.exceptions import InconsistentData from src.providers.ipfs import CID -from src.types import ( - BlockStamp, - EpochNumber, - ReferenceBlockStamp, - SlotNumber, - StakingModuleAddress, - StakingModuleId, -) from src.utils.blockstamp import build_blockstamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.slot import get_next_non_missed_slot @@ -64,9 +64,10 @@ class CSOracle(BaseModule, ConsensusModule): report_contract: CSFeeOracleContract module_id: StakingModuleId - def __init__(self, w3: Web3): + def __init__(self, w3: Web3, run_past: bool = False): self.report_contract = w3.csm.oracle self.state = State.load() + self.run_past = run_past super().__init__(w3) self.module_id = self._get_module_id() diff --git a/src/modules/csm/log.py b/src/modules/csm/log.py index f89f4ef58..d66aeb0ee 100644 --- a/src/modules/csm/log.py +++ b/src/modules/csm/log.py @@ -2,9 +2,9 @@ from collections import defaultdict from dataclasses import asdict, dataclass, field +from src.custom_types import EpochNumber, NodeOperatorId, ReferenceBlockStamp, ValidatorIndex from src.modules.csm.state import AttestationsAccumulator from src.modules.csm.types import Shares -from src.types import EpochNumber, NodeOperatorId, ReferenceBlockStamp, ValidatorIndex class LogJSONEncoder(json.JSONEncoder): ... diff --git a/src/modules/csm/state.py b/src/modules/csm/state.py index 4373f5259..f7445e391 100644 --- a/src/modules/csm/state.py +++ b/src/modules/csm/state.py @@ -7,7 +7,7 @@ from typing import Self from src import variables -from src.types import EpochNumber, ValidatorIndex +from src.custom_types import EpochNumber, ValidatorIndex from src.utils.range import sequence logger = logging.getLogger(__name__) diff --git a/src/modules/csm/types.py b/src/modules/csm/types.py index d305a6e0b..12564b6b4 100644 --- a/src/modules/csm/types.py +++ b/src/modules/csm/types.py @@ -4,8 +4,8 @@ from hexbytes import HexBytes +from src.custom_types import NodeOperatorId, SlotNumber from src.providers.ipfs import CID -from src.types import NodeOperatorId, SlotNumber logger = logging.getLogger(__name__) diff --git a/src/modules/ejector/data_encode.py b/src/modules/ejector/data_encode.py index 2ca8ad771..dc0084b27 100644 --- a/src/modules/ejector/data_encode.py +++ b/src/modules/ejector/data_encode.py @@ -1,10 +1,9 @@ from eth_typing import HexStr -from src.types import ValidatorIndex +from src.custom_types import ValidatorIndex from src.utils.types import hex_str_to_bytes from src.web3py.extensions.lido_validators import LidoValidator, NodeOperatorGlobalIndex - DATA_FORMAT_LIST = 1 MODULE_ID_LENGTH = 3 diff --git a/src/modules/ejector/ejector.py b/src/modules/ejector/ejector.py index a31b506ef..e3298420f 100644 --- a/src/modules/ejector/ejector.py +++ b/src/modules/ejector/ejector.py @@ -9,6 +9,7 @@ FAR_FUTURE_EPOCH, MIN_VALIDATOR_WITHDRAWABILITY_DELAY, ) +from src.custom_types import BlockStamp, EpochNumber, Gwei, NodeOperatorGlobalIndex, ReferenceBlockStamp from src.metrics.prometheus.business import CONTRACT_ON_PAUSE from src.metrics.prometheus.duration_meter import duration_meter from src.metrics.prometheus.ejector import ( @@ -27,7 +28,6 @@ from src.services.exit_order_iterator import ValidatorExitIterator from src.services.prediction import RewardsPredictionService from src.services.validator_state import LidoValidatorStateService -from src.types import BlockStamp, EpochNumber, Gwei, NodeOperatorGlobalIndex, ReferenceBlockStamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.units import gwei_to_wei from src.utils.validator_state import ( @@ -66,9 +66,9 @@ class Ejector(BaseModule, ConsensusModule): AVG_EXPECTING_WITHDRAWALS_SWEEP_DURATION_MULTIPLIER = 0.5 - def __init__(self, w3: Web3): + def __init__(self, w3: Web3, run_past: bool = False): self.report_contract: ExitBusOracleContract = w3.lido_contracts.validators_exit_bus_oracle - + self.run_past = run_past super().__init__(w3) self.prediction_service = RewardsPredictionService(w3) diff --git a/src/modules/ejector/sweep.py b/src/modules/ejector/sweep.py index f6ac23075..51a05f0e6 100644 --- a/src/modules/ejector/sweep.py +++ b/src/modules/ejector/sweep.py @@ -9,9 +9,9 @@ MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, MAX_WITHDRAWALS_PER_PAYLOAD, ) +from src.custom_types import Gwei from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import BeaconStateView -from src.types import Gwei from src.utils.validator_state import ( is_fully_withdrawable_validator, is_partially_withdrawable_validator, diff --git a/src/modules/ejector/types.py b/src/modules/ejector/types.py index 07c81c643..86cffc949 100644 --- a/src/modules/ejector/types.py +++ b/src/modules/ejector/types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from src.types import SlotNumber +from src.custom_types import SlotNumber @dataclass diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index 8cb802592..38b053192 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -8,6 +8,7 @@ from web3.exceptions import ContractCustomError from src import variables +from src.custom_types import BlockStamp, ReferenceBlockStamp, SlotNumber, FrameNumber from src.metrics.prometheus.basic import ORACLE_SLOT_NUMBER, ORACLE_BLOCK_NUMBER, GENESIS_TIME, ACCOUNT_BALANCE from src.metrics.prometheus.business import ( ORACLE_MEMBER_LAST_REPORT_REF_SLOT, @@ -19,7 +20,6 @@ from src.modules.submodules.types import ChainConfig, MemberInfo, ZERO_HASH, CurrentFrame, FrameConfig from src.providers.execution.contracts.base_oracle import BaseOracleContract from src.providers.execution.contracts.hash_consensus import HashConsensusContract -from src.types import BlockStamp, ReferenceBlockStamp, SlotNumber, FrameNumber from src.utils.blockstamp import build_blockstamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.slot import get_reference_blockstamp @@ -44,6 +44,7 @@ class ConsensusModule(ABC): report_contract should contain getConsensusContract method. """ report_contract: BaseOracleContract + run_past: bool = False # Contains tuple[CONTRACT_VERSION, CONSENSUS_VERSION] COMPATIBLE_ONCHAIN_VERSIONS: list[tuple[int, int]] @@ -191,10 +192,23 @@ def get_member_info(self, blockstamp: BlockStamp) -> MemberInfo: current_frame_member_report=current_frame_member_report, deadline_slot=current_frame.report_processing_deadline_slot, ) - logger.debug({'msg': 'Fetch member info.', 'value': mi}) + logger.info({'msg': 'Fetch member info.', 'value': mi}) return mi + def _calculate_reference_blockstamp(self, last_finalized_blockstamp: BlockStamp) -> ReferenceBlockStamp: + converter = self._get_web3_converter(last_finalized_blockstamp) + member_info = self.get_member_info(self._get_latest_blockstamp()) + + bs = get_reference_blockstamp( + cc=self.w3.cc, + ref_slot=member_info.current_frame_ref_slot, + ref_epoch=converter.get_epoch_by_slot(member_info.current_frame_ref_slot), + last_finalized_slot_number=last_finalized_blockstamp.slot_number, + ) + logger.info({'msg': 'Calculate blockstamp for report.', 'value': bs}) + return bs + # ----- Calculation reference slot for report ----- def get_blockstamp_for_report(self, last_finalized_blockstamp: BlockStamp) -> ReferenceBlockStamp | None: """ @@ -202,6 +216,8 @@ def get_blockstamp_for_report(self, last_finalized_blockstamp: BlockStamp) -> Re Returns: Non-missed reference slot blockstamp in case contract is reportable. """ + if self.run_past: + return self._calculate_reference_blockstamp(last_finalized_blockstamp) latest_blockstamp = self._get_latest_blockstamp() # Check if contract is currently reportable @@ -210,7 +226,6 @@ def get_blockstamp_for_report(self, last_finalized_blockstamp: BlockStamp) -> Re return None member_info = self.get_member_info(latest_blockstamp) - logger.info({'msg': 'Fetch member info.', 'value': member_info}) # Check if current slot is higher than member slot if last_finalized_blockstamp.slot_number < member_info.current_frame_ref_slot: @@ -222,17 +237,7 @@ def get_blockstamp_for_report(self, last_finalized_blockstamp: BlockStamp) -> Re logger.info({'msg': 'Deadline missed.'}) return None - converter = self._get_web3_converter(last_finalized_blockstamp) - - bs = get_reference_blockstamp( - cc=self.w3.cc, - ref_slot=member_info.current_frame_ref_slot, - ref_epoch=converter.get_epoch_by_slot(member_info.current_frame_ref_slot), - last_finalized_slot_number=last_finalized_blockstamp.slot_number, - ) - logger.info({'msg': 'Calculate blockstamp for report.', 'value': bs}) - - return bs + return self._calculate_reference_blockstamp(last_finalized_blockstamp) def _check_compatability(self, blockstamp: BlockStamp): """ diff --git a/src/modules/submodules/oracle_module.py b/src/modules/submodules/oracle_module.py index 51dd434fc..bc64c1836 100644 --- a/src/modules/submodules/oracle_module.py +++ b/src/modules/submodules/oracle_module.py @@ -11,13 +11,13 @@ from web3_multi_provider import NoActiveProviderError from src import variables +from src.custom_types import SlotNumber, BlockStamp, BlockRoot from src.metrics.healthcheck_server import pulse from src.metrics.prometheus.basic import ORACLE_BLOCK_NUMBER, ORACLE_SLOT_NUMBER from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch from src.providers.http_provider import NotOkResponse from src.providers.ipfs import IPFSError from src.providers.keys.client import KeysOutdatedException -from src.types import SlotNumber, BlockStamp, BlockRoot from src.utils.blockstamp import build_blockstamp from src.utils.cache import clear_global_cache from src.utils.slot import NoSlotsAvailable, SlotNotFinalized, InconsistentData diff --git a/src/modules/submodules/types.py b/src/modules/submodules/types.py index 5cd940b86..d7dbc987e 100644 --- a/src/modules/submodules/types.py +++ b/src/modules/submodules/types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from src.types import SlotNumber - +from src.custom_types import SlotNumber ZERO_HASH = bytes([0]*32) diff --git a/src/providers/consensus/client.py b/src/providers/consensus/client.py index 4b13d2ecc..36d417749 100644 --- a/src/providers/consensus/client.py +++ b/src/providers/consensus/client.py @@ -3,6 +3,7 @@ from json_stream.base import TransientStreamingJSONObject # type: ignore +from src.custom_types import BlockRoot, BlockStamp, SlotNumber, EpochNumber, StateRoot from src.metrics.logging import logging from src.metrics.prometheus.basic import CL_REQUESTS_DURATION from src.providers.consensus.types import ( @@ -19,9 +20,8 @@ SlotAttestationCommittee, ) from src.providers.http_provider import HTTPProvider, NotOkResponse -from src.types import BlockRoot, BlockStamp, SlotNumber, EpochNumber, StateRoot -from src.utils.dataclass import list_of_dataclasses from src.utils.cache import global_lru_cache as lru_cache +from src.utils.dataclass import list_of_dataclasses logger = logging.getLogger(__name__) diff --git a/src/providers/consensus/types.py b/src/providers/consensus/types.py index 6f35456c5..12bef72bd 100644 --- a/src/providers/consensus/types.py +++ b/src/providers/consensus/types.py @@ -6,7 +6,7 @@ from web3.types import Timestamp from src.constants import FAR_FUTURE_EPOCH -from src.types import BlockHash, BlockRoot, CommitteeIndex, EpochNumber, Gwei, SlotNumber, StateRoot, ValidatorIndex +from src.custom_types import BlockHash, BlockRoot, CommitteeIndex, EpochNumber, Gwei, SlotNumber, StateRoot, ValidatorIndex from src.utils.dataclass import FromResponse, Nested diff --git a/src/providers/execution/contracts/base_oracle.py b/src/providers/execution/contracts/base_oracle.py index 54d161b4e..bcb32b270 100644 --- a/src/providers/execution/contracts/base_oracle.py +++ b/src/providers/execution/contracts/base_oracle.py @@ -1,13 +1,12 @@ import logging -from src.utils.cache import global_lru_cache as lru_cache from eth_typing import ChecksumAddress, Hash32 from web3.contract.contract import ContractFunction from web3.types import BlockIdentifier +from src.custom_types import SlotNumber from src.providers.execution.base_interface import ContractInterface -from src.types import SlotNumber - +from src.utils.cache import global_lru_cache as lru_cache logger = logging.getLogger(__name__) diff --git a/src/providers/execution/contracts/hash_consensus.py b/src/providers/execution/contracts/hash_consensus.py index 0d909f549..77b0d9fea 100644 --- a/src/providers/execution/contracts/hash_consensus.py +++ b/src/providers/execution/contracts/hash_consensus.py @@ -1,14 +1,14 @@ import logging -from src.types import SlotNumber -from src.utils.cache import global_lru_cache as lru_cache from eth_typing import ChecksumAddress from web3.contract.contract import ContractFunction from web3.types import BlockIdentifier +from src.custom_types import SlotNumber from src.modules.submodules.types import ChainConfig, CurrentFrame, FrameConfig from src.providers.execution.base_interface import ContractInterface from src.utils.abi import named_tuple_to_dataclass +from src.utils.cache import global_lru_cache as lru_cache logger = logging.getLogger(__name__) diff --git a/src/providers/execution/contracts/lido.py b/src/providers/execution/contracts/lido.py index 856c27ed0..2b4f300c6 100644 --- a/src/providers/execution/contracts/lido.py +++ b/src/providers/execution/contracts/lido.py @@ -4,9 +4,9 @@ from web3 import Web3 from web3.types import Wei, BlockIdentifier, StateOverride, StateOverrideParams +from src.custom_types import SlotNumber from src.modules.accounting.types import LidoReportRebase, BeaconStat from src.providers.execution.base_interface import ContractInterface -from src.types import SlotNumber from src.utils.abi import named_tuple_to_dataclass from src.utils.cache import global_lru_cache as lru_cache @@ -60,7 +60,6 @@ def handle_oracle_report( 'error': repr(error), }) hex_ref_slot = HexStr(hex(ref_slot)) - return self._handle_oracle_report( timestamp, time_elapsed, diff --git a/src/providers/keys/client.py b/src/providers/keys/client.py index ae0ba65a6..554d12dca 100644 --- a/src/providers/keys/client.py +++ b/src/providers/keys/client.py @@ -1,10 +1,10 @@ from time import sleep from typing import cast, TypedDict, List +from src.custom_types import BlockStamp, StakingModuleAddress from src.metrics.prometheus.basic import KEYS_API_REQUESTS_DURATION, KEYS_API_LATEST_BLOCKNUMBER from src.providers.http_provider import HTTPProvider, NotOkResponse from src.providers.keys.types import LidoKey, KeysApiStatus -from src.types import BlockStamp, StakingModuleAddress from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/providers/keys/types.py b/src/providers/keys/types.py index 797c80b9b..f3df544db 100644 --- a/src/providers/keys/types.py +++ b/src/providers/keys/types.py @@ -3,7 +3,7 @@ from eth_typing import ChecksumAddress, HexStr -from src.types import NodeOperatorId +from src.custom_types import NodeOperatorId from src.utils.dataclass import FromResponse diff --git a/src/services/bunker.py b/src/services/bunker.py index 1294570d1..6c286ceda 100644 --- a/src/services/bunker.py +++ b/src/services/bunker.py @@ -3,6 +3,7 @@ from web3.types import Wei from src.constants import TOTAL_BASIS_POINTS +from src.custom_types import BlockStamp, ReferenceBlockStamp, Gwei from src.metrics.prometheus.duration_meter import duration_meter from src.metrics.prometheus.validators import ( ALL_VALIDATORS, @@ -16,7 +17,6 @@ from src.services.bunker_cases.midterm_slashing_penalty import MidtermSlashingPenalty from src.services.bunker_cases.types import BunkerConfig from src.services.safe_border import filter_slashed_validators -from src.types import BlockStamp, ReferenceBlockStamp, Gwei from src.utils.units import wei_to_gwei from src.utils.web3converter import Web3Converter from src.web3py.types import Web3 diff --git a/src/services/bunker_cases/abnormal_cl_rebase.py b/src/services/bunker_cases/abnormal_cl_rebase.py index 72941e5a7..744d060b3 100644 --- a/src/services/bunker_cases/abnormal_cl_rebase.py +++ b/src/services/bunker_cases/abnormal_cl_rebase.py @@ -6,11 +6,11 @@ from web3.types import EventData from src.constants import EFFECTIVE_BALANCE_INCREMENT, LIDO_DEPOSIT_AMOUNT +from src.custom_types import ReferenceBlockStamp, Gwei, BlockNumber, SlotNumber, BlockStamp, EpochNumber from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator from src.providers.keys.types import LidoKey from src.services.bunker_cases.types import BunkerConfig -from src.types import ReferenceBlockStamp, Gwei, BlockNumber, SlotNumber, BlockStamp, EpochNumber from src.utils.events import get_events_in_range from src.utils.slot import get_blockstamp, get_reference_blockstamp from src.utils.units import wei_to_gwei diff --git a/src/services/bunker_cases/midterm_slashing_penalty.py b/src/services/bunker_cases/midterm_slashing_penalty.py index 968c8e262..cb3ab600c 100644 --- a/src/services/bunker_cases/midterm_slashing_penalty.py +++ b/src/services/bunker_cases/midterm_slashing_penalty.py @@ -7,8 +7,8 @@ MIN_VALIDATOR_WITHDRAWABILITY_DELAY, PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX, ) +from src.custom_types import EpochNumber, FrameNumber, Gwei, ReferenceBlockStamp, SlotNumber from src.providers.consensus.types import Validator -from src.types import EpochNumber, FrameNumber, Gwei, ReferenceBlockStamp, SlotNumber from src.utils.validator_state import calculate_total_active_effective_balance from src.utils.web3converter import Web3Converter from src.web3py.extensions.lido_validators import LidoValidator diff --git a/src/services/exit_order_iterator.py b/src/services/exit_order_iterator.py index 2815d0737..21e7f0c26 100644 --- a/src/services/exit_order_iterator.py +++ b/src/services/exit_order_iterator.py @@ -5,10 +5,10 @@ from more_itertools import ilen from src.constants import TOTAL_BASIS_POINTS, LIDO_DEPOSIT_AMOUNT +from src.custom_types import ReferenceBlockStamp, NodeOperatorGlobalIndex, StakingModuleId, Gwei from src.metrics.prometheus.duration_meter import duration_meter from src.providers.consensus.types import Validator from src.services.validator_state import LidoValidatorStateService -from src.types import ReferenceBlockStamp, NodeOperatorGlobalIndex, StakingModuleId, Gwei from src.utils.validator_state import is_on_exit, get_validator_age from src.web3py.extensions.lido_validators import LidoValidator, StakingModule, NodeOperator, NodeOperatorLimitMode from src.web3py.types import Web3 diff --git a/src/services/prediction.py b/src/services/prediction.py index 8896b5f98..0180dfdf5 100644 --- a/src/services/prediction.py +++ b/src/services/prediction.py @@ -2,14 +2,13 @@ from web3.types import EventData, Wei +from src.custom_types import ReferenceBlockStamp from src.modules.submodules.types import ChainConfig from src.providers.execution.exceptions import InconsistentEvents -from src.types import ReferenceBlockStamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.events import get_events_in_past from src.web3py.types import Web3 - logger = logging.getLogger(__name__) diff --git a/src/services/safe_border.py b/src/services/safe_border.py index ca32108a3..728575c87 100644 --- a/src/services/safe_border.py +++ b/src/services/safe_border.py @@ -4,9 +4,9 @@ from eth_typing import HexStr from src.constants import EPOCHS_PER_SLASHINGS_VECTOR, MIN_VALIDATOR_WITHDRAWABILITY_DELAY +from src.custom_types import EpochNumber, FrameNumber, ReferenceBlockStamp, SlotNumber from src.metrics.prometheus.duration_meter import duration_meter from src.modules.submodules.consensus import ChainConfig, FrameConfig -from src.types import EpochNumber, FrameNumber, ReferenceBlockStamp, SlotNumber from src.utils.slot import get_blockstamp from src.utils.web3converter import Web3Converter from src.web3py.extensions.lido_validators import Validator diff --git a/src/services/validator_state.py b/src/services/validator_state.py index 64679606a..1ce710f0e 100644 --- a/src/services/validator_state.py +++ b/src/services/validator_state.py @@ -6,13 +6,13 @@ from more_itertools import ilen from src.constants import FAR_FUTURE_EPOCH, SHARD_COMMITTEE_PERIOD +from src.custom_types import BlockStamp, ReferenceBlockStamp, EpochNumber, OperatorsValidatorCount from src.metrics.prometheus.accounting import ( ACCOUNTING_STUCK_VALIDATORS, ACCOUNTING_EXITED_VALIDATORS, ACCOUNTING_DELAYED_VALIDATORS, ) from src.modules.submodules.types import ChainConfig -from src.types import BlockStamp, ReferenceBlockStamp, EpochNumber, OperatorsValidatorCount from src.utils.cache import global_lru_cache as lru_cache from src.utils.events import get_events_in_past from src.utils.types import bytes_to_hex_str diff --git a/src/services/withdrawal.py b/src/services/withdrawal.py index ada3d5a41..20acaceda 100644 --- a/src/services/withdrawal.py +++ b/src/services/withdrawal.py @@ -1,12 +1,12 @@ from web3.types import Wei +from src.custom_types import ReferenceBlockStamp, FinalizationBatches from src.metrics.prometheus.business import CONTRACT_ON_PAUSE +from src.modules.accounting.types import BatchState +from src.modules.submodules.consensus import ChainConfig, FrameConfig +from src.services.safe_border import SafeBorder from src.variables import FINALIZATION_BATCH_MAX_REQUEST_COUNT from src.web3py.types import Web3 -from src.types import ReferenceBlockStamp, FinalizationBatches -from src.services.safe_border import SafeBorder -from src.modules.submodules.consensus import ChainConfig, FrameConfig -from src.modules.accounting.types import BatchState class Withdrawal: diff --git a/src/utils/blockstamp.py b/src/utils/blockstamp.py index 5f268b8a4..e7f039557 100644 --- a/src/utils/blockstamp.py +++ b/src/utils/blockstamp.py @@ -1,7 +1,7 @@ from eth_utils import add_0x_prefix +from src.custom_types import BlockStamp, EpochNumber, ReferenceBlockStamp, SlotNumber from src.providers.consensus.types import BlockDetailsResponse -from src.types import BlockStamp, EpochNumber, ReferenceBlockStamp, SlotNumber def build_reference_blockstamp( diff --git a/src/utils/dataclass.py b/src/utils/dataclass.py index 417bbcdea..2cb25ee6a 100644 --- a/src/utils/dataclass.py +++ b/src/utils/dataclass.py @@ -3,7 +3,7 @@ from types import GenericAlias from typing import Callable, Self, Sequence -from src.types import BlockNumber, CommitteeIndex, EpochNumber, Gwei, SlotNumber, Timestamp, ValidatorIndex +from src.custom_types import BlockNumber, CommitteeIndex, EpochNumber, Gwei, SlotNumber, Timestamp, ValidatorIndex from src.utils.abi import named_tuple_to_dataclass diff --git a/src/utils/events.py b/src/utils/events.py index edb006c4e..b02b77641 100644 --- a/src/utils/events.py +++ b/src/utils/events.py @@ -6,8 +6,8 @@ from web3.types import EventData from src import variables +from src.custom_types import ReferenceBlockStamp from src.providers.execution.exceptions import InconsistentEvents -from src.types import ReferenceBlockStamp logger = logging.getLogger(__name__) diff --git a/src/utils/slot.py b/src/utils/slot.py index d73530351..665e1b990 100644 --- a/src/utils/slot.py +++ b/src/utils/slot.py @@ -1,11 +1,11 @@ import logging from http import HTTPStatus +from src.custom_types import EpochNumber, ReferenceBlockStamp, SlotNumber from src.providers.consensus.client import ConsensusClient from src.providers.consensus.types import BlockDetailsResponse, BlockHeaderFullResponse from src.providers.execution.exceptions import InconsistentData from src.providers.http_provider import NotOkResponse -from src.types import EpochNumber, ReferenceBlockStamp, SlotNumber from src.utils.blockstamp import build_blockstamp, build_reference_blockstamp logger = logging.getLogger(__name__) diff --git a/src/utils/units.py b/src/utils/units.py index f2ba96973..378fecbb5 100644 --- a/src/utils/units.py +++ b/src/utils/units.py @@ -3,7 +3,7 @@ from web3.types import Wei from src.constants import GWEI_TO_WEI -from src.types import Gwei +from src.custom_types import Gwei def wei_to_gwei(amount: Wei) -> Gwei: diff --git a/src/utils/validator_state.py b/src/utils/validator_state.py index 531735d68..5405d9c7e 100644 --- a/src/utils/validator_state.py +++ b/src/utils/validator_state.py @@ -14,8 +14,8 @@ MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA, SHARD_COMMITTEE_PERIOD, ) +from src.custom_types import EpochNumber, Gwei from src.providers.consensus.types import Validator, ValidatorState -from src.types import EpochNumber, Gwei def is_active_validator(validator: Validator, epoch: EpochNumber) -> bool: diff --git a/src/utils/web3converter.py b/src/utils/web3converter.py index e8706785f..33bf66196 100644 --- a/src/utils/web3converter.py +++ b/src/utils/web3converter.py @@ -1,4 +1,4 @@ -from src.types import SlotNumber, EpochNumber, FrameNumber +from src.custom_types import SlotNumber, EpochNumber, FrameNumber from src.modules.submodules.types import ChainConfig, FrameConfig diff --git a/src/variables.py b/src/variables.py index 5ee2f62ea..6fa591f17 100644 --- a/src/variables.py +++ b/src/variables.py @@ -4,7 +4,7 @@ from eth_account import Account -from src.types import OracleModule +from src.custom_types import OracleModule from src.utils.env import from_file_or_env # - Providers- diff --git a/src/web3py/extensions/contracts.py b/src/web3py/extensions/contracts.py index 39f30536c..237efa138 100644 --- a/src/web3py/extensions/contracts.py +++ b/src/web3py/extensions/contracts.py @@ -7,6 +7,7 @@ from web3.types import Wei from src import variables +from src.custom_types import BlockStamp, SlotNumber, WithdrawalVaultBalance, ELVaultBalance from src.metrics.prometheus.business import FRAME_PREV_REPORT_REF_SLOT from src.providers.execution.contracts.accounting_oracle import AccountingOracleContract from src.providers.execution.contracts.burner import BurnerContract @@ -16,12 +17,9 @@ from src.providers.execution.contracts.oracle_daemon_config import OracleDaemonConfigContract from src.providers.execution.contracts.oracle_report_sanity_checker import OracleReportSanityCheckerContract from src.providers.execution.contracts.staking_router import StakingRouterContract - from src.providers.execution.contracts.withdrawal_queue_nft import WithdrawalQueueNftContract -from src.types import BlockStamp, SlotNumber, WithdrawalVaultBalance, ELVaultBalance from src.utils.cache import global_lru_cache as lru_cache - logger = logging.getLogger(__name__) diff --git a/src/web3py/extensions/csm.py b/src/web3py/extensions/csm.py index 78e0669a2..bc305d4ae 100644 --- a/src/web3py/extensions/csm.py +++ b/src/web3py/extensions/csm.py @@ -13,13 +13,13 @@ from web3.types import BlockIdentifier, EventData from src import variables +from src.custom_types import BlockStamp, SlotNumber from src.metrics.prometheus.business import FRAME_PREV_REPORT_REF_SLOT from src.providers.execution.contracts.cs_accounting import CSAccountingContract from src.providers.execution.contracts.cs_fee_distributor import CSFeeDistributorContract from src.providers.execution.contracts.cs_fee_oracle import CSFeeOracleContract from src.providers.execution.contracts.cs_module import CSModuleContract from src.providers.ipfs import CID, CIDv0, CIDv1, is_cid_v0 -from src.types import BlockStamp, SlotNumber from src.utils.events import get_events_in_range from src.utils.lazy_object_proxy import LazyObjectProxy from src.web3py.extensions.lido_validators import NodeOperatorId diff --git a/src/web3py/extensions/lido_validators.py b/src/web3py/extensions/lido_validators.py index df071a364..dae9d1b2b 100644 --- a/src/web3py/extensions/lido_validators.py +++ b/src/web3py/extensions/lido_validators.py @@ -6,9 +6,9 @@ from eth_typing import ChecksumAddress, HexStr from web3.module import Module +from src.custom_types import BlockStamp, StakingModuleId, NodeOperatorId, NodeOperatorGlobalIndex, StakingModuleAddress from src.providers.consensus.types import Validator from src.providers.keys.types import LidoKey -from src.types import BlockStamp, StakingModuleId, NodeOperatorId, NodeOperatorGlobalIndex, StakingModuleAddress from src.utils.cache import global_lru_cache as lru_cache from src.utils.dataclass import Nested, FromResponse diff --git a/tests/conftest.py b/tests/conftest.py index 6b21f1d54..7a2f05cb7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from web3.types import Timestamp import src.variables -from src.types import BlockNumber, EpochNumber, ReferenceBlockStamp, SlotNumber +from src.custom_types import BlockNumber, EpochNumber, ReferenceBlockStamp, SlotNumber from src.variables import CONSENSUS_CLIENT_URI, EXECUTION_CLIENT_URI, KEYS_API_URI from src.web3py.contract_tweak import tweak_w3_contracts from src.web3py.extensions import LidoContracts, LidoValidatorsProvider, TransactionUtils diff --git a/tests/factory/blockstamp.py b/tests/factory/blockstamp.py index 24887cc70..1484137b9 100644 --- a/tests/factory/blockstamp.py +++ b/tests/factory/blockstamp.py @@ -1,7 +1,7 @@ from eth_typing import BlockNumber, HexStr from web3.types import Timestamp -from src.types import BlockStamp, StateRoot, SlotNumber, BlockHash, ReferenceBlockStamp, EpochNumber +from src.custom_types import BlockStamp, StateRoot, SlotNumber, BlockHash, ReferenceBlockStamp, EpochNumber from tests.factory.web3_factory import Web3DataclassFactory diff --git a/tests/factory/no_registry.py b/tests/factory/no_registry.py index df71d9c62..ca372c58e 100644 --- a/tests/factory/no_registry.py +++ b/tests/factory/no_registry.py @@ -13,9 +13,9 @@ FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE, ) +from src.custom_types import Gwei from src.providers.consensus.types import Validator, ValidatorState from src.providers.keys.types import LidoKey -from src.types import Gwei from src.web3py.extensions.lido_validators import LidoValidator, NodeOperator, StakingModule from tests.factory.web3_factory import Web3DataclassFactory diff --git a/tests/fork/conftest.py b/tests/fork/conftest.py index 8a6867e84..6dd8da56b 100644 --- a/tests/fork/conftest.py +++ b/tests/fork/conftest.py @@ -14,6 +14,7 @@ from web3_multi_provider import MultiProvider from src import variables +from src.custom_types import BlockRoot, BlockStamp, SlotNumber from src.main import ipfs_providers, logger from src.modules.submodules.consensus import ConsensusModule from src.modules.submodules.oracle_module import BaseModule @@ -23,7 +24,6 @@ from src.providers.execution.contracts.base_oracle import BaseOracleContract from src.providers.execution.contracts.hash_consensus import HashConsensusContract from src.providers.ipfs import CID, MultiIPFSProvider -from src.types import BlockRoot, BlockStamp, SlotNumber from src.utils.blockstamp import build_blockstamp from src.utils.cache import clear_global_cache from src.utils.slot import get_next_non_missed_slot diff --git a/tests/modules/accounting/bunker/conftest.py b/tests/modules/accounting/bunker/conftest.py index e9a0015e1..847be971e 100644 --- a/tests/modules/accounting/bunker/conftest.py +++ b/tests/modules/accounting/bunker/conftest.py @@ -3,13 +3,13 @@ import pytest from src.constants import FAR_FUTURE_EPOCH +from src.custom_types import BlockNumber, BlockStamp, EpochNumber, Gwei, ReferenceBlockStamp, ValidatorIndex from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState -from src.services.bunker import BunkerService from src.providers.keys.types import LidoKey +from src.services.bunker import BunkerService from src.services.bunker_cases.abnormal_cl_rebase import AbnormalClRebase from src.services.bunker_cases.types import BunkerConfig -from src.types import BlockNumber, BlockStamp, EpochNumber, Gwei, ReferenceBlockStamp, ValidatorIndex def simple_ref_blockstamp(block_number: int) -> ReferenceBlockStamp: diff --git a/tests/modules/accounting/bunker/test_bunker.py b/tests/modules/accounting/bunker/test_bunker.py index ccc732b47..59c39ed8f 100644 --- a/tests/modules/accounting/bunker/test_bunker.py +++ b/tests/modules/accounting/bunker/test_bunker.py @@ -3,10 +3,10 @@ import pytest +from src.custom_types import ReferenceBlockStamp from src.modules.accounting.types import LidoReportRebase from src.providers.consensus.types import BeaconStateView from src.services.bunker import BunkerService -from src.types import ReferenceBlockStamp from src.web3py.extensions.lido_validators import LidoValidator from tests.factory.blockstamp import ReferenceBlockStampFactory from tests.factory.configs import BunkerConfigFactory, ChainConfigFactory, FrameConfigFactory diff --git a/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py b/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py index 2bd7dfc07..cef34e6e0 100644 --- a/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py +++ b/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py @@ -3,10 +3,10 @@ import pytest from src.constants import FAR_FUTURE_EPOCH, UINT64_MAX +from src.custom_types import Gwei, ValidatorIndex, EpochNumber from src.providers.consensus.types import Validator, ValidatorState from src.services.bunker_cases.abnormal_cl_rebase import AbnormalClRebase from src.services.bunker_cases.types import BunkerConfig -from src.types import Gwei, ValidatorIndex, EpochNumber from src.web3py.extensions import LidoValidatorsProvider from src.web3py.types import Web3 from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py b/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py index 6152af407..ad777170d 100644 --- a/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py +++ b/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py @@ -6,11 +6,11 @@ MAX_EFFECTIVE_BALANCE_ELECTRA, EPOCHS_PER_SLASHINGS_VECTOR, ) +from src.custom_types import EpochNumber, Gwei, ReferenceBlockStamp, SlotNumber, ValidatorIndex from src.modules.submodules.consensus import FrameConfig from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState from src.services.bunker_cases.midterm_slashing_penalty import MidtermSlashingPenalty -from src.types import EpochNumber, Gwei, ReferenceBlockStamp, SlotNumber, ValidatorIndex from src.utils.web3converter import Web3Converter diff --git a/tests/modules/accounting/test_accounting_module.py b/tests/modules/accounting/test_accounting_module.py index 483e23f77..a65af8394 100644 --- a/tests/modules/accounting/test_accounting_module.py +++ b/tests/modules/accounting/test_accounting_module.py @@ -6,6 +6,7 @@ from web3.types import Wei from src import variables +from src.custom_types import BlockStamp, ReferenceBlockStamp from src.modules.accounting import accounting as accounting_module from src.modules.accounting.accounting import Accounting from src.modules.accounting.accounting import logger as accounting_logger @@ -14,7 +15,6 @@ from src.modules.submodules.oracle_module import ModuleExecuteDelay from src.modules.submodules.types import ChainConfig, FrameConfig, CurrentFrame, ZERO_HASH from src.services.withdrawal import Withdrawal -from src.types import BlockStamp, ReferenceBlockStamp from src.web3py.extensions.lido_validators import NodeOperatorId, StakingModule from tests.factory.base_oracle import AccountingProcessingStateFactory from tests.factory.blockstamp import BlockStampFactory, ReferenceBlockStampFactory diff --git a/tests/modules/accounting/test_safe_border_integration.py b/tests/modules/accounting/test_safe_border_integration.py index 04b584740..0aa8b0f96 100644 --- a/tests/modules/accounting/test_safe_border_integration.py +++ b/tests/modules/accounting/test_safe_border_integration.py @@ -3,8 +3,8 @@ import pytest from src.constants import EPOCHS_PER_SLASHINGS_VECTOR, MIN_VALIDATOR_WITHDRAWABILITY_DELAY +from src.custom_types import ReferenceBlockStamp from src.services.safe_border import SafeBorder -from src.types import ReferenceBlockStamp from tests.factory.blockstamp import ReferenceBlockStampFactory from tests.factory.configs import ChainConfigFactory, FrameConfigFactory, OracleReportLimitsFactory from tests.factory.no_registry import LidoValidatorFactory, ValidatorStateFactory diff --git a/tests/modules/accounting/test_validator_state.py b/tests/modules/accounting/test_validator_state.py index af308463d..8fabe9d5e 100644 --- a/tests/modules/accounting/test_validator_state.py +++ b/tests/modules/accounting/test_validator_state.py @@ -5,11 +5,11 @@ from eth_typing import HexStr from src.constants import FAR_FUTURE_EPOCH +from src.custom_types import EpochNumber, Gwei, StakingModuleId, NodeOperatorId, ValidatorIndex from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState from src.providers.keys.types import LidoKey from src.services.validator_state import LidoValidatorStateService -from src.types import EpochNumber, Gwei, StakingModuleId, NodeOperatorId, ValidatorIndex from src.web3py.extensions.lido_validators import ( NodeOperator, StakingModule, diff --git a/tests/modules/csm/test_checkpoint.py b/tests/modules/csm/test_checkpoint.py index 44f23735e..79072704c 100644 --- a/tests/modules/csm/test_checkpoint.py +++ b/tests/modules/csm/test_checkpoint.py @@ -6,6 +6,7 @@ from faker import Faker import src.modules.csm.checkpoint as checkpoint_module +from src.custom_types import EpochNumber, SlotNumber, ValidatorIndex from src.modules.csm.checkpoint import ( FrameCheckpoint, FrameCheckpointProcessor, @@ -17,7 +18,6 @@ from src.modules.submodules.types import ChainConfig, FrameConfig from src.providers.consensus.client import ConsensusClient from src.providers.consensus.types import BeaconSpecResponse, BlockAttestation, SlotAttestationCommittee -from src.types import EpochNumber, SlotNumber, ValidatorIndex from src.utils.web3converter import Web3Converter from tests.factory.bitarrays import BitListFactory from tests.factory.configs import ( diff --git a/tests/modules/csm/test_csm_module.py b/tests/modules/csm/test_csm_module.py index f74af8d69..ecf7f47bd 100644 --- a/tests/modules/csm/test_csm_module.py +++ b/tests/modules/csm/test_csm_module.py @@ -8,13 +8,13 @@ from hexbytes import HexBytes from src.constants import UINT64_MAX +from src.custom_types import EpochNumber, NodeOperatorId, SlotNumber, StakingModuleId, ValidatorIndex from src.modules.csm.csm import CSOracle from src.modules.csm.state import AttestationsAccumulator, State from src.modules.csm.tree import Tree from src.modules.submodules.oracle_module import ModuleExecuteDelay from src.modules.submodules.types import CurrentFrame, ZERO_HASH from src.providers.ipfs import CIDv0, CID -from src.types import EpochNumber, NodeOperatorId, SlotNumber, StakingModuleId, ValidatorIndex from src.web3py.extensions.csm import CSM from tests.factory.blockstamp import BlockStampFactory, ReferenceBlockStampFactory from tests.factory.configs import ChainConfigFactory, FrameConfigFactory diff --git a/tests/modules/csm/test_log.py b/tests/modules/csm/test_log.py index de52ca9ef..a768c718a 100644 --- a/tests/modules/csm/test_log.py +++ b/tests/modules/csm/test_log.py @@ -1,9 +1,10 @@ import json + import pytest +from src.custom_types import EpochNumber, NodeOperatorId, ReferenceBlockStamp from src.modules.csm.log import FramePerfLog from src.modules.csm.state import AttestationsAccumulator -from src.types import EpochNumber, NodeOperatorId, ReferenceBlockStamp from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/csm/test_state.py b/tests/modules/csm/test_state.py index 7539f7d26..b5551eb08 100644 --- a/tests/modules/csm/test_state.py +++ b/tests/modules/csm/test_state.py @@ -3,8 +3,8 @@ import pytest +from src.custom_types import EpochNumber, ValidatorIndex from src.modules.csm.state import AttestationsAccumulator, State -from src.types import EpochNumber, ValidatorIndex from src.utils.range import sequence diff --git a/tests/modules/csm/test_tree.py b/tests/modules/csm/test_tree.py index 623b4056e..8fbbd1ece 100644 --- a/tests/modules/csm/test_tree.py +++ b/tests/modules/csm/test_tree.py @@ -1,8 +1,8 @@ import pytest from src.constants import UINT64_MAX +from src.custom_types import NodeOperatorId from src.modules.csm.tree import StandardMerkleTree, Tree, TreeJSONEncoder -from src.types import NodeOperatorId @pytest.fixture() diff --git a/tests/modules/ejector/test_data_encode.py b/tests/modules/ejector/test_data_encode.py index bf368dcca..9212d9eb2 100644 --- a/tests/modules/ejector/test_data_encode.py +++ b/tests/modules/ejector/test_data_encode.py @@ -4,6 +4,7 @@ import pytest +from src.custom_types import StakingModuleId, NodeOperatorId from src.modules.ejector.data_encode import ( MODULE_ID_LENGTH, NODE_OPERATOR_ID_LENGTH, @@ -12,11 +13,9 @@ encode_data, sort_validators_to_eject, ) -from src.types import StakingModuleId, NodeOperatorId from src.web3py.extensions.lido_validators import LidoValidator from tests.factory.no_registry import LidoValidatorFactory - RECORD_LENGTH = sum( [ MODULE_ID_LENGTH, diff --git a/tests/modules/ejector/test_ejector.py b/tests/modules/ejector/test_ejector.py index a7ab9efd9..73dcf5394 100644 --- a/tests/modules/ejector/test_ejector.py +++ b/tests/modules/ejector/test_ejector.py @@ -14,6 +14,7 @@ MIN_ACTIVATION_BALANCE, MIN_VALIDATOR_WITHDRAWABILITY_DELAY, ) +from src.custom_types import BlockStamp, Gwei, ReferenceBlockStamp from src.modules.ejector import ejector as ejector_module from src.modules.ejector.ejector import ( Ejector, @@ -25,7 +26,6 @@ from src.providers.consensus.types import ( BeaconStateView, ) -from src.types import BlockStamp, Gwei, ReferenceBlockStamp from src.utils import validator_state from src.web3py.extensions.contracts import LidoContracts from src.web3py.extensions.lido_validators import NodeOperatorId, StakingModuleId diff --git a/tests/modules/ejector/test_prediction.py b/tests/modules/ejector/test_prediction.py index 9c96b99ce..8d78d72d7 100644 --- a/tests/modules/ejector/test_prediction.py +++ b/tests/modules/ejector/test_prediction.py @@ -5,9 +5,9 @@ from web3.types import Wei import src.services.prediction as prediction_module +from src.custom_types import BlockNumber, SlotNumber from src.modules.submodules.types import ChainConfig from src.services.prediction import RewardsPredictionService -from src.types import BlockNumber, SlotNumber from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/ejector/test_sweep.py b/tests/modules/ejector/test_sweep.py index 36ae5780f..9048affea 100644 --- a/tests/modules/ejector/test_sweep.py +++ b/tests/modules/ejector/test_sweep.py @@ -5,6 +5,7 @@ import src.modules.ejector.sweep as sweep_module from src.constants import MAX_WITHDRAWALS_PER_PAYLOAD, MIN_ACTIVATION_BALANCE +from src.custom_types import Gwei from src.modules.ejector.sweep import ( Withdrawal, get_pending_partial_withdrawals, @@ -13,7 +14,6 @@ ) from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import BeaconStateView, PendingPartialWithdrawal -from src.types import Gwei from tests.factory.consensus import BeaconStateViewFactory from tests.factory.no_registry import LidoValidatorFactory, ValidatorStateFactory diff --git a/tests/modules/ejector/test_validator_exit_order_iterator.py b/tests/modules/ejector/test_validator_exit_order_iterator.py index 75daf67f0..8f828b453 100644 --- a/tests/modules/ejector/test_validator_exit_order_iterator.py +++ b/tests/modules/ejector/test_validator_exit_order_iterator.py @@ -3,8 +3,8 @@ import pytest +from src.custom_types import Gwei from src.services.exit_order_iterator import ValidatorExitIterator, StakingModuleStats, NodeOperatorStats -from src.types import Gwei from src.web3py.extensions.lido_validators import NodeOperatorLimitMode from tests.factory.blockstamp import ReferenceBlockStampFactory from tests.factory.no_registry import ( diff --git a/tests/modules/submodules/consensus/conftest.py b/tests/modules/submodules/consensus/conftest.py index 8d4329c76..31943ceeb 100644 --- a/tests/modules/submodules/consensus/conftest.py +++ b/tests/modules/submodules/consensus/conftest.py @@ -1,7 +1,7 @@ import pytest +from src.custom_types import BlockStamp, ReferenceBlockStamp from src.modules.submodules.consensus import ConsensusModule -from src.types import BlockStamp, ReferenceBlockStamp class SimpleConsensusModule(ConsensusModule): diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index 5127c9027..c9209a19b 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -5,12 +5,12 @@ from web3.exceptions import ContractCustomError from src import variables +from src.custom_types import BlockStamp, ReferenceBlockStamp from src.modules.submodules import consensus as consensus_module from src.modules.submodules.consensus import ZERO_HASH, ConsensusModule, IsNotMemberException, MemberInfo from src.modules.submodules.exceptions import IncompatibleOracleVersion, ContractVersionMismatch from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import BeaconSpecResponse -from src.types import BlockStamp, ReferenceBlockStamp from tests.conftest import get_blockstamp_by_state, Account from tests.factory.blockstamp import ReferenceBlockStampFactory, BlockStampFactory from tests.factory.configs import BeaconSpecResponseFactory, ChainConfigFactory, FrameConfigFactory diff --git a/tests/modules/submodules/test_oracle_module.py b/tests/modules/submodules/test_oracle_module.py index 0eaabc791..9aab2ba0e 100644 --- a/tests/modules/submodules/test_oracle_module.py +++ b/tests/modules/submodules/test_oracle_module.py @@ -1,19 +1,17 @@ from unittest.mock import Mock, patch, MagicMock -from typing import Type import pytest from requests.exceptions import ConnectionError as RequestsConnectionError from timeout_decorator import TimeoutError as DecoratorTimeoutError - from web3_multi_provider.multi_http_provider import NoActiveProviderError +from src import variables +from src.custom_types import BlockStamp from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay from src.providers.http_provider import NotOkResponse from src.providers.keys.client import KeysOutdatedException -from src.types import BlockStamp from src.utils.slot import InconsistentData, NoSlotsAvailable, SlotNotFinalized -from src import variables from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/providers/consensus/test_consensus_client.py b/tests/providers/consensus/test_consensus_client.py index 06955dc2e..524f40819 100644 --- a/tests/providers/consensus/test_consensus_client.py +++ b/tests/providers/consensus/test_consensus_client.py @@ -5,9 +5,9 @@ import pytest +from src.custom_types import EpochNumber, SlotNumber from src.providers.consensus.client import ConsensusClient from src.providers.consensus.types import Validator -from src.types import EpochNumber, SlotNumber from src.utils.blockstamp import build_blockstamp from src.variables import CONSENSUS_CLIENT_URI from tests.factory.blockstamp import BlockStampFactory diff --git a/tests/providers_clients/test_keys_api_client.py b/tests/providers_clients/test_keys_api_client.py index 25e1bbf93..1b6a112a3 100644 --- a/tests/providers_clients/test_keys_api_client.py +++ b/tests/providers_clients/test_keys_api_client.py @@ -11,9 +11,9 @@ import src.providers.keys.client as keys_api_client_module from src import constants from src import variables +from src.custom_types import StakingModuleAddress from src.providers.keys.client import KAPIClientError, KeysAPIClient, KeysOutdatedException from src.providers.keys.types import LidoKey -from src.types import StakingModuleAddress from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/utils/test_slot.py b/tests/utils/test_slot.py index a6e7006cf..cec69c203 100644 --- a/tests/utils/test_slot.py +++ b/tests/utils/test_slot.py @@ -4,8 +4,8 @@ import pytest +from src.custom_types import SlotNumber from src.providers.http_provider import NotOkResponse -from src.types import SlotNumber from src.utils.slot import NoSlotsAvailable, get_prev_non_missed_slot, get_next_non_missed_slot from tests.conftest import get_blockstamp_by_state diff --git a/tests/utils/test_validator_state_utils.py b/tests/utils/test_validator_state_utils.py index 6e6a9acfd..4791c2e55 100644 --- a/tests/utils/test_validator_state_utils.py +++ b/tests/utils/test_validator_state_utils.py @@ -6,8 +6,8 @@ MAX_EFFECTIVE_BALANCE_ELECTRA, MIN_ACTIVATION_BALANCE, ) +from src.custom_types import EpochNumber, Gwei from src.providers.consensus.types import Validator, ValidatorState -from src.types import EpochNumber, Gwei from src.utils.validator_state import ( calculate_active_effective_balance_sum, calculate_total_active_effective_balance, diff --git a/tests/utils/test_web3_converter.py b/tests/utils/test_web3_converter.py index b091dfae3..5b9567e59 100644 --- a/tests/utils/test_web3_converter.py +++ b/tests/utils/test_web3_converter.py @@ -1,7 +1,7 @@ import pytest +from src.custom_types import EpochNumber, FrameNumber, SlotNumber from src.modules.submodules.types import ChainConfig, FrameConfig -from src.types import EpochNumber, FrameNumber, SlotNumber from src.utils.web3converter import Web3Converter from tests.factory.configs import ChainConfigFactory, FrameConfigFactory From b67919cec43b6e560fda9e227e4fd49489dc0683 Mon Sep 17 00:00:00 2001 From: hweawer Date: Tue, 15 Apr 2025 18:00:25 +0200 Subject: [PATCH 04/13] feat: refslot --- src/main.py | 12 ++++++------ src/modules/accounting/accounting.py | 5 +++-- src/modules/csm/csm.py | 6 +++--- src/modules/ejector/ejector.py | 5 +++-- src/modules/submodules/consensus.py | 16 +++++++++++----- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/main.py b/src/main.py index 561db9f10..8488e0435 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,7 @@ import argparse import os import sys -from typing import Iterator, cast +from typing import Iterator, cast, Optional import requests from packaging.version import Version @@ -81,17 +81,17 @@ def _construct_web3() -> Web3: return web3 -def _construct_module(web3: Web3, module_name: OracleModule, run_past: bool = False) -> Accounting | Ejector | CSOracle: +def _construct_module(web3: Web3, module_name: OracleModule, refslot: Optional[int] = None) -> Accounting | Ejector | CSOracle: instance: Accounting | Ejector | CSOracle if module_name == OracleModule.ACCOUNTING: logger.info({'msg': 'Initialize Accounting module.'}) - instance = Accounting(web3, run_past) + instance = Accounting(web3, refslot) elif module_name == OracleModule.EJECTOR: logger.info({'msg': 'Initialize Ejector module.'}) - instance = Ejector(web3, run_past) + instance = Ejector(web3, refslot) elif module_name == OracleModule.CSM: logger.info({'msg': 'Initialize CSM performance oracle module.'}) - instance = CSOracle(web3, run_past) + instance = CSOracle(web3, refslot) else: raise ValueError(f'Unexpected arg: {module_name=}.') @@ -169,7 +169,7 @@ def run_on_refslot(module_name: OracleModule): block_details = w3.cc.get_block_details(block_root) bs = build_blockstamp(block_details) instance.refresh_contracts_and_run_cycle(bs) - instance = _construct_module(w3, module_name, True) + instance = _construct_module(w3, module_name, refslot) def check_providers_chain_ids(web3: Web3, cc: ConsensusClientModule, kac: KeysAPIClientModule): diff --git a/src/modules/accounting/accounting.py b/src/modules/accounting/accounting.py index 8c51e43f4..6f0df1bd2 100644 --- a/src/modules/accounting/accounting.py +++ b/src/modules/accounting/accounting.py @@ -1,6 +1,7 @@ import logging from collections import defaultdict from time import sleep +from typing import Optional from hexbytes import HexBytes from web3.exceptions import ContractCustomError @@ -59,9 +60,9 @@ class Accounting(BaseModule, ConsensusModule): """ COMPATIBLE_ONCHAIN_VERSIONS = [(2, 2), (2, 3)] - def __init__(self, w3: Web3, run_past: bool = False): + def __init__(self, w3: Web3, refslot: Optional[int] = None): self.report_contract: AccountingOracleContract = w3.lido_contracts.accounting_oracle - self.run_past = run_past + self.refslot = refslot super().__init__(w3) self.lido_validator_state_service = LidoValidatorStateService(self.w3) diff --git a/src/modules/csm/csm.py b/src/modules/csm/csm.py index bc57a19ba..b7214bfa1 100644 --- a/src/modules/csm/csm.py +++ b/src/modules/csm/csm.py @@ -1,6 +1,6 @@ import logging from collections import defaultdict -from typing import Iterator +from typing import Iterator, Optional from hexbytes import HexBytes @@ -64,10 +64,10 @@ class CSOracle(BaseModule, ConsensusModule): report_contract: CSFeeOracleContract module_id: StakingModuleId - def __init__(self, w3: Web3, run_past: bool = False): + def __init__(self, w3: Web3, refslot: Optional[int] = None): self.report_contract = w3.csm.oracle self.state = State.load() - self.run_past = run_past + self.refslot = refslot super().__init__(w3) self.module_id = self._get_module_id() diff --git a/src/modules/ejector/ejector.py b/src/modules/ejector/ejector.py index e3298420f..a4478e2a8 100644 --- a/src/modules/ejector/ejector.py +++ b/src/modules/ejector/ejector.py @@ -1,5 +1,6 @@ import dataclasses import logging +from typing import Optional from web3.exceptions import ContractCustomError from web3.types import Wei @@ -66,9 +67,9 @@ class Ejector(BaseModule, ConsensusModule): AVG_EXPECTING_WITHDRAWALS_SWEEP_DURATION_MULTIPLIER = 0.5 - def __init__(self, w3: Web3, run_past: bool = False): + def __init__(self, w3: Web3, refslot: Optional[int] = None): self.report_contract: ExitBusOracleContract = w3.lido_contracts.validators_exit_bus_oracle - self.run_past = run_past + self.refslot = refslot super().__init__(w3) self.prediction_service = RewardsPredictionService(w3) diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index 38b053192..811047069 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -1,7 +1,7 @@ import logging from abc import ABC, abstractmethod from time import sleep -from typing import cast +from typing import cast, Optional from eth_abi import encode from hexbytes import HexBytes @@ -44,7 +44,7 @@ class ConsensusModule(ABC): report_contract should contain getConsensusContract method. """ report_contract: BaseOracleContract - run_past: bool = False + refslot: Optional[int] = None # Contains tuple[CONTRACT_VERSION, CONSENSUS_VERSION] COMPATIBLE_ONCHAIN_VERSIONS: list[tuple[int, int]] @@ -200,9 +200,12 @@ def _calculate_reference_blockstamp(self, last_finalized_blockstamp: BlockStamp) converter = self._get_web3_converter(last_finalized_blockstamp) member_info = self.get_member_info(self._get_latest_blockstamp()) + refslot = member_info.current_frame_ref_slot + if self.refslot: + refslot = self.refslot bs = get_reference_blockstamp( cc=self.w3.cc, - ref_slot=member_info.current_frame_ref_slot, + ref_slot=refslot, ref_epoch=converter.get_epoch_by_slot(member_info.current_frame_ref_slot), last_finalized_slot_number=last_finalized_blockstamp.slot_number, ) @@ -216,7 +219,7 @@ def get_blockstamp_for_report(self, last_finalized_blockstamp: BlockStamp) -> Re Returns: Non-missed reference slot blockstamp in case contract is reportable. """ - if self.run_past: + if self.refslot: return self._calculate_reference_blockstamp(last_finalized_blockstamp) latest_blockstamp = self._get_latest_blockstamp() @@ -410,7 +413,10 @@ def _submit_report(self, report: tuple, contract_version: int): self.w3.transaction.check_and_send_transaction(tx, variables.ACCOUNT) def _get_latest_blockstamp(self) -> BlockStamp: - root = self.w3.cc.get_block_root('head').root + if self.refslot: + root = self.w3.cc.get_block_root(SlotNumber(self.refslot + 3 * 32)).root + else: + root = self.w3.cc.get_block_root('head').root block_details = self.w3.cc.get_block_details(root) bs = build_blockstamp(block_details) logger.debug({'msg': 'Fetch latest blockstamp.', 'value': bs}) From a351df837da346d2d615f0702e02fcdd97e22085 Mon Sep 17 00:00:00 2001 From: hweawer Date: Wed, 16 Apr 2025 08:50:41 +0200 Subject: [PATCH 05/13] feat: Refslot --- src/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.py b/src/main.py index 8488e0435..e55a1269e 100644 --- a/src/main.py +++ b/src/main.py @@ -168,6 +168,7 @@ def run_on_refslot(module_name: OracleModule): block_root = BlockRoot(w3.cc.get_block_root(SlotNumber(refslot + 3 * 32)).root) block_details = w3.cc.get_block_details(block_root) bs = build_blockstamp(block_details) + instance.refslot = refslot instance.refresh_contracts_and_run_cycle(bs) instance = _construct_module(w3, module_name, refslot) From b7e48054aa6c0d35fc17a0537d2a8a42cb21857b Mon Sep 17 00:00:00 2001 From: hweawer Date: Fri, 25 Apr 2025 17:23:46 +0200 Subject: [PATCH 06/13] feat: Update logging --- src/constants.py | 2 +- src/main.py | 18 ++++++++++++------ src/modules/accounting/accounting.py | 2 +- .../accounting/third_phase/extra_data.py | 2 +- src/modules/accounting/types.py | 2 +- src/modules/checks/suites/conftest.py | 2 +- src/modules/csm/checkpoint.py | 2 +- src/modules/csm/csm.py | 2 +- src/modules/csm/log.py | 2 +- src/modules/csm/state.py | 2 +- src/modules/csm/types.py | 2 +- src/modules/ejector/data_encode.py | 2 +- src/modules/ejector/ejector.py | 2 +- src/modules/ejector/sweep.py | 2 +- src/modules/ejector/types.py | 2 +- src/modules/submodules/consensus.py | 2 +- src/modules/submodules/oracle_module.py | 2 +- src/modules/submodules/types.py | 2 +- src/providers/consensus/client.py | 2 +- src/providers/consensus/types.py | 2 +- .../execution/contracts/base_oracle.py | 2 +- .../execution/contracts/hash_consensus.py | 2 +- src/providers/execution/contracts/lido.py | 2 +- src/providers/keys/client.py | 2 +- src/providers/keys/types.py | 2 +- src/services/bunker.py | 2 +- .../bunker_cases/abnormal_cl_rebase.py | 2 +- .../bunker_cases/midterm_slashing_penalty.py | 2 +- src/services/exit_order_iterator.py | 2 +- src/services/prediction.py | 2 +- src/services/safe_border.py | 2 +- src/services/validator_state.py | 2 +- src/services/withdrawal.py | 2 +- src/{custom_types.py => types.py} | 0 src/utils/blockstamp.py | 2 +- src/utils/dataclass.py | 2 +- src/utils/events.py | 2 +- src/utils/slot.py | 2 +- src/utils/units.py | 2 +- src/utils/validator_state.py | 2 +- src/utils/web3converter.py | 2 +- src/variables.py | 2 +- src/web3py/extensions/contracts.py | 2 +- src/web3py/extensions/csm.py | 2 +- src/web3py/extensions/lido_validators.py | 2 +- tests/conftest.py | 2 +- tests/factory/blockstamp.py | 2 +- tests/factory/no_registry.py | 2 +- tests/fork/conftest.py | 2 +- tests/modules/accounting/bunker/conftest.py | 2 +- tests/modules/accounting/bunker/test_bunker.py | 2 +- .../bunker/test_bunker_abnormal_cl_rebase.py | 2 +- .../bunker/test_bunker_midterm_penalty.py | 2 +- .../accounting/test_accounting_module.py | 2 +- .../accounting/test_safe_border_integration.py | 2 +- .../modules/accounting/test_validator_state.py | 2 +- tests/modules/csm/test_checkpoint.py | 2 +- tests/modules/csm/test_csm_module.py | 2 +- tests/modules/csm/test_log.py | 2 +- tests/modules/csm/test_state.py | 2 +- tests/modules/csm/test_tree.py | 2 +- tests/modules/ejector/test_data_encode.py | 2 +- tests/modules/ejector/test_ejector.py | 2 +- tests/modules/ejector/test_prediction.py | 2 +- tests/modules/ejector/test_sweep.py | 2 +- .../test_validator_exit_order_iterator.py | 2 +- tests/modules/submodules/consensus/conftest.py | 2 +- .../submodules/consensus/test_consensus.py | 2 +- tests/modules/submodules/test_oracle_module.py | 2 +- .../consensus/test_consensus_client.py | 2 +- .../providers_clients/test_keys_api_client.py | 2 +- tests/utils/test_slot.py | 2 +- tests/utils/test_validator_state_utils.py | 2 +- tests/utils/test_web3_converter.py | 2 +- 74 files changed, 84 insertions(+), 78 deletions(-) rename src/{custom_types.py => types.py} (100%) diff --git a/src/constants.py b/src/constants.py index f19ff51c3..2ba467222 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,6 +1,6 @@ from packaging.version import Version -from src.custom_types import Gwei, SlotNumber +from src.types import Gwei, SlotNumber # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#misc GENESIS_SLOT = SlotNumber(0) diff --git a/src/main.py b/src/main.py index e55a1269e..dcadfb31e 100644 --- a/src/main.py +++ b/src/main.py @@ -4,12 +4,12 @@ from typing import Iterator, cast, Optional import requests +from hexbytes import HexBytes from packaging.version import Version from prometheus_client import start_http_server from src import constants from src import variables -from src.custom_types import OracleModule, BlockRoot, SlotNumber from src.metrics.healthcheck_server import start_pulse_server from src.metrics.logging import logging from src.metrics.prometheus.basic import ENV_VARIABLES_INFO, BUILD_INFO @@ -18,6 +18,7 @@ from src.modules.csm.csm import CSOracle from src.modules.ejector.ejector import Ejector from src.providers.ipfs import GW3, IPFSProvider, MultiIPFSProvider, Pinata, PublicIPFS +from src.types import OracleModule, BlockRoot, SlotNumber from src.utils.blockstamp import build_blockstamp from src.utils.build import get_build_info from src.utils.exception import IncompatibleException @@ -145,16 +146,16 @@ def get_transactions(contract_address, selector: str, limit: int = 10): def run_on_refslot(module_name: OracleModule): + logging.getLogger().setLevel(logging.WARNING) w3 = _construct_web3() instance: Accounting | Ejector | CSOracle = _construct_module(w3, module_name, True) instance.check_contract_configs() submit_report_fn = instance.report_contract.get_function_by_name("submitReportData") selector = '0x' + w3.keccak(text=submit_report_fn.abi_element_identifier)[:4].hex() - logger.info("Manual Selector: %s", selector) txs = get_transactions(instance.report_contract.address, selector) if not txs: - logger.info("No submitReportData transactions found!") + logger.error("No submitReportData transactions found!") sys.exit(0) logger.info("Found %d submitReportData calls", len(txs)) @@ -163,13 +164,18 @@ def run_on_refslot(module_name: OracleModule): tx_hash = tx["hash"] _, data = instance.report_contract.decode_function_input(tx["input"]) refslot = int(data['data']['refSlot']) - print(f"Tx: {tx_hash} → X: {refslot}") - logger.info(data) + print(f"Tx: {tx_hash} → slot: {refslot}") + print([HexBytes(v.hex()) if isinstance(v, bytes) else v for v in data['data'].values()]) block_root = BlockRoot(w3.cc.get_block_root(SlotNumber(refslot + 3 * 32)).root) block_details = w3.cc.get_block_details(block_root) bs = build_blockstamp(block_details) instance.refslot = refslot - instance.refresh_contracts_and_run_cycle(bs) + instance.refresh_contracts_if_address_change() + report_blockstamp = instance.get_blockstamp_for_report(bs) + report = instance.build_report(report_blockstamp) + + + print(report) instance = _construct_module(w3, module_name, refslot) diff --git a/src/modules/accounting/accounting.py b/src/modules/accounting/accounting.py index 6f0df1bd2..edcaab4a4 100644 --- a/src/modules/accounting/accounting.py +++ b/src/modules/accounting/accounting.py @@ -9,7 +9,7 @@ from src import variables from src.constants import SHARE_RATE_PRECISION_E27 -from src.custom_types import BlockStamp, Gwei, ReferenceBlockStamp, StakingModuleId, NodeOperatorGlobalIndex, FinalizationBatches +from src.types import BlockStamp, Gwei, ReferenceBlockStamp, StakingModuleId, NodeOperatorGlobalIndex, FinalizationBatches from src.metrics.prometheus.accounting import ( ACCOUNTING_IS_BUNKER, ACCOUNTING_CL_BALANCE_GWEI, diff --git a/src/modules/accounting/third_phase/extra_data.py b/src/modules/accounting/third_phase/extra_data.py index a6393ad5e..0543d773d 100644 --- a/src/modules/accounting/third_phase/extra_data.py +++ b/src/modules/accounting/third_phase/extra_data.py @@ -2,7 +2,7 @@ from itertools import groupby, batched from typing import Sequence -from src.custom_types import NodeOperatorGlobalIndex +from src.types import NodeOperatorGlobalIndex from src.modules.accounting.third_phase.types import ExtraData, ItemType, ExtraDataLengths, FormatList from src.modules.submodules.types import ZERO_HASH from src.web3py.types import Web3 diff --git a/src/modules/accounting/types.py b/src/modules/accounting/types.py index d4e3a29b8..9843c61ae 100644 --- a/src/modules/accounting/types.py +++ b/src/modules/accounting/types.py @@ -5,7 +5,7 @@ from hexbytes import HexBytes from web3.types import Wei -from src.custom_types import ( +from src.types import ( SlotNumber, Gwei, StakingModuleId, diff --git a/src/modules/checks/suites/conftest.py b/src/modules/checks/suites/conftest.py index d06e23342..c78b2a9dd 100644 --- a/src/modules/checks/suites/conftest.py +++ b/src/modules/checks/suites/conftest.py @@ -4,7 +4,7 @@ from xdist.dsession import TerminalDistReporter # type: ignore[import] from src import variables -from src.custom_types import EpochNumber, SlotNumber, BlockRoot +from src.types import EpochNumber, SlotNumber, BlockRoot from src.utils.api import opsgenie_api from src.utils.blockstamp import build_blockstamp from src.utils.slot import get_reference_blockstamp diff --git a/src/modules/csm/checkpoint.py b/src/modules/csm/checkpoint.py index 6001834f7..4f42df532 100644 --- a/src/modules/csm/checkpoint.py +++ b/src/modules/csm/checkpoint.py @@ -7,7 +7,7 @@ from src import variables from src.constants import SLOTS_PER_HISTORICAL_ROOT -from src.custom_types import BlockRoot, BlockStamp, CommitteeIndex, EpochNumber, SlotNumber, ValidatorIndex +from src.types import BlockRoot, BlockStamp, CommitteeIndex, EpochNumber, SlotNumber, ValidatorIndex from src.metrics.prometheus.csm import CSM_MIN_UNPROCESSED_EPOCH, CSM_UNPROCESSED_EPOCHS_COUNT from src.modules.csm.state import State from src.providers.consensus.client import ConsensusClient diff --git a/src/modules/csm/csm.py b/src/modules/csm/csm.py index b7214bfa1..7c8dfda0a 100644 --- a/src/modules/csm/csm.py +++ b/src/modules/csm/csm.py @@ -5,7 +5,7 @@ from hexbytes import HexBytes from src.constants import TOTAL_BASIS_POINTS, UINT64_MAX -from src.custom_types import ( +from src.types import ( BlockStamp, EpochNumber, ReferenceBlockStamp, diff --git a/src/modules/csm/log.py b/src/modules/csm/log.py index d66aeb0ee..0f90d3096 100644 --- a/src/modules/csm/log.py +++ b/src/modules/csm/log.py @@ -2,7 +2,7 @@ from collections import defaultdict from dataclasses import asdict, dataclass, field -from src.custom_types import EpochNumber, NodeOperatorId, ReferenceBlockStamp, ValidatorIndex +from src.types import EpochNumber, NodeOperatorId, ReferenceBlockStamp, ValidatorIndex from src.modules.csm.state import AttestationsAccumulator from src.modules.csm.types import Shares diff --git a/src/modules/csm/state.py b/src/modules/csm/state.py index f7445e391..4373f5259 100644 --- a/src/modules/csm/state.py +++ b/src/modules/csm/state.py @@ -7,7 +7,7 @@ from typing import Self from src import variables -from src.custom_types import EpochNumber, ValidatorIndex +from src.types import EpochNumber, ValidatorIndex from src.utils.range import sequence logger = logging.getLogger(__name__) diff --git a/src/modules/csm/types.py b/src/modules/csm/types.py index 12564b6b4..121d29dd5 100644 --- a/src/modules/csm/types.py +++ b/src/modules/csm/types.py @@ -4,7 +4,7 @@ from hexbytes import HexBytes -from src.custom_types import NodeOperatorId, SlotNumber +from src.types import NodeOperatorId, SlotNumber from src.providers.ipfs import CID logger = logging.getLogger(__name__) diff --git a/src/modules/ejector/data_encode.py b/src/modules/ejector/data_encode.py index dc0084b27..e9468a6dd 100644 --- a/src/modules/ejector/data_encode.py +++ b/src/modules/ejector/data_encode.py @@ -1,6 +1,6 @@ from eth_typing import HexStr -from src.custom_types import ValidatorIndex +from src.types import ValidatorIndex from src.utils.types import hex_str_to_bytes from src.web3py.extensions.lido_validators import LidoValidator, NodeOperatorGlobalIndex diff --git a/src/modules/ejector/ejector.py b/src/modules/ejector/ejector.py index a4478e2a8..d6e3ad709 100644 --- a/src/modules/ejector/ejector.py +++ b/src/modules/ejector/ejector.py @@ -10,7 +10,6 @@ FAR_FUTURE_EPOCH, MIN_VALIDATOR_WITHDRAWABILITY_DELAY, ) -from src.custom_types import BlockStamp, EpochNumber, Gwei, NodeOperatorGlobalIndex, ReferenceBlockStamp from src.metrics.prometheus.business import CONTRACT_ON_PAUSE from src.metrics.prometheus.duration_meter import duration_meter from src.metrics.prometheus.ejector import ( @@ -29,6 +28,7 @@ from src.services.exit_order_iterator import ValidatorExitIterator from src.services.prediction import RewardsPredictionService from src.services.validator_state import LidoValidatorStateService +from src.types import BlockStamp, EpochNumber, Gwei, NodeOperatorGlobalIndex, ReferenceBlockStamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.units import gwei_to_wei from src.utils.validator_state import ( diff --git a/src/modules/ejector/sweep.py b/src/modules/ejector/sweep.py index 51a05f0e6..e46919c75 100644 --- a/src/modules/ejector/sweep.py +++ b/src/modules/ejector/sweep.py @@ -9,7 +9,7 @@ MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, MAX_WITHDRAWALS_PER_PAYLOAD, ) -from src.custom_types import Gwei +from src.types import Gwei from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import BeaconStateView from src.utils.validator_state import ( diff --git a/src/modules/ejector/types.py b/src/modules/ejector/types.py index 86cffc949..07c81c643 100644 --- a/src/modules/ejector/types.py +++ b/src/modules/ejector/types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from src.custom_types import SlotNumber +from src.types import SlotNumber @dataclass diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index 811047069..e6a1d732f 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -8,7 +8,7 @@ from web3.exceptions import ContractCustomError from src import variables -from src.custom_types import BlockStamp, ReferenceBlockStamp, SlotNumber, FrameNumber +from src.types import BlockStamp, ReferenceBlockStamp, SlotNumber, FrameNumber from src.metrics.prometheus.basic import ORACLE_SLOT_NUMBER, ORACLE_BLOCK_NUMBER, GENESIS_TIME, ACCOUNT_BALANCE from src.metrics.prometheus.business import ( ORACLE_MEMBER_LAST_REPORT_REF_SLOT, diff --git a/src/modules/submodules/oracle_module.py b/src/modules/submodules/oracle_module.py index bc64c1836..685ece8b3 100644 --- a/src/modules/submodules/oracle_module.py +++ b/src/modules/submodules/oracle_module.py @@ -11,7 +11,7 @@ from web3_multi_provider import NoActiveProviderError from src import variables -from src.custom_types import SlotNumber, BlockStamp, BlockRoot +from src.types import SlotNumber, BlockStamp, BlockRoot from src.metrics.healthcheck_server import pulse from src.metrics.prometheus.basic import ORACLE_BLOCK_NUMBER, ORACLE_SLOT_NUMBER from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch diff --git a/src/modules/submodules/types.py b/src/modules/submodules/types.py index d7dbc987e..6b07a8ac3 100644 --- a/src/modules/submodules/types.py +++ b/src/modules/submodules/types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from src.custom_types import SlotNumber +from src.types import SlotNumber ZERO_HASH = bytes([0]*32) diff --git a/src/providers/consensus/client.py b/src/providers/consensus/client.py index 36d417749..3f1abab6b 100644 --- a/src/providers/consensus/client.py +++ b/src/providers/consensus/client.py @@ -3,7 +3,7 @@ from json_stream.base import TransientStreamingJSONObject # type: ignore -from src.custom_types import BlockRoot, BlockStamp, SlotNumber, EpochNumber, StateRoot +from src.types import BlockRoot, BlockStamp, SlotNumber, EpochNumber, StateRoot from src.metrics.logging import logging from src.metrics.prometheus.basic import CL_REQUESTS_DURATION from src.providers.consensus.types import ( diff --git a/src/providers/consensus/types.py b/src/providers/consensus/types.py index 12bef72bd..6f35456c5 100644 --- a/src/providers/consensus/types.py +++ b/src/providers/consensus/types.py @@ -6,7 +6,7 @@ from web3.types import Timestamp from src.constants import FAR_FUTURE_EPOCH -from src.custom_types import BlockHash, BlockRoot, CommitteeIndex, EpochNumber, Gwei, SlotNumber, StateRoot, ValidatorIndex +from src.types import BlockHash, BlockRoot, CommitteeIndex, EpochNumber, Gwei, SlotNumber, StateRoot, ValidatorIndex from src.utils.dataclass import FromResponse, Nested diff --git a/src/providers/execution/contracts/base_oracle.py b/src/providers/execution/contracts/base_oracle.py index bcb32b270..bcaa36b08 100644 --- a/src/providers/execution/contracts/base_oracle.py +++ b/src/providers/execution/contracts/base_oracle.py @@ -4,7 +4,7 @@ from web3.contract.contract import ContractFunction from web3.types import BlockIdentifier -from src.custom_types import SlotNumber +from src.types import SlotNumber from src.providers.execution.base_interface import ContractInterface from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/providers/execution/contracts/hash_consensus.py b/src/providers/execution/contracts/hash_consensus.py index 77b0d9fea..049be2635 100644 --- a/src/providers/execution/contracts/hash_consensus.py +++ b/src/providers/execution/contracts/hash_consensus.py @@ -4,7 +4,7 @@ from web3.contract.contract import ContractFunction from web3.types import BlockIdentifier -from src.custom_types import SlotNumber +from src.types import SlotNumber from src.modules.submodules.types import ChainConfig, CurrentFrame, FrameConfig from src.providers.execution.base_interface import ContractInterface from src.utils.abi import named_tuple_to_dataclass diff --git a/src/providers/execution/contracts/lido.py b/src/providers/execution/contracts/lido.py index 2b4f300c6..ae86275c9 100644 --- a/src/providers/execution/contracts/lido.py +++ b/src/providers/execution/contracts/lido.py @@ -4,7 +4,7 @@ from web3 import Web3 from web3.types import Wei, BlockIdentifier, StateOverride, StateOverrideParams -from src.custom_types import SlotNumber +from src.types import SlotNumber from src.modules.accounting.types import LidoReportRebase, BeaconStat from src.providers.execution.base_interface import ContractInterface from src.utils.abi import named_tuple_to_dataclass diff --git a/src/providers/keys/client.py b/src/providers/keys/client.py index 554d12dca..95b6b2205 100644 --- a/src/providers/keys/client.py +++ b/src/providers/keys/client.py @@ -1,7 +1,7 @@ from time import sleep from typing import cast, TypedDict, List -from src.custom_types import BlockStamp, StakingModuleAddress +from src.types import BlockStamp, StakingModuleAddress from src.metrics.prometheus.basic import KEYS_API_REQUESTS_DURATION, KEYS_API_LATEST_BLOCKNUMBER from src.providers.http_provider import HTTPProvider, NotOkResponse from src.providers.keys.types import LidoKey, KeysApiStatus diff --git a/src/providers/keys/types.py b/src/providers/keys/types.py index f3df544db..797c80b9b 100644 --- a/src/providers/keys/types.py +++ b/src/providers/keys/types.py @@ -3,7 +3,7 @@ from eth_typing import ChecksumAddress, HexStr -from src.custom_types import NodeOperatorId +from src.types import NodeOperatorId from src.utils.dataclass import FromResponse diff --git a/src/services/bunker.py b/src/services/bunker.py index 6c286ceda..0b6141e3d 100644 --- a/src/services/bunker.py +++ b/src/services/bunker.py @@ -3,7 +3,7 @@ from web3.types import Wei from src.constants import TOTAL_BASIS_POINTS -from src.custom_types import BlockStamp, ReferenceBlockStamp, Gwei +from src.types import BlockStamp, ReferenceBlockStamp, Gwei from src.metrics.prometheus.duration_meter import duration_meter from src.metrics.prometheus.validators import ( ALL_VALIDATORS, diff --git a/src/services/bunker_cases/abnormal_cl_rebase.py b/src/services/bunker_cases/abnormal_cl_rebase.py index 744d060b3..3861ebfbe 100644 --- a/src/services/bunker_cases/abnormal_cl_rebase.py +++ b/src/services/bunker_cases/abnormal_cl_rebase.py @@ -6,7 +6,7 @@ from web3.types import EventData from src.constants import EFFECTIVE_BALANCE_INCREMENT, LIDO_DEPOSIT_AMOUNT -from src.custom_types import ReferenceBlockStamp, Gwei, BlockNumber, SlotNumber, BlockStamp, EpochNumber +from src.types import ReferenceBlockStamp, Gwei, BlockNumber, SlotNumber, BlockStamp, EpochNumber from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator from src.providers.keys.types import LidoKey diff --git a/src/services/bunker_cases/midterm_slashing_penalty.py b/src/services/bunker_cases/midterm_slashing_penalty.py index cb3ab600c..4aba79952 100644 --- a/src/services/bunker_cases/midterm_slashing_penalty.py +++ b/src/services/bunker_cases/midterm_slashing_penalty.py @@ -7,7 +7,7 @@ MIN_VALIDATOR_WITHDRAWABILITY_DELAY, PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX, ) -from src.custom_types import EpochNumber, FrameNumber, Gwei, ReferenceBlockStamp, SlotNumber +from src.types import EpochNumber, FrameNumber, Gwei, ReferenceBlockStamp, SlotNumber from src.providers.consensus.types import Validator from src.utils.validator_state import calculate_total_active_effective_balance from src.utils.web3converter import Web3Converter diff --git a/src/services/exit_order_iterator.py b/src/services/exit_order_iterator.py index 21e7f0c26..b43e08cbb 100644 --- a/src/services/exit_order_iterator.py +++ b/src/services/exit_order_iterator.py @@ -5,7 +5,7 @@ from more_itertools import ilen from src.constants import TOTAL_BASIS_POINTS, LIDO_DEPOSIT_AMOUNT -from src.custom_types import ReferenceBlockStamp, NodeOperatorGlobalIndex, StakingModuleId, Gwei +from src.types import ReferenceBlockStamp, NodeOperatorGlobalIndex, StakingModuleId, Gwei from src.metrics.prometheus.duration_meter import duration_meter from src.providers.consensus.types import Validator from src.services.validator_state import LidoValidatorStateService diff --git a/src/services/prediction.py b/src/services/prediction.py index 0180dfdf5..7bb7c20be 100644 --- a/src/services/prediction.py +++ b/src/services/prediction.py @@ -2,7 +2,7 @@ from web3.types import EventData, Wei -from src.custom_types import ReferenceBlockStamp +from src.types import ReferenceBlockStamp from src.modules.submodules.types import ChainConfig from src.providers.execution.exceptions import InconsistentEvents from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/services/safe_border.py b/src/services/safe_border.py index 728575c87..ca32108a3 100644 --- a/src/services/safe_border.py +++ b/src/services/safe_border.py @@ -4,9 +4,9 @@ from eth_typing import HexStr from src.constants import EPOCHS_PER_SLASHINGS_VECTOR, MIN_VALIDATOR_WITHDRAWABILITY_DELAY -from src.custom_types import EpochNumber, FrameNumber, ReferenceBlockStamp, SlotNumber from src.metrics.prometheus.duration_meter import duration_meter from src.modules.submodules.consensus import ChainConfig, FrameConfig +from src.types import EpochNumber, FrameNumber, ReferenceBlockStamp, SlotNumber from src.utils.slot import get_blockstamp from src.utils.web3converter import Web3Converter from src.web3py.extensions.lido_validators import Validator diff --git a/src/services/validator_state.py b/src/services/validator_state.py index 1ce710f0e..f1ae69281 100644 --- a/src/services/validator_state.py +++ b/src/services/validator_state.py @@ -6,7 +6,7 @@ from more_itertools import ilen from src.constants import FAR_FUTURE_EPOCH, SHARD_COMMITTEE_PERIOD -from src.custom_types import BlockStamp, ReferenceBlockStamp, EpochNumber, OperatorsValidatorCount +from src.types import BlockStamp, ReferenceBlockStamp, EpochNumber, OperatorsValidatorCount from src.metrics.prometheus.accounting import ( ACCOUNTING_STUCK_VALIDATORS, ACCOUNTING_EXITED_VALIDATORS, diff --git a/src/services/withdrawal.py b/src/services/withdrawal.py index 20acaceda..b682bf70c 100644 --- a/src/services/withdrawal.py +++ b/src/services/withdrawal.py @@ -1,6 +1,6 @@ from web3.types import Wei -from src.custom_types import ReferenceBlockStamp, FinalizationBatches +from src.types import ReferenceBlockStamp, FinalizationBatches from src.metrics.prometheus.business import CONTRACT_ON_PAUSE from src.modules.accounting.types import BatchState from src.modules.submodules.consensus import ChainConfig, FrameConfig diff --git a/src/custom_types.py b/src/types.py similarity index 100% rename from src/custom_types.py rename to src/types.py diff --git a/src/utils/blockstamp.py b/src/utils/blockstamp.py index e7f039557..e58f83556 100644 --- a/src/utils/blockstamp.py +++ b/src/utils/blockstamp.py @@ -1,6 +1,6 @@ from eth_utils import add_0x_prefix -from src.custom_types import BlockStamp, EpochNumber, ReferenceBlockStamp, SlotNumber +from src.types import BlockStamp, EpochNumber, ReferenceBlockStamp, SlotNumber from src.providers.consensus.types import BlockDetailsResponse diff --git a/src/utils/dataclass.py b/src/utils/dataclass.py index 2cb25ee6a..417bbcdea 100644 --- a/src/utils/dataclass.py +++ b/src/utils/dataclass.py @@ -3,7 +3,7 @@ from types import GenericAlias from typing import Callable, Self, Sequence -from src.custom_types import BlockNumber, CommitteeIndex, EpochNumber, Gwei, SlotNumber, Timestamp, ValidatorIndex +from src.types import BlockNumber, CommitteeIndex, EpochNumber, Gwei, SlotNumber, Timestamp, ValidatorIndex from src.utils.abi import named_tuple_to_dataclass diff --git a/src/utils/events.py b/src/utils/events.py index b02b77641..aa02658cd 100644 --- a/src/utils/events.py +++ b/src/utils/events.py @@ -6,7 +6,7 @@ from web3.types import EventData from src import variables -from src.custom_types import ReferenceBlockStamp +from src.types import ReferenceBlockStamp from src.providers.execution.exceptions import InconsistentEvents logger = logging.getLogger(__name__) diff --git a/src/utils/slot.py b/src/utils/slot.py index 665e1b990..d73530351 100644 --- a/src/utils/slot.py +++ b/src/utils/slot.py @@ -1,11 +1,11 @@ import logging from http import HTTPStatus -from src.custom_types import EpochNumber, ReferenceBlockStamp, SlotNumber from src.providers.consensus.client import ConsensusClient from src.providers.consensus.types import BlockDetailsResponse, BlockHeaderFullResponse from src.providers.execution.exceptions import InconsistentData from src.providers.http_provider import NotOkResponse +from src.types import EpochNumber, ReferenceBlockStamp, SlotNumber from src.utils.blockstamp import build_blockstamp, build_reference_blockstamp logger = logging.getLogger(__name__) diff --git a/src/utils/units.py b/src/utils/units.py index 378fecbb5..f2ba96973 100644 --- a/src/utils/units.py +++ b/src/utils/units.py @@ -3,7 +3,7 @@ from web3.types import Wei from src.constants import GWEI_TO_WEI -from src.custom_types import Gwei +from src.types import Gwei def wei_to_gwei(amount: Wei) -> Gwei: diff --git a/src/utils/validator_state.py b/src/utils/validator_state.py index 5405d9c7e..9d6d1c9b6 100644 --- a/src/utils/validator_state.py +++ b/src/utils/validator_state.py @@ -14,7 +14,7 @@ MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA, SHARD_COMMITTEE_PERIOD, ) -from src.custom_types import EpochNumber, Gwei +from src.types import EpochNumber, Gwei from src.providers.consensus.types import Validator, ValidatorState diff --git a/src/utils/web3converter.py b/src/utils/web3converter.py index 33bf66196..e8706785f 100644 --- a/src/utils/web3converter.py +++ b/src/utils/web3converter.py @@ -1,4 +1,4 @@ -from src.custom_types import SlotNumber, EpochNumber, FrameNumber +from src.types import SlotNumber, EpochNumber, FrameNumber from src.modules.submodules.types import ChainConfig, FrameConfig diff --git a/src/variables.py b/src/variables.py index 6fa591f17..5ee2f62ea 100644 --- a/src/variables.py +++ b/src/variables.py @@ -4,7 +4,7 @@ from eth_account import Account -from src.custom_types import OracleModule +from src.types import OracleModule from src.utils.env import from_file_or_env # - Providers- diff --git a/src/web3py/extensions/contracts.py b/src/web3py/extensions/contracts.py index 237efa138..8924d4081 100644 --- a/src/web3py/extensions/contracts.py +++ b/src/web3py/extensions/contracts.py @@ -7,7 +7,7 @@ from web3.types import Wei from src import variables -from src.custom_types import BlockStamp, SlotNumber, WithdrawalVaultBalance, ELVaultBalance +from src.types import BlockStamp, SlotNumber, WithdrawalVaultBalance, ELVaultBalance from src.metrics.prometheus.business import FRAME_PREV_REPORT_REF_SLOT from src.providers.execution.contracts.accounting_oracle import AccountingOracleContract from src.providers.execution.contracts.burner import BurnerContract diff --git a/src/web3py/extensions/csm.py b/src/web3py/extensions/csm.py index bc305d4ae..63afa4d6e 100644 --- a/src/web3py/extensions/csm.py +++ b/src/web3py/extensions/csm.py @@ -13,7 +13,7 @@ from web3.types import BlockIdentifier, EventData from src import variables -from src.custom_types import BlockStamp, SlotNumber +from src.types import BlockStamp, SlotNumber from src.metrics.prometheus.business import FRAME_PREV_REPORT_REF_SLOT from src.providers.execution.contracts.cs_accounting import CSAccountingContract from src.providers.execution.contracts.cs_fee_distributor import CSFeeDistributorContract diff --git a/src/web3py/extensions/lido_validators.py b/src/web3py/extensions/lido_validators.py index dae9d1b2b..6cdb6f37c 100644 --- a/src/web3py/extensions/lido_validators.py +++ b/src/web3py/extensions/lido_validators.py @@ -6,7 +6,7 @@ from eth_typing import ChecksumAddress, HexStr from web3.module import Module -from src.custom_types import BlockStamp, StakingModuleId, NodeOperatorId, NodeOperatorGlobalIndex, StakingModuleAddress +from src.types import BlockStamp, StakingModuleId, NodeOperatorId, NodeOperatorGlobalIndex, StakingModuleAddress from src.providers.consensus.types import Validator from src.providers.keys.types import LidoKey from src.utils.cache import global_lru_cache as lru_cache diff --git a/tests/conftest.py b/tests/conftest.py index 7a2f05cb7..6b21f1d54 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from web3.types import Timestamp import src.variables -from src.custom_types import BlockNumber, EpochNumber, ReferenceBlockStamp, SlotNumber +from src.types import BlockNumber, EpochNumber, ReferenceBlockStamp, SlotNumber from src.variables import CONSENSUS_CLIENT_URI, EXECUTION_CLIENT_URI, KEYS_API_URI from src.web3py.contract_tweak import tweak_w3_contracts from src.web3py.extensions import LidoContracts, LidoValidatorsProvider, TransactionUtils diff --git a/tests/factory/blockstamp.py b/tests/factory/blockstamp.py index 1484137b9..24887cc70 100644 --- a/tests/factory/blockstamp.py +++ b/tests/factory/blockstamp.py @@ -1,7 +1,7 @@ from eth_typing import BlockNumber, HexStr from web3.types import Timestamp -from src.custom_types import BlockStamp, StateRoot, SlotNumber, BlockHash, ReferenceBlockStamp, EpochNumber +from src.types import BlockStamp, StateRoot, SlotNumber, BlockHash, ReferenceBlockStamp, EpochNumber from tests.factory.web3_factory import Web3DataclassFactory diff --git a/tests/factory/no_registry.py b/tests/factory/no_registry.py index ca372c58e..ad8c16c8a 100644 --- a/tests/factory/no_registry.py +++ b/tests/factory/no_registry.py @@ -13,7 +13,7 @@ FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE, ) -from src.custom_types import Gwei +from src.types import Gwei from src.providers.consensus.types import Validator, ValidatorState from src.providers.keys.types import LidoKey from src.web3py.extensions.lido_validators import LidoValidator, NodeOperator, StakingModule diff --git a/tests/fork/conftest.py b/tests/fork/conftest.py index 6dd8da56b..f54c53160 100644 --- a/tests/fork/conftest.py +++ b/tests/fork/conftest.py @@ -14,7 +14,7 @@ from web3_multi_provider import MultiProvider from src import variables -from src.custom_types import BlockRoot, BlockStamp, SlotNumber +from src.types import BlockRoot, BlockStamp, SlotNumber from src.main import ipfs_providers, logger from src.modules.submodules.consensus import ConsensusModule from src.modules.submodules.oracle_module import BaseModule diff --git a/tests/modules/accounting/bunker/conftest.py b/tests/modules/accounting/bunker/conftest.py index 847be971e..0487e7239 100644 --- a/tests/modules/accounting/bunker/conftest.py +++ b/tests/modules/accounting/bunker/conftest.py @@ -3,7 +3,7 @@ import pytest from src.constants import FAR_FUTURE_EPOCH -from src.custom_types import BlockNumber, BlockStamp, EpochNumber, Gwei, ReferenceBlockStamp, ValidatorIndex +from src.types import BlockNumber, BlockStamp, EpochNumber, Gwei, ReferenceBlockStamp, ValidatorIndex from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState from src.providers.keys.types import LidoKey diff --git a/tests/modules/accounting/bunker/test_bunker.py b/tests/modules/accounting/bunker/test_bunker.py index 59c39ed8f..ccc732b47 100644 --- a/tests/modules/accounting/bunker/test_bunker.py +++ b/tests/modules/accounting/bunker/test_bunker.py @@ -3,10 +3,10 @@ import pytest -from src.custom_types import ReferenceBlockStamp from src.modules.accounting.types import LidoReportRebase from src.providers.consensus.types import BeaconStateView from src.services.bunker import BunkerService +from src.types import ReferenceBlockStamp from src.web3py.extensions.lido_validators import LidoValidator from tests.factory.blockstamp import ReferenceBlockStampFactory from tests.factory.configs import BunkerConfigFactory, ChainConfigFactory, FrameConfigFactory diff --git a/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py b/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py index cef34e6e0..0b6aaa7fa 100644 --- a/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py +++ b/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py @@ -3,7 +3,7 @@ import pytest from src.constants import FAR_FUTURE_EPOCH, UINT64_MAX -from src.custom_types import Gwei, ValidatorIndex, EpochNumber +from src.types import Gwei, ValidatorIndex, EpochNumber from src.providers.consensus.types import Validator, ValidatorState from src.services.bunker_cases.abnormal_cl_rebase import AbnormalClRebase from src.services.bunker_cases.types import BunkerConfig diff --git a/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py b/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py index ad777170d..6836fa68c 100644 --- a/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py +++ b/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py @@ -6,7 +6,7 @@ MAX_EFFECTIVE_BALANCE_ELECTRA, EPOCHS_PER_SLASHINGS_VECTOR, ) -from src.custom_types import EpochNumber, Gwei, ReferenceBlockStamp, SlotNumber, ValidatorIndex +from src.types import EpochNumber, Gwei, ReferenceBlockStamp, SlotNumber, ValidatorIndex from src.modules.submodules.consensus import FrameConfig from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState diff --git a/tests/modules/accounting/test_accounting_module.py b/tests/modules/accounting/test_accounting_module.py index a65af8394..fab99e3f8 100644 --- a/tests/modules/accounting/test_accounting_module.py +++ b/tests/modules/accounting/test_accounting_module.py @@ -6,7 +6,7 @@ from web3.types import Wei from src import variables -from src.custom_types import BlockStamp, ReferenceBlockStamp +from src.types import BlockStamp, ReferenceBlockStamp from src.modules.accounting import accounting as accounting_module from src.modules.accounting.accounting import Accounting from src.modules.accounting.accounting import logger as accounting_logger diff --git a/tests/modules/accounting/test_safe_border_integration.py b/tests/modules/accounting/test_safe_border_integration.py index 0aa8b0f96..04b584740 100644 --- a/tests/modules/accounting/test_safe_border_integration.py +++ b/tests/modules/accounting/test_safe_border_integration.py @@ -3,8 +3,8 @@ import pytest from src.constants import EPOCHS_PER_SLASHINGS_VECTOR, MIN_VALIDATOR_WITHDRAWABILITY_DELAY -from src.custom_types import ReferenceBlockStamp from src.services.safe_border import SafeBorder +from src.types import ReferenceBlockStamp from tests.factory.blockstamp import ReferenceBlockStampFactory from tests.factory.configs import ChainConfigFactory, FrameConfigFactory, OracleReportLimitsFactory from tests.factory.no_registry import LidoValidatorFactory, ValidatorStateFactory diff --git a/tests/modules/accounting/test_validator_state.py b/tests/modules/accounting/test_validator_state.py index 8fabe9d5e..dca8fa852 100644 --- a/tests/modules/accounting/test_validator_state.py +++ b/tests/modules/accounting/test_validator_state.py @@ -5,7 +5,7 @@ from eth_typing import HexStr from src.constants import FAR_FUTURE_EPOCH -from src.custom_types import EpochNumber, Gwei, StakingModuleId, NodeOperatorId, ValidatorIndex +from src.types import EpochNumber, Gwei, StakingModuleId, NodeOperatorId, ValidatorIndex from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState from src.providers.keys.types import LidoKey diff --git a/tests/modules/csm/test_checkpoint.py b/tests/modules/csm/test_checkpoint.py index 79072704c..94888df8d 100644 --- a/tests/modules/csm/test_checkpoint.py +++ b/tests/modules/csm/test_checkpoint.py @@ -6,7 +6,7 @@ from faker import Faker import src.modules.csm.checkpoint as checkpoint_module -from src.custom_types import EpochNumber, SlotNumber, ValidatorIndex +from src.types import EpochNumber, SlotNumber, ValidatorIndex from src.modules.csm.checkpoint import ( FrameCheckpoint, FrameCheckpointProcessor, diff --git a/tests/modules/csm/test_csm_module.py b/tests/modules/csm/test_csm_module.py index ecf7f47bd..cd246c22d 100644 --- a/tests/modules/csm/test_csm_module.py +++ b/tests/modules/csm/test_csm_module.py @@ -8,7 +8,7 @@ from hexbytes import HexBytes from src.constants import UINT64_MAX -from src.custom_types import EpochNumber, NodeOperatorId, SlotNumber, StakingModuleId, ValidatorIndex +from src.types import EpochNumber, NodeOperatorId, SlotNumber, StakingModuleId, ValidatorIndex from src.modules.csm.csm import CSOracle from src.modules.csm.state import AttestationsAccumulator, State from src.modules.csm.tree import Tree diff --git a/tests/modules/csm/test_log.py b/tests/modules/csm/test_log.py index a768c718a..5fcbbb038 100644 --- a/tests/modules/csm/test_log.py +++ b/tests/modules/csm/test_log.py @@ -2,7 +2,7 @@ import pytest -from src.custom_types import EpochNumber, NodeOperatorId, ReferenceBlockStamp +from src.types import EpochNumber, NodeOperatorId, ReferenceBlockStamp from src.modules.csm.log import FramePerfLog from src.modules.csm.state import AttestationsAccumulator from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/csm/test_state.py b/tests/modules/csm/test_state.py index b5551eb08..7e945ac0f 100644 --- a/tests/modules/csm/test_state.py +++ b/tests/modules/csm/test_state.py @@ -3,7 +3,7 @@ import pytest -from src.custom_types import EpochNumber, ValidatorIndex +from src.types import EpochNumber, ValidatorIndex from src.modules.csm.state import AttestationsAccumulator, State from src.utils.range import sequence diff --git a/tests/modules/csm/test_tree.py b/tests/modules/csm/test_tree.py index 8fbbd1ece..35b28233e 100644 --- a/tests/modules/csm/test_tree.py +++ b/tests/modules/csm/test_tree.py @@ -1,7 +1,7 @@ import pytest from src.constants import UINT64_MAX -from src.custom_types import NodeOperatorId +from src.types import NodeOperatorId from src.modules.csm.tree import StandardMerkleTree, Tree, TreeJSONEncoder diff --git a/tests/modules/ejector/test_data_encode.py b/tests/modules/ejector/test_data_encode.py index 9212d9eb2..173c5132c 100644 --- a/tests/modules/ejector/test_data_encode.py +++ b/tests/modules/ejector/test_data_encode.py @@ -4,7 +4,7 @@ import pytest -from src.custom_types import StakingModuleId, NodeOperatorId +from src.types import StakingModuleId, NodeOperatorId from src.modules.ejector.data_encode import ( MODULE_ID_LENGTH, NODE_OPERATOR_ID_LENGTH, diff --git a/tests/modules/ejector/test_ejector.py b/tests/modules/ejector/test_ejector.py index 73dcf5394..f81a01f5b 100644 --- a/tests/modules/ejector/test_ejector.py +++ b/tests/modules/ejector/test_ejector.py @@ -14,7 +14,7 @@ MIN_ACTIVATION_BALANCE, MIN_VALIDATOR_WITHDRAWABILITY_DELAY, ) -from src.custom_types import BlockStamp, Gwei, ReferenceBlockStamp +from src.types import BlockStamp, Gwei, ReferenceBlockStamp from src.modules.ejector import ejector as ejector_module from src.modules.ejector.ejector import ( Ejector, diff --git a/tests/modules/ejector/test_prediction.py b/tests/modules/ejector/test_prediction.py index 8d78d72d7..5b31ed7d3 100644 --- a/tests/modules/ejector/test_prediction.py +++ b/tests/modules/ejector/test_prediction.py @@ -5,7 +5,7 @@ from web3.types import Wei import src.services.prediction as prediction_module -from src.custom_types import BlockNumber, SlotNumber +from src.types import BlockNumber, SlotNumber from src.modules.submodules.types import ChainConfig from src.services.prediction import RewardsPredictionService from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/ejector/test_sweep.py b/tests/modules/ejector/test_sweep.py index 9048affea..36ae5780f 100644 --- a/tests/modules/ejector/test_sweep.py +++ b/tests/modules/ejector/test_sweep.py @@ -5,7 +5,6 @@ import src.modules.ejector.sweep as sweep_module from src.constants import MAX_WITHDRAWALS_PER_PAYLOAD, MIN_ACTIVATION_BALANCE -from src.custom_types import Gwei from src.modules.ejector.sweep import ( Withdrawal, get_pending_partial_withdrawals, @@ -14,6 +13,7 @@ ) from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import BeaconStateView, PendingPartialWithdrawal +from src.types import Gwei from tests.factory.consensus import BeaconStateViewFactory from tests.factory.no_registry import LidoValidatorFactory, ValidatorStateFactory diff --git a/tests/modules/ejector/test_validator_exit_order_iterator.py b/tests/modules/ejector/test_validator_exit_order_iterator.py index 8f828b453..fc27173b9 100644 --- a/tests/modules/ejector/test_validator_exit_order_iterator.py +++ b/tests/modules/ejector/test_validator_exit_order_iterator.py @@ -3,7 +3,7 @@ import pytest -from src.custom_types import Gwei +from src.types import Gwei from src.services.exit_order_iterator import ValidatorExitIterator, StakingModuleStats, NodeOperatorStats from src.web3py.extensions.lido_validators import NodeOperatorLimitMode from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/submodules/consensus/conftest.py b/tests/modules/submodules/consensus/conftest.py index 31943ceeb..62f120781 100644 --- a/tests/modules/submodules/consensus/conftest.py +++ b/tests/modules/submodules/consensus/conftest.py @@ -1,6 +1,6 @@ import pytest -from src.custom_types import BlockStamp, ReferenceBlockStamp +from src.types import BlockStamp, ReferenceBlockStamp from src.modules.submodules.consensus import ConsensusModule diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index c9209a19b..6c6e57e23 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -5,7 +5,7 @@ from web3.exceptions import ContractCustomError from src import variables -from src.custom_types import BlockStamp, ReferenceBlockStamp +from src.types import BlockStamp, ReferenceBlockStamp from src.modules.submodules import consensus as consensus_module from src.modules.submodules.consensus import ZERO_HASH, ConsensusModule, IsNotMemberException, MemberInfo from src.modules.submodules.exceptions import IncompatibleOracleVersion, ContractVersionMismatch diff --git a/tests/modules/submodules/test_oracle_module.py b/tests/modules/submodules/test_oracle_module.py index 9aab2ba0e..5794890f1 100644 --- a/tests/modules/submodules/test_oracle_module.py +++ b/tests/modules/submodules/test_oracle_module.py @@ -6,7 +6,7 @@ from web3_multi_provider.multi_http_provider import NoActiveProviderError from src import variables -from src.custom_types import BlockStamp +from src.types import BlockStamp from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay from src.providers.http_provider import NotOkResponse diff --git a/tests/providers/consensus/test_consensus_client.py b/tests/providers/consensus/test_consensus_client.py index 524f40819..f0192309f 100644 --- a/tests/providers/consensus/test_consensus_client.py +++ b/tests/providers/consensus/test_consensus_client.py @@ -5,7 +5,7 @@ import pytest -from src.custom_types import EpochNumber, SlotNumber +from src.types import EpochNumber, SlotNumber from src.providers.consensus.client import ConsensusClient from src.providers.consensus.types import Validator from src.utils.blockstamp import build_blockstamp diff --git a/tests/providers_clients/test_keys_api_client.py b/tests/providers_clients/test_keys_api_client.py index 1b6a112a3..c16bdcb02 100644 --- a/tests/providers_clients/test_keys_api_client.py +++ b/tests/providers_clients/test_keys_api_client.py @@ -11,7 +11,7 @@ import src.providers.keys.client as keys_api_client_module from src import constants from src import variables -from src.custom_types import StakingModuleAddress +from src.types import StakingModuleAddress from src.providers.keys.client import KAPIClientError, KeysAPIClient, KeysOutdatedException from src.providers.keys.types import LidoKey from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/utils/test_slot.py b/tests/utils/test_slot.py index cec69c203..bcba7954b 100644 --- a/tests/utils/test_slot.py +++ b/tests/utils/test_slot.py @@ -4,7 +4,7 @@ import pytest -from src.custom_types import SlotNumber +from src.types import SlotNumber from src.providers.http_provider import NotOkResponse from src.utils.slot import NoSlotsAvailable, get_prev_non_missed_slot, get_next_non_missed_slot from tests.conftest import get_blockstamp_by_state diff --git a/tests/utils/test_validator_state_utils.py b/tests/utils/test_validator_state_utils.py index 4791c2e55..4f35f9742 100644 --- a/tests/utils/test_validator_state_utils.py +++ b/tests/utils/test_validator_state_utils.py @@ -6,7 +6,7 @@ MAX_EFFECTIVE_BALANCE_ELECTRA, MIN_ACTIVATION_BALANCE, ) -from src.custom_types import EpochNumber, Gwei +from src.types import EpochNumber, Gwei from src.providers.consensus.types import Validator, ValidatorState from src.utils.validator_state import ( calculate_active_effective_balance_sum, diff --git a/tests/utils/test_web3_converter.py b/tests/utils/test_web3_converter.py index 5b9567e59..2835902af 100644 --- a/tests/utils/test_web3_converter.py +++ b/tests/utils/test_web3_converter.py @@ -1,6 +1,6 @@ import pytest -from src.custom_types import EpochNumber, FrameNumber, SlotNumber +from src.types import EpochNumber, FrameNumber, SlotNumber from src.modules.submodules.types import ChainConfig, FrameConfig from src.utils.web3converter import Web3Converter from tests.factory.configs import ChainConfigFactory, FrameConfigFactory From eb2fa3a2d033b9840aece2e23d743b13558d95ef Mon Sep 17 00:00:00 2001 From: hweawer Date: Fri, 25 Apr 2025 17:56:01 +0200 Subject: [PATCH 07/13] feat: Imports --- src/modules/csm/checkpoint.py | 3 +-- src/modules/submodules/consensus.py | 2 +- src/providers/consensus/client.py | 1 - tests/providers/consensus/test_consensus_client.py | 1 - 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/modules/csm/checkpoint.py b/src/modules/csm/checkpoint.py index bbb4ff9e3..35d926ada 100644 --- a/src/modules/csm/checkpoint.py +++ b/src/modules/csm/checkpoint.py @@ -7,11 +7,10 @@ from src import variables from src.constants import SLOTS_PER_HISTORICAL_ROOT -from src.types import BlockRoot, BlockStamp, CommitteeIndex, EpochNumber, SlotNumber, ValidatorIndex from src.metrics.prometheus.csm import CSM_MIN_UNPROCESSED_EPOCH, CSM_UNPROCESSED_EPOCHS_COUNT from src.modules.csm.state import State from src.providers.consensus.client import ConsensusClient -from src.providers.consensus.types import BlockAttestation, BlockAttestationEIP7549 +from src.providers.consensus.types import BlockAttestation from src.types import BlockRoot, BlockStamp, CommitteeIndex, EpochNumber, SlotNumber, ValidatorIndex from src.utils.range import sequence from src.utils.timeit import timeit diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index 0aaef5bb3..8098713e7 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -8,7 +8,6 @@ from web3.exceptions import ContractCustomError from src import variables -from src.types import BlockStamp, ReferenceBlockStamp, SlotNumber, FrameNumber from src.metrics.prometheus.basic import ORACLE_SLOT_NUMBER, ORACLE_BLOCK_NUMBER, GENESIS_TIME, ACCOUNT_BALANCE from src.metrics.prometheus.business import ( ORACLE_MEMBER_LAST_REPORT_REF_SLOT, @@ -20,6 +19,7 @@ from src.modules.submodules.types import ChainConfig, MemberInfo, ZERO_HASH, CurrentFrame, FrameConfig from src.providers.execution.contracts.base_oracle import BaseOracleContract from src.providers.execution.contracts.hash_consensus import HashConsensusContract +from src.types import BlockStamp, ReferenceBlockStamp, SlotNumber, FrameNumber from src.utils.blockstamp import build_blockstamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.slot import get_reference_blockstamp diff --git a/src/providers/consensus/client.py b/src/providers/consensus/client.py index fe766e89c..d6d5f9aa3 100644 --- a/src/providers/consensus/client.py +++ b/src/providers/consensus/client.py @@ -3,7 +3,6 @@ from json_stream.base import TransientStreamingJSONObject # type: ignore -from src.types import BlockRoot, BlockStamp, SlotNumber, EpochNumber, StateRoot from src.metrics.logging import logging from src.metrics.prometheus.basic import CL_REQUESTS_DURATION from src.providers.consensus.types import ( diff --git a/tests/providers/consensus/test_consensus_client.py b/tests/providers/consensus/test_consensus_client.py index a5530bce4..6e375b225 100644 --- a/tests/providers/consensus/test_consensus_client.py +++ b/tests/providers/consensus/test_consensus_client.py @@ -4,7 +4,6 @@ import pytest -from src.types import EpochNumber, SlotNumber from src.providers.consensus.client import ConsensusClient from src.providers.consensus.types import Validator from src.types import SlotNumber From b36b4863ea874865d2f26dfcd58354d8d18fa085 Mon Sep 17 00:00:00 2001 From: hweawer Date: Mon, 28 Apr 2025 12:12:42 +0200 Subject: [PATCH 08/13] refactor: Fix imports --- src/modules/accounting/accounting.py | 24 ++++--------------- .../accounting/third_phase/extra_data.py | 6 ++--- src/modules/csm/csm.py | 16 ++++++------- src/modules/csm/log.py | 2 +- src/modules/csm/types.py | 4 ++-- src/modules/ejector/sweep.py | 16 ++++--------- src/modules/submodules/oracle_module.py | 8 +++---- .../execution/contracts/base_oracle.py | 2 +- .../execution/contracts/hash_consensus.py | 2 +- src/providers/execution/contracts/lido.py | 6 ++--- src/providers/keys/client.py | 8 +++---- src/services/bunker.py | 11 +++------ .../bunker_cases/abnormal_cl_rebase.py | 4 ++-- .../bunker_cases/midterm_slashing_penalty.py | 2 +- src/services/exit_order_iterator.py | 8 +++---- src/services/prediction.py | 2 +- src/services/validator_state.py | 14 ++++------- src/services/withdrawal.py | 2 +- src/utils/blockstamp.py | 2 +- src/utils/events.py | 2 +- src/utils/validator_state.py | 2 +- src/web3py/extensions/contracts.py | 2 +- src/web3py/extensions/csm.py | 4 ++-- src/web3py/extensions/lido_validators.py | 4 ++-- tests/factory/no_registry.py | 2 +- tests/fork/conftest.py | 2 +- tests/modules/accounting/bunker/conftest.py | 2 +- .../bunker/test_bunker_abnormal_cl_rebase.py | 6 ++--- .../bunker/test_bunker_midterm_penalty.py | 4 ++-- .../accounting/test_accounting_module.py | 11 ++++----- .../accounting/test_validator_state.py | 9 ++----- tests/modules/csm/test_checkpoint.py | 2 -- tests/modules/csm/test_csm_module.py | 6 ++--- tests/modules/csm/test_log.py | 2 +- tests/modules/csm/test_state.py | 2 +- tests/modules/csm/test_tree.py | 2 +- tests/modules/ejector/test_data_encode.py | 6 ++--- tests/modules/ejector/test_ejector.py | 9 +++---- tests/modules/ejector/test_prediction.py | 2 +- .../test_validator_exit_order_iterator.py | 8 ++----- .../modules/submodules/consensus/conftest.py | 2 +- .../submodules/consensus/test_consensus.py | 10 ++++---- .../modules/submodules/test_oracle_module.py | 6 ++--- .../providers_clients/test_keys_api_client.py | 5 ++-- tests/utils/test_slot.py | 4 ++-- tests/utils/test_validator_state_utils.py | 2 +- tests/utils/test_web3_converter.py | 2 +- 47 files changed, 105 insertions(+), 154 deletions(-) diff --git a/src/modules/accounting/accounting.py b/src/modules/accounting/accounting.py index bc4b29564..76db8040a 100644 --- a/src/modules/accounting/accounting.py +++ b/src/modules/accounting/accounting.py @@ -9,28 +9,13 @@ from src import variables from src.constants import SHARE_RATE_PRECISION_E27 -from src.types import BlockStamp, Gwei, ReferenceBlockStamp, StakingModuleId, NodeOperatorGlobalIndex, FinalizationBatches -from src.metrics.prometheus.accounting import ( - ACCOUNTING_IS_BUNKER, - ACCOUNTING_CL_BALANCE_GWEI, - ACCOUNTING_EL_REWARDS_VAULT_BALANCE_WEI, - ACCOUNTING_WITHDRAWAL_VAULT_BALANCE_WEI -) +from src.metrics.prometheus.accounting import (ACCOUNTING_CL_BALANCE_GWEI, ACCOUNTING_EL_REWARDS_VAULT_BALANCE_WEI, ACCOUNTING_IS_BUNKER, + ACCOUNTING_WITHDRAWAL_VAULT_BALANCE_WEI) from src.metrics.prometheus.duration_meter import duration_meter from src.modules.accounting.third_phase.extra_data import ExtraDataService from src.modules.accounting.third_phase.types import ExtraData, FormatList -from src.modules.accounting.types import ( - ReportData, - LidoReportRebase, - GenericExtraData, - WqReport, - RebaseReport, - BunkerMode, - FinalizationShareRate, - ValidatorsCount, - ValidatorsBalance, - AccountingProcessingState, -) +from src.modules.accounting.types import (AccountingProcessingState, BunkerMode, FinalizationShareRate, GenericExtraData, LidoReportRebase, + RebaseReport, ReportData, ValidatorsBalance, ValidatorsCount, WqReport) from src.modules.submodules.consensus import ConsensusModule, InitialEpochIsYetToArriveRevert from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay from src.modules.submodules.types import ZERO_HASH @@ -38,6 +23,7 @@ from src.services.bunker import BunkerService from src.services.validator_state import LidoValidatorStateService from src.services.withdrawal import Withdrawal +from src.types import BlockStamp, FinalizationBatches, Gwei, NodeOperatorGlobalIndex, ReferenceBlockStamp, StakingModuleId from src.utils.cache import global_lru_cache as lru_cache from src.utils.units import gwei_to_wei from src.variables import ALLOW_REPORTING_IN_BUNKER_MODE diff --git a/src/modules/accounting/third_phase/extra_data.py b/src/modules/accounting/third_phase/extra_data.py index 0543d773d..fa74b2f2b 100644 --- a/src/modules/accounting/third_phase/extra_data.py +++ b/src/modules/accounting/third_phase/extra_data.py @@ -1,10 +1,10 @@ from dataclasses import dataclass -from itertools import groupby, batched +from itertools import batched, groupby from typing import Sequence -from src.types import NodeOperatorGlobalIndex -from src.modules.accounting.third_phase.types import ExtraData, ItemType, ExtraDataLengths, FormatList +from src.modules.accounting.third_phase.types import ExtraData, ExtraDataLengths, FormatList, ItemType from src.modules.submodules.types import ZERO_HASH +from src.types import NodeOperatorGlobalIndex from src.web3py.types import Web3 diff --git a/src/modules/csm/csm.py b/src/modules/csm/csm.py index 6a9ded27b..30d26b3b8 100644 --- a/src/modules/csm/csm.py +++ b/src/modules/csm/csm.py @@ -5,14 +5,6 @@ from hexbytes import HexBytes from src.constants import TOTAL_BASIS_POINTS, UINT64_MAX -from src.types import ( - BlockStamp, - EpochNumber, - ReferenceBlockStamp, - SlotNumber, - StakingModuleAddress, - StakingModuleId, -) from src.metrics.prometheus.business import CONTRACT_ON_PAUSE from src.metrics.prometheus.csm import ( CSM_CURRENT_FRAME_RANGE_L_EPOCH, @@ -30,6 +22,14 @@ from src.providers.execution.contracts.cs_fee_oracle import CSFeeOracleContract from src.providers.execution.exceptions import InconsistentData from src.providers.ipfs import CID +from src.types import ( + BlockStamp, + EpochNumber, + ReferenceBlockStamp, + SlotNumber, + StakingModuleAddress, + StakingModuleId, +) from src.utils.blockstamp import build_blockstamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.slot import get_next_non_missed_slot diff --git a/src/modules/csm/log.py b/src/modules/csm/log.py index 0f90d3096..f89f4ef58 100644 --- a/src/modules/csm/log.py +++ b/src/modules/csm/log.py @@ -2,9 +2,9 @@ from collections import defaultdict from dataclasses import asdict, dataclass, field -from src.types import EpochNumber, NodeOperatorId, ReferenceBlockStamp, ValidatorIndex from src.modules.csm.state import AttestationsAccumulator from src.modules.csm.types import Shares +from src.types import EpochNumber, NodeOperatorId, ReferenceBlockStamp, ValidatorIndex class LogJSONEncoder(json.JSONEncoder): ... diff --git a/src/modules/csm/types.py b/src/modules/csm/types.py index 121d29dd5..3478a6399 100644 --- a/src/modules/csm/types.py +++ b/src/modules/csm/types.py @@ -1,11 +1,11 @@ import logging from dataclasses import dataclass -from typing import TypeAlias, Literal +from typing import Literal, TypeAlias from hexbytes import HexBytes -from src.types import NodeOperatorId, SlotNumber from src.providers.ipfs import CID +from src.types import NodeOperatorId, SlotNumber logger = logging.getLogger(__name__) diff --git a/src/modules/ejector/sweep.py b/src/modules/ejector/sweep.py index e46919c75..f26fe16f9 100644 --- a/src/modules/ejector/sweep.py +++ b/src/modules/ejector/sweep.py @@ -3,20 +3,12 @@ from dataclasses import dataclass from typing import List -from src.constants import ( - FAR_FUTURE_EPOCH, - MIN_ACTIVATION_BALANCE, - MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, - MAX_WITHDRAWALS_PER_PAYLOAD, -) -from src.types import Gwei +from src.constants import (FAR_FUTURE_EPOCH, MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, MAX_WITHDRAWALS_PER_PAYLOAD, + MIN_ACTIVATION_BALANCE) from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import BeaconStateView -from src.utils.validator_state import ( - is_fully_withdrawable_validator, - is_partially_withdrawable_validator, - get_max_effective_balance, -) +from src.types import Gwei +from src.utils.validator_state import (get_max_effective_balance, is_fully_withdrawable_validator, is_partially_withdrawable_validator) from src.utils.web3converter import epoch_from_slot diff --git a/src/modules/submodules/oracle_module.py b/src/modules/submodules/oracle_module.py index 685ece8b3..da81a8c54 100644 --- a/src/modules/submodules/oracle_module.py +++ b/src/modules/submodules/oracle_module.py @@ -1,7 +1,7 @@ import logging import time import traceback -from abc import abstractmethod, ABC +from abc import ABC, abstractmethod from dataclasses import asdict from enum import Enum @@ -11,16 +11,16 @@ from web3_multi_provider import NoActiveProviderError from src import variables -from src.types import SlotNumber, BlockStamp, BlockRoot from src.metrics.healthcheck_server import pulse from src.metrics.prometheus.basic import ORACLE_BLOCK_NUMBER, ORACLE_SLOT_NUMBER -from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch +from src.modules.submodules.exceptions import ContractVersionMismatch, IncompatibleOracleVersion, IsNotMemberException from src.providers.http_provider import NotOkResponse from src.providers.ipfs import IPFSError from src.providers.keys.client import KeysOutdatedException +from src.types import BlockRoot, BlockStamp, SlotNumber from src.utils.blockstamp import build_blockstamp from src.utils.cache import clear_global_cache -from src.utils.slot import NoSlotsAvailable, SlotNotFinalized, InconsistentData +from src.utils.slot import InconsistentData, NoSlotsAvailable, SlotNotFinalized from src.web3py.extensions.lido_validators import CountOfKeysDiffersException from src.web3py.types import Web3 diff --git a/src/providers/execution/contracts/base_oracle.py b/src/providers/execution/contracts/base_oracle.py index bcaa36b08..e580fc275 100644 --- a/src/providers/execution/contracts/base_oracle.py +++ b/src/providers/execution/contracts/base_oracle.py @@ -4,8 +4,8 @@ from web3.contract.contract import ContractFunction from web3.types import BlockIdentifier -from src.types import SlotNumber from src.providers.execution.base_interface import ContractInterface +from src.types import SlotNumber from src.utils.cache import global_lru_cache as lru_cache logger = logging.getLogger(__name__) diff --git a/src/providers/execution/contracts/hash_consensus.py b/src/providers/execution/contracts/hash_consensus.py index 049be2635..a697c03bd 100644 --- a/src/providers/execution/contracts/hash_consensus.py +++ b/src/providers/execution/contracts/hash_consensus.py @@ -4,9 +4,9 @@ from web3.contract.contract import ContractFunction from web3.types import BlockIdentifier -from src.types import SlotNumber from src.modules.submodules.types import ChainConfig, CurrentFrame, FrameConfig from src.providers.execution.base_interface import ContractInterface +from src.types import SlotNumber from src.utils.abi import named_tuple_to_dataclass from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/providers/execution/contracts/lido.py b/src/providers/execution/contracts/lido.py index 8e3f9a0ce..5b0513022 100644 --- a/src/providers/execution/contracts/lido.py +++ b/src/providers/execution/contracts/lido.py @@ -3,11 +3,11 @@ from eth_typing import ChecksumAddress, HexStr from web3 import Web3 from web3.exceptions import Web3RPCError -from web3.types import Wei, BlockIdentifier, StateOverride, StateOverrideParams +from web3.types import BlockIdentifier, StateOverride, StateOverrideParams, Wei -from src.types import SlotNumber -from src.modules.accounting.types import LidoReportRebase, BeaconStat +from src.modules.accounting.types import BeaconStat, LidoReportRebase from src.providers.execution.base_interface import ContractInterface +from src.types import SlotNumber from src.utils.abi import named_tuple_to_dataclass from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/providers/keys/client.py b/src/providers/keys/client.py index 95b6b2205..f527930cf 100644 --- a/src/providers/keys/client.py +++ b/src/providers/keys/client.py @@ -1,10 +1,10 @@ from time import sleep -from typing import cast, TypedDict, List +from typing import cast, List, TypedDict -from src.types import BlockStamp, StakingModuleAddress -from src.metrics.prometheus.basic import KEYS_API_REQUESTS_DURATION, KEYS_API_LATEST_BLOCKNUMBER +from src.metrics.prometheus.basic import KEYS_API_LATEST_BLOCKNUMBER, KEYS_API_REQUESTS_DURATION from src.providers.http_provider import HTTPProvider, NotOkResponse -from src.providers.keys.types import LidoKey, KeysApiStatus +from src.providers.keys.types import KeysApiStatus, LidoKey +from src.types import BlockStamp, StakingModuleAddress from src.utils.cache import global_lru_cache as lru_cache diff --git a/src/services/bunker.py b/src/services/bunker.py index 0b6141e3d..4895e5491 100644 --- a/src/services/bunker.py +++ b/src/services/bunker.py @@ -3,20 +3,15 @@ from web3.types import Wei from src.constants import TOTAL_BASIS_POINTS -from src.types import BlockStamp, ReferenceBlockStamp, Gwei from src.metrics.prometheus.duration_meter import duration_meter -from src.metrics.prometheus.validators import ( - ALL_VALIDATORS, - LIDO_VALIDATORS, - ALL_SLASHED_VALIDATORS, - LIDO_SLASHED_VALIDATORS, -) +from src.metrics.prometheus.validators import (ALL_SLASHED_VALIDATORS, ALL_VALIDATORS, LIDO_SLASHED_VALIDATORS, LIDO_VALIDATORS) from src.modules.accounting.types import LidoReportRebase -from src.modules.submodules.consensus import FrameConfig, ChainConfig +from src.modules.submodules.consensus import ChainConfig, FrameConfig from src.services.bunker_cases.abnormal_cl_rebase import AbnormalClRebase from src.services.bunker_cases.midterm_slashing_penalty import MidtermSlashingPenalty from src.services.bunker_cases.types import BunkerConfig from src.services.safe_border import filter_slashed_validators +from src.types import BlockStamp, Gwei, ReferenceBlockStamp from src.utils.units import wei_to_gwei from src.utils.web3converter import Web3Converter from src.web3py.types import Web3 diff --git a/src/services/bunker_cases/abnormal_cl_rebase.py b/src/services/bunker_cases/abnormal_cl_rebase.py index 3861ebfbe..5da205c2c 100644 --- a/src/services/bunker_cases/abnormal_cl_rebase.py +++ b/src/services/bunker_cases/abnormal_cl_rebase.py @@ -1,16 +1,16 @@ import logging import math -from typing import Sequence, cast +from typing import cast, Sequence from web3.contract.contract import ContractEvent from web3.types import EventData from src.constants import EFFECTIVE_BALANCE_INCREMENT, LIDO_DEPOSIT_AMOUNT -from src.types import ReferenceBlockStamp, Gwei, BlockNumber, SlotNumber, BlockStamp, EpochNumber from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator from src.providers.keys.types import LidoKey from src.services.bunker_cases.types import BunkerConfig +from src.types import BlockNumber, BlockStamp, EpochNumber, Gwei, ReferenceBlockStamp, SlotNumber from src.utils.events import get_events_in_range from src.utils.slot import get_blockstamp, get_reference_blockstamp from src.utils.units import wei_to_gwei diff --git a/src/services/bunker_cases/midterm_slashing_penalty.py b/src/services/bunker_cases/midterm_slashing_penalty.py index 4aba79952..968c8e262 100644 --- a/src/services/bunker_cases/midterm_slashing_penalty.py +++ b/src/services/bunker_cases/midterm_slashing_penalty.py @@ -7,8 +7,8 @@ MIN_VALIDATOR_WITHDRAWABILITY_DELAY, PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX, ) -from src.types import EpochNumber, FrameNumber, Gwei, ReferenceBlockStamp, SlotNumber from src.providers.consensus.types import Validator +from src.types import EpochNumber, FrameNumber, Gwei, ReferenceBlockStamp, SlotNumber from src.utils.validator_state import calculate_total_active_effective_balance from src.utils.web3converter import Web3Converter from src.web3py.extensions.lido_validators import LidoValidator diff --git a/src/services/exit_order_iterator.py b/src/services/exit_order_iterator.py index 499759e02..32495bf81 100644 --- a/src/services/exit_order_iterator.py +++ b/src/services/exit_order_iterator.py @@ -1,13 +1,13 @@ import logging from dataclasses import dataclass -from src.constants import TOTAL_BASIS_POINTS, LIDO_DEPOSIT_AMOUNT -from src.types import ReferenceBlockStamp, NodeOperatorGlobalIndex, StakingModuleId, Gwei +from src.constants import LIDO_DEPOSIT_AMOUNT, TOTAL_BASIS_POINTS from src.metrics.prometheus.duration_meter import duration_meter from src.providers.consensus.types import Validator from src.services.validator_state import LidoValidatorStateService -from src.utils.validator_state import is_on_exit, get_validator_age -from src.web3py.extensions.lido_validators import LidoValidator, StakingModule, NodeOperator, NodeOperatorLimitMode +from src.types import Gwei, NodeOperatorGlobalIndex, ReferenceBlockStamp, StakingModuleId +from src.utils.validator_state import get_validator_age, is_on_exit +from src.web3py.extensions.lido_validators import LidoValidator, NodeOperator, NodeOperatorLimitMode, StakingModule from src.web3py.types import Web3 logger = logging.getLogger(__name__) diff --git a/src/services/prediction.py b/src/services/prediction.py index 7bb7c20be..9ac3576f9 100644 --- a/src/services/prediction.py +++ b/src/services/prediction.py @@ -2,9 +2,9 @@ from web3.types import EventData, Wei -from src.types import ReferenceBlockStamp from src.modules.submodules.types import ChainConfig from src.providers.execution.exceptions import InconsistentEvents +from src.types import ReferenceBlockStamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.events import get_events_in_past from src.web3py.types import Web3 diff --git a/src/services/validator_state.py b/src/services/validator_state.py index 39b9a3bde..ca7c33291 100644 --- a/src/services/validator_state.py +++ b/src/services/validator_state.py @@ -5,20 +5,14 @@ from eth_typing import HexStr from src.constants import FAR_FUTURE_EPOCH, SHARD_COMMITTEE_PERIOD -from src.types import BlockStamp, ReferenceBlockStamp, EpochNumber, OperatorsValidatorCount -from src.metrics.prometheus.accounting import ( - ACCOUNTING_STUCK_VALIDATORS, - ACCOUNTING_EXITED_VALIDATORS, -) +from src.metrics.prometheus.accounting import (ACCOUNTING_EXITED_VALIDATORS, ACCOUNTING_STUCK_VALIDATORS) from src.modules.submodules.types import ChainConfig +from src.types import BlockStamp, EpochNumber, OperatorsValidatorCount, ReferenceBlockStamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.events import get_events_in_past from src.utils.types import bytes_to_hex_str -from src.utils.validator_state import is_exited_validator, is_validator_eligible_to_exit, is_on_exit -from src.web3py.extensions.lido_validators import ( - NodeOperatorGlobalIndex, - LidoValidator, -) +from src.utils.validator_state import is_exited_validator, is_on_exit, is_validator_eligible_to_exit +from src.web3py.extensions.lido_validators import (LidoValidator, NodeOperatorGlobalIndex) from src.web3py.types import Web3 logger = logging.getLogger(__name__) diff --git a/src/services/withdrawal.py b/src/services/withdrawal.py index b682bf70c..f458f2533 100644 --- a/src/services/withdrawal.py +++ b/src/services/withdrawal.py @@ -1,10 +1,10 @@ from web3.types import Wei -from src.types import ReferenceBlockStamp, FinalizationBatches from src.metrics.prometheus.business import CONTRACT_ON_PAUSE from src.modules.accounting.types import BatchState from src.modules.submodules.consensus import ChainConfig, FrameConfig from src.services.safe_border import SafeBorder +from src.types import FinalizationBatches, ReferenceBlockStamp from src.variables import FINALIZATION_BATCH_MAX_REQUEST_COUNT from src.web3py.types import Web3 diff --git a/src/utils/blockstamp.py b/src/utils/blockstamp.py index e58f83556..5f268b8a4 100644 --- a/src/utils/blockstamp.py +++ b/src/utils/blockstamp.py @@ -1,7 +1,7 @@ from eth_utils import add_0x_prefix -from src.types import BlockStamp, EpochNumber, ReferenceBlockStamp, SlotNumber from src.providers.consensus.types import BlockDetailsResponse +from src.types import BlockStamp, EpochNumber, ReferenceBlockStamp, SlotNumber def build_reference_blockstamp( diff --git a/src/utils/events.py b/src/utils/events.py index aa02658cd..edb006c4e 100644 --- a/src/utils/events.py +++ b/src/utils/events.py @@ -6,8 +6,8 @@ from web3.types import EventData from src import variables -from src.types import ReferenceBlockStamp from src.providers.execution.exceptions import InconsistentEvents +from src.types import ReferenceBlockStamp logger = logging.getLogger(__name__) diff --git a/src/utils/validator_state.py b/src/utils/validator_state.py index 9d6d1c9b6..531735d68 100644 --- a/src/utils/validator_state.py +++ b/src/utils/validator_state.py @@ -14,8 +14,8 @@ MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA, SHARD_COMMITTEE_PERIOD, ) -from src.types import EpochNumber, Gwei from src.providers.consensus.types import Validator, ValidatorState +from src.types import EpochNumber, Gwei def is_active_validator(validator: Validator, epoch: EpochNumber) -> bool: diff --git a/src/web3py/extensions/contracts.py b/src/web3py/extensions/contracts.py index 8924d4081..e861ddb96 100644 --- a/src/web3py/extensions/contracts.py +++ b/src/web3py/extensions/contracts.py @@ -7,7 +7,6 @@ from web3.types import Wei from src import variables -from src.types import BlockStamp, SlotNumber, WithdrawalVaultBalance, ELVaultBalance from src.metrics.prometheus.business import FRAME_PREV_REPORT_REF_SLOT from src.providers.execution.contracts.accounting_oracle import AccountingOracleContract from src.providers.execution.contracts.burner import BurnerContract @@ -18,6 +17,7 @@ from src.providers.execution.contracts.oracle_report_sanity_checker import OracleReportSanityCheckerContract from src.providers.execution.contracts.staking_router import StakingRouterContract from src.providers.execution.contracts.withdrawal_queue_nft import WithdrawalQueueNftContract +from src.types import BlockStamp, ELVaultBalance, SlotNumber, WithdrawalVaultBalance from src.utils.cache import global_lru_cache as lru_cache logger = logging.getLogger(__name__) diff --git a/src/web3py/extensions/csm.py b/src/web3py/extensions/csm.py index 63afa4d6e..cb8e15cfa 100644 --- a/src/web3py/extensions/csm.py +++ b/src/web3py/extensions/csm.py @@ -2,7 +2,7 @@ from functools import partial from itertools import groupby from time import sleep -from typing import Callable, Iterator, cast +from typing import Callable, cast, Iterator from eth_typing import BlockNumber from hexbytes import HexBytes @@ -13,13 +13,13 @@ from web3.types import BlockIdentifier, EventData from src import variables -from src.types import BlockStamp, SlotNumber from src.metrics.prometheus.business import FRAME_PREV_REPORT_REF_SLOT from src.providers.execution.contracts.cs_accounting import CSAccountingContract from src.providers.execution.contracts.cs_fee_distributor import CSFeeDistributorContract from src.providers.execution.contracts.cs_fee_oracle import CSFeeOracleContract from src.providers.execution.contracts.cs_module import CSModuleContract from src.providers.ipfs import CID, CIDv0, CIDv1, is_cid_v0 +from src.types import BlockStamp, SlotNumber from src.utils.events import get_events_in_range from src.utils.lazy_object_proxy import LazyObjectProxy from src.web3py.extensions.lido_validators import NodeOperatorId diff --git a/src/web3py/extensions/lido_validators.py b/src/web3py/extensions/lido_validators.py index 6cdb6f37c..2b643be28 100644 --- a/src/web3py/extensions/lido_validators.py +++ b/src/web3py/extensions/lido_validators.py @@ -6,11 +6,11 @@ from eth_typing import ChecksumAddress, HexStr from web3.module import Module -from src.types import BlockStamp, StakingModuleId, NodeOperatorId, NodeOperatorGlobalIndex, StakingModuleAddress from src.providers.consensus.types import Validator from src.providers.keys.types import LidoKey +from src.types import BlockStamp, NodeOperatorGlobalIndex, NodeOperatorId, StakingModuleAddress, StakingModuleId from src.utils.cache import global_lru_cache as lru_cache -from src.utils.dataclass import Nested, FromResponse +from src.utils.dataclass import FromResponse, Nested logger = logging.getLogger(__name__) diff --git a/tests/factory/no_registry.py b/tests/factory/no_registry.py index ad8c16c8a..df71d9c62 100644 --- a/tests/factory/no_registry.py +++ b/tests/factory/no_registry.py @@ -13,9 +13,9 @@ FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE, ) -from src.types import Gwei from src.providers.consensus.types import Validator, ValidatorState from src.providers.keys.types import LidoKey +from src.types import Gwei from src.web3py.extensions.lido_validators import LidoValidator, NodeOperator, StakingModule from tests.factory.web3_factory import Web3DataclassFactory diff --git a/tests/fork/conftest.py b/tests/fork/conftest.py index f54c53160..8a6867e84 100644 --- a/tests/fork/conftest.py +++ b/tests/fork/conftest.py @@ -14,7 +14,6 @@ from web3_multi_provider import MultiProvider from src import variables -from src.types import BlockRoot, BlockStamp, SlotNumber from src.main import ipfs_providers, logger from src.modules.submodules.consensus import ConsensusModule from src.modules.submodules.oracle_module import BaseModule @@ -24,6 +23,7 @@ from src.providers.execution.contracts.base_oracle import BaseOracleContract from src.providers.execution.contracts.hash_consensus import HashConsensusContract from src.providers.ipfs import CID, MultiIPFSProvider +from src.types import BlockRoot, BlockStamp, SlotNumber from src.utils.blockstamp import build_blockstamp from src.utils.cache import clear_global_cache from src.utils.slot import get_next_non_missed_slot diff --git a/tests/modules/accounting/bunker/conftest.py b/tests/modules/accounting/bunker/conftest.py index 0487e7239..640b56f9a 100644 --- a/tests/modules/accounting/bunker/conftest.py +++ b/tests/modules/accounting/bunker/conftest.py @@ -3,13 +3,13 @@ import pytest from src.constants import FAR_FUTURE_EPOCH -from src.types import BlockNumber, BlockStamp, EpochNumber, Gwei, ReferenceBlockStamp, ValidatorIndex from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState from src.providers.keys.types import LidoKey from src.services.bunker import BunkerService from src.services.bunker_cases.abnormal_cl_rebase import AbnormalClRebase from src.services.bunker_cases.types import BunkerConfig +from src.types import BlockNumber, BlockStamp, EpochNumber, Gwei, ReferenceBlockStamp, ValidatorIndex def simple_ref_blockstamp(block_number: int) -> ReferenceBlockStamp: diff --git a/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py b/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py index bed21508c..9d226b680 100644 --- a/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py +++ b/tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py @@ -3,16 +3,16 @@ import pytest from src.constants import FAR_FUTURE_EPOCH, UINT64_MAX -from src.types import Gwei, ValidatorIndex, EpochNumber from src.providers.consensus.types import Validator, ValidatorState from src.services.bunker_cases.abnormal_cl_rebase import AbnormalClRebase from src.services.bunker_cases.types import BunkerConfig +from src.types import EpochNumber, Gwei, ValidatorIndex from src.web3py.extensions import LidoValidatorsProvider from src.web3py.types import Web3 from tests.factory.blockstamp import ReferenceBlockStampFactory -from tests.factory.configs import ChainConfigFactory, BunkerConfigFactory +from tests.factory.configs import BunkerConfigFactory, ChainConfigFactory from tests.factory.no_registry import LidoValidatorFactory -from tests.modules.accounting.bunker.conftest import simple_ref_blockstamp, simple_key, simple_blockstamp +from tests.modules.accounting.bunker.conftest import simple_blockstamp, simple_key, simple_ref_blockstamp def simple_validators( diff --git a/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py b/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py index 6836fa68c..68d00203f 100644 --- a/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py +++ b/tests/modules/accounting/bunker/test_bunker_midterm_penalty.py @@ -1,16 +1,16 @@ import pytest from src.constants import ( + EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE, MAX_EFFECTIVE_BALANCE_ELECTRA, - EPOCHS_PER_SLASHINGS_VECTOR, ) -from src.types import EpochNumber, Gwei, ReferenceBlockStamp, SlotNumber, ValidatorIndex from src.modules.submodules.consensus import FrameConfig from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState from src.services.bunker_cases.midterm_slashing_penalty import MidtermSlashingPenalty +from src.types import EpochNumber, Gwei, ReferenceBlockStamp, SlotNumber, ValidatorIndex from src.utils.web3converter import Web3Converter diff --git a/tests/modules/accounting/test_accounting_module.py b/tests/modules/accounting/test_accounting_module.py index 30f50eff8..de241fc19 100644 --- a/tests/modules/accounting/test_accounting_module.py +++ b/tests/modules/accounting/test_accounting_module.py @@ -1,4 +1,4 @@ -from typing import Iterable, cast +from typing import cast, Iterable from unittest.mock import Mock, patch import pytest @@ -6,15 +6,14 @@ from web3.types import Wei from src import variables -from src.types import BlockStamp, ReferenceBlockStamp from src.modules.accounting import accounting as accounting_module -from src.modules.accounting.accounting import Accounting -from src.modules.accounting.accounting import logger as accounting_logger +from src.modules.accounting.accounting import Accounting, logger as accounting_logger from src.modules.accounting.third_phase.types import FormatList -from src.modules.accounting.types import LidoReportRebase, AccountingProcessingState +from src.modules.accounting.types import AccountingProcessingState, LidoReportRebase from src.modules.submodules.oracle_module import ModuleExecuteDelay -from src.modules.submodules.types import ChainConfig, FrameConfig, CurrentFrame, ZERO_HASH +from src.modules.submodules.types import ChainConfig, CurrentFrame, FrameConfig, ZERO_HASH from src.services.withdrawal import Withdrawal +from src.types import BlockStamp, ReferenceBlockStamp from src.web3py.extensions.lido_validators import NodeOperatorId, StakingModule from tests.factory.base_oracle import AccountingProcessingStateFactory from tests.factory.blockstamp import BlockStampFactory, ReferenceBlockStampFactory diff --git a/tests/modules/accounting/test_validator_state.py b/tests/modules/accounting/test_validator_state.py index dca8fa852..1a23c2360 100644 --- a/tests/modules/accounting/test_validator_state.py +++ b/tests/modules/accounting/test_validator_state.py @@ -5,17 +5,12 @@ from eth_typing import HexStr from src.constants import FAR_FUTURE_EPOCH -from src.types import EpochNumber, Gwei, StakingModuleId, NodeOperatorId, ValidatorIndex from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import Validator, ValidatorState from src.providers.keys.types import LidoKey from src.services.validator_state import LidoValidatorStateService -from src.web3py.extensions.lido_validators import ( - NodeOperator, - StakingModule, - LidoValidatorsProvider, - LidoValidator, -) +from src.types import EpochNumber, Gwei, NodeOperatorId, StakingModuleId, ValidatorIndex +from src.web3py.extensions.lido_validators import LidoValidator, LidoValidatorsProvider, NodeOperator, StakingModule from tests.factory.blockstamp import ReferenceBlockStampFactory TESTING_REF_EPOCH = 100 diff --git a/tests/modules/csm/test_checkpoint.py b/tests/modules/csm/test_checkpoint.py index 6eb996d29..ae20f82dc 100644 --- a/tests/modules/csm/test_checkpoint.py +++ b/tests/modules/csm/test_checkpoint.py @@ -5,8 +5,6 @@ import pytest from faker import Faker -import src.modules.csm.checkpoint as checkpoint_module -from src.types import EpochNumber, SlotNumber, ValidatorIndex from src.modules.csm.checkpoint import ( FrameCheckpoint, FrameCheckpointProcessor, diff --git a/tests/modules/csm/test_csm_module.py b/tests/modules/csm/test_csm_module.py index 3b554405d..ed1cdc457 100644 --- a/tests/modules/csm/test_csm_module.py +++ b/tests/modules/csm/test_csm_module.py @@ -1,20 +1,20 @@ import logging from collections import defaultdict from dataclasses import dataclass -from typing import NoReturn, Iterable, Literal, Type +from typing import Iterable, Literal, NoReturn, Type from unittest.mock import Mock, patch, PropertyMock import pytest from hexbytes import HexBytes from src.constants import UINT64_MAX -from src.types import EpochNumber, NodeOperatorId, SlotNumber, StakingModuleId, ValidatorIndex from src.modules.csm.csm import CSOracle from src.modules.csm.state import AttestationsAccumulator, State from src.modules.csm.tree import Tree from src.modules.submodules.oracle_module import ModuleExecuteDelay from src.modules.submodules.types import CurrentFrame, ZERO_HASH -from src.providers.ipfs import CIDv0, CID +from src.providers.ipfs import CID, CIDv0 +from src.types import EpochNumber, NodeOperatorId, SlotNumber, StakingModuleId, ValidatorIndex from src.web3py.extensions.csm import CSM from tests.factory.blockstamp import BlockStampFactory, ReferenceBlockStampFactory from tests.factory.configs import ChainConfigFactory, FrameConfigFactory diff --git a/tests/modules/csm/test_log.py b/tests/modules/csm/test_log.py index 5fcbbb038..22b69bf85 100644 --- a/tests/modules/csm/test_log.py +++ b/tests/modules/csm/test_log.py @@ -2,9 +2,9 @@ import pytest -from src.types import EpochNumber, NodeOperatorId, ReferenceBlockStamp from src.modules.csm.log import FramePerfLog from src.modules.csm.state import AttestationsAccumulator +from src.types import EpochNumber, NodeOperatorId, ReferenceBlockStamp from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/csm/test_state.py b/tests/modules/csm/test_state.py index 6e62eaca0..b3b0a733d 100644 --- a/tests/modules/csm/test_state.py +++ b/tests/modules/csm/test_state.py @@ -3,8 +3,8 @@ import pytest -from src.types import EpochNumber, ValidatorIndex from src.modules.csm.state import AttestationsAccumulator, State +from src.types import EpochNumber, ValidatorIndex from src.utils.range import sequence diff --git a/tests/modules/csm/test_tree.py b/tests/modules/csm/test_tree.py index 35b28233e..623b4056e 100644 --- a/tests/modules/csm/test_tree.py +++ b/tests/modules/csm/test_tree.py @@ -1,8 +1,8 @@ import pytest from src.constants import UINT64_MAX -from src.types import NodeOperatorId from src.modules.csm.tree import StandardMerkleTree, Tree, TreeJSONEncoder +from src.types import NodeOperatorId @pytest.fixture() diff --git a/tests/modules/ejector/test_data_encode.py b/tests/modules/ejector/test_data_encode.py index 173c5132c..6ebeea923 100644 --- a/tests/modules/ejector/test_data_encode.py +++ b/tests/modules/ejector/test_data_encode.py @@ -4,15 +4,15 @@ import pytest -from src.types import StakingModuleId, NodeOperatorId from src.modules.ejector.data_encode import ( + encode_data, MODULE_ID_LENGTH, NODE_OPERATOR_ID_LENGTH, + sort_validators_to_eject, VALIDATOR_INDEX_LENGTH, VALIDATOR_PUB_KEY_LENGTH, - encode_data, - sort_validators_to_eject, ) +from src.types import NodeOperatorId, StakingModuleId from src.web3py.extensions.lido_validators import LidoValidator from tests.factory.no_registry import LidoValidatorFactory diff --git a/tests/modules/ejector/test_ejector.py b/tests/modules/ejector/test_ejector.py index deef6378b..3f2e1ed1c 100644 --- a/tests/modules/ejector/test_ejector.py +++ b/tests/modules/ejector/test_ejector.py @@ -1,4 +1,4 @@ -from typing import Iterable, cast +from typing import cast, Iterable from unittest.mock import Mock import pytest @@ -14,18 +14,15 @@ MIN_ACTIVATION_BALANCE, MIN_VALIDATOR_WITHDRAWABILITY_DELAY, ) -from src.types import BlockStamp, Gwei, ReferenceBlockStamp from src.modules.ejector import ejector as ejector_module -from src.modules.ejector.ejector import ( - Ejector, -) -from src.modules.ejector.ejector import logger as ejector_logger +from src.modules.ejector.ejector import Ejector, logger as ejector_logger from src.modules.ejector.types import EjectorProcessingState from src.modules.submodules.oracle_module import ModuleExecuteDelay from src.modules.submodules.types import ChainConfig, CurrentFrame from src.providers.consensus.types import ( BeaconStateView, ) +from src.types import BlockStamp, Gwei, ReferenceBlockStamp from src.utils import validator_state from src.web3py.extensions.contracts import LidoContracts from src.web3py.extensions.lido_validators import NodeOperatorId, StakingModuleId diff --git a/tests/modules/ejector/test_prediction.py b/tests/modules/ejector/test_prediction.py index 5b31ed7d3..9c96b99ce 100644 --- a/tests/modules/ejector/test_prediction.py +++ b/tests/modules/ejector/test_prediction.py @@ -5,9 +5,9 @@ from web3.types import Wei import src.services.prediction as prediction_module -from src.types import BlockNumber, SlotNumber from src.modules.submodules.types import ChainConfig from src.services.prediction import RewardsPredictionService +from src.types import BlockNumber, SlotNumber from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/modules/ejector/test_validator_exit_order_iterator.py b/tests/modules/ejector/test_validator_exit_order_iterator.py index 6b77e4bc1..e42140445 100644 --- a/tests/modules/ejector/test_validator_exit_order_iterator.py +++ b/tests/modules/ejector/test_validator_exit_order_iterator.py @@ -3,15 +3,11 @@ import pytest +from src.services.exit_order_iterator import NodeOperatorStats, StakingModuleStats, ValidatorExitIterator from src.types import Gwei -from src.services.exit_order_iterator import ValidatorExitIterator, StakingModuleStats, NodeOperatorStats from src.web3py.extensions.lido_validators import NodeOperatorLimitMode from tests.factory.blockstamp import ReferenceBlockStampFactory -from tests.factory.no_registry import ( - NodeOperatorFactory, - StakingModuleFactory, - LidoValidatorFactory, -) +from tests.factory.no_registry import LidoValidatorFactory, NodeOperatorFactory, StakingModuleFactory from tests.factory.web3_factory import Web3DataclassFactory diff --git a/tests/modules/submodules/consensus/conftest.py b/tests/modules/submodules/consensus/conftest.py index f171ba3a0..3f1934702 100644 --- a/tests/modules/submodules/consensus/conftest.py +++ b/tests/modules/submodules/consensus/conftest.py @@ -1,7 +1,7 @@ import pytest -from src.types import BlockStamp, ReferenceBlockStamp from src.modules.submodules.consensus import ConsensusModule +from src.types import BlockStamp, ReferenceBlockStamp class SimpleConsensusModule(ConsensusModule): diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index ebf73bdf8..fefd54b18 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -5,14 +5,14 @@ from web3.exceptions import ContractCustomError from src import variables -from src.types import BlockStamp, ReferenceBlockStamp from src.modules.submodules import consensus as consensus_module -from src.modules.submodules.consensus import ZERO_HASH, ConsensusModule, IsNotMemberException, MemberInfo -from src.modules.submodules.exceptions import IncompatibleOracleVersion, ContractVersionMismatch +from src.modules.submodules.consensus import ConsensusModule, IsNotMemberException, MemberInfo, ZERO_HASH +from src.modules.submodules.exceptions import ContractVersionMismatch, IncompatibleOracleVersion from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import BeaconSpecResponse -from tests.conftest import get_blockstamp_by_state, Account -from tests.factory.blockstamp import ReferenceBlockStampFactory, BlockStampFactory +from src.types import BlockStamp, ReferenceBlockStamp +from tests.conftest import Account, get_blockstamp_by_state +from tests.factory.blockstamp import BlockStampFactory, ReferenceBlockStampFactory from tests.factory.configs import BeaconSpecResponseFactory, ChainConfigFactory, FrameConfigFactory diff --git a/tests/modules/submodules/test_oracle_module.py b/tests/modules/submodules/test_oracle_module.py index 5794890f1..8f789a244 100644 --- a/tests/modules/submodules/test_oracle_module.py +++ b/tests/modules/submodules/test_oracle_module.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock, patch, MagicMock +from unittest.mock import MagicMock, Mock, patch import pytest from requests.exceptions import ConnectionError as RequestsConnectionError @@ -6,11 +6,11 @@ from web3_multi_provider.multi_http_provider import NoActiveProviderError from src import variables -from src.types import BlockStamp -from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion +from src.modules.submodules.exceptions import IncompatibleOracleVersion, IsNotMemberException from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay from src.providers.http_provider import NotOkResponse from src.providers.keys.client import KeysOutdatedException +from src.types import BlockStamp from src.utils.slot import InconsistentData, NoSlotsAvailable, SlotNotFinalized from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/providers_clients/test_keys_api_client.py b/tests/providers_clients/test_keys_api_client.py index c16bdcb02..78ca46aaa 100644 --- a/tests/providers_clients/test_keys_api_client.py +++ b/tests/providers_clients/test_keys_api_client.py @@ -9,11 +9,10 @@ from web3 import Web3 import src.providers.keys.client as keys_api_client_module -from src import constants -from src import variables -from src.types import StakingModuleAddress +from src import constants, variables from src.providers.keys.client import KAPIClientError, KeysAPIClient, KeysOutdatedException from src.providers.keys.types import LidoKey +from src.types import StakingModuleAddress from tests.factory.blockstamp import ReferenceBlockStampFactory diff --git a/tests/utils/test_slot.py b/tests/utils/test_slot.py index bcba7954b..0f66d9ae6 100644 --- a/tests/utils/test_slot.py +++ b/tests/utils/test_slot.py @@ -4,9 +4,9 @@ import pytest -from src.types import SlotNumber from src.providers.http_provider import NotOkResponse -from src.utils.slot import NoSlotsAvailable, get_prev_non_missed_slot, get_next_non_missed_slot +from src.types import SlotNumber +from src.utils.slot import get_next_non_missed_slot, get_prev_non_missed_slot, NoSlotsAvailable from tests.conftest import get_blockstamp_by_state diff --git a/tests/utils/test_validator_state_utils.py b/tests/utils/test_validator_state_utils.py index 4f35f9742..6e6a9acfd 100644 --- a/tests/utils/test_validator_state_utils.py +++ b/tests/utils/test_validator_state_utils.py @@ -6,8 +6,8 @@ MAX_EFFECTIVE_BALANCE_ELECTRA, MIN_ACTIVATION_BALANCE, ) -from src.types import EpochNumber, Gwei from src.providers.consensus.types import Validator, ValidatorState +from src.types import EpochNumber, Gwei from src.utils.validator_state import ( calculate_active_effective_balance_sum, calculate_total_active_effective_balance, diff --git a/tests/utils/test_web3_converter.py b/tests/utils/test_web3_converter.py index 2835902af..b091dfae3 100644 --- a/tests/utils/test_web3_converter.py +++ b/tests/utils/test_web3_converter.py @@ -1,7 +1,7 @@ import pytest -from src.types import EpochNumber, FrameNumber, SlotNumber from src.modules.submodules.types import ChainConfig, FrameConfig +from src.types import EpochNumber, FrameNumber, SlotNumber from src.utils.web3converter import Web3Converter from tests.factory.configs import ChainConfigFactory, FrameConfigFactory From e336313a356e5c6ecce6898c146e553942b1bf45 Mon Sep 17 00:00:00 2001 From: hweawer Date: Tue, 6 May 2025 10:00:19 +0200 Subject: [PATCH 09/13] fix: Rewrite to remove etherscan dependency --- src/main.py | 73 +++++++++++++++++++++++++---------------------------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/src/main.py b/src/main.py index ab4cbb445..cb8fcd9de 100644 --- a/src/main.py +++ b/src/main.py @@ -1,37 +1,28 @@ import argparse -import os import sys -from typing import Iterator, cast, Optional +from typing import cast, Iterator, Optional -import requests from hexbytes import HexBytes from packaging.version import Version from prometheus_client import start_http_server -from src import constants -from src import variables +from src import constants, variables from src.metrics.healthcheck_server import start_pulse_server from src.metrics.logging import logging -from src.metrics.prometheus.basic import ENV_VARIABLES_INFO, BUILD_INFO +from src.metrics.prometheus.basic import BUILD_INFO, ENV_VARIABLES_INFO from src.modules.accounting.accounting import Accounting from src.modules.checks.checks_module import execute_checks from src.modules.csm.csm import CSOracle from src.modules.ejector.ejector import Ejector +from src.providers.execution.contracts.base_oracle import BaseOracleContract from src.providers.ipfs import GW3, IPFSProvider, Kubo, MultiIPFSProvider, Pinata, PublicIPFS -from src.types import OracleModule, BlockRoot, SlotNumber +from src.types import BlockRoot, OracleModule, SlotNumber from src.utils.blockstamp import build_blockstamp from src.utils.build import get_build_info from src.utils.exception import IncompatibleException from src.web3py.contract_tweak import tweak_w3_contracts -from src.web3py.extensions import ( - LidoContracts, - TransactionUtils, - ConsensusClientModule, - KeysAPIClientModule, - LidoValidatorsProvider, - FallbackProviderModule, - LazyCSM, -) +from src.web3py.extensions import (ConsensusClientModule, FallbackProviderModule, KeysAPIClientModule, LazyCSM, LidoContracts, + LidoValidatorsProvider, TransactionUtils) from src.web3py.middleware import add_requests_metric_middleware from src.web3py.types import Web3 @@ -127,22 +118,34 @@ def main(module_name: OracleModule): instance.cycle_handler() -def get_transactions(contract_address, selector: str, limit: int = 10): - etherscan_key = os.getenv('ETHERSCAN_API_KEY') - if etherscan_key is None: - raise ValueError('ETHERSCAN_API_KEY is not set') - url = f"https://api.etherscan.io/api?module=account&action=txlist&address={contract_address}&sort=desc&apikey={etherscan_key}" +def get_transactions(w3, contract: BaseOracleContract, limit: int = 10): + default_block_offset = 100_000 + event_abi = "ProcessingStarted(uint256,bytes32)" + event_topic = '0x' + Web3.keccak(text=event_abi).hex() - response = requests.get(url, timeout=30) - data = response.json() + def get_processing_started_logs(from_block: int, to_block: int): + return w3.eth.get_logs({ + "fromBlock": from_block, + "toBlock": to_block, + "address": contract.address, + "topics": [event_topic], + }) - if data["status"] != "1": - logger.error("Error fetching transactions: %s", data["message"]) - return [] + latest = w3.eth.block_number + logs = get_processing_started_logs(from_block=latest - default_block_offset, to_block=latest) + results = [log["transactionHash"].hex() for log in logs] - transactions = data["result"] - successful_transactions = [tx for tx in transactions if tx.get("txreceipt_status") == "1"] - return [tx for tx in successful_transactions if tx["input"].startswith(selector)][:limit] + txs = [] + for tx_hash in results[:limit]: + tx = w3.eth.get_transaction(tx_hash) + _, params = contract.decode_function_input(tx["input"]) + data = { + k: HexBytes(v.hex()) if isinstance(v, (bytes, bytearray)) else v + for k, v in params["data"].items() + } + txs.append(data) + + return txs def run_on_refslot(module_name: OracleModule): @@ -150,10 +153,8 @@ def run_on_refslot(module_name: OracleModule): w3 = _construct_web3() instance: Accounting | Ejector | CSOracle = _construct_module(w3, module_name, True) instance.check_contract_configs() - submit_report_fn = instance.report_contract.get_function_by_name("submitReportData") - selector = '0x' + w3.keccak(text=submit_report_fn.abi_element_identifier)[:4].hex() - txs = get_transactions(instance.report_contract.address, selector) + txs = get_transactions(w3, instance.report_contract) if not txs: logger.error("No submitReportData transactions found!") sys.exit(0) @@ -161,11 +162,8 @@ def run_on_refslot(module_name: OracleModule): logger.info("Found %d submitReportData calls", len(txs)) for tx in txs: - tx_hash = tx["hash"] - _, data = instance.report_contract.decode_function_input(tx["input"]) - refslot = int(data['data']['refSlot']) - print(f"Tx: {tx_hash} → slot: {refslot}") - print([HexBytes(v.hex()) if isinstance(v, bytes) else v for v in data['data'].values()]) + refslot = int(tx['refSlot']) + print("Input data:", tx) block_root = BlockRoot(w3.cc.get_block_root(SlotNumber(refslot + 3 * 32)).root) block_details = w3.cc.get_block_details(block_root) bs = build_blockstamp(block_details) @@ -174,7 +172,6 @@ def run_on_refslot(module_name: OracleModule): report_blockstamp = instance.get_blockstamp_for_report(bs) report = instance.build_report(report_blockstamp) - print(report) instance = _construct_module(w3, module_name, refslot) From 6e9047c7e1434f52ecbbd25ff5fdc89a021c775d Mon Sep 17 00:00:00 2001 From: hweawer Date: Mon, 12 May 2025 10:23:21 +0200 Subject: [PATCH 10/13] fix: Remove log --- src/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main.py b/src/main.py index cb8fcd9de..a2325bbb3 100644 --- a/src/main.py +++ b/src/main.py @@ -104,8 +104,6 @@ def main(module_name: OracleModule): }) ENV_VARIABLES_INFO.info(variables.PUBLIC_ENV_VARS) BUILD_INFO.info(build_info) - - logger.info({'msg': f'Start healthcheck server for Docker container on port {variables.HEALTHCHECK_SERVER_PORT}'}) start_pulse_server() logger.info({'msg': f'Start http server with prometheus metrics on port {variables.PROMETHEUS_PORT}'}) From a6d1072b41d9ad76e5e90c8be4005b7a47bc0584 Mon Sep 17 00:00:00 2001 From: hweawer Date: Mon, 12 May 2025 12:30:24 +0200 Subject: [PATCH 11/13] fix: import --- tests/modules/submodules/consensus/test_consensus.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index 0693eaaaf..feace5480 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -1,5 +1,5 @@ -from typing import cast from dataclasses import dataclass +from typing import cast from unittest.mock import Mock import pytest @@ -14,9 +14,7 @@ from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import BeaconSpecResponse from src.types import BlockStamp, ReferenceBlockStamp -from tests.conftest import Account from tests.factory.blockstamp import BlockStampFactory, ReferenceBlockStampFactory - from tests.factory.configs import ( BeaconSpecResponseFactory, BlockDetailsResponseFactory, From 0cf28af4259e45b05b9419db9af5a0628006528c Mon Sep 17 00:00:00 2001 From: hweawer Date: Mon, 12 May 2025 21:01:05 +0200 Subject: [PATCH 12/13] fix: Enhance report --- poetry.lock | 38 +++++++++- pyproject.toml | 1 + src/main.py | 71 ++++++++++++++----- src/modules/accounting/accounting.py | 20 +++--- src/modules/accounting/types.py | 35 ++++----- src/modules/csm/csm.py | 8 +-- src/modules/ejector/ejector.py | 28 ++------ src/modules/ejector/types.py | 4 ++ src/modules/submodules/consensus.py | 28 ++++---- .../accounting/test_accounting_module.py | 2 +- tests/modules/csm/test_csm_module.py | 3 +- tests/modules/ejector/test_ejector.py | 60 ++-------------- .../modules/submodules/consensus/conftest.py | 6 +- .../submodules/consensus/test_consensus.py | 2 +- .../submodules/consensus/test_reports.py | 5 +- 15 files changed, 158 insertions(+), 153 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7d6ad8ff0..ab54b3ea5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -856,6 +856,29 @@ files = [ {file = "decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360"}, ] +[[package]] +name = "deepdiff" +version = "8.5.0" +description = "Deep Difference and Search of any Python object/data. Recreate objects by adding adding deltas to each other." +optional = false +python-versions = ">=3.9" +files = [ + {file = "deepdiff-8.5.0-py3-none-any.whl", hash = "sha256:d4599db637f36a1c285f5fdfc2cd8d38bde8d8be8636b65ab5e425b67c54df26"}, + {file = "deepdiff-8.5.0.tar.gz", hash = "sha256:a4dd3529fa8d4cd5b9cbb6e3ea9c95997eaa919ba37dac3966c1b8f872dc1cd1"}, +] + +[package.dependencies] +orderly-set = ">=5.4.1,<6" + +[package.extras] +cli = ["click (>=8.1.0,<8.2.0)", "pyyaml (>=6.0.0,<6.1.0)"] +coverage = ["coverage (>=7.6.0,<7.7.0)"] +dev = ["bump2version (>=1.0.0,<1.1.0)", "ipdb (>=0.13.0,<0.14.0)", "jsonpickle (>=4.0.0,<4.1.0)", "nox (==2025.5.1)", "numpy (>=2.0,<3.0)", "numpy (>=2.2.0,<2.3.0)", "orjson (>=3.10.0,<3.11.0)", "pandas (>=2.2.0,<2.3.0)", "polars (>=1.21.0,<1.22.0)", "python-dateutil (>=2.9.0,<2.10.0)", "tomli (>=2.2.0,<2.3.0)", "tomli-w (>=1.2.0,<1.3.0)"] +docs = ["Sphinx (>=6.2.0,<6.3.0)", "sphinx-sitemap (>=2.6.0,<2.7.0)", "sphinxemoji (>=0.3.0,<0.4.0)"] +optimize = ["orjson"] +static = ["flake8 (>=7.1.0,<7.2.0)", "flake8-pyproject (>=1.2.3,<1.3.0)", "pydantic (>=2.10.0,<2.11.0)"] +test = ["pytest (>=8.3.0,<8.4.0)", "pytest-benchmark (>=5.1.0,<5.2.0)", "pytest-cov (>=6.0.0,<6.1.0)", "python-dotenv (>=1.0.0,<1.1.0)"] + [[package]] name = "dill" version = "0.4.0" @@ -1813,6 +1836,17 @@ files = [ {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] +[[package]] +name = "orderly-set" +version = "5.4.1" +description = "Orderly set" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orderly_set-5.4.1-py3-none-any.whl", hash = "sha256:b5e21d21680bd9ef456885db800c5cb4f76a03879880c0175e1b077fb166fd83"}, + {file = "orderly_set-5.4.1.tar.gz", hash = "sha256:a1fb5a4fdc5e234e9e8d8e5c1bbdbc4540f4dfe50d12bf17c8bc5dbf1c9c878d"}, +] + [[package]] name = "oz-merkle-tree" version = "0.1.0" @@ -3212,4 +3246,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "bf7db38e210a00035d54a5f082f8d28f8af28fa70e248a4a0788ea8cc6a8e2dd" +content-hash = "8e648f9fe72065ce44ea1ff52fa873ffc611b2b385f89809e93b17420630a02b" diff --git a/pyproject.toml b/pyproject.toml index 94014e09b..01da5a1c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ json-stream = "^2.3.2" oz-merkle-tree = { git = "https://github.com/lidofinance/oz-merkle-tree" } py-multiformats-cid = "^0.4.4" conventional-pre-commit = "^4.0.0" +deepdiff = "^8.5.0" [tool.poetry.group.dev.dependencies] base58 = "^2.1.1" diff --git a/src/main.py b/src/main.py index a2325bbb3..dee284982 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,10 @@ import argparse +import re import sys -from typing import cast, Iterator, Optional +from dataclasses import asdict +from typing import Any, cast, Dict, Iterator, Optional +from deepdiff import DeepDiff from hexbytes import HexBytes from packaging.version import Version from prometheus_client import start_http_server @@ -116,10 +119,9 @@ def main(module_name: OracleModule): instance.cycle_handler() -def get_transactions(w3, contract: BaseOracleContract, limit: int = 10): - default_block_offset = 100_000 +def get_transactions(w3, contract: BaseOracleContract, limit: int, block_offset: int): event_abi = "ProcessingStarted(uint256,bytes32)" - event_topic = '0x' + Web3.keccak(text=event_abi).hex() + event_topic = Web3.to_hex(primitive=Web3.keccak(text=event_abi)) def get_processing_started_logs(from_block: int, to_block: int): return w3.eth.get_logs({ @@ -130,34 +132,44 @@ def get_processing_started_logs(from_block: int, to_block: int): }) latest = w3.eth.block_number - logs = get_processing_started_logs(from_block=latest - default_block_offset, to_block=latest) - results = [log["transactionHash"].hex() for log in logs] + logs = get_processing_started_logs(from_block=latest - block_offset, to_block=latest) + tx_hashes = [log['transactionHash'].hex() for log in logs] + print(f"Found {len(tx_hashes)} submitReportData transactions in latest {block_offset} blocks.") txs = [] - for tx_hash in results[:limit]: + for tx_hash in tx_hashes[:limit]: tx = w3.eth.get_transaction(tx_hash) - _, params = contract.decode_function_input(tx["input"]) + _, params = contract.decode_function_input(tx['input']) data = { k: HexBytes(v.hex()) if isinstance(v, (bytes, bytearray)) else v - for k, v in params["data"].items() + for k, v in params['data'].items() } txs.append(data) + print(f"Will process {len(txs)} transactions") return txs -def run_on_refslot(module_name: OracleModule): - logging.getLogger().setLevel(logging.WARNING) +def run_on_refslot(module_name: OracleModule, limit, block_offset): + #logging.getLogger().setLevel(logging.WARNING) w3 = _construct_web3() instance: Accounting | Ejector | CSOracle = _construct_module(w3, module_name, True) instance.check_contract_configs() - txs = get_transactions(w3, instance.report_contract) + txs = get_transactions(w3, instance.report_contract, limit, block_offset) if not txs: logger.error("No submitReportData transactions found!") sys.exit(0) - logger.info("Found %d submitReportData calls", len(txs)) + _camel_to_snake_pattern = re.compile(r'(.)([A-Z][a-z]+)') + _camel_to_snake_pattern2 = re.compile(r'([a-z0-9])([A-Z])') + + def camel_to_snake(name: str) -> str: + s1 = _camel_to_snake_pattern.sub(r'\1_\2', name) + return _camel_to_snake_pattern2.sub(r'\1_\2', s1).lower() + + def normalize_keys(d: Dict[str, Any]) -> Dict[str, Any]: + return {camel_to_snake(k): v for k, v in d.items()} for tx in txs: refslot = int(tx['refSlot']) @@ -170,7 +182,19 @@ def run_on_refslot(module_name: OracleModule): report_blockstamp = instance.get_blockstamp_for_report(bs) report = instance.build_report(report_blockstamp) - print(report) + normalized_input = normalize_keys(tx) + report_dict = asdict(report) + report_dict = { + k: HexBytes(v.hex()) if isinstance(v, (bytes, bytearray)) else v + for k, v in report_dict.items() + } + print("Output data:", report_dict) + diff = DeepDiff(normalized_input, report_dict, ignore_order=True) + if diff: + print("🚨 Differences found:") + print(diff) + else: + print("✅ All fields match!") instance = _construct_module(w3, module_name, refslot) @@ -232,10 +256,25 @@ def parse_args(): default=None, help="Module name to check for a refslot execution." ) + check_parser.add_argument( + "--limit", + type=int, + default=10, + help="Maximum number of items to process." + ) + check_parser.add_argument( + "--offset", + type=int, + default=100_000, + help="Starting index offset for processing." + ) for mod in OracleModule: if mod == OracleModule.CHECK: continue - subparsers.add_parser(mod.value, help=f"Run the {mod.value} module.") + subparsers.add_parser( + mod.value, + help=f"Run the {mod.value} module." + ) return parser.parse_args() @@ -256,7 +295,7 @@ def parse_args(): else: errors = variables.check_all_required_variables(module) variables.raise_from_errors(errors) - run_on_refslot(args.name) + run_on_refslot(args.name, args.limit, args.offset) sys.exit(0) errors = variables.check_all_required_variables(module) diff --git a/src/modules/accounting/accounting.py b/src/modules/accounting/accounting.py index 76db8040a..dd21985a5 100644 --- a/src/modules/accounting/accounting.py +++ b/src/modules/accounting/accounting.py @@ -16,7 +16,7 @@ from src.modules.accounting.third_phase.types import ExtraData, FormatList from src.modules.accounting.types import (AccountingProcessingState, BunkerMode, FinalizationShareRate, GenericExtraData, LidoReportRebase, RebaseReport, ReportData, ValidatorsBalance, ValidatorsCount, WqReport) -from src.modules.submodules.consensus import ConsensusModule, InitialEpochIsYetToArriveRevert +from src.modules.submodules.consensus import ConsensusModule, InitialEpochIsYetToArriveRevert, Report from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay from src.modules.submodules.types import ZERO_HASH from src.providers.execution.contracts.accounting_oracle import AccountingOracleContract @@ -101,10 +101,10 @@ def _submit_extra_data(self, blockstamp: ReferenceBlockStamp) -> None: @lru_cache(maxsize=1) @duration_meter() - def build_report(self, blockstamp: ReferenceBlockStamp) -> tuple: + def build_report(self, blockstamp: ReferenceBlockStamp) -> Report: report_data = self._calculate_report(blockstamp) logger.info({'msg': 'Calculate report for accounting module.', 'value': report_data}) - return report_data.as_tuple() + return report_data def is_main_data_submitted(self, blockstamp: BlockStamp) -> bool: # Consensus module: if contract got report data (second phase) @@ -154,7 +154,7 @@ def _get_processing_state(self, blockstamp: BlockStamp) -> AccountingProcessingS ) # ---------------------------------------- Build report ---------------------------------------- - def _calculate_report(self, blockstamp: ReferenceBlockStamp): + def _calculate_report(self, blockstamp: ReferenceBlockStamp) -> ReportData: consensus_version = self.get_consensus_version(blockstamp) logger.info({'msg': 'Building the report', 'consensus_version': consensus_version}) rebase_part = self._calculate_rebase_report(blockstamp) @@ -358,7 +358,7 @@ def _calculate_extra_data_report(self, blockstamp: ReferenceBlockStamp) -> Extra @staticmethod def _update_metrics(report_data: ReportData): - ACCOUNTING_IS_BUNKER.set(report_data.is_bunker) + ACCOUNTING_IS_BUNKER.set(report_data.is_bunker_mode) ACCOUNTING_CL_BALANCE_GWEI.set(report_data.cl_balance_gwei) ACCOUNTING_EL_REWARDS_VAULT_BALANCE_WEI.set(report_data.el_rewards_vault_balance) ACCOUNTING_WITHDRAWAL_VAULT_BALANCE_WEI.set(report_data.withdrawal_vault_balance) @@ -378,16 +378,16 @@ def _combine_report_parts( return ReportData( consensus_version=consensus_version, ref_slot=blockstamp.ref_slot, - validators_count=validators_count, + num_validators=validators_count, cl_balance_gwei=cl_balance, - staking_module_ids_with_exited_validators=staking_module_ids_list, - count_exited_validators_by_staking_module=exit_validators_count_list, + staking_module_ids_with_newly_exited_validators=staking_module_ids_list, + num_exited_validators_by_staking_module=exit_validators_count_list, withdrawal_vault_balance=withdrawal_vault_balance, el_rewards_vault_balance=el_rewards_vault_balance, shares_requested_to_burn=shares_requested_to_burn, withdrawal_finalization_batches=finalization_batches, - finalization_share_rate=finalization_share_rate, - is_bunker=is_bunker, + simulated_share_rate=finalization_share_rate, + is_bunker_mode=is_bunker, extra_data_format=extra_data.format, extra_data_hash=extra_data.data_hash, extra_data_items_count=extra_data.items_count, diff --git a/src/modules/accounting/types.py b/src/modules/accounting/types.py index 9843c61ae..898b3774e 100644 --- a/src/modules/accounting/types.py +++ b/src/modules/accounting/types.py @@ -1,54 +1,47 @@ from dataclasses import dataclass -from typing import Self, NewType +from typing import NewType, Self from eth_typing import ChecksumAddress from hexbytes import HexBytes from web3.types import Wei -from src.types import ( - SlotNumber, - Gwei, - StakingModuleId, - FinalizationBatches, - ELVaultBalance, - WithdrawalVaultBalance, - OperatorsValidatorCount, -) +from src.types import (ELVaultBalance, FinalizationBatches, Gwei, OperatorsValidatorCount, SlotNumber, StakingModuleId, + WithdrawalVaultBalance) @dataclass class ReportData: consensus_version: int ref_slot: SlotNumber - validators_count: int + num_validators: int cl_balance_gwei: Gwei - staking_module_ids_with_exited_validators: list[StakingModuleId] - count_exited_validators_by_staking_module: list[int] + staking_module_ids_with_newly_exited_validators: list[StakingModuleId] + num_exited_validators_by_staking_module: list[int] withdrawal_vault_balance: Wei el_rewards_vault_balance: Wei shares_requested_to_burn: int withdrawal_finalization_batches: list[int] - finalization_share_rate: int - is_bunker: bool + simulated_share_rate: int + is_bunker_mode: bool extra_data_format: int extra_data_hash: bytes extra_data_items_count: int - def as_tuple(self): + def as_tuple(self) -> tuple: # Tuple with report in correct order return ( self.consensus_version, self.ref_slot, - self.validators_count, + self.num_validators, self.cl_balance_gwei, - self.staking_module_ids_with_exited_validators, - self.count_exited_validators_by_staking_module, + self.staking_module_ids_with_newly_exited_validators, + self.num_exited_validators_by_staking_module, self.withdrawal_vault_balance, self.el_rewards_vault_balance, self.shares_requested_to_burn, self.withdrawal_finalization_batches, - self.finalization_share_rate, - self.is_bunker, + self.simulated_share_rate, + self.is_bunker_mode, self.extra_data_format, self.extra_data_hash, self.extra_data_items_count, diff --git a/src/modules/csm/csm.py b/src/modules/csm/csm.py index 30d26b3b8..78de29673 100644 --- a/src/modules/csm/csm.py +++ b/src/modules/csm/csm.py @@ -16,7 +16,7 @@ from src.modules.csm.state import State from src.modules.csm.tree import Tree from src.modules.csm.types import ReportData, Shares -from src.modules.submodules.consensus import ConsensusModule +from src.modules.submodules.consensus import ConsensusModule, Report from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay from src.modules.submodules.types import ZERO_HASH from src.providers.execution.contracts.cs_fee_oracle import CSFeeOracleContract @@ -94,7 +94,7 @@ def execute_module(self, last_finalized_blockstamp: BlockStamp) -> ModuleExecute @lru_cache(maxsize=1) @duration_meter() - def build_report(self, blockstamp: ReferenceBlockStamp) -> tuple: + def build_report(self, blockstamp: ReferenceBlockStamp) -> Report: self.validate_state(blockstamp) prev_root = self.w3.csm.get_csm_tree_root(blockstamp) @@ -119,7 +119,7 @@ def build_report(self, blockstamp: ReferenceBlockStamp) -> tuple: tree_cid=prev_cid or "", log_cid=log_cid, distributed=0, - ).as_tuple() + ) if prev_cid and prev_root != ZERO_HASH: # Update cumulative amount of shares for all operators. @@ -138,7 +138,7 @@ def build_report(self, blockstamp: ReferenceBlockStamp) -> tuple: tree_cid=tree_cid, log_cid=log_cid, distributed=distributed, - ).as_tuple() + ) def is_main_data_submitted(self, blockstamp: BlockStamp) -> bool: last_ref_slot = self.w3.csm.get_csm_last_processing_ref_slot(blockstamp) diff --git a/src/modules/ejector/ejector.py b/src/modules/ejector/ejector.py index c523be6e1..00dd52792 100644 --- a/src/modules/ejector/ejector.py +++ b/src/modules/ejector/ejector.py @@ -1,4 +1,3 @@ -import dataclasses import logging from typing import Optional @@ -20,10 +19,10 @@ from src.modules.ejector.data_encode import encode_data from src.modules.ejector.sweep import get_sweep_delay_in_epochs from src.modules.ejector.types import EjectorProcessingState, ReportData -from src.modules.submodules.consensus import ConsensusModule, InitialEpochIsYetToArriveRevert +from src.modules.submodules.consensus import ConsensusModule, InitialEpochIsYetToArriveRevert, Report from src.modules.submodules.oracle_module import BaseModule, ModuleExecuteDelay from src.modules.submodules.types import ZERO_HASH -from src.providers.consensus.types import Validator, BeaconStateView +from src.providers.consensus.types import BeaconStateView, Validator from src.providers.execution.contracts.exit_bus_oracle import ExitBusOracleContract from src.services.exit_order_iterator import ValidatorExitIterator from src.services.prediction import RewardsPredictionService @@ -31,14 +30,8 @@ from src.types import BlockStamp, EpochNumber, Gwei, NodeOperatorGlobalIndex, ReferenceBlockStamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.units import gwei_to_wei -from src.utils.validator_state import ( - compute_activation_exit_epoch, - get_activation_exit_churn_limit, - get_validator_churn_limit, - get_max_effective_balance, - is_active_validator, - is_fully_withdrawable_validator, -) +from src.utils.validator_state import (compute_activation_exit_epoch, get_activation_exit_churn_limit, get_max_effective_balance, + is_active_validator, is_fully_withdrawable_validator) from src.web3py.extensions.lido_validators import LidoValidator from src.web3py.types import Web3 @@ -90,7 +83,7 @@ def execute_module(self, last_finalized_blockstamp: BlockStamp) -> ModuleExecute @lru_cache(maxsize=1) @duration_meter() - def build_report(self, blockstamp: ReferenceBlockStamp) -> tuple: + def build_report(self, blockstamp: ReferenceBlockStamp) -> Report: # For metrics only self.w3.lido_contracts.get_ejector_last_processing_ref_slot(blockstamp) @@ -111,8 +104,7 @@ def build_report(self, blockstamp: ReferenceBlockStamp) -> tuple: ) EJECTOR_VALIDATORS_COUNT_TO_EJECT.set(report_data.requests_count) - - return dataclasses.astuple(report_data) + return report_data def get_validators_to_eject(self, blockstamp: ReferenceBlockStamp) -> list[tuple[NodeOperatorGlobalIndex, LidoValidator]]: to_withdraw_amount = self.w3.lido_contracts.withdrawal_queue_nft.unfinalized_steth(blockstamp.block_hash) @@ -291,14 +283,6 @@ def _get_sweep_delay_in_epochs(self, blockstamp: ReferenceBlockStamp) -> int: state = self.w3.cc.get_state_view(blockstamp) return get_sweep_delay_in_epochs(state, chain_config) - @lru_cache(maxsize=1) - def _get_churn_limit(self, blockstamp: ReferenceBlockStamp) -> int: - total_active_validators = len(self._get_active_validators(blockstamp)) - logger.info({'msg': 'Calculate total active validators.', 'value': total_active_validators}) - churn_limit = get_validator_churn_limit(total_active_validators) - logger.info({'msg': 'Calculate churn limit.', 'value': churn_limit}) - return churn_limit - # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#get_total_active_balance def _get_total_active_balance(self, blockstamp: ReferenceBlockStamp) -> Gwei: active_validators = self._get_active_validators(blockstamp) diff --git a/src/modules/ejector/types.py b/src/modules/ejector/types.py index 07c81c643..3d354b556 100644 --- a/src/modules/ejector/types.py +++ b/src/modules/ejector/types.py @@ -1,3 +1,4 @@ +import dataclasses from dataclasses import dataclass from src.types import SlotNumber @@ -21,3 +22,6 @@ class ReportData: requests_count: int data_format: int data: bytes + + def as_tuple(self) -> tuple: + return dataclasses.astuple(self) diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index 8098713e7..332ed11ec 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -1,25 +1,21 @@ import logging from abc import ABC, abstractmethod from time import sleep -from typing import cast, Optional +from typing import cast, Optional, Protocol from eth_abi import encode from hexbytes import HexBytes from web3.exceptions import ContractCustomError from src import variables -from src.metrics.prometheus.basic import ORACLE_SLOT_NUMBER, ORACLE_BLOCK_NUMBER, GENESIS_TIME, ACCOUNT_BALANCE -from src.metrics.prometheus.business import ( - ORACLE_MEMBER_LAST_REPORT_REF_SLOT, - FRAME_CURRENT_REF_SLOT, - FRAME_DEADLINE_SLOT, - ORACLE_MEMBER_INFO -) -from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch -from src.modules.submodules.types import ChainConfig, MemberInfo, ZERO_HASH, CurrentFrame, FrameConfig +from src.metrics.prometheus.basic import ACCOUNT_BALANCE, GENESIS_TIME, ORACLE_BLOCK_NUMBER, ORACLE_SLOT_NUMBER +from src.metrics.prometheus.business import (FRAME_CURRENT_REF_SLOT, FRAME_DEADLINE_SLOT, ORACLE_MEMBER_INFO, + ORACLE_MEMBER_LAST_REPORT_REF_SLOT) +from src.modules.submodules.exceptions import ContractVersionMismatch, IncompatibleOracleVersion, IsNotMemberException +from src.modules.submodules.types import ChainConfig, CurrentFrame, FrameConfig, MemberInfo, ZERO_HASH from src.providers.execution.contracts.base_oracle import BaseOracleContract from src.providers.execution.contracts.hash_consensus import HashConsensusContract -from src.types import BlockStamp, ReferenceBlockStamp, SlotNumber, FrameNumber +from src.types import BlockStamp, FrameNumber, ReferenceBlockStamp, SlotNumber from src.utils.blockstamp import build_blockstamp from src.utils.cache import global_lru_cache as lru_cache from src.utils.slot import get_reference_blockstamp @@ -32,6 +28,11 @@ InitialEpochIsYetToArriveRevert = Web3.to_hex(primitive=Web3.keccak(text="InitialEpochIsYetToArrive()")[:4]) +class Report(Protocol): + def as_tuple(self) -> tuple: + ... + + class ConsensusModule(ABC): """ Module that works with Hash Consensus Contract. @@ -279,7 +280,8 @@ def _check_compatability(self, blockstamp: BlockStamp) -> bool: # ----- Working with report ----- def process_report(self, blockstamp: ReferenceBlockStamp) -> None: """Builds and sends report for current frame with provided blockstamp.""" - report_data = self.build_report(blockstamp) + x = self.build_report(blockstamp) + report_data = x.as_tuple() logger.info({'msg': 'Build report.', 'value': report_data}) report_hash = self._encode_data_hash(report_data) @@ -465,7 +467,7 @@ def _get_web3_converter(self, blockstamp: BlockStamp) -> Web3Converter: @abstractmethod @lru_cache(maxsize=1) - def build_report(self, blockstamp: ReferenceBlockStamp) -> tuple: + def build_report(self, blockstamp: ReferenceBlockStamp) -> Report: """Returns ReportData struct with calculated data.""" @abstractmethod diff --git a/tests/modules/accounting/test_accounting_module.py b/tests/modules/accounting/test_accounting_module.py index f34d6b6dd..6ee9c537d 100644 --- a/tests/modules/accounting/test_accounting_module.py +++ b/tests/modules/accounting/test_accounting_module.py @@ -377,7 +377,7 @@ def test_build_report( accounting._calculate_report = Mock(return_value=Mock(as_tuple=Mock(return_value=REPORT))) - report = accounting.build_report(ref_bs) + report = accounting.build_report(ref_bs).as_tuple() assert report is REPORT, "build_report returned unexpected value" accounting._calculate_report.assert_called_once_with(ref_bs) diff --git a/tests/modules/csm/test_csm_module.py b/tests/modules/csm/test_csm_module.py index df0c6ee1c..4114695dd 100644 --- a/tests/modules/csm/test_csm_module.py +++ b/tests/modules/csm/test_csm_module.py @@ -15,7 +15,6 @@ from src.modules.submodules.types import CurrentFrame, ZERO_HASH from src.providers.ipfs import CID, CIDv0 from src.types import EpochNumber, NodeOperatorId, SlotNumber, StakingModuleId, ValidatorIndex -from src.web3py.extensions.csm import CSM from tests.factory.blockstamp import BlockStampFactory, ReferenceBlockStampFactory from tests.factory.configs import ChainConfigFactory, FrameConfigFactory @@ -707,7 +706,7 @@ def test_build_report(module: CSOracle, param: BuildReportTestParam): module.publish_tree = Mock(return_value=param.curr_tree_cid) module.publish_log = Mock(return_value=param.curr_log_cid) - report = module.build_report(blockstamp=Mock(ref_slot=100500)) + report = module.build_report(blockstamp=Mock(ref_slot=100500)).as_tuple() assert module.make_tree.call_args == param.expected_make_tree_call_args assert report == param.expected_func_result diff --git a/tests/modules/ejector/test_ejector.py b/tests/modules/ejector/test_ejector.py index 3949d4bcb..894826108 100644 --- a/tests/modules/ejector/test_ejector.py +++ b/tests/modules/ejector/test_ejector.py @@ -1,11 +1,9 @@ -from typing import cast, Iterable +from typing import cast from unittest.mock import Mock import pytest from web3.exceptions import ContractCustomError -from src.providers.execution.contracts.exit_bus_oracle import ExitBusOracleContract -from src import constants from src.constants import ( EFFECTIVE_BALANCE_INCREMENT, GWEI_TO_WEI, @@ -23,9 +21,8 @@ from src.providers.consensus.types import ( BeaconStateView, ) +from src.providers.execution.contracts.exit_bus_oracle import ExitBusOracleContract from src.types import BlockStamp, Gwei, ReferenceBlockStamp, SlotNumber -from src.utils import validator_state -from src.web3py.extensions.contracts import LidoContracts from src.web3py.extensions.lido_validators import NodeOperatorId, StakingModuleId from src.web3py.types import Web3 from tests.factory.base_oracle import EjectorProcessingStateFactory @@ -33,6 +30,7 @@ from tests.factory.configs import ChainConfigFactory from tests.factory.no_registry import LidoValidatorFactory from tests.modules.accounting.test_safe_border_unit import FAR_FUTURE_EPOCH +from types import SimpleNamespace @pytest.fixture(autouse=True) @@ -90,7 +88,7 @@ def test_ejector_execute_module_on_pause(ejector: Ejector, blockstamp: BlockStam ejector.w3.lido_contracts.validators_exit_bus_oracle.get_contract_version = Mock(return_value=1) ejector.w3.lido_contracts.validators_exit_bus_oracle.get_consensus_version = Mock(return_value=3) ejector.get_blockstamp_for_report = Mock(return_value=blockstamp) - ejector.build_report = Mock(return_value=(1, 294271, 0, 1, b'')) + ejector.build_report = ejector.build_report = lambda bs: SimpleNamespace(as_tuple=lambda: (1, 294271, 0, 1, b'')) ejector.w3.lido_contracts.validators_exit_bus_oracle.is_paused = Mock(return_value=True) result = ejector.execute_module(last_finalized_blockstamp=blockstamp) @@ -103,7 +101,7 @@ def test_ejector_build_report(ejector: Ejector, ref_blockstamp: ReferenceBlockSt ejector.get_validators_to_eject = Mock(return_value=[]) ejector.w3.lido_contracts.validators_exit_bus_oracle.get_last_processing_ref_slot.return_value = SlotNumber(0) - _, ref_slot, _, _, data = ejector.build_report(ref_blockstamp) + _, ref_slot, _, _, data = ejector.build_report(ref_blockstamp).as_tuple() ejector.build_report(ref_blockstamp) assert ref_slot == ref_blockstamp.ref_slot, "Unexpected blockstamp.ref_slot" @@ -419,54 +417,6 @@ def test_get_total_balance(ejector: Ejector, blockstamp: BlockStamp) -> None: ejector.w3.lido_contracts.lido.get_buffered_ether.assert_called_once_with(blockstamp.block_hash) -@pytest.mark.unit -class TestChurnLimit: - """_get_churn_limit tests""" - - @pytest.fixture(autouse=True) - def mock_is_active_validator(self, monkeypatch: pytest.MonkeyPatch) -> Iterable: - with monkeypatch.context() as m: - m.setattr( - ejector_module, - "is_active_validator", - Mock(side_effect=lambda v, _: bool(v)), - ) - yield - - def test_get_churn_limit_no_validators(self, ejector: Ejector, ref_blockstamp: ReferenceBlockStamp) -> None: - ejector.w3.cc.get_validators = Mock(return_value=[]) - result = ejector._get_churn_limit(ref_blockstamp) - assert result == constants.MIN_PER_EPOCH_CHURN_LIMIT, "Unexpected churn limit" - ejector.w3.cc.get_validators.assert_called_once_with(ref_blockstamp) - - def test_get_churn_limit_validators_less_than_min_churn( - self, - ejector: Ejector, - ref_blockstamp: ReferenceBlockStamp, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - with monkeypatch.context() as m: - ejector.w3.cc.get_validators = Mock(return_value=[1, 1, 0]) - result = ejector._get_churn_limit(ref_blockstamp) - assert result == 4, "Unexpected churn limit" - ejector.w3.cc.get_validators.assert_called_once_with(ref_blockstamp) - - def test_get_churn_limit_basic( - self, - ejector: Ejector, - ref_blockstamp: ReferenceBlockStamp, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - with monkeypatch.context() as m: - ejector.w3.cc.get_validators = Mock(return_value=[1] * 99) - m.setattr(validator_state, "MIN_PER_EPOCH_CHURN_LIMIT", 0) - m.setattr(validator_state, "CHURN_LIMIT_QUOTIENT", 2) - result = ejector._get_churn_limit(ref_blockstamp) - assert result == 49, "Unexpected churn limit" - ejector._get_churn_limit(ref_blockstamp) - ejector.w3.cc.get_validators.assert_called_once_with(ref_blockstamp) - - @pytest.mark.unit def test_get_latest_exit_epoch(ejector: Ejector, blockstamp: BlockStamp) -> None: ejector.w3.cc.get_validators = Mock( diff --git a/tests/modules/submodules/consensus/conftest.py b/tests/modules/submodules/consensus/conftest.py index 39658b72a..0c8a38e9b 100644 --- a/tests/modules/submodules/consensus/conftest.py +++ b/tests/modules/submodules/consensus/conftest.py @@ -1,6 +1,6 @@ import pytest -from src.modules.submodules.consensus import ConsensusModule +from src.modules.submodules.consensus import ConsensusModule, Report from src.types import BlockStamp, ReferenceBlockStamp @@ -12,8 +12,8 @@ def __init__(self, w3): self.report_contract = w3.lido_contracts.accounting_oracle super().__init__(w3) - def build_report(self, blockstamp: ReferenceBlockStamp) -> tuple: - return tuple() + def build_report(self, blockstamp: ReferenceBlockStamp) -> Report: + return type("", (), {"as_tuple": lambda self: ()})() def is_main_data_submitted(self, blockstamp: BlockStamp) -> bool: return True diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index feace5480..c72cad70e 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -327,7 +327,7 @@ class ConsensusImpl(ConsensusModule): """Consensus module implementation for testing purposes""" def build_report(self, _: ReferenceBlockStamp) -> tuple: - return tuple() + return type("", (), {"as_tuple": lambda self: ()})() def is_main_data_submitted(self, _: BlockStamp) -> bool: return True diff --git a/tests/modules/submodules/consensus/test_reports.py b/tests/modules/submodules/consensus/test_reports.py index d2952dd86..b412b0340 100644 --- a/tests/modules/submodules/consensus/test_reports.py +++ b/tests/modules/submodules/consensus/test_reports.py @@ -1,15 +1,14 @@ +from dataclasses import dataclass from unittest.mock import Mock import pytest from eth_typing import ChecksumAddress from hexbytes import HexBytes -from dataclasses import dataclass from src import variables from src.modules.accounting.accounting import Accounting from src.modules.accounting.types import ReportData from src.modules.submodules.types import ChainConfig, FrameConfig, ZERO_HASH - from tests.factory.blockstamp import ReferenceBlockStampFactory from tests.factory.member_info import MemberInfoFactory @@ -54,7 +53,7 @@ def test_process_report_main(consensus, caplog): consensus._send_report_hash = Mock() report_data = ReportData( 1, 2, 3, 4, [5, 6], [7, 8], 9, 10, 11, [12], 13, True, 13, HexBytes(int.to_bytes(14, 32)), 15 - ).as_tuple() + ) consensus.build_report = Mock(return_value=report_data) consensus.process_report(blockstamp) From 93f20e9fc924bf516b5b107fcd0355be0ab18c95 Mon Sep 17 00:00:00 2001 From: hweawer Date: Tue, 13 May 2025 09:44:53 +0200 Subject: [PATCH 13/13] fix: logging --- src/main.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/main.py b/src/main.py index dee284982..52bbbcebf 100644 --- a/src/main.py +++ b/src/main.py @@ -2,7 +2,7 @@ import re import sys from dataclasses import asdict -from typing import Any, cast, Dict, Iterator, Optional +from typing import Any, Dict, Iterator, Optional, cast from deepdiff import DeepDiff from hexbytes import HexBytes @@ -18,14 +18,28 @@ from src.modules.csm.csm import CSOracle from src.modules.ejector.ejector import Ejector from src.providers.execution.contracts.base_oracle import BaseOracleContract -from src.providers.ipfs import GW3, IPFSProvider, Kubo, MultiIPFSProvider, Pinata, PublicIPFS +from src.providers.ipfs import ( + GW3, + IPFSProvider, + Kubo, + MultiIPFSProvider, + Pinata, + PublicIPFS, +) from src.types import BlockRoot, OracleModule, SlotNumber from src.utils.blockstamp import build_blockstamp from src.utils.build import get_build_info from src.utils.exception import IncompatibleException from src.web3py.contract_tweak import tweak_w3_contracts -from src.web3py.extensions import (ConsensusClientModule, FallbackProviderModule, KeysAPIClientModule, LazyCSM, LidoContracts, - LidoValidatorsProvider, TransactionUtils) +from src.web3py.extensions import ( + ConsensusClientModule, + FallbackProviderModule, + KeysAPIClientModule, + LazyCSM, + LidoContracts, + LidoValidatorsProvider, + TransactionUtils, +) from src.web3py.middleware import add_requests_metric_middleware from src.web3py.types import Web3 @@ -151,7 +165,7 @@ def get_processing_started_logs(from_block: int, to_block: int): def run_on_refslot(module_name: OracleModule, limit, block_offset): - #logging.getLogger().setLevel(logging.WARNING) + logging.getLogger().setLevel(logging.WARNING) w3 = _construct_web3() instance: Accounting | Ejector | CSOracle = _construct_module(w3, module_name, True) instance.check_contract_configs()