From 611717555fbcd379d81a92957474fc6b72ea2482 Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Mon, 13 Oct 2025 16:58:14 +0300 Subject: [PATCH 1/6] feat(orc-460): Add pinata dedicated gateways support --- README.md | 80 ++++++++++++++++++------------------ src/main.py | 4 +- src/providers/ipfs/pinata.py | 31 ++++++++++++-- src/variables.py | 3 ++ 4 files changed, 75 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 047a1afaa..7ad2b37bf 100644 --- a/README.md +++ b/README.md @@ -177,45 +177,47 @@ In manual mode all sleeps are disabled and `ALLOW_REPORTING_IN_BUNKER_MODE` is T ## Env variables -| Name | Description | Required | Example value | -|--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------------------------------------------| -| `EXECUTION_CLIENT_URI` | URI of the Execution Layer client | True | `http://localhost:8545` | -| `CONSENSUS_CLIENT_URI` | URI of the Consensus Layer client | True | `http://localhost:5052` | -| `KEYS_API_URI` | URI of the Keys API | True | `http://localhost:8080` | -| `LIDO_LOCATOR_ADDRESS` | Address of the Lido contract | True | `0x1...` | -| `CSM_MODULE_ADDRESS` | Address of the CSModule contract | CSM only | `0x1...` | -| `MEMBER_PRIV_KEY` | Private key of the Oracle member account | False | `0x1...` | -| `MEMBER_PRIV_KEY_FILE` | A path to the file contained the private key of the Oracle member account. It takes precedence over `MEMBER_PRIV_KEY` | False | `/app/private_key` | -| `PINATA_JWT` | JWT token to access pinata.cloud IPFS provider | CSM only | `aBcD1234...` | -| `PINATA_JWT_FILE` | A path to a file with a JWT token to access pinata.cloud IPFS provider | CSM only | `/app/pintata_secret` | -| `KUBO_HOST` | Host to access running Kubo IPFS node | CSM only | `localhost` | -| `KUBO_RPC_PORT` | Port to access RPC provided by Kubo IPFS node | CSM only | `5001` | -| `KUBO_GATEWAY_PORT` | Port to access gateway provided by Kubo IPFS node | CSM only | `8080` | -| `FINALIZATION_BATCH_MAX_REQUEST_COUNT` | The size of the batch to be finalized per request (The larger the batch size, the more memory of the contract is used but the fewer requests are needed) | False | `1000` | -| `EL_REQUESTS_BATCH_SIZE` | The amount of entities that would be fetched in one request to EL | False | `1000` | -| `ALLOW_REPORTING_IN_BUNKER_MODE` | Allow the Oracle to do report if bunker mode is active | False | `True` | -| `DAEMON` | If False Oracle runs one cycle and ask for manual input to send report. | False | `True` | -| `TX_GAS_ADDITION` | Used to modify gas parameter that used in transaction. (gas = estimated_gas + TX_GAS_ADDITION) | False | `100000` | -| `CYCLE_SLEEP_IN_SECONDS` | The time between cycles of the oracle's activity | False | `12` | -| `MAX_CYCLE_LIFETIME_IN_SECONDS` | The maximum time for a cycle to continue | False | `3000` | -| `SUBMIT_DATA_DELAY_IN_SLOTS` | The difference in slots between submit data transactions from Oracles. It is used to prevent simultaneous sending of transactions and, as a result, transactions revert. | False | `6` | -| `HTTP_REQUEST_TIMEOUT_EXECUTION` | Timeout for HTTP execution layer requests | False | `120` | -| `HTTP_REQUEST_TIMEOUT_CONSENSUS` | Timeout for HTTP consensus layer requests | False | `300` | -| `HTTP_REQUEST_RETRY_COUNT_CONSENSUS` | Total number of retries to fetch data from endpoint for consensus layer requests | False | `5` | -| `HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_CONSENSUS` | The delay http provider sleeps if API is stuck for consensus layer | False | `12` | -| `HTTP_REQUEST_TIMEOUT_KEYS_API` | Timeout for HTTP keys api requests | False | `120` | -| `HTTP_REQUEST_RETRY_COUNT_KEYS_API` | Total number of retries to fetch data from endpoint for keys api requests | False | `300` | -| `HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_KEYS_API` | The delay http provider sleeps if API is stuck for keys api | False | `300` | -| `HTTP_REQUEST_TIMEOUT_IPFS` | Timeout for HTTP requests to an IPFS provider | False | `30` | -| `HTTP_REQUEST_RETRY_COUNT_IPFS` | Total number of retries to fetch data from an IPFS provider | False | `3` | -| `EVENTS_SEARCH_STEP` | Maximum length of a range for eth_getLogs method calls | False | `10000` | -| `PRIORITY_FEE_PERCENTILE` | Priority fee percentile from prev block that would be used to send tx | False | `3` | -| `MIN_PRIORITY_FEE` | Min priority fee that would be used to send tx | False | `50000000` | -| `MAX_PRIORITY_FEE` | Max priority fee that would be used to send tx | False | `100000000000` | -| `CSM_ORACLE_MAX_CONCURRENCY` | Max count of dedicated workers for CSM module | False | `2` | -| `CACHE_PATH` | Directory to store cache for CSM module | False | `.` | -| `OPSGENIE_API_KEY` | OpsGenie API key for authentication with the OpsGenie API. Used to send alerts from lido-oracle health-checks. | False | `` | -| `OPSGENIE_API_URL` | Base URL for the OpsGenie API. | False | `http://localhost:8080` | +| Name | Description | Required | Example value | +|--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------------------------------------| +| `EXECUTION_CLIENT_URI` | URI of the Execution Layer client | True | `http://localhost:8545` | +| `CONSENSUS_CLIENT_URI` | URI of the Consensus Layer client | True | `http://localhost:5052` | +| `KEYS_API_URI` | URI of the Keys API | True | `http://localhost:8080` | +| `LIDO_LOCATOR_ADDRESS` | Address of the Lido contract | True | `0x1...` | +| `CSM_MODULE_ADDRESS` | Address of the CSModule contract | CSM only | `0x1...` | +| `MEMBER_PRIV_KEY` | Private key of the Oracle member account | False | `0x1...` | +| `MEMBER_PRIV_KEY_FILE` | A path to the file contained the private key of the Oracle member account. It takes precedence over `MEMBER_PRIV_KEY` | False | `/app/private_key` | +| `PINATA_JWT` | JWT token to access pinata.cloud IPFS provider | CSM only | `aBcD1234...` | +| `PINATA_JWT_FILE` | A path to a file with a JWT token to access pinata.cloud IPFS provider | CSM only | `/app/pintata_secret` | +| `PINATA_DEDICATED_GATEWAY_URL` | URL of the dedicated Pinata gateway (required for Pinata provider, fallback to public gateway if dedicated fails) | CSM only | `https://gateway.pinata.cloud` | +| `PINATA_DEDICATED_GATEWAY_TOKEN` | Token for accessing dedicated Pinata gateway (required for Pinata provider) | CSM only | `gAT_abc123...` | +| `KUBO_HOST` | Host to access running Kubo IPFS node | CSM only | `localhost` | +| `KUBO_RPC_PORT` | Port to access RPC provided by Kubo IPFS node | CSM only | `5001` | +| `KUBO_GATEWAY_PORT` | Port to access gateway provided by Kubo IPFS node | CSM only | `8080` | +| `FINALIZATION_BATCH_MAX_REQUEST_COUNT` | The size of the batch to be finalized per request (The larger the batch size, the more memory of the contract is used but the fewer requests are needed) | False | `1000` | +| `EL_REQUESTS_BATCH_SIZE` | The amount of entities that would be fetched in one request to EL | False | `1000` | +| `ALLOW_REPORTING_IN_BUNKER_MODE` | Allow the Oracle to do report if bunker mode is active | False | `True` | +| `DAEMON` | If False Oracle runs one cycle and ask for manual input to send report. | False | `True` | +| `TX_GAS_ADDITION` | Used to modify gas parameter that used in transaction. (gas = estimated_gas + TX_GAS_ADDITION) | False | `100000` | +| `CYCLE_SLEEP_IN_SECONDS` | The time between cycles of the oracle's activity | False | `12` | +| `MAX_CYCLE_LIFETIME_IN_SECONDS` | The maximum time for a cycle to continue | False | `3000` | +| `SUBMIT_DATA_DELAY_IN_SLOTS` | The difference in slots between submit data transactions from Oracles. It is used to prevent simultaneous sending of transactions and, as a result, transactions revert. | False | `6` | +| `HTTP_REQUEST_TIMEOUT_EXECUTION` | Timeout for HTTP execution layer requests | False | `120` | +| `HTTP_REQUEST_TIMEOUT_CONSENSUS` | Timeout for HTTP consensus layer requests | False | `300` | +| `HTTP_REQUEST_RETRY_COUNT_CONSENSUS` | Total number of retries to fetch data from endpoint for consensus layer requests | False | `5` | +| `HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_CONSENSUS` | The delay http provider sleeps if API is stuck for consensus layer | False | `12` | +| `HTTP_REQUEST_TIMEOUT_KEYS_API` | Timeout for HTTP keys api requests | False | `120` | +| `HTTP_REQUEST_RETRY_COUNT_KEYS_API` | Total number of retries to fetch data from endpoint for keys api requests | False | `300` | +| `HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_KEYS_API` | The delay http provider sleeps if API is stuck for keys api | False | `300` | +| `HTTP_REQUEST_TIMEOUT_IPFS` | Timeout for HTTP requests to an IPFS provider | False | `30` | +| `HTTP_REQUEST_RETRY_COUNT_IPFS` | Total number of retries to fetch data from an IPFS provider | False | `3` | +| `EVENTS_SEARCH_STEP` | Maximum length of a range for eth_getLogs method calls | False | `10000` | +| `PRIORITY_FEE_PERCENTILE` | Priority fee percentile from prev block that would be used to send tx | False | `3` | +| `MIN_PRIORITY_FEE` | Min priority fee that would be used to send tx | False | `50000000` | +| `MAX_PRIORITY_FEE` | Max priority fee that would be used to send tx | False | `100000000000` | +| `CSM_ORACLE_MAX_CONCURRENCY` | Max count of dedicated workers for CSM module | False | `2` | +| `CACHE_PATH` | Directory to store cache for CSM module | False | `.` | +| `OPSGENIE_API_KEY` | OpsGenie API key for authentication with the OpsGenie API. Used to send alerts from lido-oracle health-checks. | False | `` | +| `OPSGENIE_API_URL` | Base URL for the OpsGenie API. | False | `http://localhost:8080` | ### Mainnet variables > LIDO_LOCATOR_ADDRESS=0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb diff --git a/src/main.py b/src/main.py index 44c39ea5b..0e507a744 100644 --- a/src/main.py +++ b/src/main.py @@ -144,10 +144,12 @@ def ipfs_providers() -> Iterator[IPFSProvider]: timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS, ) - if variables.PINATA_JWT: + if variables.PINATA_JWT and variables.PINATA_DEDICATED_GATEWAY_URL and variables.PINATA_DEDICATED_GATEWAY_TOKEN: yield Pinata( variables.PINATA_JWT, timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS, + dedicated_gateway_url=variables.PINATA_DEDICATED_GATEWAY_URL, + dedicated_gateway_token=variables.PINATA_DEDICATED_GATEWAY_TOKEN, ) yield PublicIPFS(timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS) diff --git a/src/providers/ipfs/pinata.py b/src/providers/ipfs/pinata.py index d7de7632f..2ee59ec4d 100644 --- a/src/providers/ipfs/pinata.py +++ b/src/providers/ipfs/pinata.py @@ -1,5 +1,6 @@ import logging from json import JSONDecodeError +from urllib.parse import urljoin import requests @@ -15,17 +16,41 @@ class Pinata(IPFSProvider): """pinata.cloud IPFS provider""" API_ENDPOINT = "https://api.pinata.cloud" - GATEWAY = "https://gateway.pinata.cloud" + PUBLIC_GATEWAY = "https://gateway.pinata.cloud" - def __init__(self, jwt_token: str, *, timeout: int) -> None: + def __init__(self, jwt_token: str, *, timeout: int, dedicated_gateway_url: str, dedicated_gateway_token: str) -> None: super().__init__() validate_jwt(jwt_token) self.timeout = timeout self.session = requests.Session() self.session.headers["Authorization"] = f"Bearer {jwt_token}" + self.dedicated_gateway_url = dedicated_gateway_url + self.dedicated_gateway_token = dedicated_gateway_token + self.max_dedicated_gateway_failures = 2 def fetch(self, cid: CID) -> bytes: - url = f"{self.GATEWAY}/ipfs/{cid}" + for attempt in range(self.max_dedicated_gateway_failures): + try: + return self._fetch_from_dedicated_gateway(cid) + except requests.RequestException as ex: + logger.warning({ + "msg": "Dedicated gateway failed, trying public gateway", + "error": str(ex), + "failures": attempt + 1 + }) + + return self._fetch_from_public_gateway(cid) + + def _fetch_from_dedicated_gateway(self, cid: CID) -> bytes: + url = urljoin(self.dedicated_gateway_url, f"/ipfs/{cid}") + headers = {"x-pinata-gateway-token": self.dedicated_gateway_token} + + resp = requests.get(url, headers=headers, timeout=self.timeout) + resp.raise_for_status() + return resp.content + + def _fetch_from_public_gateway(self, cid: CID) -> bytes: + url = f"{self.PUBLIC_GATEWAY}/ipfs/{cid}" try: resp = requests.get(url, timeout=self.timeout) resp.raise_for_status() diff --git a/src/variables.py b/src/variables.py index 685ead3c1..4b1098249 100644 --- a/src/variables.py +++ b/src/variables.py @@ -13,6 +13,8 @@ KEYS_API_URI: Final = os.getenv('KEYS_API_URI', '').split(',') PINATA_JWT: Final = from_file_or_env('PINATA_JWT') +PINATA_DEDICATED_GATEWAY_URL: Final = os.getenv('PINATA_DEDICATED_GATEWAY_URL') +PINATA_DEDICATED_GATEWAY_TOKEN: Final = from_file_or_env('PINATA_DEDICATED_GATEWAY_TOKEN') KUBO_HOST: Final = os.getenv('KUBO_HOST') KUBO_GATEWAY_PORT: Final = int(os.getenv('KUBO_GATEWAY_PORT', 8080)) KUBO_RPC_PORT: Final = int(os.getenv('KUBO_RPC_PORT', 5001)) @@ -157,6 +159,7 @@ def raise_from_errors(errors): 'CONSENSUS_CLIENT_URI': CONSENSUS_CLIENT_URI, 'KEYS_API_URI': KEYS_API_URI, 'PINATA_JWT': PINATA_JWT, + 'PINATA_DEDICATED_GATEWAY_TOKEN': PINATA_DEDICATED_GATEWAY_TOKEN, 'MEMBER_PRIV_KEY': MEMBER_PRIV_KEY, 'OPSGENIE_API_KEY': OPSGENIE_API_KEY, 'OPSGENIE_API_URL': OPSGENIE_API_URL, From 3a92c1d16afaa08cec657e42d839840181a6a1b1 Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Mon, 13 Oct 2025 18:40:06 +0300 Subject: [PATCH 2/6] feat(orc-460): Add pinata dedicated gateway tests --- src/providers/ipfs/pinata.py | 5 +- tests/providers/test_pinata.py | 86 ++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 tests/providers/test_pinata.py diff --git a/src/providers/ipfs/pinata.py b/src/providers/ipfs/pinata.py index 2ee59ec4d..a7e93995f 100644 --- a/src/providers/ipfs/pinata.py +++ b/src/providers/ipfs/pinata.py @@ -17,6 +17,7 @@ class Pinata(IPFSProvider): API_ENDPOINT = "https://api.pinata.cloud" PUBLIC_GATEWAY = "https://gateway.pinata.cloud" + MAX_DEDICATED_GATEWAY_FAILURES = 2 def __init__(self, jwt_token: str, *, timeout: int, dedicated_gateway_url: str, dedicated_gateway_token: str) -> None: super().__init__() @@ -26,10 +27,9 @@ def __init__(self, jwt_token: str, *, timeout: int, dedicated_gateway_url: str, self.session.headers["Authorization"] = f"Bearer {jwt_token}" self.dedicated_gateway_url = dedicated_gateway_url self.dedicated_gateway_token = dedicated_gateway_token - self.max_dedicated_gateway_failures = 2 def fetch(self, cid: CID) -> bytes: - for attempt in range(self.max_dedicated_gateway_failures): + for attempt in range(self.MAX_DEDICATED_GATEWAY_FAILURES): try: return self._fetch_from_dedicated_gateway(cid) except requests.RequestException as ex: @@ -44,7 +44,6 @@ def fetch(self, cid: CID) -> bytes: def _fetch_from_dedicated_gateway(self, cid: CID) -> bytes: url = urljoin(self.dedicated_gateway_url, f"/ipfs/{cid}") headers = {"x-pinata-gateway-token": self.dedicated_gateway_token} - resp = requests.get(url, headers=headers, timeout=self.timeout) resp.raise_for_status() return resp.content diff --git a/tests/providers/test_pinata.py b/tests/providers/test_pinata.py new file mode 100644 index 000000000..746a8f788 --- /dev/null +++ b/tests/providers/test_pinata.py @@ -0,0 +1,86 @@ +import pytest +import responses +from unittest.mock import patch + +from src.providers.ipfs.pinata import Pinata +from src.providers.ipfs.types import FetchError + + +@pytest.fixture +def pinata_provider(): + return Pinata( + jwt_token="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyX2lkIjoxLCJ1c2VybmFtZSI6InRlc3QiLCJleHAiOjk5OTk5OTk5OTl9.Ps6jFKniFhNMYr_4WgETZP_LcXEfSzg3yUhNBn6Xgok", + timeout=30, + dedicated_gateway_url="https://dedicated.gateway.com", + dedicated_gateway_token="dedicated_token_123", + ) + + +@pytest.mark.unit +@responses.activate +def test_fetch__dedicated_gateway_available__returns_content_from_dedicated(pinata_provider): + responses.add(responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", body=b'test content', status=200) + + result = pinata_provider.fetch("QmTest123") + + assert result == b'test content' + assert len(responses.calls) == 1 + request_headers = responses.calls[0].request.headers + assert request_headers.get("x-pinata-gateway-token") == "dedicated_token_123" + + +@pytest.mark.unit +@responses.activate +def test_fetch__dedicated_gateway_fails_max_attempts__falls_back_to_public(pinata_provider): + responses.add( + responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Gateway error"}, status=500 + ) + responses.add( + responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Gateway error"}, status=500 + ) + responses.add(responses.GET, "https://gateway.pinata.cloud/ipfs/QmTest123", body=b'public content', status=200) + + with patch('src.providers.ipfs.pinata.logger') as mock_logger: + result = pinata_provider.fetch("QmTest123") + + assert result == b'public content' + assert len(responses.calls) == 3 + assert responses.calls[0].request.headers.get("x-pinata-gateway-token") == "dedicated_token_123" + assert responses.calls[1].request.headers.get("x-pinata-gateway-token") == "dedicated_token_123" + assert "x-pinata-gateway-token" not in responses.calls[2].request.headers + assert mock_logger.warning.call_count == 2 + + +@pytest.mark.unit +@responses.activate +def test_fetch__dedicated_gateway_fails_once__retries_and_succeeds(pinata_provider): + responses.add( + responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "First failure"}, status=500 + ) + responses.add(responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", body=b'dedicated success', status=200) + + with patch('src.providers.ipfs.pinata.logger') as mock_logger: + result = pinata_provider.fetch("QmTest123") + + assert result == b'dedicated success' + assert len(responses.calls) == 2 + assert mock_logger.warning.call_count == 1 + + +@pytest.mark.unit +@responses.activate +def test_fetch__both_gateways_fail__raises_fetch_error(pinata_provider): + responses.add( + responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Dedicated error"}, status=500 + ) + responses.add( + responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Dedicated error"}, status=500 + ) + responses.add( + responses.GET, "https://gateway.pinata.cloud/ipfs/QmTest123", json={"error": "Public error"}, status=500 + ) + + with pytest.raises(FetchError): + pinata_provider.fetch("QmTest123") + + assert len(responses.calls) == 3 From e8dbac0f82e3061e2f592fa1b0d49051888c2977 Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Mon, 13 Oct 2025 23:44:47 +0300 Subject: [PATCH 3/6] feat(orc-460): fix flaky test --- .../integration/contracts/test_cs_parameters_registry.py | 8 ++++---- tests/integration/contracts/test_lido.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/contracts/test_cs_parameters_registry.py b/tests/integration/contracts/test_cs_parameters_registry.py index accd2eedd..bc571df85 100644 --- a/tests/integration/contracts/test_cs_parameters_registry.py +++ b/tests/integration/contracts/test_cs_parameters_registry.py @@ -15,10 +15,10 @@ def test_cs_parameters_registry(cs_params_contract, caplog): check_contract( cs_params_contract, [ - ("get_performance_coefficients", None, check_is_instance_of(PerformanceCoefficients)), - ("get_reward_share_data", None, check_is_instance_of(KeyNumberValueIntervalList)), - ("get_performance_leeway_data", None, check_is_instance_of(KeyNumberValueIntervalList)), - ("get_strikes_params", None, check_is_instance_of(StrikesParams)), + ("get_performance_coefficients", (0,), check_is_instance_of(PerformanceCoefficients)), + ("get_reward_share_data", (0,), check_is_instance_of(KeyNumberValueIntervalList)), + ("get_performance_leeway_data", (0,), check_is_instance_of(KeyNumberValueIntervalList)), + ("get_strikes_params", (0,), check_is_instance_of(StrikesParams)), ], caplog, ) diff --git a/tests/integration/contracts/test_lido.py b/tests/integration/contracts/test_lido.py index b76e72b40..99fb00bd8 100644 --- a/tests/integration/contracts/test_lido.py +++ b/tests/integration/contracts/test_lido.py @@ -17,15 +17,15 @@ def test_lido_contract_call(lido_contract, accounting_oracle_contract, burner_co ( 1746275159, # timestamp 86400, - 389746, - 9190764598468942000000000, + 403105, # Updated to match current beacon_validators count + 8461615483077294000000000, # Updated to match current beacon_balance 13771995248000000000, 478072602914417566, 0, accounting_oracle_contract.address, 11620928, # Call depends on contract state - '0xffa34bcc5a08c92272a62e591f7afb9cb839134aa08c091ae0c95682fba35da9', + '0x4b7cf7dbb70179f2e0b6891972fa55903577a11008f6c87a309183829c8aebf1', ), lambda response: check_value_type(response, LidoReportRebase), ), From 89e4ca79bf546adf7ed88a96c0c0d2d929666a81 Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Tue, 14 Oct 2025 14:47:42 +0300 Subject: [PATCH 4/6] feat(orc-460): moved to urlib Retry --- src/providers/ipfs/pinata.py | 36 ++++++++++++++++++++++------------ tests/providers/test_pinata.py | 26 +++++++++++++++++------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/providers/ipfs/pinata.py b/src/providers/ipfs/pinata.py index a7e93995f..207f39023 100644 --- a/src/providers/ipfs/pinata.py +++ b/src/providers/ipfs/pinata.py @@ -3,6 +3,8 @@ from urllib.parse import urljoin import requests +from requests.adapters import HTTPAdapter +from urllib3 import Retry from src.utils.jwt import validate_jwt @@ -23,28 +25,36 @@ def __init__(self, jwt_token: str, *, timeout: int, dedicated_gateway_url: str, super().__init__() validate_jwt(jwt_token) self.timeout = timeout + self.session = requests.Session() self.session.headers["Authorization"] = f"Bearer {jwt_token}" + + dedicated_adapter = HTTPAdapter(max_retries=Retry( + total=self.MAX_DEDICATED_GATEWAY_FAILURES - 1, + status_forcelist=list(range(400, 600)), + backoff_factor=3.0, + )) + self.dedicated_session = requests.Session() + self.dedicated_session.headers["x-pinata-gateway-token"] = dedicated_gateway_token + self.dedicated_session.mount("https://", dedicated_adapter) + self.dedicated_session.mount("http://", dedicated_adapter) + self.dedicated_gateway_url = dedicated_gateway_url self.dedicated_gateway_token = dedicated_gateway_token def fetch(self, cid: CID) -> bytes: - for attempt in range(self.MAX_DEDICATED_GATEWAY_FAILURES): - try: - return self._fetch_from_dedicated_gateway(cid) - except requests.RequestException as ex: - logger.warning({ - "msg": "Dedicated gateway failed, trying public gateway", - "error": str(ex), - "failures": attempt + 1 - }) - - return self._fetch_from_public_gateway(cid) + try: + return self._fetch_from_dedicated_gateway(cid) + except requests.RequestException as ex: + logger.warning({ + "msg": "Dedicated gateway failed after retries, trying public gateway", + "error": str(ex) + }) + return self._fetch_from_public_gateway(cid) def _fetch_from_dedicated_gateway(self, cid: CID) -> bytes: url = urljoin(self.dedicated_gateway_url, f"/ipfs/{cid}") - headers = {"x-pinata-gateway-token": self.dedicated_gateway_token} - resp = requests.get(url, headers=headers, timeout=self.timeout) + resp = self.dedicated_session.get(url, timeout=self.timeout) resp.raise_for_status() return resp.content diff --git a/tests/providers/test_pinata.py b/tests/providers/test_pinata.py index 746a8f788..e7f4d8a94 100644 --- a/tests/providers/test_pinata.py +++ b/tests/providers/test_pinata.py @@ -1,6 +1,5 @@ import pytest import responses -from unittest.mock import patch from src.providers.ipfs.pinata import Pinata from src.providers.ipfs.types import FetchError @@ -40,15 +39,13 @@ def test_fetch__dedicated_gateway_fails_max_attempts__falls_back_to_public(pinat ) responses.add(responses.GET, "https://gateway.pinata.cloud/ipfs/QmTest123", body=b'public content', status=200) - with patch('src.providers.ipfs.pinata.logger') as mock_logger: - result = pinata_provider.fetch("QmTest123") + result = pinata_provider.fetch("QmTest123") assert result == b'public content' assert len(responses.calls) == 3 assert responses.calls[0].request.headers.get("x-pinata-gateway-token") == "dedicated_token_123" assert responses.calls[1].request.headers.get("x-pinata-gateway-token") == "dedicated_token_123" assert "x-pinata-gateway-token" not in responses.calls[2].request.headers - assert mock_logger.warning.call_count == 2 @pytest.mark.unit @@ -59,12 +56,10 @@ def test_fetch__dedicated_gateway_fails_once__retries_and_succeeds(pinata_provid ) responses.add(responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", body=b'dedicated success', status=200) - with patch('src.providers.ipfs.pinata.logger') as mock_logger: - result = pinata_provider.fetch("QmTest123") + result = pinata_provider.fetch("QmTest123") assert result == b'dedicated success' assert len(responses.calls) == 2 - assert mock_logger.warning.call_count == 1 @pytest.mark.unit @@ -84,3 +79,20 @@ def test_fetch__both_gateways_fail__raises_fetch_error(pinata_provider): pinata_provider.fetch("QmTest123") assert len(responses.calls) == 3 + + +@pytest.mark.unit +@responses.activate +def test_fetch__dedicated_gateway_429_rate_limit__retries_and_falls_back_to_public(pinata_provider): + responses.add( + responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Rate limit exceeded"}, status=429 + ) + responses.add( + responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Rate limit exceeded"}, status=429 + ) + responses.add(responses.GET, "https://gateway.pinata.cloud/ipfs/QmTest123", body=b'public content', status=200) + + result = pinata_provider.fetch("QmTest123") + + assert result == b'public content' + assert len(responses.calls) == 3 From 72699d6c466e5153c9089f18e5e4c579cef46909 Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Tue, 14 Oct 2025 17:16:00 +0300 Subject: [PATCH 5/6] feat(orc-460): fix flaky test --- tests/integration/contracts/test_lido.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/integration/contracts/test_lido.py b/tests/integration/contracts/test_lido.py index 99fb00bd8..49cf2406c 100644 --- a/tests/integration/contracts/test_lido.py +++ b/tests/integration/contracts/test_lido.py @@ -18,14 +18,13 @@ def test_lido_contract_call(lido_contract, accounting_oracle_contract, burner_co 1746275159, # timestamp 86400, 403105, # Updated to match current beacon_validators count - 8461615483077294000000000, # Updated to match current beacon_balance + 8462132592019028000000000, # Updated to match current beacon_balance 13771995248000000000, 478072602914417566, 0, accounting_oracle_contract.address, - 11620928, - # Call depends on contract state - '0x4b7cf7dbb70179f2e0b6891972fa55903577a11008f6c87a309183829c8aebf1', + 11620928, # ref_slot + '0x9bad2cb4e0ef017912b8c77e9ce1c6ec52a6b79013fe8d0d099a65a51ee4a66e', # block_identifier ), lambda response: check_value_type(response, LidoReportRebase), ), From fc721b9811ca45b62296b5b737ac03223f1d83b9 Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Wed, 15 Oct 2025 13:09:21 +0300 Subject: [PATCH 6/6] feat(orc-460): use urljoins --- src/providers/ipfs/pinata.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/providers/ipfs/pinata.py b/src/providers/ipfs/pinata.py index 207f39023..423dfe24b 100644 --- a/src/providers/ipfs/pinata.py +++ b/src/providers/ipfs/pinata.py @@ -19,7 +19,7 @@ class Pinata(IPFSProvider): API_ENDPOINT = "https://api.pinata.cloud" PUBLIC_GATEWAY = "https://gateway.pinata.cloud" - MAX_DEDICATED_GATEWAY_FAILURES = 2 + MAX_DEDICATED_GATEWAY_RETRIES = 1 def __init__(self, jwt_token: str, *, timeout: int, dedicated_gateway_url: str, dedicated_gateway_token: str) -> None: super().__init__() @@ -30,7 +30,7 @@ def __init__(self, jwt_token: str, *, timeout: int, dedicated_gateway_url: str, self.session.headers["Authorization"] = f"Bearer {jwt_token}" dedicated_adapter = HTTPAdapter(max_retries=Retry( - total=self.MAX_DEDICATED_GATEWAY_FAILURES - 1, + total=self.MAX_DEDICATED_GATEWAY_RETRIES, status_forcelist=list(range(400, 600)), backoff_factor=3.0, )) @@ -59,7 +59,7 @@ def _fetch_from_dedicated_gateway(self, cid: CID) -> bytes: return resp.content def _fetch_from_public_gateway(self, cid: CID) -> bytes: - url = f"{self.PUBLIC_GATEWAY}/ipfs/{cid}" + url = urljoin(self.PUBLIC_GATEWAY, f'/ipfs/{cid}') try: resp = requests.get(url, timeout=self.timeout) resp.raise_for_status() @@ -74,8 +74,7 @@ def publish(self, content: bytes, name: str | None = None) -> CID: def _upload(self, content: bytes, name: str | None = None) -> str: """Pinata has no dedicated endpoint for uploading, so pinFileToIPFS is used""" - - url = f"{self.API_ENDPOINT}/pinning/pinFileToIPFS" + url = urljoin(self.API_ENDPOINT, '/pinning/pinFileToIPFS') try: with self.session as s: resp = s.post(url, files={"file": content})