diff --git a/public/images/quickstarts/feature/QuickStarts-DataStreams-Transmitter.webp b/public/images/quickstarts/feature/QuickStarts-DataStreams-Transmitter.webp new file mode 100644 index 00000000000..d0ba1372256 Binary files /dev/null and b/public/images/quickstarts/feature/QuickStarts-DataStreams-Transmitter.webp differ diff --git a/public/images/quickstarts/transmitter/transmitter_home.png b/public/images/quickstarts/transmitter/transmitter_home.png new file mode 100644 index 00000000000..12377f1ba70 Binary files /dev/null and b/public/images/quickstarts/transmitter/transmitter_home.png differ diff --git a/src/content.config.ts b/src/content.config.ts index 0da9cf5065d..649edc3f1ae 100644 --- a/src/content.config.ts +++ b/src/content.config.ts @@ -8,6 +8,7 @@ enum Products { FUNCTIONS = "functions", VRF = "vrf", FEEDS = "feeds", + STREAMS = "streams", GENERAL = "general", CHAINLINK_LOCAL = "chainlink-local", } @@ -18,6 +19,7 @@ export const productsInfo: Record = { functions: { name: "Functions", slug: "chainlink-functions" }, vrf: { name: "VRF", slug: "vrf" }, feeds: { name: "Data Feeds", slug: "data-feeds" }, + streams: { name: "Data Streams", slug: "data-streams" }, general: { name: "General", slug: "/" }, "chainlink-local": { name: "Chainlink Local", slug: "chainlink-local" }, } diff --git a/src/content/data-streams/index.mdx b/src/content/data-streams/index.mdx index 89392d9487f..39c930d4bc7 100644 --- a/src/content/data-streams/index.mdx +++ b/src/content/data-streams/index.mdx @@ -91,8 +91,27 @@ Access data directly through REST APIs or WebSocket connections using our SDKs: 1. Implement Verification: Add [onchain verification](/data-streams/reference/onchain-verification) to ensure data authenticity in your smart contracts. -### Streams Trade: An alternative implementation +## Alternative implementations + +### Streams Trade For applications that require automated data retrieval and execution, Streams Trade combines Chainlink Data Streams with [Chainlink Automation](/chainlink-automation) to deliver automated trade execution with frontrunning mitigation. This approach suits dApps that require automated, trust-minimized trade execution and high-frequency market data. [Learn more about Streams Trade](/data-streams/streams-trade). + +### Data Streams Transmitter + + + +While Data Streams provides low-latency data offchain, bringing verified reports onchain requires developers to build and maintain a custom service to monitor the feed and push updates. + +[Data Streams Transmitter](https://github.com/hackbg/chainlink-datastreams-transmitter) is a configurable service that you operate on your own infrastructure. It offers a simplified process for DevOps teams and Infrastructure Managers, handling the core logic of connecting to the off-chain Data Streams network via a WebSocket, monitoring for new reports, and relaying them to your smart contracts. The service also includes a UI for on-the-fly configuration and monitoring. + +Based on predefined conditions like time intervals or price deviations, the Transmitter bundles and sends a report for onchain verification and consumption. This approach suits dApps that need fresh data pushed onchain based on custom logic, without the overhead of building and maintaining the necessary infrastructure. + +[See our quickstart guide for Data Streams Transmitter](/quickstarts/transmitter). diff --git a/src/content/quickstarts/transmitter.mdx b/src/content/quickstarts/transmitter.mdx new file mode 100644 index 00000000000..f435bcbf5e3 --- /dev/null +++ b/src/content/quickstarts/transmitter.mdx @@ -0,0 +1,443 @@ +--- +title: "Automated Data Relay with Data Streams Transmitter" +description: "Set up and manage a self-hosted, automated relay to push offchain Data Streams to your smart contracts." +githubSourceCodeUrl: "https://github.com/hackbg/chainlink-datastreams-transmitter" +image: "QuickStarts-DataStreams-Transmitter.webp" +products: ["streams"] +time: "25 minutes" +requires: "Foundry, Docker, Node.js, pnpm, Redis, Wallet with AVAX, Contract funded with LINK, Data Streams credentials." +--- + +import { Accordion, Address, Aside, ClickToZoom, CodeSample, CopyText } from "@components" +import { TabsContent } from "@components/Tabs" + +import DataStreams from "@features/data-streams/common/DataStreams.astro" + +{/* Common Links */} + +[DOCKER_INSTALL]: https://docs.docker.com/get-started/get-docker/ +[NODE_INSTALL]: https://nodejs.org/en/download/ +[REDIS_INSTALL]: https://redis.io/docs/latest/operate/oss_and_stack/install/archive/install-redis/ +[GITHUB_REPO]: https://github.com/hackbg/chainlink-datastreams-transmitter +[DATA_STREAMS_CONTACT]: https://chainlinkcommunity.typeform.com/datastreams?typeform-source=docs.chain.link#ref_id=docs +[FOUNDRY_INSTALL]: https://book.getfoundry.sh/getting-started/installation + +## Overview + + + +While [Chainlink Data Streams](/data-streams/index) provide access to low-latency market data offchain, the process of bringing this data onchain requires significant engineering. Developers must build, deploy, and maintain a custom infrastructure service that can reliably connect to the Data Streams API, implement robust WebSocket reconnection logic, and manage the onchain transaction logic for when and how to push updates. + +The [Data Streams Transmitter][GITHUB_REPO], a community-led solution, provides a foundational framework for the DevOps teams and Infrastructure Managers tasked with building these systems. Transmitter is a configurable service that handles the core logic of monitoring offchain Data Streams and pushing values to your smart contracts based on predefined conditions, such as time intervals or price deviations. It also includes a web interface to manage your feeds and monitor their status. + +This guide walks you through deploying a `DataStreamsFeed` consumer contract, setting up the Data Streams Transmitter, and configuring it to monitor a specific Data Stream and relay it to your onchain contract. + +--- + +## High-level Architecture + + + +Transmitter relays data from the offchain Chainlink network to your onchain contracts in the following sequence: + +1. **Offchain | Data Generation:** A Chainlink Decentralized Oracle Network (DON) generates and signs low-latency data reports, making them available in the offchain Data Streams Aggregation Network. + +1. **Offchain | Monitoring:** Transmitter, which you operate, continuously monitors this offchain network for new reports via a WebSocket connection. + +1. **Onchain | Transmission:** Based on your predefined conditions (such as a time interval or price deviation), Transmitter bundles the latest report into a transaction and submits it to your designated Oracle Contract. + +1. **Onchain | Verification:** Within that single transaction, your Oracle Contract immediately calls the Chainlink Verifier Contract, which uses its `LINK` allowance to submit the verification request. The Verifier authenticates the report by checking the DON's cryptographic signature. If verification fails, the entire transaction reverts, ensuring no state change occurs. + +1. **Onchain | Consumption:** Upon successful verification, the data is written to your Oracle Contract's storage. This freshly verified, low-latency data is now available onchain for your dApp or other smart contracts to consume. + + + +--- + +## Requirements + + + +Before you start this tutorial, ensure you have the following prerequisites: + +- **Deployment Host**: A stable server environment (cloud or on-premise) is required for a production deployment of the Transmitter service. This guide uses your local machine for initial setup and testing. +- **Software Dependencies**: [Foundry][FOUNDRY_INSTALL], [Docker][DOCKER_INSTALL], [Node.js][NODE_INSTALL], and [Redis][REDIS_INSTALL] installed on your deployment host. This guide uses `pnpm` for package management. +- **Development Wallet**: This guide requires a funded development wallet and its **private key** to send transactions. + - **Network Configuration**: This guide uses **Avalanche Fuji**. You can find RPC URLs on [Chainlist](https://chainlist.org). +- **Testnet Funds**: The wallet must be funded with testnet `AVAX` to pay for transaction fees on the **Avalanche Fuji** network. Testnet `LINK` must be provided to the deployed contract to pay for onchain verification fees. Both are available from the [Chainlink Faucet](https://faucets.chain.link/). +- **API Credentials**: Access to Data Streams requires API credentials. If you haven't already, [contact us][DATA_STREAMS_CONTACT] to request Data Streams access. +- **Web3 Foundation**: A basic understanding of [smart contracts and blockchain development](/quickstarts/deploy-your-first-contract). + +--- + +## Steps to Implement + + +First, deploy a [`DataStreamsFeed.sol` contract](https://github.com/woogieboogie-jl/chainlink-transmitter-contract/blob/main/src/feed/DataStreamsFeed.sol), which receives, verifies, and stores the data relayed by the Transmitter. + +1. Install [Foundry][FOUNDRY_INSTALL], or update it to the latest version by running `foundryup` in your terminal. + +1. Clone the `chainlink-transmitter-contract` repository and navigate into the directory: + + ```bash + git clone https://github.com/woogieboogie-jl/chainlink-transmitter-contract.git + cd chainlink-transmitter-contract + ``` + +1. Install the dependencies using `pnpm` and install/initialize Foundry: + + ```bash + pnpm add --save-dev @openzeppelin/contracts @chainlink/contracts && curl -L https://foundry.paradigm.xyz | bash && forge install + ``` + +1. Create a `.env` file in the project's root directory to store your private key, RPC URL, and link token address: + + ```bash + cat >.env <` with the [RPC URL for the Avalanche Fuji network](https://build.avax.network/docs/quick-start/networks/fuji-testnet#add-avalanche-c-chain-testnet-to-wallet). + + The `LINK_TOKEN` address for Avalanche Fuji is `0x5947BB275c521040051D82396192181b413227A3`. Refer to the [Chainlink documentation for `LINK` token addresses on other networks](/resources/link-token-contracts). + +1. Load the environment variables into your current shell session: + + ```bash + source .env + ``` + +1. Configure `script/DeployDataStreamsFeed.s.sol`. + + This script deploys the `DataStreamsFeed` contract, which will receive data from the Transmitter. + + You must set the constructor arguments for the contract: + + | Parameter | Value | Description | + | :--------------------- | :------------------------------------------------------------------- | :----------------------------------------- | + | `verifierProxyAddress` | `0x2bf612C65f5a4d388E687948bb2CF842FFb8aBB3` | Avalanche Fuji Verifier | + | `feedId` | `0x000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba782` | ETH/USD | + | `decimals` | `18` | The number of decimals for the data. | + | `description` | `"ETH / USD Feed"` | A human-readable description for the feed. | + +1. Deploy the contract via the command line. + + Run the deployment script to deploy to **Avalanche Fuji**: + ```sh + forge script script/DeployDataStreamsFeed.s.sol:DeployDataStreamsFeed \ + --rpc-url $RPC_URL_AVAX_FUJI \ + --private-key $PRIVATE_KEY \ + --broadcast + ``` + After deployment, copy the **Deployed Contract Address** and **ABI** from the output. You will need them to configure the Transmitter. + - **Contract Address:** Find the contract address in your terminal output under `deployedAddress`. + - **Contract ABI:** Foundry saves the ABI in `out/DataStreamsFeed.sol/DataStreamsFeed.json`. + + Before we can use the Transmitter, we need to fund this contract with `LINK`. + +1. Fund the contract with `LINK`. + + After deployment, transfer some testnet `LINK` to the feed contract address, which allows the contract to pay for verification fees. + + ```sh + # Example using cast to transfer 5 LINK (18 decimals) + cast send $LINK_TOKEN "transfer(address,uint256)" \ + 5000000000000000000 \ + --rpc-url $RPC_URL_AVAX_FUJI \ + --private-key $PRIVATE_KEY + ``` + + + + +With a consumer contract onchain, you can now set up the Transmitter service to relay data to it. + +1. In a new terminal window, navigate to your development space and clone the [Data Stream Transmitter repository][GITHUB_REPO]: + + ```bash + git clone https://github.com/hackbg/chainlink-datastreams-transmitter.git + cd chainlink-datastreams-transmitter + ``` + +1. Install the project dependencies using `npm`: + + ```bash + npm install + ``` + +1. Copy the example environment file: + + ```bash + cp .env.example .env + ``` + +1. Open your `.env` file in a text editor and provide the required values: + | Variable | Description | + | :-------------------------- | :------------------------------------------------------------------------------------ | + | `PRIVATE_KEY` | The private key of your developer wallet (ensure this is kept secure and not shared). | + | `DATASTREAMS_CLIENT_ID` | Your Data Streams client ID. | + | `DATASTREAMS_CLIENT_SECRET` | Your Data Streams client secret. | + + + +1. Create the runtime configuration file by copying the example: + + ```bash + cp config-chainlink-verify-example.yml config.yml + ``` + +1. Open `config.yml` in a text editor. Update the `targetChains` section to point to the contract you deployed on **Avalanche Fuji**. + + ```yaml + feeds: + - name: "ETH/USD" + feedId: "0x000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba782" + + # --- Default Global Settings --- + + # The default chainId to use for transactions if not specified in a target. + chainId: 43113 # Default to Avalanche Fuji + + # The maximum gas limit you are willing to spend on a transaction. + gasCap: "250000" + + # Cron expression defining the data update frequency. + # This example runs every 30 seconds. + interval: "*/30 * * * * *" + + # The minimum price change percentage to trigger an on-chain update. + # This example is set to 0.1% + priceDeltaPercentage: 0.001 + + # --- Chain & Verifier Definitions --- + + # A list of all supported blockchain networks with their RPC URLs. + chains: + - id: 43113 + name: "Avalanche Fuji Testnet" + currencyName: "Fuji AVAX" + currencySymbol: "AVAX" + currencyDecimals: 18 + rpc: "https://api.avax-test.network/ext/bc/C/rpc" # <-- TODO: Replace with your own reliable RPC URL + testnet: true + - id: 421614 + name: "Arbitrum Sepolia" + currencyName: "Arbitrum Sepolia Ether" + currencySymbol: "ETH" + currencyDecimals: 18 + rpc: "https://sepolia-rollup.arbitrum.io/rpc" # <-- TODO: Replace with your own reliable RPC URL + testnet: true + + # The addresses of the official Chainlink Verifier contracts on each network. + verifierAddresses: + - chainId: 43113 + address: "0x2bf612C65f5a4d388E687948bb2CF842FFb8aBB3" + - chainId: 421614 + address: "0x2ff010DEbC1297f19579B4246cad07bd24F2488A" + + # --- On-Chain Target Configurations --- + + # This section defines which smart contracts to call for each feed on each chain. + # You can have multiple targets for the same feed. + targetChains: + - chainId: 43113 # Target is on Avalanche Fuji + targetContracts: + # This configuration sends ETH/USD data to a contract on Fuji + - feedId: "0x000359843a543ee2fe414dc14c7e7920ef10f4372990b79d6361cdc0dd1ba782" + address: "0xYourDataStreamsFeedContractOnFuji" # <-- TODO: Update with your deployed contract address + # The name of the function to call on your smart contract + functionName: "verifyAndUpdateReport" + # The arguments the Transmitter should prepare and send to the function + # â€ĸ rawReport = the unverified payload from the Data Streams websocket + # â€ĸ parameterPayload = abi.encode(address feeToken) – produced automatically by the transmitter + functionArgs: ["rawReport", "parameterPayload"] + # The ABI for the target function, required to encode the transaction + abi: + - name: "verifyAndUpdateReport" + type: "function" + stateMutability: "nonpayable" + inputs: + - { "internalType": "bytes", "name": "unverifiedReportData", "type": "bytes" } + - { "internalType": "bytes", "name": "parameterPayload", "type": "bytes" } + outputs: [] + # Off-chain verification must run so keep skipVerify false (default) + skipVerify: false + ``` + + - Replace `0xYOUR_AVALANCHE_FUJI_CONTRACT_ADDRESS` with the address of the contract you deployed in the previous step. + - The `feedId` should match the one you set in the contract deployment. + - Each target block must keep `skipVerify: false`, `functionName: verifyAndUpdateReport`, and `functionArgs: ["rawReport", "parameterPayload"]` to ensure the Transmitter correctly prepares the transaction for onchain verification. + +1. Start the Docker daemon. + + The Docker daemon is required to run the Redis instance that Transmitter uses. Starting the daemon depends on your operating system. Please refer to the [Docker documentation](https://docs.docker.com/engine/daemon/start/) for instructions. + +1. Run the following command to start the Transmitter and its services in the background: + + ```bash + docker compose up -d + ``` + +1. Open your web browser and navigate to `http://localhost:3000` to access the Transmitter UI. Because you pre-configured the contract in `config.yml`, the `ETH/USD` stream on the dashboard should show a `Running` status. + + + + +--- + +## UI Overview + +The Transmitter UI lets you manage your feeds, monitor their status, and define how and where data is pushed onchain. Although we configured the basics in `config.yml`, you can manage the service from this dashboard. + + + +The optional sections below provide a detailed overview of the Transmitter UI, explaining each section and its purpose. + + + +On the upper-right corner of the page you will find the **Chain** section. Since the default chain is configured to `Avalanche Fuji`, you should see that network listed along with details for your wallet. + +| Field | Description | +| :----------------------- | :---------------------------------------------------------------------------------- | +| **Current Chain** | The current chain, which is where data will be pushed. | +| **Chain ID** | The unique numeric identifier for the current chain. | +| **Wallet Address** | Your unique wallet address. | +| **Native Token Balance** | Your native token balance, which is used to pay for onchain transaction fees (gas). | +| **LINK Balance** | Your `LINK` token balance, used to pay for Chainlink Data Streams transactions. | + +Your balance should reflect the amount of testnet LINK and AVAX you have in your wallet. If you need more, you can use the [Chainlink Faucet](https://faucets.chain.link/). + + + + + +On the top-right corner of the page, you will find the **Logs** section, which contains details for all Transmitter actions. When operating correctly, the log should reflect a clear, step-by-step process for each onchain update. + +Example log output: + +```bash +[2025-07-01 16:48:44] info: đŸŠē Health check endpoint running at http://localhost:8081/ready +[2025-07-01 16:48:44] info: 🚀 running at http://localhost:3000 +[2025-07-01 16:48:44] warn: ℹī¸ App already configured +[2025-07-01 16:48:44] info: 🏁 Streams have been started +[2025-07-01 16:50:00] info: 🚨 Price deviation detected | ETH/USD: 3450.06$ | 📉 -0.2135% (-7.36$) +[2025-07-01 16:50:00] info: 📆 Scheduled for writing onchain +[2025-07-01 16:50:00] info: â›Ŋī¸ Estimated fee: 0.024395523392104215 LINK # This fee is deducted from the contract's LINK allowance, NOT the sender's balance +[2025-07-01 16:50:04] info: ✅ Report verified +[2025-07-01 16:50:06] info: 📝 Prepared verification transaction +[2025-07-01 16:50:07] info: ⌛ī¸ Sending transaction 0xa59aa3e9c7a2cd020d79bd73e6c7f1aca9a576b77edb43ee35180001709fdf43 +[2025-07-01 16:50:08] info: ℹī¸ Transaction status: success +[2025-07-01 16:50:08] info: 💾 Price stored | ETH/USD: 3450.06$ +``` + + + + + +The **Data Streams** section lists all Data Streams, their status, and any related contracts. You should see the `ETH/USD` feed listed here, which was set in the `config.yml` file. + +| Field | Description | +| :---------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Stream** | The name of the stream. This is the name set by the user for easier tracking and feed identification. | +| **Feed ID** | The feed ID. Check [Chainlink Data Streams Documentation](/data-streams/crypto-streams) for a list of supported streams. | +| **Contract** | Target contract for the feed on the current chain. Click to open a configuration page where you can define the target contract, functions, and ABI. | +| **Saved price** | The latest price recorded onchain. | +| **Last reported** | The latest price reported by the stream. | +| **Status** | Current status of the stream. Possible states include: `Running`, `Connecting`, `Stopping` or `Stopped`. | +| **Remove** | Click to remove the stream. | + + + + + +The **Schedule** section allows you to define how often the Transmitter should push data onchain, using a cron expression. We have already defined this in the `config.yml` file, but you can change it here. + + + + + +The **Verifier Contracts** section lets you specify contracts that will verify the data being pushed onchain. The verifier contract for your current network will be automatically selected based on `config.yml`. + + + + + +The **Price Delta** section lets you define the price change threshold that will trigger an onchain update. We set this in the `config.yml` file, but you can change this value here. + + + + + +The **Gas Cap** section lets you set the maximum gas limit for transactions. We set this in the `config.yml` file, but you can change it here. + + + +--- + +## Troubleshooting + +If you run into issues during setup or operation, refer to these common problems and their solutions. + +--- + +### Transmitter Service Issues + +- **Service fails to start or crashes immediately** + + - Double-check that your `chainlink-datastreams-transmitter/.env` file is present and correctly populated with your `PRIVATE_KEY` and Data Streams credentials. + - Run `docker compose logs -f` in the `chainlink-datastreams-transmitter` directory to view real-time logs and identify specific error messages. + +- **Data is not updating on your contract** + - Review the logs in Transmitter for specific error messages related to the feed or contract interaction. + - **Authentication:** Verify that your `DATASTREAMS_CLIENT_ID` and `DATASTREAMS_CLIENT_SECRET` are correct and do not have extra spaces or characters. + - **LINK Balance:** Ensure the contract holds sufficient `LINK` balance to pay for onchain verification fees. Top up the contract if needed -- testnet `LINK` can be found at the [Chainlink Faucet](https://faucets.chain.link/). + - **RPC Endpoint:** Confirm the RPC URL in your `config.yml` is correct and the node is reachable. + +--- + +### Contract Deployment & Onchain Errors + +- **`Gas estimation too high` error** + + - When deploying with Foundry, you can adjust the `gas_limit` in your `foundry.toml` file. + - For the transmitter, you can increase the `gasCap` value in your `config.yml` file. You can also update it in the Transmitter UI under the **Gas Cap** section. + +- **Missing private key** + - `forge script` will silently fall back to Foundry's default dev key and assign contract roles to `0x1804c8AB1F12E6bbf3894d4083f33e07309d1f38`. + - Always pass your key via `--private-key` or export it in `.env/shell`. + +--- + +## Conclusion + +Congratulations! 🎉 You have successfully deployed a `DataStreamsFeed` contract, set up the Data Streams Transmitter, and linked them together. + +You can now monitor offchain Data Streams and push updates to your smart contracts without writing any code or managing your own backend infrastructure. The Transmitter handles all the complexity of connecting to Data Streams, monitoring data changes, and pushing updates onchain. + +For more advanced usage, troubleshooting, or to contribute, visit the [GitHub repository][GITHUB_REPO] or check out the [official Data Streams documentation](/data-streams/index). + +Happy building!