diff --git a/src/providers/core/createGetCheckpointsRange.ts b/src/providers/core/createGetCheckpointsRange.ts new file mode 100644 index 0000000..01f9341 --- /dev/null +++ b/src/providers/core/createGetCheckpointsRange.ts @@ -0,0 +1,106 @@ +import { CheckpointRecord } from '../../stores/checkpoints'; + +type CacheEntry = { + /** + * Block range for which cache record is valid. This is inclusive on both ends. + */ + range: [number, number]; + /** + * Checkpoint records for the source in the given range. + */ + records: CheckpointRecord[]; +}; + +/** + * Options for createGetCheckpointsRange. + */ +type Options = { + /** + * Function to retrieve sources based on the block number. + * + * @param blockNumber Block number. + * @returns Array of sources. + */ + sourcesFn: (blockNumber: number) => T[]; + /** + * Function to extract a key from the source. This key is used for cache. + * + * @param source Source. + * @returns Key. + */ + keyFn: (source: T) => string; + /** + * Function to query checkpoint records for a source within given block range. + * + * @param fromBlock Starting block number. + * @param toBlock Ending block number. + * @param source The source to query. + * @returns Promise resolving to an array of CheckpointRecords. + */ + querySourceFn: (fromBlock: number, toBlock: number, source: T) => Promise; +}; + +/** + * Creates a getCheckpointsRange function. + * + * This function has a cache to avoid querying the same source for the same block range multiple times. + * This cache automatically evicts entries outside of the last queried range. + * + * @param options + * @returns A function that retrieves checkpoint records for a given block range. + */ +export function createGetCheckpointsRange(options: Options) { + const cache = new Map(); + + return async (fromBlock: number, toBlock: number): Promise => { + const sources = options.sourcesFn(fromBlock); + + let events: CheckpointRecord[] = []; + for (const source of sources) { + let sourceEvents: CheckpointRecord[] = []; + + const key = options.keyFn(source); + + const cacheEntry = cache.get(key); + if (!cacheEntry) { + const events = await options.querySourceFn(fromBlock, toBlock, source); + sourceEvents = sourceEvents.concat(events); + } else { + const [cacheStart, cacheEnd] = cacheEntry.range; + + const cacheEntries = cacheEntry.records.filter( + ({ blockNumber }) => blockNumber >= fromBlock && blockNumber <= toBlock + ); + + sourceEvents = sourceEvents.concat(cacheEntries); + + const bottomHalfStart = fromBlock; + const bottomHalfEnd = Math.min(toBlock, cacheStart - 1); + + const topHalfStart = Math.max(fromBlock, cacheEnd + 1); + const topHalfEnd = toBlock; + + if (bottomHalfStart <= bottomHalfEnd) { + const events = await options.querySourceFn(bottomHalfStart, bottomHalfEnd, source); + sourceEvents = sourceEvents.concat(events); + } + + if (topHalfStart <= topHalfEnd) { + const events = await options.querySourceFn(topHalfStart, topHalfEnd, source); + sourceEvents = sourceEvents.concat(events); + } + } + + sourceEvents.sort((a, b) => a.blockNumber - b.blockNumber); + + cache.set(key, { + range: [fromBlock, toBlock], + records: sourceEvents + }); + + events = events.concat(sourceEvents); + } + + return events; + }; +} diff --git a/src/providers/evm/provider.ts b/src/providers/evm/provider.ts index b7c028b..618b768 100644 --- a/src/providers/evm/provider.ts +++ b/src/providers/evm/provider.ts @@ -4,10 +4,10 @@ import { Log, Provider, StaticJsonRpcProvider } from '@ethersproject/providers'; import { Interface, LogDescription } from '@ethersproject/abi'; import { keccak256 } from '@ethersproject/keccak256'; import { toUtf8Bytes } from '@ethersproject/strings'; -import { CheckpointRecord } from '../../stores/checkpoints'; import { Writer } from './types'; import { ContractSourceConfig } from '../../types'; import { sleep } from '../../utils/helpers'; +import { createGetCheckpointsRange } from '../core/createGetCheckpointsRange'; type BlockWithTransactions = Awaited>; type Transaction = BlockWithTransactions['transactions'][number]; @@ -32,6 +32,13 @@ export class EvmProvider extends BaseProvider { this.provider = new StaticJsonRpcProvider(this.instance.config.network_node_url); this.writers = writers; + + this.getCheckpointsRange = createGetCheckpointsRange({ + sourcesFn: blockNumber => this.instance.getCurrentSources(blockNumber), + keyFn: source => source.contract, + querySourceFn: async (fromBlock, toBlock, source) => + this.getLogs(fromBlock, toBlock, source.contract) + }); } formatAddresses(addresses: string[]): string[] { @@ -274,17 +281,6 @@ export class EvmProvider extends BaseProvider { })); } - async getCheckpointsRange(fromBlock: number, toBlock: number): Promise { - let events: CheckpointRecord[] = []; - - for (const source of this.instance.getCurrentSources(fromBlock)) { - const addressEvents = await this.getLogs(fromBlock, toBlock, source.contract); - events = events.concat(addressEvents); - } - - return events; - } - getEventHash(eventName: string) { if (!this.sourceHashes.has(eventName)) { this.sourceHashes.set(eventName, keccak256(toUtf8Bytes(eventName))); diff --git a/src/providers/index.ts b/src/providers/index.ts index d496696..908035a 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -1,3 +1,4 @@ export * from './base'; export * as starknet from './starknet'; export * as evm from './evm'; +export * from './core/createGetCheckpointsRange'; diff --git a/src/providers/starknet/provider.ts b/src/providers/starknet/provider.ts index ab6b6aa..e765103 100644 --- a/src/providers/starknet/provider.ts +++ b/src/providers/starknet/provider.ts @@ -16,6 +16,7 @@ import { } from './types'; import { ContractSourceConfig } from '../../types'; import { sleep } from '../../utils/helpers'; +import { createGetCheckpointsRange } from '../core/createGetCheckpointsRange'; export class StarknetProvider extends BaseProvider { private readonly provider: RpcProvider; @@ -39,6 +40,18 @@ export class StarknetProvider extends BaseProvider { nodeUrl: this.instance.config.network_node_url }); this.writers = writers; + + this.getCheckpointsRangeForAddress = createGetCheckpointsRange({ + sourcesFn: blockNumber => this.instance.getCurrentSources(blockNumber), + keyFn: source => source.contract, + querySourceFn: async (fromBlock, toBlock, source) => + this.getCheckpointsRangeForAddress( + fromBlock, + toBlock, + source.contract, + source.events.map(event => event.name) + ) + }); } public async init() { diff --git a/test/unit/providers/core/createGetCheckpointsRange.test.ts b/test/unit/providers/core/createGetCheckpointsRange.test.ts new file mode 100644 index 0000000..46f91fd --- /dev/null +++ b/test/unit/providers/core/createGetCheckpointsRange.test.ts @@ -0,0 +1,161 @@ +import { createGetCheckpointsRange } from '../../../../src/providers/core/createGetCheckpointsRange'; + +it('should call querySourceFn for every source', async () => { + const sources = ['a', 'b', 'c']; + + const mockFunction = jest.fn().mockReturnValue([]); + + const getCheckpointsRange = createGetCheckpointsRange({ + sourcesFn: () => sources, + keyFn: source => source, + querySourceFn: mockFunction + }); + + await getCheckpointsRange(10, 20); + expect(mockFunction).toHaveBeenCalledTimes(sources.length); + for (const source of sources) { + expect(mockFunction).toHaveBeenCalledWith(10, 20, source); + } +}); + +it('should return value for requested source', async () => { + let sources = ['a', 'b']; + + const mockFunction = jest.fn().mockImplementation((fromBlock, toBlock, source) => { + return { + blockNumber: 14, + contractAddress: source + }; + }); + + const getCheckpointsRange = createGetCheckpointsRange({ + sourcesFn: () => sources, + keyFn: source => source, + querySourceFn: mockFunction + }); + + let result = await getCheckpointsRange(10, 20); + expect(result).toEqual([ + { + blockNumber: 14, + contractAddress: 'a' + }, + { + blockNumber: 14, + contractAddress: 'b' + } + ]); + + sources = ['b']; + result = await getCheckpointsRange(10, 20); + expect(result).toEqual([ + { + blockNumber: 14, + contractAddress: 'b' + } + ]); +}); + +describe('cache', () => { + const mockFunction = jest.fn().mockResolvedValue([]); + + function getCheckpointQuery() { + return createGetCheckpointsRange({ + sourcesFn: () => ['a'], + keyFn: source => source, + querySourceFn: mockFunction + }); + } + + beforeEach(() => { + mockFunction.mockClear(); + }); + + // Case 1: + // Cache exists and we are fetching the same range again + // This triggers no queryFn calls + it('exact cache match', async () => { + const getCheckpointsRange = getCheckpointQuery(); + + await getCheckpointsRange(10, 20); + expect(mockFunction).toHaveBeenCalledTimes(1); + + mockFunction.mockClear(); + + await getCheckpointsRange(10, 20); + expect(mockFunction).toHaveBeenCalledTimes(0); + }); + + // Case 2: + // Cache exists but we are fetching blocks outside of cache range + // This triggers single queryFn call + test('cache outside of the range', async () => { + const getCheckpointsRange = getCheckpointQuery(); + + await getCheckpointsRange(10, 20); + expect(mockFunction).toHaveBeenCalledTimes(1); + + // Case 2a: Cache exists but we are fetching blocks further than the cache + mockFunction.mockClear(); + + await getCheckpointsRange(21, 31); + expect(mockFunction).toHaveBeenCalledTimes(1); + expect(mockFunction).toHaveBeenCalledWith(21, 31, 'a'); + + // Case 2b: Cache exists but we are fetching blocks before the cache + mockFunction.mockClear(); + + await getCheckpointsRange(0, 9); + expect(mockFunction).toHaveBeenCalledTimes(1); + expect(mockFunction).toHaveBeenCalledWith(0, 9, 'a'); + }); + + // Case 3: + // Part of the range is cached and part of the range is not cached + // This triggers two queryFn calls (one to fetch block before cache and one to fetch block after cache) + test('cache is fully inside the range', async () => { + const getCheckpointsRange = getCheckpointQuery(); + + await getCheckpointsRange(10, 20); + expect(mockFunction).toHaveBeenCalledTimes(1); + + mockFunction.mockClear(); + + await getCheckpointsRange(5, 25); + expect(mockFunction).toHaveBeenCalledTimes(2); + expect(mockFunction).toHaveBeenCalledWith(5, 9, 'a'); + expect(mockFunction).toHaveBeenCalledWith(21, 25, 'a'); + }); + + // Case 4: + // Cache covers bottom part of the range + // This triggers single queryFn call to fetch the top part of the range + test('cache covers bottom part of range', async () => { + const getCheckpointsRange = getCheckpointQuery(); + + await getCheckpointsRange(10, 20); + expect(mockFunction).toHaveBeenCalledTimes(1); + + mockFunction.mockClear(); + + await getCheckpointsRange(15, 25); + expect(mockFunction).toHaveBeenCalledTimes(1); + expect(mockFunction).toHaveBeenCalledWith(21, 25, 'a'); + }); + + // Case 5: + // Cache covers top part of the range + // This triggers single queryFn call to fetch the bottom part of the range + test('cache covers top part of range', async () => { + const getCheckpointsRange = getCheckpointQuery(); + + await getCheckpointsRange(10, 20); + expect(mockFunction).toHaveBeenCalledTimes(1); + + mockFunction.mockClear(); + + await getCheckpointsRange(0, 15); + expect(mockFunction).toHaveBeenCalledTimes(1); + expect(mockFunction).toHaveBeenCalledWith(0, 9, 'a'); + }); +});