From 6235e0ab5d1b936575949656d8eddba1015a8ba6 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Fri, 28 Jan 2022 13:27:54 -0500 Subject: [PATCH 1/2] Moved to one SubgraphManifest schema file per protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will make it easier to add new network and will improve error messages when someone will be using fields that do not exist on a specific network but does in other (`temlates` for example used in NEAR were given a bunch of errors like the support was possible while it should have displayed an error on the `templates` node itself). Using `templates` on NEAR before this PR: ``` ✔ Apply migrations ✖ Failed to load subgraph from ../graph-node-dev/subgraphs/near/data-source/testnet.yaml: Error in ../graph-node-dev/subgraphs/near/data-source/testnet.yaml: Path: templates > 0 > source No value provided Path: templates > 0 > mapping Unexpected key in map: receiptHandlers Path: templates > 0 > mapping > abis No value provided ``` And with this PR: ``` $ ./bin/graph build ../graph-node-dev/subgraphs/near/data-source/testnet.yaml ✖ Failed to load subgraph from ../graph-node-dev/subgraphs/near/data-source/testnet.yaml: Error in ../graph-node-dev/subgraphs/near/data-source/testnet.yaml: Path: / Unexpected key in map: templates ``` --- manifest-schema.graphql | 122 ------------------ schemas/ethereum.graphql | 86 ++++++++++++ schemas/near.graphql | 51 ++++++++ src/protocols/index.js | 25 +++- src/subgraph.js | 31 +++-- src/validation/manifest.js | 20 +-- .../validation/example-values-found.stderr | 4 +- tests/cli/validation/invalid-manifest.stderr | 30 +---- 8 files changed, 189 insertions(+), 180 deletions(-) delete mode 100644 manifest-schema.graphql create mode 100644 schemas/ethereum.graphql create mode 100644 schemas/near.graphql diff --git a/manifest-schema.graphql b/manifest-schema.graphql deleted file mode 100644 index f4cf25c46..000000000 --- a/manifest-schema.graphql +++ /dev/null @@ -1,122 +0,0 @@ -scalar String -scalar File -scalar BigInt - -type Schema { - file: File! -} - -union DataSource = - EthereumContractDataSource | - NearContractDataSource - -union DataSourceTemplate = EthereumContractDataSourceTemplate - -type EthereumContractDataSource { - kind: String! - name: String! - network: String - source: EthereumContractSource! - mapping: EthereumContractMapping! -} - -type NearContractDataSource { - kind: String! - name: String! - network: String - source: NearContractSource! - mapping: NearContractMapping! -} - -type EthereumContractSource { - address: String - abi: String! - startBlock: BigInt -} - -type NearContractSource { - account: String - startBlock: BigInt -} - -type EthereumContractMapping { - kind: String - apiVersion: String! - language: String! - file: File! - entities: [String!]! - abis: [EthereumContractAbi!]! - blockHandlers: [EthereumBlockHandler!] - callHandlers: [EthereumCallHandler!] - eventHandlers: [EthereumContractEventHandler!] -} - -type NearContractMapping { - apiVersion: String! - language: String! - file: File! - entities: [String!]! - blockHandlers: [NearBlockHandler!] - receiptHandlers: [NearReceiptHandler!] -} - -type EthereumContractAbi { - name: String! - file: File! -} - -type EthereumBlockHandler { - handler: String! - filter: EthereumBlockFilter -} - -type EthereumBlockFilter { - kind: String! -} - -type NearBlockHandler { - handler: String! -} - -type EthereumCallHandler { - function: String! - handler: String! -} - -type NearReceiptHandler { - handler: String! -} - -type EthereumContractEventHandler { - event: String! - topic0: String - handler: String! -} - -type Graft { - base: String! - block: BigInt! -} - -type SubgraphManifest { - specVersion: String! - features: [String!] - schema: Schema! - description: String - repository: String - graft: Graft - dataSources: [DataSource!]! - templates: [DataSourceTemplate!] -} - -type EthereumContractDataSourceTemplate { - kind: String! - name: String! - network: String - source: EthereumContractSourceTemplate! - mapping: EthereumContractMapping! -} - -type EthereumContractSourceTemplate { - abi: String! -} diff --git a/schemas/ethereum.graphql b/schemas/ethereum.graphql new file mode 100644 index 000000000..1b16d31dd --- /dev/null +++ b/schemas/ethereum.graphql @@ -0,0 +1,86 @@ +scalar String +scalar File +scalar BigInt + +type SubgraphManifest { + specVersion: String! + features: [String!] + schema: Schema! + description: String + repository: String + graft: Graft + dataSources: [DataSource!]! + templates: [DataSourceTemplate!] +} + +type Schema { + file: File! +} + +type DataSource { + kind: String! + name: String! + network: String + source: ContractSource! + mapping: ContractMapping! +} + +type ContractSource { + address: String + abi: String! + startBlock: BigInt +} + +type ContractMapping { + kind: String + apiVersion: String! + language: String! + file: File! + entities: [String!]! + abis: [ContractAbi!]! + blockHandlers: [BlockHandler!] + callHandlers: [CallHandler!] + eventHandlers: [ContractEventHandler!] +} + +type ContractAbi { + name: String! + file: File! +} + +type BlockHandler { + handler: String! + filter: BlockFilter +} + +type BlockFilter { + kind: String! +} + +type CallHandler { + function: String! + handler: String! +} + +type ContractEventHandler { + event: String! + topic0: String + handler: String! +} + +type Graft { + base: String! + block: BigInt! +} + +type DataSourceTemplate { + kind: String! + name: String! + network: String + source: ContractSourceTemplate! + mapping: ContractMapping! +} + +type ContractSourceTemplate { + abi: String! +} diff --git a/schemas/near.graphql b/schemas/near.graphql new file mode 100644 index 000000000..b40ebd848 --- /dev/null +++ b/schemas/near.graphql @@ -0,0 +1,51 @@ +scalar String +scalar File +scalar BigInt + +type SubgraphManifest { + specVersion: String! + schema: Schema! + description: String + repository: String + graft: Graft + dataSources: [DataSource!]! +} + +type Schema { + file: File! +} + +type DataSource { + kind: String! + name: String! + network: String + source: ContractSource! + mapping: ContractMapping! +} + +type ContractSource { + account: String + startBlock: BigInt +} + +type ContractMapping { + apiVersion: String! + language: String! + file: File! + entities: [String!]! + blockHandlers: [BlockHandler!] + receiptHandlers: [ReceiptHandler!] +} + +type BlockHandler { + handler: String! +} + +type ReceiptHandler { + handler: String! +} + +type Graft { + base: String! + block: BigInt! +} diff --git a/src/protocols/index.js b/src/protocols/index.js index 8bd77ecab..05cc73499 100644 --- a/src/protocols/index.js +++ b/src/protocols/index.js @@ -1,6 +1,7 @@ const immutable = require('immutable') const EthereumTypeGenerator = require('./ethereum/type-generator') const EthereumTemplateCodeGen = require('./ethereum/codegen/template') +const NearTemplateCodeGen = require('./near/codegen/template') const EthereumABI = require('./ethereum/abi') const EthereumSubgraph = require('./ethereum/subgraph') const NearSubgraph = require('./near/subgraph') @@ -60,10 +61,7 @@ module.exports = class Protocol { 'aurora', 'aurora-testnet', ], - near: [ - 'near-mainnet', - 'near-testnet' - ], + near: ['near-mainnet', 'near-testnet'], }) } @@ -108,6 +106,15 @@ module.exports = class Protocol { } } + hasTemplates() { + switch (this.name) { + case 'ethereum': + return true + case 'near': + return false + } + } + getTypeGenerator(options) { switch (this.name) { case 'ethereum': @@ -118,13 +125,17 @@ module.exports = class Protocol { } getTemplateCodeGen(template) { + if (!this.hasTemplates()) { + throw new Error( + `Template data sources with kind '${this.name}' are not supported yet`, + ) + } + switch (this.name) { case 'ethereum': return new EthereumTemplateCodeGen(template) default: - throw new Error( - `Template data sources with kind '${this.name}' are not supported yet`, - ) + throw new Error(`Template data sources with kind '${this.name}' is unknown`) } } diff --git a/src/subgraph.js b/src/subgraph.js index 99f61b89a..86dc8c746 100644 --- a/src/subgraph.js +++ b/src/subgraph.js @@ -27,7 +27,7 @@ const buildCombinedWarning = (filename, warnings) => ? warnings.reduce( (msg, w) => `${msg} - + Path: ${w.get('path').size === 0 ? '/' : w.get('path').join(' > ')} ${w .get('message') @@ -39,9 +39,21 @@ const buildCombinedWarning = (filename, warnings) => module.exports = class Subgraph { static async validate(data, protocol, { resolveFile }) { + if (protocol.name == null) { + return immutable.fromJS([ + { + path: [], + message: `Unable to determine for which protocol manifest file is built for. Ensure you have at least one 'dataSources' and/or 'templates' elements defined in your subgraph.`, + }, + ]) + } + // Parse the default subgraph schema let schema = graphql.parse( - await fs.readFile(path.join(__dirname, '..', 'manifest-schema.graphql'), 'utf-8'), + await fs.readFile( + path.join(__dirname, '..', 'schemas', `${protocol.name}.graphql`), + 'utf-8', + ), ) // Obtain the root `SubgraphManifest` type from the schema @@ -146,10 +158,7 @@ At least one such handler must be defined.`, } static validateContractValues(manifest, protocol) { - return validation.validateContractValues( - manifest, - protocol, - ) + return validation.validateContractValues(manifest, protocol) } // Validate that data source names are unique, so they don't overwrite each other. @@ -200,17 +209,13 @@ More than one template named '${name}', template names must be unique.`, return yaml.stringify(manifest.toJS()) } - static async load( - filename, - { protocol, skipValidation } = { skipValidation: false } - ) { + static async load(filename, { protocol, skipValidation } = { skipValidation: false }) { // Load and validate the manifest let data = null - if(filename.match(/.js$/)) { + if (filename.match(/.js$/)) { data = require(path.resolve(filename)) - } - else { + } else { data = yaml.parse(await fs.readFile(filename, 'utf-8')) } diff --git a/src/validation/manifest.js b/src/validation/manifest.js index 66c865bf5..1de702a54 100644 --- a/src/validation/manifest.js +++ b/src/validation/manifest.js @@ -52,8 +52,7 @@ const validators = immutable.fromJS({ validators.get(ctx.getIn(['type', 'name', 'value']))(value, ctx), UnionTypeDefinition: (value, ctx) => { - const unionVariants = ctx - .getIn(['type', 'types']) + const unionVariants = ctx.getIn(['type', 'types']) let errors = List() @@ -78,7 +77,10 @@ const validators = immutable.fromJS({ NonNullType: (value, ctx) => value !== null && value !== undefined - ? validateValue(value, ctx.update('type', type => type.get('type'))) + ? validateValue( + value, + ctx.update('type', type => type.get('type')), + ) : immutable.fromJS([ { path: ctx.get('path'), @@ -126,7 +128,7 @@ const validators = immutable.fromJS({ ), ) : errors.push( - key == 'templates' + key == 'templates' && ctx.get('protocol').hasTemplates() ? immutable.fromJS({ path: ctx.get('path'), message: @@ -211,7 +213,8 @@ const validateValue = (value, ctx) => { } const validateDataSourceForNetwork = (dataSources, protocol) => - dataSources.filter(dataSource => protocol.isValidKindName(dataSource.kind)) + dataSources + .filter(dataSource => protocol.isValidKindName(dataSource.kind)) .reduce( (networks, dataSource) => networks.update(dataSource.network, dataSources => @@ -235,11 +238,11 @@ const validateDataSourceNetworks = (value, protocol) => { ${networks .map( (dataSources, network) => - ` ${ + ` ${ network === undefined ? 'Data sources and templates having no network set' - : `Data sources and templates using '${network}'` - }:\n${dataSources.map(ds => ` - ${ds}`).join('\n')}`, + : `Data sources and templates using '${network}'` + }:\n${dataSources.map(ds => ` - ${ds}`).join('\n')}`, ) .join('\n')} Recommendation: Make all data sources and templates use the same network name.`, @@ -262,6 +265,7 @@ const validateManifest = (value, type, schema, protocol, { resolveFile }) => { path: [], errors: [], resolveFile, + protocol, }), ) : immutable.fromJS([ diff --git a/tests/cli/validation/example-values-found.stderr b/tests/cli/validation/example-values-found.stderr index 600bf2fcf..524095619 100644 --- a/tests/cli/validation/example-values-found.stderr +++ b/tests/cli/validation/example-values-found.stderr @@ -1,10 +1,10 @@ - Load subgraph from subgraph.yaml ⚠ Warnings while loading subgraph from subgraph.yaml: Warnings in subgraph.yaml: - + Path: repository The repository is still set to https://github.com/graphprotocol/example-subgraph. Please replace it with a link to your subgraph source code. - + Path: description The description is still the one from the example subgraph. Please update it to tell users more about your subgraph. diff --git a/tests/cli/validation/invalid-manifest.stderr b/tests/cli/validation/invalid-manifest.stderr index f7794fb77..6c3bc469e 100644 --- a/tests/cli/validation/invalid-manifest.stderr +++ b/tests/cli/validation/invalid-manifest.stderr @@ -1,31 +1,5 @@ - Load subgraph from subgraph.yaml ✖ Failed to load subgraph from subgraph.yaml: Error in subgraph.yaml: - Path: specVersion - No value provided - - Path: schema > file - File does not exist: non-existent.grapqhl - - Path: dataSources > 0 > kind - No value provided - - Path: dataSources > 0 > name - Expected string, found number: - 5 - - Path: dataSources > 0 > source - No value provided - - Path: dataSources > 0 > mapping - Expected map, found list: - - 12 - - 13 - - 14 - - Path: dataSources > 0 - Unexpected key in map: abis - - Path: templates - Expected list, found map: - field: foo + Path: / + Unable to determine for which protocol manifest file is built for. Ensure you have at least one 'dataSources' and/or 'templates' elements defined in your subgraph. From ef18430b2fdf74636f3391213e95ea0a5af321ee Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Fri, 28 Jan 2022 22:07:26 -0500 Subject: [PATCH 2/2] Added support for NEAR templates --- schemas/near.graphql | 8 ++++++ src/protocols/index.js | 4 ++- src/protocols/near/codegen/template.js | 40 ++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 src/protocols/near/codegen/template.js diff --git a/schemas/near.graphql b/schemas/near.graphql index b40ebd848..b0b97c6f0 100644 --- a/schemas/near.graphql +++ b/schemas/near.graphql @@ -9,6 +9,7 @@ type SubgraphManifest { repository: String graft: Graft dataSources: [DataSource!]! + templates: [DataSourceTemplate!] } type Schema { @@ -49,3 +50,10 @@ type Graft { base: String! block: BigInt! } + +type DataSourceTemplate { + kind: String! + name: String! + network: String + mapping: ContractMapping! +} diff --git a/src/protocols/index.js b/src/protocols/index.js index 05cc73499..63362848f 100644 --- a/src/protocols/index.js +++ b/src/protocols/index.js @@ -111,7 +111,7 @@ module.exports = class Protocol { case 'ethereum': return true case 'near': - return false + return true } } @@ -134,6 +134,8 @@ module.exports = class Protocol { switch (this.name) { case 'ethereum': return new EthereumTemplateCodeGen(template) + case 'near': + return new NearTemplateCodeGen(template) default: throw new Error(`Template data sources with kind '${this.name}' is unknown`) } diff --git a/src/protocols/near/codegen/template.js b/src/protocols/near/codegen/template.js new file mode 100644 index 000000000..d4bf1a256 --- /dev/null +++ b/src/protocols/near/codegen/template.js @@ -0,0 +1,40 @@ +const tsCodegen = require('../../../codegen/typescript') + +module.exports = class NearTemplateCodeGen { + constructor(template) { + this.template = template + } + + generateModuleImports() { + return [] + } + + generateCreateMethod() { + const name = this.template.get('name') + + return tsCodegen.staticMethod( + 'create', + [tsCodegen.param('account', tsCodegen.namedType('string'))], + tsCodegen.namedType('void'), + ` + DataSourceTemplate.create('${name}', [account]) + `, + ) + } + + generateCreateWithContextMethod() { + const name = this.template.get('name') + + return tsCodegen.staticMethod( + 'createWithContext', + [ + tsCodegen.param('account', tsCodegen.namedType('string')), + tsCodegen.param('context', tsCodegen.namedType('DataSourceContext')), + ], + tsCodegen.namedType('void'), + ` + DataSourceTemplate.createWithContext('${name}', [account], context) + `, + ) + } +}