diff --git a/.eslintrc.json b/.eslintrc.json index a9c0d4f59..26e435bc5 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -61,7 +61,8 @@ "@typescript-eslint/no-unused-vars": [ "error", { "ignoreRestSiblings": true, "argsIgnorePattern": "^_" } ], "@typescript-eslint/no-explicit-any": "off", "@typescript-eslint/ban-ts-comment": "warn", - "eol-last": [ "error", "always" ] + "eol-last": [ "error", "always" ], + "@typescript-eslint/no-empty-interface": "off" } } ] diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42ae49fb9..78eed8770 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,14 +38,7 @@ jobs: strategy: matrix: database: [ - "postgres", "postgres13", "postgres12", "postgres11", "postgres10", "postgres9", - "spanner", - "mysql", "mysql5", - "mssql", "mssql17", - "mongo", "mongo4", - "firestore", - "dynamodb", - "google-sheets" + "mysql", "mysql5" ] env: diff --git a/apps/velo-external-db/src/app.ts b/apps/velo-external-db/src/app.ts index c7d14fd46..2fa51abfb 100644 --- a/apps/velo-external-db/src/app.ts +++ b/apps/velo-external-db/src/app.ts @@ -4,10 +4,20 @@ import { ExternalDbRouter, Hooks } from '@wix-velo/velo-external-db-core' import { engineConnectorFor } from './storage/factory' -const initConnector = async(hooks?: Hooks) => { - const { vendor, type: adapterType } = readCommonConfig() +process.env.CLOUD_VENDOR = 'azure' +process.env.TYPE = 'mysql' +process.env.EXTERNAL_DATABASE_ID = '' +process.env.ALLOWED_METASITES = '' +process.env['TYPE'] = 'mysql' +process.env['HOST'] = 'localhost' +process.env['USER'] = 'test-user' +process.env['PASSWORD'] = 'password' +process.env['DB'] = 'test-db' + +const initConnector = async(wixDataBaseUrl?: string, hooks?: Hooks) => { + const { vendor, type: adapterType, externalDatabaseId, allowedMetasites } = readCommonConfig() const configReader = create() - const { authorization, secretKey, ...dbConfig } = await configReader.readConfig() + const { authorization, ...dbConfig } = await configReader.readConfig() const { connector: engineConnector, providers, cleanup } = await engineConnectorFor(adapterType, dbConfig) @@ -17,10 +27,12 @@ const initConnector = async(hooks?: Hooks) => { authorization: { roleConfig: authorization }, - secretKey, + externalDatabaseId, + allowedMetasites, vendor, adapterType, - commonExtended: true + commonExtended: true, + wixDataBaseUrl: wixDataBaseUrl || 'https://www.wixapis.com/wix-data' }, hooks }) @@ -28,11 +40,11 @@ const initConnector = async(hooks?: Hooks) => { return { externalDbRouter, cleanup: async() => await cleanup(), schemaProvider: providers.schemaProvider } } -export const createApp = async() => { +export const createApp = async(wixDataBaseUrl?: string) => { const app = express() - const initConnectorResponse = await initConnector() + const initConnectorResponse = await initConnector(wixDataBaseUrl) app.use(initConnectorResponse.externalDbRouter.router) const server = app.listen(8080, () => console.log('Connector listening on port 8080')) - return { server, ...initConnectorResponse, reload: () => initConnector() } + return { server, ...initConnectorResponse, reload: () => initConnector(wixDataBaseUrl) } } diff --git a/apps/velo-external-db/src/storage/factory.ts b/apps/velo-external-db/src/storage/factory.ts index 9948269b8..1d05933ab 100644 --- a/apps/velo-external-db/src/storage/factory.ts +++ b/apps/velo-external-db/src/storage/factory.ts @@ -7,42 +7,42 @@ export const engineConnectorFor = async(_type: string, config: any): Promise await axios.post('/data/insert/bulk', { collectionName: collectionName, items: items }, auth) +export const insertRequest = (collectionName: string, items: Item[], overwriteExisting: boolean): dataSpi.InsertRequest => ({ + collectionId: collectionName, + items: items, + overwriteExisting, + options: { + consistentRead: false, + appOptions: {}, + } +}) + +export const updateRequest = (collectionName: string, items: Item[]): dataSpi.UpdateRequest => ({ + collectionId: collectionName, + items: items, + options: { + consistentRead: false, + appOptions: {}, + } +}) + +export const countRequest = (collectionName: string): dataSpi.CountRequest => ({ + collectionId: collectionName, + filter: '', + options: { + consistentRead: false, + appOptions: {}, + }, +}) + +export const queryRequest = (collectionName: string, sort: dataSpi.Sorting[], fields: string[], filter?: dataSpi.Filter): dataSpi.QueryRequest => ({ + collectionId: collectionName, + query: { + filter: filter ?? '', + sort: sort, + fields: fields, + fieldsets: undefined, + paging: { + limit: 25, + offset: 0, + }, + cursorPaging: null + }, + includeReferencedItems: [], + options: { + consistentRead: false, + appOptions: {}, + }, + omitTotalCount: false +}) + + +export const queryCollectionAsArray = (collectionName: string, sort: dataSpi.Sorting[], fields: string[], auth: any, filter?: dataSpi.Filter) => + axiosInstance.post('/data/query', + queryRequest(collectionName, sort, fields, filter), { responseType: 'stream', transformRequest: auth.transformRequest }) + .then(response => streamToArray(response.data)) + + +export const pagingMetadata = (total: number, count: number): dataSpi.QueryResponsePart => ({ pagingMetadata: { count: count, offset: 0, total: total, tooManyToCount: false } }) + -export const expectAllDataIn = async(collectionName: string, auth: any) => (await axios.post('/data/find', { collectionName: collectionName, filter: '', sort: '', skip: 0, limit: 25 }, auth)).data +export const givenItems = async(items: Item[], collectionName: string, auth: any) => + await axiosInstance.post('/data/insert', insertRequest(collectionName, items, false), { responseType: 'stream', transformRequest: auth.transformRequest }) diff --git a/apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts b/apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts index c7b6c014e..fd2f6665e 100644 --- a/apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts +++ b/apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts @@ -1,5 +1,6 @@ import { SystemFields, asWixSchemaHeaders } from '@wix-velo/velo-external-db-commons' import { InputField } from '@wix-velo/velo-external-db-types' +import { schemaUtils } from '@wix-velo/velo-external-db-core' export const responseWith = (matcher: any) => expect.objectContaining( { data: matcher } ) @@ -40,3 +41,35 @@ const toHaveCollections = (collections: string[]) => expect.objectContaining( { const listToHaveCollection = (collectionName: string) => expect.objectContaining( { schemas: expect.arrayContaining( [ expect.objectContaining( { id: collectionName } ) ] ) } ) + +const collectionCapabilities = (_collectionOperations: any[], _dataOperations: any[], _fieldTypes: any[]) => ({ + collectionOperations: expect.any(Array), + dataOperations: expect.any(Array), + fieldTypes: expect.any(Array) +}) + +const fieldCapabilitiesMatcher = () => expect.objectContaining({ + queryOperators: expect.any(Array), + sortable: expect.any(Boolean), +}) + +const filedMatcher = (field: InputField) => ({ + key: field.name, + capabilities: fieldCapabilitiesMatcher(), + encrypted: expect.any(Boolean), + type: schemaUtils.fieldTypeToWixDataEnum(field.type) +}) + +const fieldsMatcher = (fields: InputField[]) => expect.toIncludeSameMembers(fields.map(filedMatcher)) + +export const collectionResponsesWith = (collectionName: string, fields: InputField[]) => ({ + id: collectionName, + capabilities: collectionCapabilities([], [], []), + fields: fieldsMatcher(fields), +}) + +export const createCollectionResponse = (collectionName: string, fields: InputField[]) => ({ + id: collectionName, + capabilities: collectionCapabilities([], [], []), + fields: fieldsMatcher(fields), +}) diff --git a/apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts b/apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts index bf492fc90..558cc26b6 100644 --- a/apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts +++ b/apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts @@ -1,14 +1,34 @@ +import axios from 'axios' import { InputField } from '@wix-velo/velo-external-db-types' +import { streamToArray } from '@wix-velo/test-commons' +import { schemaUtils } from '@wix-velo/velo-external-db-core' -const axios = require('axios').create({ + +const axiosClient = axios.create({ baseURL: 'http://localhost:8080' }) export const givenCollection = async(name: string, columns: InputField[], auth: any) => { - await axios.post('/schemas/create', { collectionName: name }, auth) - for (const column of columns) { - await axios.post('/schemas/column/add', { collectionName: name, column: column }, auth) + const collection = { + id: name, + fields: columns.map(schemaUtils.InputFieldToWixFormatField) } + await axiosClient.post('/collections/create', { collection }, { ...auth, responseType: 'stream' }) } -export const retrieveSchemaFor = async(collectionName: string, auth: any) => axios.post('/schemas/find', { schemaIds: [collectionName] }, auth) +export const deleteAllCollections = async(auth: any) => { + const res = await axiosClient.post('/collections/get', { collectionIds: [] }, { ...auth, responseType: 'stream' }) + const dataRes = await streamToArray(res.data) as any [] + const collectionIds = dataRes.map(d => d.id) + + for (const collectionId of collectionIds) { + await axiosClient.post('/collections/delete', { collectionId }, { ...auth, responseType: 'stream' }) + } + +} + +export const retrieveSchemaFor = async(collectionName: string, auth: any) => { + const collectionGetStream = await axiosClient.post('/collections/get', { collectionIds: [collectionName] }, { ...auth, responseType: 'stream' }) + const [collectionGetRes] = await streamToArray(collectionGetStream.data) as any[] + return collectionGetRes +} diff --git a/apps/velo-external-db/test/drivers/schema_provider_matchers.ts b/apps/velo-external-db/test/drivers/schema_provider_matchers.ts index 6e41f7eed..33e553d43 100644 --- a/apps/velo-external-db/test/drivers/schema_provider_matchers.ts +++ b/apps/velo-external-db/test/drivers/schema_provider_matchers.ts @@ -1,6 +1,18 @@ -export const hasSameSchemaFieldsLike = (fields: {field: string, [x: string]: any}[]) => expect.arrayContaining( fields.map((f: any) => expect.objectContaining( f ) )) +import { SystemFields } from '@wix-velo/velo-external-db-commons' +import { ResponseField } from '@wix-velo/velo-external-db-types' -export const collectionWithDefaultFields = () => hasSameSchemaFieldsLike([ { field: '_id', type: 'text' }, - { field: '_createdDate', type: 'datetime' }, - { field: '_updatedDate', type: 'datetime' }, - { field: '_owner', type: 'text' } ]) +export const hasSameSchemaFieldsLike = (fields: ResponseField[]) => expect.arrayContaining(fields.map((f) => expect.objectContaining( f ))) + +export const toContainDefaultFields = () => hasSameSchemaFieldsLike(SystemFields.map(f => ({ field: f.name, type: f.type }))) + +export const collectionToContainFields = (collectionName: string, fields: ResponseField[], capabilities: any) => ({ + id: collectionName, + fields: hasSameSchemaFieldsLike(fields), + capabilities: { + collectionOperations: capabilities.CollectionOperations, + dataOperations: capabilities.ReadWriteOperations, + fieldTypes: capabilities.FieldTypes + } +}) + +export const toBeDefaultCollectionWith = (collectionName: string, capabilities: any) => collectionToContainFields(collectionName, SystemFields.map(f => ({ field: f.name, type: f.type })), capabilities) diff --git a/apps/velo-external-db/test/drivers/wix_data_resources.ts b/apps/velo-external-db/test/drivers/wix_data_resources.ts new file mode 100644 index 000000000..02b136867 --- /dev/null +++ b/apps/velo-external-db/test/drivers/wix_data_resources.ts @@ -0,0 +1,17 @@ +import { Server } from 'http' +import { app as mockServer } from './wix_data_testkit' + +let _server: Server +const PORT = 9001 + +export const initWixDataEnv = async() => { + _server = mockServer.listen(PORT) +} + +export const shutdownWixDataEnv = async() => { + _server.close() +} + +export const wixDataBaseUrl = () => { + return `http://localhost:${PORT}` +} diff --git a/apps/velo-external-db/test/drivers/wix_data_testkit.ts b/apps/velo-external-db/test/drivers/wix_data_testkit.ts new file mode 100644 index 000000000..ebe170fcc --- /dev/null +++ b/apps/velo-external-db/test/drivers/wix_data_testkit.ts @@ -0,0 +1,33 @@ +import { authConfig } from '@wix-velo/test-commons' +import * as express from 'express' + +export const app = express() + +app.set('case sensitive routing', true) + +app.use(express.json()) + +app.get('/v1/external-databases/:externalDatabaseId/public-keys', (_req, res) => { + res.json({ + publicKeys: [ + { id: authConfig.kid, publicKeyPem: authConfig.authPublicKey }, + ] + }) +}) + +app.use((_req, res) => { + res.status(404) + res.json({ error: 'NOT_FOUND' }) +}) + +app.use((err, _req, res, next) => { + res.status(err.status) + res.json({ + error: { + message: err.message, + status: err.status, + error: err.error + } + }) + next() +}) diff --git a/apps/velo-external-db/test/e2e/app.e2e.spec.ts b/apps/velo-external-db/test/e2e/app.e2e.spec.ts index d8f673173..e303a2419 100644 --- a/apps/velo-external-db/test/e2e/app.e2e.spec.ts +++ b/apps/velo-external-db/test/e2e/app.e2e.spec.ts @@ -1,6 +1,7 @@ import { authOwner } from '@wix-velo/external-db-testkit' import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName } from '../resources/e2e_resources' +import { CollectionCapability } from '@wix-velo/velo-external-db-core' const axios = require('axios').create({ baseURL: 'http://localhost:8080' }) @@ -19,7 +20,16 @@ describe(`Velo External DB: ${currentDbImplementationName()}`, () => { }) test('answer provision with stub response', async() => { - expect((await axios.post('/provision', { }, authOwner)).data).toEqual(expect.objectContaining({ protocolVersion: 2, vendor: 'azure' })) + expect((await axios.post('/provision', { }, authOwner)).data).toEqual(expect.objectContaining({ protocolVersion: 3, vendor: 'azure' })) + }) + + test('answer capability', async() => { + + expect((await axios.get('/capabilities', { }, authOwner)).data).toEqual(expect.objectContaining({ + capabilities: { + collection: [CollectionCapability.CREATE] + } + })) }) diff --git a/apps/velo-external-db/test/e2e/app_auth.e2e._spec.ts b/apps/velo-external-db/test/e2e/app_auth.e2e._spec.ts new file mode 100644 index 000000000..5a582be11 --- /dev/null +++ b/apps/velo-external-db/test/e2e/app_auth.e2e._spec.ts @@ -0,0 +1,36 @@ +import { Uninitialized, gen } from '@wix-velo/test-commons' +import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName } from '../resources/e2e_resources' + + +describe(`Velo External DB authorization: ${currentDbImplementationName()}`, () => { + beforeAll(async() => { + await setupDb() + + await initApp() + }, 20000) + + afterAll(async() => { + await dbTeardown() + }, 20000) + + // each(['data/query', 'data/aggregate', 'data/insert', 'data/insert/bulk', 'data/get', 'data/update', + // 'data/update/bulk', 'data/remove', 'data/remove/bulk', 'data/count']) + // .test('should throw 401 on a request to %s without the appropriate role', async(api) => { + // return expect(() => axios.post(api, { collectionName: ctx.collectionName }, authVisitor)).rejects.toThrow('401') + // }) + + // test('wrong secretKey will throw an appropriate error with the right format', async() => { + // return expect(() => axios.post('/schemas/list', {}, authOwnerWithoutSecretKey)).rejects.toMatchObject(errorResponseWith(401, 'You are not authorized')) + // }) + + + const ctx = { + collectionName: Uninitialized, + } + + beforeEach(async() => { + ctx.collectionName = gen.randomCollectionName() + }) + + afterAll(async() => await teardownApp()) +}) diff --git a/apps/velo-external-db/test/e2e/app_auth.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_auth.e2e.spec.ts deleted file mode 100644 index fd533b696..000000000 --- a/apps/velo-external-db/test/e2e/app_auth.e2e.spec.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Uninitialized, gen } from '@wix-velo/test-commons' -import { authVisitor, authOwnerWithoutSecretKey, errorResponseWith } from '@wix-velo/external-db-testkit' -import each from 'jest-each' -import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName } from '../resources/e2e_resources' - - - -const axios = require('axios').create({ - baseURL: 'http://localhost:8080' -}) - -describe(`Velo External DB authorization: ${currentDbImplementationName()}`, () => { - beforeAll(async() => { - await setupDb() - - await initApp() - }, 20000) - - afterAll(async() => { - await dbTeardown() - }, 20000) - - each(['data/find', 'data/aggregate', 'data/insert', 'data/insert/bulk', 'data/get', 'data/update', - 'data/update/bulk', 'data/remove', 'data/remove/bulk', 'data/count']) - .test('should throw 401 on a request to %s without the appropriate role', async(api) => { - return expect(() => axios.post(api, { collectionName: ctx.collectionName }, authVisitor)).rejects.toThrow('401') - }) - - test('wrong secretKey will throw an appropriate error with the right format', async() => { - return expect(() => axios.post('/schemas/list', {}, authOwnerWithoutSecretKey)).rejects.toMatchObject(errorResponseWith(401, 'You are not authorized')) - }) - - const ctx = { - collectionName: Uninitialized, - } - - beforeEach(async() => { - ctx.collectionName = gen.randomCollectionName() - }) - - afterAll(async() => await teardownApp()) -}) diff --git a/apps/velo-external-db/test/e2e/app_data.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_data.e2e.spec.ts index 229e26b5c..bc5e59cab 100644 --- a/apps/velo-external-db/test/e2e/app_data.e2e.spec.ts +++ b/apps/velo-external-db/test/e2e/app_data.e2e.spec.ts @@ -1,19 +1,20 @@ -import { Uninitialized, gen as genCommon } from '@wix-velo/test-commons' +import axios from 'axios' +import Chance = require('chance') +import { Uninitialized, gen as genCommon, testIfSupportedOperationsIncludes, streamToArray } from '@wix-velo/test-commons' import { SchemaOperations } from '@wix-velo/velo-external-db-types' -const { UpdateImmediately, DeleteImmediately, Truncate, Aggregate, FindWithSort, Projection, FilterByEveryField } = SchemaOperations -import { testIfSupportedOperationsIncludes } from '@wix-velo/test-commons' +import { dataSpi } from '@wix-velo/velo-external-db-core' +import { authAdmin, authOwner, authVisitor } from '@wix-velo/external-db-testkit' import * as gen from '../gen' import * as schema from '../drivers/schema_api_rest_test_support' -import * as data from '../drivers/data_api_rest_test_support' import * as matchers from '../drivers/schema_api_rest_matchers' -import { authAdmin, authOwner, authVisitor } from '@wix-velo/external-db-testkit' +import * as data from '../drivers/data_api_rest_test_support' import * as authorization from '../drivers/authorization_test_support' -import Chance = require('chance') import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, supportedOperations } from '../resources/e2e_resources' +const { UpdateImmediately, DeleteImmediately, Truncate, Aggregate, FindWithSort, Projection, FilterByEveryField } = SchemaOperations const chance = Chance() -const axios = require('axios').create({ +const axiosInstance = axios.create({ baseURL: 'http://localhost:8080' }) @@ -31,11 +32,12 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, () await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item, ctx.anotherItem], ctx.collectionName, authAdmin) await authorization.givenCollectionWithVisitorReadPolicy(ctx.collectionName) - await expect( axios.post('/data/find', { collectionName: ctx.collectionName, filter: '', sort: [{ fieldName: ctx.column.name }], skip: 0, limit: 25 }, authVisitor) ).resolves.toEqual( - expect.objectContaining({ data: { - items: [ ctx.item, ctx.anotherItem ].sort((a, b) => (a[ctx.column.name] > b[ctx.column.name]) ? 1 : -1), - totalCount: 2 - } })) + + const itemsByOrder = [ctx.item, ctx.anotherItem].sort((a, b) => (a[ctx.column.name] > b[ctx.column.name]) ? 1 : -1).map(item => ({ item })) + + await expect(data.queryCollectionAsArray(ctx.collectionName, [{ fieldName: ctx.column.name, order: dataSpi.SortOrder.ASC }], undefined, authVisitor)).resolves.toEqual( + ([...itemsByOrder, data.pagingMetadata(2, 2)]) + ) }) testIfSupportedOperationsIncludes(supportedOperations, [FilterByEveryField])('find api - filter by date', async() => { @@ -45,164 +47,211 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, () _createdDate: { $gte: ctx.pastVeloDate } } - await expect( axios.post('/data/find', { collectionName: ctx.collectionName, filter: filterByDate, skip: 0, limit: 25 }, authOwner) ).resolves.toEqual( - expect.objectContaining({ data: { - items: [ ctx.item ], - totalCount: 1 - } })) + + await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner, filterByDate)).resolves.toEqual( + expect.toIncludeSameMembers([{ item: ctx.item }, data.pagingMetadata(1, 1)])) }) testIfSupportedOperationsIncludes(supportedOperations, [ Projection ])('find api with projection', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item ], ctx.collectionName, authAdmin) - await expect( axios.post('/data/find', { collectionName: ctx.collectionName, filter: '', skip: 0, limit: 25, projection: [ctx.column.name] }, authOwner) ).resolves.toEqual( - expect.objectContaining({ data: { - items: [ ctx.item ].map(item => ({ [ctx.column.name]: item[ctx.column.name] })), - totalCount: 1 - } })) + + await expect(data.queryCollectionAsArray(ctx.collectionName, [], [ctx.column.name], authOwner)).resolves.toEqual( + expect.toIncludeSameMembers([{ item: { [ctx.column.name]: ctx.item[ctx.column.name] } }, data.pagingMetadata(1, 1)]) + ) }) //todo: create another test without sort for these implementations test('insert api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await axios.post('/data/insert', { collectionName: ctx.collectionName, item: ctx.item }, authAdmin) - await expect( data.expectAllDataIn(ctx.collectionName, authAdmin) ).resolves.toEqual({ items: [ctx.item], totalCount: 1 }) + const response = await axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, ctx.items, false), { responseType: 'stream', ...authAdmin }) + + const expectedItems = ctx.items.map(item => ({ item })) + + await expect(streamToArray(response.data)).resolves.toEqual(expectedItems) + await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeSameMembers( + [ + ...expectedItems, + data.pagingMetadata(ctx.items.length, ctx.items.length) + ]) + ) }) - test('bulk insert api', async() => { + test('insert api should fail if item already exists', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems([ ctx.items[1] ], ctx.collectionName, authAdmin) + + const response = axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, ctx.items, false), { responseType: 'stream', ...authAdmin }) - await axios.post('/data/insert/bulk', { collectionName: ctx.collectionName, items: ctx.items }, authAdmin) + const expectedItems = [dataSpi.QueryResponsePart.item(ctx.items[1])] - await expect( data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual( { items: expect.arrayContaining(ctx.items), totalCount: ctx.items.length }) + await expect(response).rejects.toThrow('409') + + await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeAllMembers( + [ + ...expectedItems, + data.pagingMetadata(expectedItems.length, expectedItems.length) + ]) + ) + }) + + test('insert api should succeed if item already exists and overwriteExisting is on', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems([ ctx.item ], ctx.collectionName, authAdmin) + + const response = await axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, [ctx.modifiedItem], true), { responseType: 'stream', ...authOwner }) + const expectedItems = [dataSpi.QueryResponsePart.item(ctx.modifiedItem)] + + await expect(streamToArray(response.data)).resolves.toEqual(expectedItems) + await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeAllMembers( + [ + ...expectedItems, + data.pagingMetadata(expectedItems.length, expectedItems.length) + ]) + ) }) testIfSupportedOperationsIncludes(supportedOperations, [ Aggregate ])('aggregate api', async() => { + await schema.givenCollection(ctx.collectionName, ctx.numberColumns, authOwner) - await data.givenItems([ctx.numberItem, ctx.anotherNumberItem], ctx.collectionName, authAdmin) - - await expect( axios.post('/data/aggregate', - { - collectionName: ctx.collectionName, - filter: { _id: { $eq: ctx.numberItem._id } }, - processingStep: { - _id: { - field1: '$_id', - field2: '$_owner', + await data.givenItems([ctx.numberItem, ctx.anotherNumberItem], ctx.collectionName, authOwner) + const response = await axiosInstance.post('/data/aggregate', + { + collectionId: ctx.collectionName, + initialFilter: { _id: { $eq: ctx.numberItem._id } }, + group: { + by: ['_id', '_owner'], aggregation: [ + { + name: 'myAvg', + avg: ctx.numberColumns[0].name }, - myAvg: { - $avg: `$${ctx.numberColumns[0].name}` - }, - mySum: { - $sum: `$${ctx.numberColumns[1].name}` + { + name: 'mySum', + sum: ctx.numberColumns[1].name } - }, - postFilteringStep: { - $and: [ - { myAvg: { $gt: 0 } }, - { mySum: { $gt: 0 } } - ], - }, - }, authAdmin) ).resolves.toEqual(matchers.responseWith({ items: [ { _id: ctx.numberItem._id, _owner: ctx.numberItem._owner, myAvg: ctx.numberItem[ctx.numberColumns[0].name], mySum: ctx.numberItem[ctx.numberColumns[1].name] } ], - totalCount: 0 })) - }) - - testIfSupportedOperationsIncludes(supportedOperations, [ DeleteImmediately ])('delete one api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - - await axios.post('/data/remove', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) - - await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ ], totalCount: 0 }) + ] + }, + finalFilter: { + $and: [ + { myAvg: { $gt: 0 } }, + { mySum: { $gt: 0 } } + ], + }, + }, { responseType: 'stream', ...authOwner }) + + await expect(streamToArray(response.data)).resolves.toEqual( + expect.toIncludeSameMembers([{ item: { + _id: ctx.numberItem._id, + _owner: ctx.numberItem._owner, + myAvg: ctx.numberItem[ctx.numberColumns[0].name], + mySum: ctx.numberItem[ctx.numberColumns[1].name] + } }, + data.pagingMetadata(1, 1) + ])) }) testIfSupportedOperationsIncludes(supportedOperations, [ DeleteImmediately ])('bulk delete api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems(ctx.items, ctx.collectionName, authAdmin) - await axios.post('/data/remove/bulk', { collectionName: ctx.collectionName, itemIds: ctx.items.map((i: { _id: any }) => i._id) }, authAdmin) + const response = await axiosInstance.post('/data/remove', { + collectionId: ctx.collectionName, itemIds: ctx.items.map(i => i._id) + }, { responseType: 'stream', ...authAdmin }) - await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ ], totalCount: 0 }) - }) + const expectedItems = ctx.items.map(item => ({ item })) - test('get by id api', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - - await expect( axios.post('/data/get', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) ).resolves.toEqual(matchers.responseWith({ item: ctx.item })) + await expect(streamToArray(response.data)).resolves.toEqual(expect.toIncludeSameMembers(expectedItems)) + await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual([data.pagingMetadata(0, 0)]) }) - test('get by id api should throw 404 if not found', async() => { + test('query by id api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - await data.givenItems([ctx.item], ctx.collectionName, authAdmin) + await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - await expect( axios.post('/data/get', { collectionName: ctx.collectionName, itemId: 'wrong' }, authAdmin) ).rejects.toThrow('404') + const filter = { + _id: { $eq: ctx.item._id } + } + + await expect(data.queryCollectionAsArray(ctx.collectionName, undefined, undefined, authOwner, filter)).resolves.toEqual(expect.toIncludeSameMembers( + [...[dataSpi.QueryResponsePart.item(ctx.item)], data.pagingMetadata(1, 1)]) + ) }) - testIfSupportedOperationsIncludes(supportedOperations, [ Projection ])('get by id api with projection', async() => { + test('query by id api should return empty result if not found', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - await expect(axios.post('/data/get', { collectionName: ctx.collectionName, itemId: ctx.item._id, projection: [ctx.column.name] }, authAdmin)).resolves.toEqual( - matchers.responseWith({ - item: { [ctx.column.name]: ctx.item[ctx.column.name] } - })) + const filter = { + _id: { $eq: 'wrong' } + } + + await expect(data.queryCollectionAsArray(ctx.collectionName, undefined, undefined, authOwner, filter)).resolves.toEqual( + ([data.pagingMetadata(0, 0)]) + ) }) - testIfSupportedOperationsIncludes(supportedOperations, [ UpdateImmediately ])('update api', async() => { + testIfSupportedOperationsIncludes(supportedOperations, [ Projection ])('query by id api with projection', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item], ctx.collectionName, authAdmin) - await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.modifiedItem }, authAdmin) + const filter = { + _id: { $eq: ctx.item._id } + } - await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ctx.modifiedItem], totalCount: 1 }) + await expect(data.queryCollectionAsArray(ctx.collectionName, undefined, [ctx.column.name], authOwner, filter)).resolves.toEqual(expect.toIncludeSameMembers( + [dataSpi.QueryResponsePart.item({ [ctx.column.name]: ctx.item[ctx.column.name] }), data.pagingMetadata(1, 1)]) + ) }) - testIfSupportedOperationsIncludes(supportedOperations, [ UpdateImmediately ])('bulk update api', async() => { + testIfSupportedOperationsIncludes(supportedOperations, [ UpdateImmediately ])('update api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems(ctx.items, ctx.collectionName, authAdmin) + const response = await axiosInstance.post('/data/update', data.updateRequest(ctx.collectionName, ctx.modifiedItems), { responseType: 'stream', ...authAdmin }) - await axios.post('/data/update/bulk', { collectionName: ctx.collectionName, items: ctx.modifiedItems }, authAdmin) + const expectedItems = ctx.modifiedItems.map(dataSpi.QueryResponsePart.item) - await expect( data.expectAllDataIn(ctx.collectionName, authAdmin) ).resolves.toEqual( { items: expect.arrayContaining(ctx.modifiedItems), totalCount: ctx.modifiedItems.length }) + await expect(streamToArray(response.data)).resolves.toEqual(expectedItems) + + await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeSameMembers( + [ + ...expectedItems, + data.pagingMetadata(ctx.modifiedItems.length, ctx.modifiedItems.length) + ])) }) test('count api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item, ctx.anotherItem], ctx.collectionName, authAdmin) - - await expect( axios.post('/data/count', { collectionName: ctx.collectionName, filter: '' }, authAdmin) ).resolves.toEqual(matchers.responseWith( { totalCount: 2 } )) + await expect( axiosInstance.post('/data/count', data.countRequest(ctx.collectionName), authAdmin) ).resolves.toEqual( + matchers.responseWith( { totalCount: 2 } )) }) testIfSupportedOperationsIncludes(supportedOperations, [ Truncate ])('truncate api', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ctx.item, ctx.anotherItem], ctx.collectionName, authAdmin) - - await axios.post('/data/truncate', { collectionName: ctx.collectionName }, authAdmin) - - await expect( data.expectAllDataIn(ctx.collectionName, authAdmin) ).resolves.toEqual({ items: [ ], totalCount: 0 }) + await axiosInstance.post('/data/truncate', { collectionId: ctx.collectionName }, authAdmin) + await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual([data.pagingMetadata(0, 0)]) }) test('insert undefined to number columns should inserted as null', async() => { await schema.givenCollection(ctx.collectionName, ctx.numberColumns, authOwner) delete ctx.numberItem[ctx.numberColumns[0].name] delete ctx.numberItem[ctx.numberColumns[1].name] + + await axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, [ctx.numberItem], false), { responseType: 'stream', ...authAdmin }) - await axios.post('/data/insert', { collectionName: ctx.collectionName, item: ctx.numberItem }, authAdmin) - - - await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ - items: [ - { + await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeSameMembers( + [ + dataSpi.QueryResponsePart.item({ ...ctx.numberItem, [ctx.numberColumns[0].name]: null, [ctx.numberColumns[1].name]: null, - } - ], totalCount: 1 - }) + }), + data.pagingMetadata(1, 1) + ])) }) @@ -212,17 +261,34 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, () ctx.numberItem[ctx.numberColumns[0].name] = null ctx.numberItem[ctx.numberColumns[1].name] = null - await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.numberItem }, authAdmin) + await axiosInstance.post('/data/update', data.updateRequest(ctx.collectionName, [ctx.numberItem]), { responseType: 'stream', ...authAdmin }) + - await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ - items: [ - { + await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeSameMembers( + [ + dataSpi.QueryResponsePart.item({ ...ctx.numberItem, [ctx.numberColumns[0].name]: null, [ctx.numberColumns[1].name]: null, - } - ], totalCount: 1 - }) + }), + data.pagingMetadata(1, 1) + ])) + }) + + test('count api on non existing collection should fail with 404', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems([ctx.item, ctx.anotherItem], ctx.collectionName, authAdmin) + await expect( + axiosInstance.post('/data/count', data.countRequest(gen.randomCollectionName()), authAdmin) + ).rejects.toThrow('404') + }) + + test('insert api on non existing collection should fail with 404', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + + const response = axiosInstance.post('/data/insert', data.insertRequest(gen.randomCollectionName(), ctx.items, false), { responseType: 'stream', transformRequest: authAdmin.transformRequest }) + + await expect(response).rejects.toThrow('404') }) diff --git a/apps/velo-external-db/test/e2e/app_data_hooks.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_data_hooks.e2e.spec.ts index 1e8341b5b..4e930f0e7 100644 --- a/apps/velo-external-db/test/e2e/app_data_hooks.e2e.spec.ts +++ b/apps/velo-external-db/test/e2e/app_data_hooks.e2e.spec.ts @@ -15,7 +15,7 @@ const axios = require('axios').create({ baseURL: 'http://localhost:8080' }) -describe(`Velo External DB Data Hooks: ${currentDbImplementationName()}`, () => { +describe.skip(`Velo External DB Data Hooks: ${currentDbImplementationName()}`, () => { beforeAll(async() => { await setupDb() diff --git a/apps/velo-external-db/test/e2e/app_schema.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_schema.e2e.spec.ts index 975ef72ca..44a980065 100644 --- a/apps/velo-external-db/test/e2e/app_schema.e2e.spec.ts +++ b/apps/velo-external-db/test/e2e/app_schema.e2e.spec.ts @@ -1,19 +1,22 @@ +import { SystemFields } from '@wix-velo/velo-external-db-commons' import { Uninitialized, gen as genCommon, testIfSupportedOperationsIncludes } from '@wix-velo/test-commons' -import { SchemaOperations } from '@wix-velo/velo-external-db-types' -const { RemoveColumn } = SchemaOperations +import { InputField, SchemaOperations } from '@wix-velo/velo-external-db-types' +const { RemoveColumn, ChangeColumnType } = SchemaOperations import * as schema from '../drivers/schema_api_rest_test_support' import * as matchers from '../drivers/schema_api_rest_matchers' +import { schemaUtils } from '@wix-velo/velo-external-db-core' import { authOwner } from '@wix-velo/external-db-testkit' import * as gen from '../gen' import Chance = require('chance') +import axios from 'axios' import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, supportedOperations } from '../resources/e2e_resources' const chance = Chance() -const axios = require('axios').create({ +const axiosClient = axios.create({ baseURL: 'http://localhost:8080' }) -describe(`Velo External DB Schema REST API: ${currentDbImplementationName()}`, () => { +describe(`Schema REST API: ${currentDbImplementationName()}`, () => { beforeAll(async() => { await setupDb() @@ -23,47 +26,97 @@ describe(`Velo External DB Schema REST API: ${currentDbImplementationName()}`, afterAll(async() => { await dbTeardown() }, 20000) + + describe('Velo External DB Collections REST API', () => { + beforeEach(async() => { + await schema.deleteAllCollections(authOwner) + }) - test('list', async() => { - await expect( axios.post('/schemas/list', {}, authOwner) ).resolves.toEqual( matchers.collectionResponseWithNoCollections() ) - }) + test('collection get', async() => { + await schema.givenCollection(ctx.collectionName, [], authOwner) - test('list headers', async() => { - await schema.givenCollection(ctx.collectionName, [], authOwner) + await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.collectionResponsesWith(ctx.collectionName, [...SystemFields])) + }) - await expect( axios.post('/schemas/list/headers', {}, authOwner) ).resolves.toEqual( matchers.collectionResponseWithCollections([ctx.collectionName]) ) - }) + test('collection create - collection without fields', async() => { + const collection = { + id: ctx.collectionName, + fields: [] + } + await axiosClient.post('/collections/create', { collection }, { ...authOwner, responseType: 'stream' }) - test('create', async() => { - await axios.post('/schemas/create', { collectionName: ctx.collectionName }, authOwner) + await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.createCollectionResponse(ctx.collectionName, [...SystemFields])) + }) - await expect( schema.retrieveSchemaFor(ctx.collectionName, authOwner) ).resolves.toEqual( matchers.collectionResponseWithDefaultFieldsFor(ctx.collectionName) ) - }) + test('collection create - collection with fields', async() => { + const collection = { + id: ctx.collectionName, + fields: [ctx.column].map(schemaUtils.InputFieldToWixFormatField) + } - test('find', async() => { - await schema.givenCollection(ctx.collectionName, [], authOwner) + await axiosClient.post('/collections/create', { collection }, { ...authOwner, responseType: 'stream' }) - await expect( axios.post('/schemas/find', { schemaIds: [ctx.collectionName] }, authOwner)).resolves.toEqual( matchers.collectionResponseWithDefaultFieldsFor(ctx.collectionName) ) - }) + await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.createCollectionResponse(ctx.collectionName, [...SystemFields, ctx.column])) + }) - test('add column', async() => { - await schema.givenCollection(ctx.collectionName, [], authOwner) + test('collection update - add column', async() => { + await schema.givenCollection(ctx.collectionName, [], authOwner) - await axios.post('/schemas/column/add', { collectionName: ctx.collectionName, column: ctx.column }, authOwner) + const collection: any = await schema.retrieveSchemaFor(ctx.collectionName, authOwner) - await expect( schema.retrieveSchemaFor(ctx.collectionName, authOwner) ).resolves.toEqual( matchers.collectionResponseHasField( ctx.column ) ) - }) + collection.fields.push(schemaUtils.InputFieldToWixFormatField(ctx.column)) + + await axiosClient.post('/collections/update', { collection }, { ...authOwner, responseType: 'stream' }) + + await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.collectionResponsesWith(ctx.collectionName, [...SystemFields, ctx.column])) + }) + + testIfSupportedOperationsIncludes(supportedOperations, [ RemoveColumn ])('collection update - remove column', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) - testIfSupportedOperationsIncludes(supportedOperations, [ RemoveColumn ])('remove column', async() => { - await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + const collection: any = await schema.retrieveSchemaFor(ctx.collectionName, authOwner) - await axios.post('/schemas/column/remove', { collectionName: ctx.collectionName, columnName: ctx.column.name }, authOwner) + const systemFieldsNames = SystemFields.map(f => f.name) + collection.fields = collection.fields.filter((f: any) => systemFieldsNames.includes(f.key)) - await expect( schema.retrieveSchemaFor(ctx.collectionName, authOwner) ).resolves.not.toEqual( matchers.collectionResponseHasField( ctx.column ) ) + await axiosClient.post('/collections/update', { collection }, { ...authOwner, responseType: 'stream' }) + + await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.collectionResponsesWith(ctx.collectionName, [...SystemFields])) + }) + + testIfSupportedOperationsIncludes(supportedOperations, [ ChangeColumnType ])('collection update - change column type', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + const collection: any = await schema.retrieveSchemaFor(ctx.collectionName, authOwner) + + const columnIndex = collection.fields.findIndex((f: any) => f.key === ctx.column.name) + collection.fields[columnIndex].type = schemaUtils.fieldTypeToWixDataEnum('number') + + await axiosClient.post('/collections/update', { collection }, { ...authOwner, responseType: 'stream' }) + + await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.createCollectionResponse(ctx.collectionName, [...SystemFields, { name: ctx.column.name, type: 'number' }])) + }) + + test('collection delete', async() => { + await schema.givenCollection(ctx.collectionName, [], authOwner) + await axiosClient.post('/collections/delete', { collectionId: ctx.collectionName }, { ...authOwner, responseType: 'stream' }) + await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).rejects.toThrow('404') + }) }) + interface Ctx { + collectionName: string + column: InputField + numberColumns: InputField[], + item: { [x: string]: any } + items: { [x: string]: any}[] + modifiedItem: { [x: string]: any } + modifiedItems: { [x: string]: any } + anotherItem: { [x: string]: any } + numberItem: { [x: string]: any } + anotherNumberItem: { [x: string]: any } + } - const ctx = { + const ctx: Ctx = { collectionName: Uninitialized, column: Uninitialized, numberColumns: Uninitialized, diff --git a/apps/velo-external-db/test/e2e/app_schema_hooks.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_schema_hooks.e2e.spec.ts index ae371394a..20b1498f2 100644 --- a/apps/velo-external-db/test/e2e/app_schema_hooks.e2e.spec.ts +++ b/apps/velo-external-db/test/e2e/app_schema_hooks.e2e.spec.ts @@ -17,7 +17,8 @@ const axios = require('axios').create({ baseURL: 'http://localhost:8080' }) -describe(`Velo External DB Schema Hooks: ${currentDbImplementationName()}`, () => { +// eslint-disable-next-line jest/no-disabled-tests +describe.skip(`Velo External DB Schema Hooks: ${currentDbImplementationName()}`, () => { beforeAll(async() => { await setupDb() diff --git a/apps/velo-external-db/test/env/env.db.setup.js b/apps/velo-external-db/test/env/env.db.setup.js index 22b402c47..facb620ad 100644 --- a/apps/velo-external-db/test/env/env.db.setup.js +++ b/apps/velo-external-db/test/env/env.db.setup.js @@ -3,16 +3,16 @@ import { registerTsProject } from 'nx/src/utils/register' registerTsProject('.', 'tsconfig.base.json') -const { testResources: postgres } = require ('@wix-velo/external-db-postgres') +// const { testResources: postgres } = require ('@wix-velo/external-db-postgres') const { testResources: mysql } = require ('@wix-velo/external-db-mysql') -const { testResources: spanner } = require ('@wix-velo/external-db-spanner') -const { testResources: firestore } = require ('@wix-velo/external-db-firestore') -const { testResources: mssql } = require ('@wix-velo/external-db-mssql') -const { testResources: mongo } = require ('@wix-velo/external-db-mongo') -const { testResources: googleSheet } = require('@wix-velo/external-db-google-sheets') -const { testResources: airtable } = require('@wix-velo/external-db-airtable') -const { testResources: dynamoDb } = require('@wix-velo/external-db-dynamodb') -const { testResources: bigquery } = require('@wix-velo/external-db-bigquery') +// const { testResources: spanner } = require ('@wix-velo/external-db-spanner') +// const { testResources: firestore } = require ('@wix-velo/external-db-firestore') +// const { testResources: mssql } = require ('@wix-velo/external-db-mssql') +// const { testResources: mongo } = require ('@wix-velo/external-db-mongo') +// const { testResources: googleSheet } = require('@wix-velo/external-db-google-sheets') +// const { testResources: airtable } = require('@wix-velo/external-db-airtable') +// const { testResources: dynamoDb } = require('@wix-velo/external-db-dynamodb') +// const { testResources: bigquery } = require('@wix-velo/external-db-bigquery') const { sleep } = require('@wix-velo/test-commons') const ci = require('./ci_utils') @@ -23,40 +23,40 @@ const initEnv = async(testEngine) => { await mysql.initEnv() break - case 'spanner': - await spanner.initEnv() - break + // case 'spanner': + // await spanner.initEnv() + // break - case 'postgres': - await postgres.initEnv() - break + // case 'postgres': + // await postgres.initEnv() + // break - case 'firestore': - await firestore.initEnv() - break + // case 'firestore': + // await firestore.initEnv() + // break - case 'mssql': - await mssql.initEnv() - break + // case 'mssql': + // await mssql.initEnv() + // break - case 'mongo': - await mongo.initEnv() - break - case 'google-sheet': - await googleSheet.initEnv() - break + // case 'mongo': + // await mongo.initEnv() + // break + // case 'google-sheet': + // await googleSheet.initEnv() + // break - case 'airtable': - await airtable.initEnv() - break + // case 'airtable': + // await airtable.initEnv() + // break - case 'dynamodb': - await dynamoDb.initEnv() - break + // case 'dynamodb': + // await dynamoDb.initEnv() + // break - case 'bigquery': - await bigquery.initEnv() - break + // case 'bigquery': + // await bigquery.initEnv() + // break } } @@ -66,37 +66,37 @@ const cleanup = async(testEngine) => { await mysql.cleanup() break - case 'spanner': - await spanner.cleanup() - break + // case 'spanner': + // await spanner.cleanup() + // break - case 'postgres': - await postgres.cleanup() - break + // case 'postgres': + // await postgres.cleanup() + // break - case 'firestore': - await firestore.cleanup() - break + // case 'firestore': + // await firestore.cleanup() + // break - case 'mssql': - await mssql.cleanup() - break + // case 'mssql': + // await mssql.cleanup() + // break - case 'google-sheet': - await googleSheet.cleanup() - break + // case 'google-sheet': + // await googleSheet.cleanup() + // break - case 'mongo': - await mongo.cleanup() - break + // case 'mongo': + // await mongo.cleanup() + // break - case 'dynamodb': - await dynamoDb.cleanup() - break + // case 'dynamodb': + // await dynamoDb.cleanup() + // break - case 'bigquery': - await bigquery.cleanup() - break + // case 'bigquery': + // await bigquery.cleanup() + // break } } diff --git a/apps/velo-external-db/test/env/env.db.teardown.js b/apps/velo-external-db/test/env/env.db.teardown.js index 63a4e09c7..9e1f1e6f4 100644 --- a/apps/velo-external-db/test/env/env.db.teardown.js +++ b/apps/velo-external-db/test/env/env.db.teardown.js @@ -1,13 +1,13 @@ -const { testResources: postgres } = require ('@wix-velo/external-db-postgres') +// const { testResources: postgres } = require ('@wix-velo/external-db-postgres') const { testResources: mysql } = require ('@wix-velo/external-db-mysql') -const { testResources: spanner } = require ('@wix-velo/external-db-spanner') -const { testResources: firestore } = require ('@wix-velo/external-db-firestore') -const { testResources: mssql } = require ('@wix-velo/external-db-mssql') -const { testResources: mongo } = require ('@wix-velo/external-db-mongo') -const { testResources: googleSheet } = require('@wix-velo/external-db-google-sheets') -const { testResources: airtable } = require('@wix-velo/external-db-airtable') -const { testResources: dynamo } = require('@wix-velo/external-db-dynamodb') -const { testResources: bigquery } = require('@wix-velo/external-db-bigquery') +// const { testResources: spanner } = require ('@wix-velo/external-db-spanner') +// const { testResources: firestore } = require ('@wix-velo/external-db-firestore') +// const { testResources: mssql } = require ('@wix-velo/external-db-mssql') +// const { testResources: mongo } = require ('@wix-velo/external-db-mongo') +// const { testResources: googleSheet } = require('@wix-velo/external-db-google-sheets') +// const { testResources: airtable } = require('@wix-velo/external-db-airtable') +// const { testResources: dynamo } = require('@wix-velo/external-db-dynamodb') +// const { testResources: bigquery } = require('@wix-velo/external-db-bigquery') const ci = require('./ci_utils') @@ -17,41 +17,41 @@ const shutdownEnv = async(testEngine) => { await mysql.shutdownEnv() break - case 'spanner': - await spanner.shutdownEnv() - break + // case 'spanner': + // await spanner.shutdownEnv() + // break - case 'postgres': - await postgres.shutdownEnv() - break + // case 'postgres': + // await postgres.shutdownEnv() + // break - case 'firestore': - await firestore.shutdownEnv() - break + // case 'firestore': + // await firestore.shutdownEnv() + // break - case 'mssql': - await mssql.shutdownEnv() - break + // case 'mssql': + // await mssql.shutdownEnv() + // break - case 'google-sheet': - await googleSheet.shutdownEnv() - break + // case 'google-sheet': + // await googleSheet.shutdownEnv() + // break - case 'airtable': - await airtable.shutdownEnv() - break + // case 'airtable': + // await airtable.shutdownEnv() + // break - case 'dynamodb': - await dynamo.shutdownEnv() - break + // case 'dynamodb': + // await dynamo.shutdownEnv() + // break - case 'mongo': - await mongo.shutdownEnv() - break + // case 'mongo': + // await mongo.shutdownEnv() + // break - case 'bigquery': - await bigquery.shutdownEnv() - break + // case 'bigquery': + // await bigquery.shutdownEnv() + // break } } diff --git a/apps/velo-external-db/test/gen.ts b/apps/velo-external-db/test/gen.ts index b6429d074..6343144ae 100644 --- a/apps/velo-external-db/test/gen.ts +++ b/apps/velo-external-db/test/gen.ts @@ -73,12 +73,12 @@ export const randomObjectDbEntity = (columns: InputField[]) => { return entity } -export const randomNumberColumns = () => { +export const randomNumberColumns = (): InputField[] => { return [ { name: chance.word(), type: 'number', subtype: 'int', isPrimary: false }, { name: chance.word(), type: 'number', subtype: 'decimal', precision: '10,2', isPrimary: false } ] } -export const randomColumn = () => ( { name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false } ) +export const randomColumn = (): InputField => ( { name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false } ) export const randomObjectColumn = () => ( { name: chance.word(), type: 'object' } ) diff --git a/apps/velo-external-db/test/resources/e2e_resources.ts b/apps/velo-external-db/test/resources/e2e_resources.ts index 9111033a6..be63ea1cb 100644 --- a/apps/velo-external-db/test/resources/e2e_resources.ts +++ b/apps/velo-external-db/test/resources/e2e_resources.ts @@ -16,6 +16,7 @@ import { Uninitialized } from '@wix-velo/test-commons' import { ExternalDbRouter } from '@wix-velo/velo-external-db-core' import { Server } from 'http' import { ConnectionCleanUp, ISchemaProvider } from '@wix-velo/velo-external-db-types' +import { initWixDataEnv, shutdownWixDataEnv, wixDataBaseUrl } from '../drivers/wix_data_resources' interface App { server: Server; @@ -40,8 +41,10 @@ export let env:{ externalDbRouter: Uninitialized } +const createAppWithWixDataBaseUrl = createApp.bind(null, wixDataBaseUrl()) + const testSuits = { - mysql: new E2EResources(mysql, createApp), + mysql: new E2EResources(mysql, createAppWithWixDataBaseUrl), postgres: new E2EResources(postgres, createApp), spanner: new E2EResources(spanner, createApp), firestore: new E2EResources(firestore, createApp), @@ -56,10 +59,16 @@ const testSuits = { export const testedSuit = () => testSuits[process.env.TEST_ENGINE] export const supportedOperations = testedSuit().supportedOperations -export const setupDb = () => testedSuit().setUpDb() -export const currentDbImplementationName = () => testedSuit().name +export const setupDb = async() => { + await initWixDataEnv() + await testedSuit().setUpDb() +} +export const currentDbImplementationName = () => testedSuit().currentDbImplementationName export const initApp = async() => { env = await testedSuit().initApp() } -export const teardownApp = async() => testedSuit().teardownApp() +export const teardownApp = async() => { + await testedSuit().teardownApp() + await shutdownWixDataEnv() +} export const dbTeardown = async() => testedSuit().dbTeardown() diff --git a/apps/velo-external-db/test/resources/provider_resources.ts b/apps/velo-external-db/test/resources/provider_resources.ts index 977246e8b..4972d008f 100644 --- a/apps/velo-external-db/test/resources/provider_resources.ts +++ b/apps/velo-external-db/test/resources/provider_resources.ts @@ -31,11 +31,13 @@ export const env: { schemaProvider: ISchemaProvider cleanup: ConnectionCleanUp driver: AnyFixMe + capabilities: any } = { dataProvider: Uninitialized, schemaProvider: Uninitialized, cleanup: Uninitialized, driver: Uninitialized, + capabilities: Uninitialized, } const dbInit = async(impl: any) => { @@ -48,6 +50,7 @@ const dbInit = async(impl: any) => { env.dataProvider = new impl.DataProvider(pool, driver.filterParser) env.schemaProvider = new impl.SchemaProvider(pool, testResources.schemaProviderTestVariables?.() ) env.driver = driver + env.capabilities = impl.testResources.capabilities env.cleanup = cleanup } @@ -56,6 +59,7 @@ export const dbTeardown = async() => { env.dataProvider = Uninitialized env.schemaProvider = Uninitialized env.driver = Uninitialized + env.capabilities = Uninitialized } const postgresTestEnvInit = async() => await dbInit(postgres) @@ -70,7 +74,7 @@ const bigqueryTestEnvInit = async() => await dbInit(bigquery) const googleSheetTestEnvInit = async() => await dbInit(googleSheet) const testSuits = { - mysql: suiteDef('MySql', mysqlTestEnvInit, mysql.testResources.supportedOperations), + mysql: suiteDef('MySql', mysqlTestEnvInit, mysql.testResources), postgres: suiteDef('Postgres', postgresTestEnvInit, postgres.testResources.supportedOperations), spanner: suiteDef('Spanner', spannerTestEnvInit, spanner.testResources.supportedOperations), firestore: suiteDef('Firestore', firestoreTestEnvInit, firestore.testResources.supportedOperations), diff --git a/apps/velo-external-db/test/resources/test_suite_definition.ts b/apps/velo-external-db/test/resources/test_suite_definition.ts index 45275b36a..09c7598e0 100644 --- a/apps/velo-external-db/test/resources/test_suite_definition.ts +++ b/apps/velo-external-db/test/resources/test_suite_definition.ts @@ -1 +1,6 @@ -export const suiteDef = (name: string, setup: any, supportedOperations: any) => ( { name, setup, supportedOperations } ) +export const suiteDef = (name: string, setup: any, testResources: any) => ({ + name, + setup, + supportedOperations: testResources.supportedOperations, + capabilities: testResources.capabilities + }) diff --git a/apps/velo-external-db/test/storage/data_provider.spec.ts b/apps/velo-external-db/test/storage/data_provider.spec.ts index 06a820f86..7bd11a1c0 100644 --- a/apps/velo-external-db/test/storage/data_provider.spec.ts +++ b/apps/velo-external-db/test/storage/data_provider.spec.ts @@ -220,8 +220,9 @@ describe(`Data API: ${currentDbImplementationName()}`, () => { env.driver.stubEmptyFilterFor(ctx.filter) env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 1) + env.driver.stubEmptyOrderByFor(ctx.sort) - await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) ).resolves.toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }, + await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit) ).resolves.toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }, { _id: ctx.anotherNumberEntity._id, [ctx.aliasColumns[0]]: ctx.anotherNumberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.anotherNumberEntity[ctx.numericColumns[1].name] } ])) }) @@ -232,8 +233,9 @@ describe(`Data API: ${currentDbImplementationName()}`, () => { env.driver.stubEmptyFilterFor(ctx.filter) env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 1) + env.driver.stubEmptyOrderByFor(ctx.sort) - await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) ).resolves.toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }, + await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit) ).resolves.toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }, { _id: ctx.anotherNumberEntity._id, [ctx.aliasColumns[0]]: ctx.anotherNumberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.anotherNumberEntity[ctx.numericColumns[1].name] } ])) }) @@ -244,8 +246,9 @@ describe(`Data API: ${currentDbImplementationName()}`, () => { env.driver.givenFilterByIdWith(ctx.numberEntity._id, ctx.filter) env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 2) + env.driver.stubEmptyOrderByFor(ctx.sort) - await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) ).resolves.toEqual([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }]) + await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit) ).resolves.toEqual([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }]) }) diff --git a/apps/velo-external-db/test/storage/schema_provider.spec.ts b/apps/velo-external-db/test/storage/schema_provider.spec.ts index 1177dc726..e2265e35e 100644 --- a/apps/velo-external-db/test/storage/schema_provider.spec.ts +++ b/apps/velo-external-db/test/storage/schema_provider.spec.ts @@ -3,7 +3,7 @@ import { errors, SystemFields } from '@wix-velo/velo-external-db-commons' import { SchemaOperations } from '@wix-velo/velo-external-db-types' import { Uninitialized, gen, testIfSupportedOperationsIncludes } from '@wix-velo/test-commons' import { env, dbTeardown, setupDb, currentDbImplementationName, supportedOperations } from '../resources/provider_resources' -import { collectionWithDefaultFields, hasSameSchemaFieldsLike } from '../drivers/schema_provider_matchers' +import { toContainDefaultFields, collectionToContainFields, toBeDefaultCollectionWith, hasSameSchemaFieldsLike } from '../drivers/schema_provider_matchers' const chance = new Chance() const { CollectionDoesNotExists, FieldAlreadyExists, CannotModifySystemField, FieldDoesNotExist } = errors const { RemoveColumn } = SchemaOperations @@ -39,11 +39,11 @@ describe(`Schema API: ${currentDbImplementationName()}`, () => { await expect( env.schemaProvider.list() ).resolves.toEqual(expect.arrayContaining([ expect.objectContaining({ id: ctx.collectionName, - fields: collectionWithDefaultFields() + fields: toContainDefaultFields() }), expect.objectContaining({ id: ctx.anotherCollectionName, - fields: collectionWithDefaultFields() + fields: toContainDefaultFields() }) ])) }) @@ -51,7 +51,7 @@ describe(`Schema API: ${currentDbImplementationName()}`, () => { test('create collection with default columns', async() => { await env.schemaProvider.create(ctx.collectionName) - await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(collectionWithDefaultFields()) + await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(toBeDefaultCollectionWith(ctx.collectionName, env.capabilities)) }) test('drop collection', async() => { @@ -65,13 +65,13 @@ describe(`Schema API: ${currentDbImplementationName()}`, () => { test('collection name and variables are case sensitive', async() => { await env.schemaProvider.create(ctx.collectionName.toUpperCase()) - await expect( env.schemaProvider.describeCollection(ctx.collectionName.toUpperCase()) ).resolves.toEqual(collectionWithDefaultFields()) + await expect( env.schemaProvider.describeCollection(ctx.collectionName.toUpperCase()) ).resolves.toEqual(toBeDefaultCollectionWith(ctx.collectionName.toUpperCase(), env.capabilities)) }) test('retrieve collection data by collection name', async() => { await env.schemaProvider.create(ctx.collectionName) - await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(collectionWithDefaultFields()) + await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(toBeDefaultCollectionWith(ctx.collectionName, env.capabilities)) }) test('create collection twice will do nothing', async() => { @@ -87,7 +87,7 @@ describe(`Schema API: ${currentDbImplementationName()}`, () => { test('add column on a an existing collection', async() => { await env.schemaProvider.create(ctx.collectionName, []) await env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' }) - await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual( hasSameSchemaFieldsLike([{ field: ctx.columnName }])) + await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(collectionToContainFields(ctx.collectionName, [{ field: ctx.columnName, type: 'datetime' }], env.capabilities)) }) test('add duplicate column will fail', async() => { diff --git a/libs/external-db-config/src/readers/aws_config_reader.ts b/libs/external-db-config/src/readers/aws_config_reader.ts index 260472499..a8d02e3d5 100644 --- a/libs/external-db-config/src/readers/aws_config_reader.ts +++ b/libs/external-db-config/src/readers/aws_config_reader.ts @@ -13,8 +13,8 @@ export class AwsConfigReader implements IConfigReader { async readConfig() { const { config } = await this.readExternalAndLocalConfig() - const { host, username, password, DB, SECRET_KEY } = config - return { host: host, user: username, password: password, db: DB, secretKey: SECRET_KEY } + const { host, username, password, DB, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = config + return { host: host, user: username, password: password, db: DB, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES } } async readExternalConfig() { @@ -29,8 +29,8 @@ export class AwsConfigReader implements IConfigReader { async readExternalAndLocalConfig() { const { externalConfig, secretMangerError }: {[key: string]: any} = await this.readExternalConfig() - const { host, username, password, DB, SECRET_KEY, HOST, PASSWORD, USER }: {[key: string]: string} = { ...process.env, ...externalConfig } - const config = { host: host || HOST, username: username || USER, password: password || PASSWORD, DB, SECRET_KEY } + const { host, username, password, DB, EXTERNAL_DATABASE_ID, ALLOWED_METASITES, HOST, PASSWORD, USER }: {[key: string]: string} = { ...process.env, ...externalConfig } + const config = { host: host || HOST, username: username || USER, password: password || PASSWORD, DB, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } return { config, secretMangerError } } } @@ -46,15 +46,15 @@ export class AwsDynamoConfigReader implements IConfigReader { async readConfig() { const { config } = await this.readExternalAndLocalConfig() if (process.env['NODE_ENV'] === 'test') { - return { region: this.region, secretKey: config.SECRET_KEY, endpoint: process.env['ENDPOINT_URL'] } + return { region: this.region, externalDatabaseId: config.EXTERNAL_DATABASE_ID, endpoint: process.env['ENDPOINT_URL'] } } - return { region: this.region, secretKey: config.SECRET_KEY } + return { region: this.region, externalDatabaseId: config.EXTERNAL_DATABASE_ID, allowedMetasites: config.ALLOWED_METASITES } } async readExternalAndLocalConfig() { const { externalConfig, secretMangerError }: {[key: string]: any} = await this.readExternalConfig() - const { SECRET_KEY = undefined } = { ...process.env, ...externalConfig } - const config = { SECRET_KEY } + const { EXTERNAL_DATABASE_ID = undefined, ALLOWED_METASITES = undefined } = { ...process.env, ...externalConfig } + const config = { EXTERNAL_DATABASE_ID, ALLOWED_METASITES } return { config, secretMangerError: secretMangerError } } @@ -90,8 +90,8 @@ export class AwsMongoConfigReader implements IConfigReader { async readExternalAndLocalConfig() { const { externalConfig, secretMangerError } :{[key: string]: any} = await this.readExternalConfig() - const { SECRET_KEY, URI }: {SECRET_KEY: string, URI: string} = { ...process.env, ...externalConfig } - const config = { SECRET_KEY, URI } + const { EXTERNAL_DATABASE_ID, ALLOWED_METASITES, URI }: {EXTERNAL_DATABASE_ID: string, ALLOWED_METASITES: string, URI: string} = { ...process.env, ...externalConfig } + const config = { EXTERNAL_DATABASE_ID, ALLOWED_METASITES, URI } return { config, secretMangerError: secretMangerError } } @@ -99,8 +99,8 @@ export class AwsMongoConfigReader implements IConfigReader { async readConfig() { const { config } = await this.readExternalAndLocalConfig() - const { SECRET_KEY, URI } = config + const { EXTERNAL_DATABASE_ID, ALLOWED_METASITES, URI } = config - return { secretKey: SECRET_KEY, connectionUri: URI } + return { externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES, connectionUri: URI } } } diff --git a/libs/external-db-config/src/readers/azure_config_reader.ts b/libs/external-db-config/src/readers/azure_config_reader.ts index bd26cbe0d..d4bee7764 100644 --- a/libs/external-db-config/src/readers/azure_config_reader.ts +++ b/libs/external-db-config/src/readers/azure_config_reader.ts @@ -5,7 +5,7 @@ export class AzureConfigReader implements IConfigReader { } async readConfig() { - const { HOST, USER, PASSWORD, DB, SECRET_KEY, UNSECURED_ENV } = process.env - return { host: HOST, user: USER, password: PASSWORD, db: DB, secretKey: SECRET_KEY, unsecuredEnv: UNSECURED_ENV } + const { HOST, USER, PASSWORD, DB, EXTERNAL_DATABASE_ID, ALLOWED_METASITES, UNSECURED_ENV } = process.env + return { host: HOST, user: USER, password: PASSWORD, db: DB, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES, unsecuredEnv: UNSECURED_ENV } } } diff --git a/libs/external-db-config/src/readers/common_config_reader.ts b/libs/external-db-config/src/readers/common_config_reader.ts index 73664fe69..fcd541782 100644 --- a/libs/external-db-config/src/readers/common_config_reader.ts +++ b/libs/external-db-config/src/readers/common_config_reader.ts @@ -4,7 +4,7 @@ export default class CommonConfigReader implements IConfigReader { constructor() { } readConfig() { - const { CLOUD_VENDOR, TYPE, REGION, SECRET_NAME } = process.env - return { vendor: CLOUD_VENDOR, type: TYPE, region: REGION, secretId: SECRET_NAME } + const { CLOUD_VENDOR, TYPE, REGION, SECRET_NAME, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env + return { vendor: CLOUD_VENDOR, type: TYPE, region: REGION, secretId: SECRET_NAME, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES } } } diff --git a/libs/external-db-config/src/readers/gcp_config_reader.ts b/libs/external-db-config/src/readers/gcp_config_reader.ts index d83a5c63c..d4e0c484b 100644 --- a/libs/external-db-config/src/readers/gcp_config_reader.ts +++ b/libs/external-db-config/src/readers/gcp_config_reader.ts @@ -5,8 +5,8 @@ export class GcpConfigReader implements IConfigReader { } async readConfig() { - const { CLOUD_SQL_CONNECTION_NAME, USER, PASSWORD, DB, SECRET_KEY } = process.env - return { cloudSqlConnectionName: CLOUD_SQL_CONNECTION_NAME, user: USER, password: PASSWORD, db: DB, secretKey: SECRET_KEY } + const { CLOUD_SQL_CONNECTION_NAME, USER, PASSWORD, DB, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env + return { cloudSqlConnectionName: CLOUD_SQL_CONNECTION_NAME, user: USER, password: PASSWORD, db: DB, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES } } } @@ -16,8 +16,8 @@ export class GcpSpannerConfigReader implements IConfigReader { } async readConfig() { - const { PROJECT_ID, INSTANCE_ID, DATABASE_ID, SECRET_KEY } = process.env - return { projectId: PROJECT_ID, instanceId: INSTANCE_ID, databaseId: DATABASE_ID, secretKey: SECRET_KEY } + const { PROJECT_ID, INSTANCE_ID, DATABASE_ID, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env + return { projectId: PROJECT_ID, instanceId: INSTANCE_ID, databaseId: DATABASE_ID, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES } } @@ -27,8 +27,8 @@ export class GcpFirestoreConfigReader implements IConfigReader { constructor() { } async readConfig() { - const { PROJECT_ID, SECRET_KEY } = process.env - return { projectId: PROJECT_ID, secretKey: SECRET_KEY } + const { PROJECT_ID, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env + return { projectId: PROJECT_ID, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES } } @@ -38,8 +38,8 @@ export class GcpGoogleSheetsConfigReader implements IConfigReader { constructor() { } async readConfig() { - const { CLIENT_EMAIL, SHEET_ID, API_PRIVATE_KEY, SECRET_KEY } = process.env - return { clientEmail: CLIENT_EMAIL, apiPrivateKey: API_PRIVATE_KEY, sheetId: SHEET_ID, secretKey: SECRET_KEY } + const { CLIENT_EMAIL, SHEET_ID, API_PRIVATE_KEY, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env + return { clientEmail: CLIENT_EMAIL, apiPrivateKey: API_PRIVATE_KEY, sheetId: SHEET_ID, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES } } } @@ -48,8 +48,8 @@ export class GcpMongoConfigReader implements IConfigReader { constructor() { } async readConfig() { - const { URI, SECRET_KEY } = process.env - return { connectionUri: URI, secretKey: SECRET_KEY } + const { URI, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env + return { connectionUri: URI, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES } } } @@ -57,8 +57,8 @@ export class GcpAirtableConfigReader implements IConfigReader { constructor() { } async readConfig() { - const { AIRTABLE_API_KEY, META_API_KEY, BASE_ID, SECRET_KEY, BASE_URL } = process.env - return { apiPrivateKey: AIRTABLE_API_KEY, metaApiKey: META_API_KEY, baseId: BASE_ID, secretKey: SECRET_KEY, baseUrl: BASE_URL } + const { AIRTABLE_API_KEY, META_API_KEY, BASE_ID, EXTERNAL_DATABASE_ID, ALLOWED_METASITES, BASE_URL } = process.env + return { apiPrivateKey: AIRTABLE_API_KEY, metaApiKey: META_API_KEY, baseId: BASE_ID, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES, baseUrl: BASE_URL } } } @@ -67,7 +67,7 @@ export class GcpBigQueryConfigReader implements IConfigReader { } async readConfig() { - const { PROJECT_ID, DATABASE_ID, SECRET_KEY } = process.env - return { projectId: PROJECT_ID, databaseId: DATABASE_ID, secretKey: SECRET_KEY } + const { PROJECT_ID, DATABASE_ID, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env + return { projectId: PROJECT_ID, databaseId: DATABASE_ID, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES } } } diff --git a/libs/external-db-config/src/service/config_validator.spec.ts b/libs/external-db-config/src/service/config_validator.spec.ts index d59ad6856..2d36fd2fd 100644 --- a/libs/external-db-config/src/service/config_validator.spec.ts +++ b/libs/external-db-config/src/service/config_validator.spec.ts @@ -10,7 +10,7 @@ describe('Config Reader Client', () => { test('read config will retrieve config from secret provider and validate retrieved data', async() => { driver.givenConfig(ctx.config) - driver.givenCommonConfig(ctx.secretKey) + driver.givenCommonConfig(ctx.externalDatabaseId, ctx.allowedMetasites) driver.givenAuthorizationConfig(ctx.authorizationConfig) expect( env.configValidator.readConfig() ).toEqual(matchers.configResponseFor(ctx.config, ctx.authorizationConfig)) @@ -85,7 +85,8 @@ describe('Config Reader Client', () => { configStatus: Uninitialized, missingProperties: Uninitialized, moreMissingProperties: Uninitialized, - secretKey: Uninitialized, + externalDatabaseId: Uninitialized, + allowedMetasites: Uninitialized, authorizationConfig: Uninitialized, } @@ -102,7 +103,8 @@ describe('Config Reader Client', () => { ctx.configStatus = gen.randomConfig() ctx.missingProperties = Array.from({ length: 5 }, () => chance.word()) ctx.moreMissingProperties = Array.from({ length: 5 }, () => chance.word()) - ctx.secretKey = chance.guid() + ctx.externalDatabaseId = chance.guid() + ctx.allowedMetasites = chance.guid() env.configValidator = new ConfigValidator(driver.configValidator, driver.authorizationConfigValidator, driver.commonConfigValidator) }) }) diff --git a/libs/external-db-config/src/validators/common_config_validator.spec.ts b/libs/external-db-config/src/validators/common_config_validator.spec.ts index ec61b8494..42ea22149 100644 --- a/libs/external-db-config/src/validators/common_config_validator.spec.ts +++ b/libs/external-db-config/src/validators/common_config_validator.spec.ts @@ -11,9 +11,9 @@ describe('MySqlConfigValidator', () => { expect(env.CommonConfigValidator.validate()).toEqual({ missingRequiredSecretsKeys: [] }) }) - test('not extended common config validator will return if secretKey is missing', () => { + test('not extended common config validator will return if externalDatabaseId or allowedMetasites are missing', () => { env.CommonConfigValidator = new CommonConfigValidator({}) - expect(env.CommonConfigValidator.validate()).toEqual({ missingRequiredSecretsKeys: ['secretKey'] }) + expect(env.CommonConfigValidator.validate()).toEqual({ missingRequiredSecretsKeys: ['externalDatabaseId', 'allowedMetasites'] }) }) each( diff --git a/libs/external-db-config/src/validators/common_config_validator.ts b/libs/external-db-config/src/validators/common_config_validator.ts index c82ec8f06..d7ef19835 100644 --- a/libs/external-db-config/src/validators/common_config_validator.ts +++ b/libs/external-db-config/src/validators/common_config_validator.ts @@ -22,7 +22,7 @@ export class CommonConfigValidator { validateBasic() { return { - missingRequiredSecretsKeys: checkRequiredKeys(this.config, ['secretKey']) + missingRequiredSecretsKeys: checkRequiredKeys(this.config, ['externalDatabaseId', 'allowedMetasites']) } } @@ -32,7 +32,7 @@ export class CommonConfigValidator { return { validType, validVendor, - missingRequiredSecretsKeys: checkRequiredKeys(this.config, ['type', 'vendor', 'secretKey']) + missingRequiredSecretsKeys: checkRequiredKeys(this.config, ['type', 'vendor', 'externalDatabaseId', 'allowedMetasites']) } } } diff --git a/libs/external-db-config/test/drivers/aws_mongo_config_test_support.ts b/libs/external-db-config/test/drivers/aws_mongo_config_test_support.ts index ee8138bbd..533249adb 100644 --- a/libs/external-db-config/test/drivers/aws_mongo_config_test_support.ts +++ b/libs/external-db-config/test/drivers/aws_mongo_config_test_support.ts @@ -17,8 +17,11 @@ export const defineValidConfig = (config: MongoConfig) => { if (config.connectionUri) { awsConfig['URI'] = config.connectionUri } - if (config.secretKey) { - awsConfig['SECRET_KEY'] = config.secretKey + if (config.externalDatabaseId) { + awsConfig['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId + } + if (config.allowedMetasites) { + awsConfig['ALLOWED_METASITES'] = config.allowedMetasites } if (config.authorization) { awsConfig['PERMISSIONS'] = JSON.stringify( config.authorization ) @@ -30,8 +33,11 @@ const defineLocalEnvs = (config: MongoConfig) => { if (config.connectionUri) { process.env['URI'] = config.connectionUri } - if (config.secretKey) { - process.env['SECRET_KEY'] = config.secretKey + if (config.externalDatabaseId) { + process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId + } + if (config.allowedMetasites) { + process.env['ALLOWED_METASITES'] = config.allowedMetasites } if (config.authorization) { process.env['PERMISSIONS'] = JSON.stringify( config.authorization ) @@ -42,7 +48,8 @@ export const defineInvalidConfig = () => defineValidConfig({}) export const validConfig = () => ({ connectionUri: chance.word(), - secretKey: chance.word() + externalDatabaseId: chance.word(), + allowedMetasites: chance.word() }) export const defineSplittedConfig = (config: MongoConfig) => { @@ -56,8 +63,8 @@ export const validConfigWithAuthorization = () => ({ authorization: validAuthorizationConfig.collectionPermissions }) -export const ExpectedProperties = ['URI', 'SECRET_KEY', 'PERMISSIONS'] -export const RequiredProperties = ['URI', 'SECRET_KEY'] +export const ExpectedProperties = ['URI', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'PERMISSIONS'] +export const RequiredProperties = ['URI', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES'] export const reset = () => { mockedAwsSdk.reset() diff --git a/libs/external-db-config/test/drivers/aws_mysql_config_test_support.ts b/libs/external-db-config/test/drivers/aws_mysql_config_test_support.ts index 84f091a30..02f87ea69 100644 --- a/libs/external-db-config/test/drivers/aws_mysql_config_test_support.ts +++ b/libs/external-db-config/test/drivers/aws_mysql_config_test_support.ts @@ -26,8 +26,11 @@ export const defineValidConfig = (config: MySqlConfig) => { if (config.db) { awsConfig['DB'] = config.db } - if (config.secretKey) { - awsConfig['SECRET_KEY'] = config.secretKey + if (config.externalDatabaseId) { + awsConfig['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId + } + if (config.allowedMetasites) { + awsConfig['ALLOWED_METASITES'] = config.allowedMetasites } if (config.authorization) { awsConfig['PERMISSIONS'] = JSON.stringify(config.authorization) @@ -48,8 +51,11 @@ const defineLocalEnvs = (config: MySqlConfig) => { if (config.db) { process.env['DB'] = config.db } - if (config.secretKey) { - process.env['SECRET_KEY'] = config.secretKey + if (config.externalDatabaseId) { + process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId + } + if (config.allowedMetasites) { + process.env['ALLOWED_METASITES'] = config.allowedMetasites } if (config.authorization) { process.env['PERMISSIONS'] = JSON.stringify(config.authorization) @@ -70,7 +76,8 @@ export const validConfig = (): MySqlConfig => ({ user: chance.word(), password: chance.word(), db: chance.word(), - secretKey: chance.word(), + externalDatabaseId: chance.word(), + allowedMetasites: chance.word(), }) export const validConfigWithAuthorization = (): MySqlConfig => ({ @@ -89,8 +96,8 @@ export const validConfigWithAuthConfig = () => ({ } }) -export const ExpectedProperties = ['host', 'username', 'password', 'DB', 'SECRET_KEY', 'PERMISSIONS'] -export const RequiredProperties = ['host', 'username', 'password', 'DB', 'SECRET_KEY'] +export const ExpectedProperties = ['host', 'username', 'password', 'DB', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'PERMISSIONS'] +export const RequiredProperties = ['host', 'username', 'password', 'DB', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES'] export const reset = () => { mockedAwsSdk.reset() diff --git a/libs/external-db-config/test/drivers/azure_mysql_config_test_support.ts b/libs/external-db-config/test/drivers/azure_mysql_config_test_support.ts index 80e94b845..d2ef52a80 100644 --- a/libs/external-db-config/test/drivers/azure_mysql_config_test_support.ts +++ b/libs/external-db-config/test/drivers/azure_mysql_config_test_support.ts @@ -17,8 +17,11 @@ export const defineValidConfig = (config: MySqlConfig) => { if (config.db) { process.env['DB'] = config.db } - if (config.secretKey) { - process.env['SECRET_KEY'] = config.secretKey + if (config.externalDatabaseId) { + process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId + } + if (config.allowedMetasites) { + process.env['ALLOWED_METASITES'] = config.allowedMetasites } if (config.authorization) { process.env['PERMISSIONS'] = JSON.stringify( config.authorization ) @@ -39,7 +42,8 @@ export const validConfig = (): MySqlConfig => ({ user: chance.word(), password: chance.word(), db: chance.word(), - secretKey: chance.word(), + externalDatabaseId: chance.word(), + allowedMetasites: chance.word(), }) export const validConfigWithAuthorization = (): MySqlConfig => ({ @@ -58,7 +62,7 @@ export const validConfigWithAuthConfig = () => ({ export const defineInvalidConfig = () => defineValidConfig({}) -export const ExpectedProperties = ['HOST', 'USER', 'PASSWORD', 'DB', 'SECRET_KEY', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS'] +export const ExpectedProperties = ['HOST', 'USER', 'PASSWORD', 'DB', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS'] export const reset = () => ExpectedProperties.forEach(p => delete process.env[p]) diff --git a/libs/external-db-config/test/drivers/external_db_config_test_support.ts b/libs/external-db-config/test/drivers/external_db_config_test_support.ts index 1717ed094..20da29647 100644 --- a/libs/external-db-config/test/drivers/external_db_config_test_support.ts +++ b/libs/external-db-config/test/drivers/external_db_config_test_support.ts @@ -27,9 +27,9 @@ export const givenValidConfig = () => when(configValidator.validate).calledWith() .mockReturnValue({ missingRequiredSecretsKeys: [] }) -export const givenCommonConfig = (secretKey: any) => +export const givenCommonConfig = (externalDatabaseId: any, allowedMetasites: any) => when(commonConfigValidator.readConfig).calledWith() - .mockReturnValue({ secretKey }) + .mockReturnValue({ externalDatabaseId, allowedMetasites }) export const givenValidCommonConfig = () => when(commonConfigValidator.validate).calledWith() diff --git a/libs/external-db-config/test/drivers/gcp_firestore_config_test_support.ts b/libs/external-db-config/test/drivers/gcp_firestore_config_test_support.ts index c21cff85d..97998f70f 100644 --- a/libs/external-db-config/test/drivers/gcp_firestore_config_test_support.ts +++ b/libs/external-db-config/test/drivers/gcp_firestore_config_test_support.ts @@ -8,8 +8,11 @@ export const defineValidConfig = (config: FiresStoreConfig) => { if (config.projectId) { process.env['PROJECT_ID'] = config.projectId } - if (config.secretKey) { - process.env['SECRET_KEY'] = config.secretKey + if (config.externalDatabaseId) { + process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId + } + if (config.allowedMetasites) { + process.env['ALLOWED_METASITES'] = config.allowedMetasites } if (config.authorization) { process.env['PERMISSIONS'] = JSON.stringify( config.authorization ) @@ -27,7 +30,8 @@ export const defineValidConfig = (config: FiresStoreConfig) => { export const validConfig = (): FiresStoreConfig => ({ projectId: chance.word(), - secretKey: chance.word(), + externalDatabaseId: chance.word(), + allowedMetasites: chance.word(), }) export const validConfigWithAuthorization = () => ({ @@ -46,7 +50,7 @@ export const validConfigWithAuthConfig = () => ({ export const defineInvalidConfig = () => defineValidConfig({}) -export const ExpectedProperties = ['PROJECT_ID', 'SECRET_KEY', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS'] +export const ExpectedProperties = ['PROJECT_ID', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS'] export const reset = () => ExpectedProperties.forEach(p => delete process.env[p]) diff --git a/libs/external-db-config/test/drivers/gcp_mysql_config_test_support.ts b/libs/external-db-config/test/drivers/gcp_mysql_config_test_support.ts index d582c6f96..bcdca9889 100644 --- a/libs/external-db-config/test/drivers/gcp_mysql_config_test_support.ts +++ b/libs/external-db-config/test/drivers/gcp_mysql_config_test_support.ts @@ -17,8 +17,11 @@ export const defineValidConfig = (config: MySqlConfig) => { if (config.db) { process.env['DB'] = config.db } - if (config.secretKey) { - process.env['SECRET_KEY'] = config.secretKey + if (config.externalDatabaseId) { + process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId + } + if (config.allowedMetasites) { + process.env['ALLOWED_METASITES'] = config.allowedMetasites } if (config.authorization) { process.env['PERMISSIONS'] = JSON.stringify( config.authorization ) @@ -39,7 +42,8 @@ export const validConfig = (): MySqlConfig => ({ user: chance.word(), password: chance.word(), db: chance.word(), - secretKey: chance.word(), + externalDatabaseId: chance.word(), + allowedMetasites: chance.word(), }) export const validConfigWithAuthorization = () => ({ @@ -56,7 +60,7 @@ export const validConfigWithAuthConfig = () => ({ } }) -export const ExpectedProperties = ['CLOUD_SQL_CONNECTION_NAME', 'USER', 'PASSWORD', 'DB', 'SECRET_KEY', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS'] +export const ExpectedProperties = ['CLOUD_SQL_CONNECTION_NAME', 'USER', 'PASSWORD', 'DB', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS'] export const defineInvalidConfig = () => defineValidConfig({}) diff --git a/libs/external-db-config/test/drivers/gcp_spanner_config_test_support.ts b/libs/external-db-config/test/drivers/gcp_spanner_config_test_support.ts index 110a41d70..0b41c36d2 100644 --- a/libs/external-db-config/test/drivers/gcp_spanner_config_test_support.ts +++ b/libs/external-db-config/test/drivers/gcp_spanner_config_test_support.ts @@ -14,8 +14,11 @@ export const defineValidConfig = (config: SpannerConfig) => { if (config.databaseId) { process.env['DATABASE_ID'] = config.databaseId } - if (config.secretKey) { - process.env['SECRET_KEY'] = config.secretKey + if (config.externalDatabaseId) { + process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId + } + if (config.allowedMetasites) { + process.env['ALLOWED_METASITES'] = config.allowedMetasites } if (config.authorization) { process.env['PERMISSIONS'] = JSON.stringify( config.authorization ) @@ -35,7 +38,8 @@ export const validConfig = (): SpannerConfig => ({ projectId: chance.word(), instanceId: chance.word(), databaseId: chance.word(), - secretKey: chance.word(), + externalDatabaseId: chance.word(), + allowedMetasites: chance.word(), }) export const validConfigWithAuthorization = (): SpannerConfig => ({ @@ -56,7 +60,7 @@ export const validConfigWithAuthConfig = () => ({ export const defineInvalidConfig = () => defineValidConfig({}) -export const ExpectedProperties = ['PROJECT_ID', 'INSTANCE_ID', 'DATABASE_ID', 'SECRET_KEY', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS'] +export const ExpectedProperties = ['PROJECT_ID', 'INSTANCE_ID', 'DATABASE_ID', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS'] export const reset = () => ExpectedProperties.forEach(p => delete process.env[p]) diff --git a/libs/external-db-config/test/gen.ts b/libs/external-db-config/test/gen.ts index 93418a488..ec2c0ca62 100644 --- a/libs/external-db-config/test/gen.ts +++ b/libs/external-db-config/test/gen.ts @@ -10,11 +10,13 @@ export const randomConfig = () => ({ }) export const randomCommonConfig = () => ({ - secretKey: chance.guid(), + externalDatabaseId: chance.guid(), + allowedMetasites: chance.guid(), }) export const randomExtendedCommonConfig = () => ({ - secretKey: chance.guid(), + externalDatabaseId: chance.guid(), + allowedMetasites: chance.guid(), vendor: chance.pickone(supportedVendors), type: chance.pickone(supportedDBs), }) diff --git a/libs/external-db-config/test/test_types.ts b/libs/external-db-config/test/test_types.ts index 24a8ba120..4ecae852d 100644 --- a/libs/external-db-config/test/test_types.ts +++ b/libs/external-db-config/test/test_types.ts @@ -1,13 +1,15 @@ export interface MongoConfig { connectionUri?: string - secretKey?: string + externalDatabaseId?: string + allowedMetasites?: string authorization?: any } export interface MongoAwsConfig { URI?: string - SECRET_KEY?: string + EXTERNAL_DATABASE_ID?: string + ALLOWED_METASITES?: string PERMISSIONS?: string } @@ -17,7 +19,8 @@ export interface MySqlConfig { user?: string password?: string db?: string - secretKey?: string + externalDatabaseId?: string + allowedMetasites?: string authorization?: any auth?: any } @@ -27,19 +30,22 @@ export interface AwsMysqlConfig { username?: string password?: string DB?: string - SECRET_KEY?: string + EXTERNAL_DATABASE_ID?: string + ALLOWED_METASITES?: string PERMISSIONS?: string } export interface CommonConfig { type?: string vendor?: string - secretKey?: string + externalDatabaseId?: string + allowedMetasites?: string } export interface FiresStoreConfig { projectId?: string - secretKey?: string + externalDatabaseId?: string + allowedMetasites?: string authorization?: any auth?: any } @@ -48,7 +54,8 @@ export interface SpannerConfig { projectId?: string instanceId?: string databaseId?: string - secretKey?: string + externalDatabaseId?: string + allowedMetasites?: string authorization?: any auth?: any } diff --git a/libs/external-db-config/test/test_utils.ts b/libs/external-db-config/test/test_utils.ts index a50ed9ee4..b411917db 100644 --- a/libs/external-db-config/test/test_utils.ts +++ b/libs/external-db-config/test/test_utils.ts @@ -23,4 +23,4 @@ export const splitConfig = (config: {[key: string]: any}) => { return { firstPart, secondPart } } -export const extendedCommonConfigRequiredProperties = ['secretKey', 'vendor', 'type'] +export const extendedCommonConfigRequiredProperties = ['externalDatabaseId', 'allowedMetasites', 'vendor', 'type'] diff --git a/libs/external-db-mysql/src/mysql_capabilities.ts b/libs/external-db-mysql/src/mysql_capabilities.ts new file mode 100644 index 000000000..c1db9dd87 --- /dev/null +++ b/libs/external-db-mysql/src/mysql_capabilities.ts @@ -0,0 +1,17 @@ +import { + CollectionOperation, + DataOperation, + FieldType, +} from '@wix-velo/velo-external-db-types' + +const { + query, + count, + queryReferenced, + aggregate, +} = DataOperation + +export const ReadWriteOperations = Object.values(DataOperation) +export const ReadOnlyOperations = [query, count, queryReferenced, aggregate] +export const FieldTypes = Object.values(FieldType) +export const CollectionOperations = Object.values(CollectionOperation) diff --git a/libs/external-db-mysql/src/mysql_data_provider.ts b/libs/external-db-mysql/src/mysql_data_provider.ts index 603f950f5..23e384536 100644 --- a/libs/external-db-mysql/src/mysql_data_provider.ts +++ b/libs/external-db-mysql/src/mysql_data_provider.ts @@ -4,7 +4,7 @@ import { promisify } from 'util' import { asParamArrays, updateFieldsFor } from '@wix-velo/velo-external-db-commons' import { translateErrorCodes } from './sql_exception_translator' import { wildCardWith } from './mysql_utils' -import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item } from '@wix-velo/velo-external-db-types' +import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item, Sort } from '@wix-velo/velo-external-db-types' import { IMySqlFilterParser } from './sql_filter_transformer' import { MySqlQuery } from './types' @@ -26,7 +26,7 @@ export default class DataProvider implements IDataProvider { const sql = `SELECT ${projectionExpr} FROM ${escapeTable(collectionName)} ${filterExpr} ${sortExpr} LIMIT ?, ?` const resultset = await this.query(sql, [...parameters, skip, limit]) - .catch( translateErrorCodes ) + .catch( err => translateErrorCodes(err, collectionName) ) return resultset } @@ -34,17 +34,18 @@ export default class DataProvider implements IDataProvider { const { filterExpr, parameters } = this.filterParser.transform(filter) const sql = `SELECT COUNT(*) AS num FROM ${escapeTable(collectionName)} ${filterExpr}` const resultset = await this.query(sql, parameters) - .catch( translateErrorCodes ) + .catch( err => translateErrorCodes(err, collectionName) ) return resultset[0]['num'] } - async insert(collectionName: string, items: Item[], fields: any[]): Promise { + async insert(collectionName: string, items: Item[], fields: any[], upsert?: boolean): Promise { const escapedFieldsNames = fields.map( (f: { field: any }) => escapeId(f.field)).join(', ') - const sql = `INSERT INTO ${escapeTable(collectionName)} (${escapedFieldsNames}) VALUES ?` + const op = upsert ? 'REPLACE' : 'INSERT' + const sql = `${op} INTO ${escapeTable(collectionName)} (${escapedFieldsNames}) VALUES ?` const data = items.map((item: Item) => asParamArrays( patchItem(item) ) ) const resultset = await this.query(sql, [data]) - .catch( translateErrorCodes ) + .catch( err => translateErrorCodes(err, collectionName) ) return resultset.affectedRows } @@ -57,7 +58,7 @@ export default class DataProvider implements IDataProvider { // @ts-ignore const resultset = await this.query(queries, [].concat(...updatables)) - .catch( translateErrorCodes ) + .catch( err => translateErrorCodes(err, collectionName) ) return Array.isArray(resultset) ? resultset.reduce((s, r) => s + r.changedRows, 0) : resultset.changedRows } @@ -65,21 +66,22 @@ export default class DataProvider implements IDataProvider { async delete(collectionName: string, itemIds: string[]): Promise { const sql = `DELETE FROM ${escapeTable(collectionName)} WHERE _id IN (${wildCardWith(itemIds.length, '?')})` const rs = await this.query(sql, itemIds) - .catch( translateErrorCodes ) + .catch( err => translateErrorCodes(err, collectionName) ) return rs.affectedRows } async truncate(collectionName: string): Promise { - await this.query(`TRUNCATE ${escapeTable(collectionName)}`).catch( translateErrorCodes ) + await this.query(`TRUNCATE ${escapeTable(collectionName)}`).catch( err => translateErrorCodes(err, collectionName) ) } - async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation): Promise { + async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort: Sort[], skip: number, limit: number): Promise { const { filterExpr: whereFilterExpr, parameters: whereParameters } = this.filterParser.transform(filter) const { fieldsStatement, groupByColumns, havingFilter, parameters } = this.filterParser.parseAggregation(aggregation) + const { sortExpr } = this.filterParser.orderBy(sort) - const sql = `SELECT ${fieldsStatement} FROM ${escapeTable(collectionName)} ${whereFilterExpr} GROUP BY ${groupByColumns.map( escapeId ).join(', ')} ${havingFilter}` - const resultset = await this.query(sql, [...whereParameters, ...parameters]) - .catch( translateErrorCodes ) + const sql = `SELECT ${fieldsStatement} FROM ${escapeTable(collectionName)} ${whereFilterExpr} GROUP BY ${groupByColumns.map( escapeId ).join(', ')} ${havingFilter} ${sortExpr} LIMIT ?, ?` + const resultset = await this.query(sql, [...whereParameters, ...parameters, skip, limit]) + .catch( err => translateErrorCodes(err, collectionName) ) return resultset } } diff --git a/libs/external-db-mysql/src/mysql_operations.ts b/libs/external-db-mysql/src/mysql_operations.ts index ffb378fa0..34cc19064 100644 --- a/libs/external-db-mysql/src/mysql_operations.ts +++ b/libs/external-db-mysql/src/mysql_operations.ts @@ -13,6 +13,6 @@ export default class DatabaseOperations implements IDatabaseOperations { async validateConnection() { return await this.query('SELECT 1').then(() => { return { valid: true } }) - .catch((e: any) => { return { valid: false, error: notThrowingTranslateErrorCodes(e) } }) + .catch((e: any) => { return { valid: false, error: notThrowingTranslateErrorCodes(e, '') } }) } } diff --git a/libs/external-db-mysql/src/mysql_schema_provider.ts b/libs/external-db-mysql/src/mysql_schema_provider.ts index 48438db9c..85be2e592 100644 --- a/libs/external-db-mysql/src/mysql_schema_provider.ts +++ b/libs/external-db-mysql/src/mysql_schema_provider.ts @@ -1,11 +1,12 @@ import { promisify } from 'util' import { translateErrorCodes } from './sql_exception_translator' import SchemaColumnTranslator, { IMySqlSchemaColumnTranslator } from './sql_schema_translator' -import { escapeId, escapeTable } from './mysql_utils' +import { escapeId, escapeTable, columnCapabilitiesFor } from './mysql_utils' import { SystemFields, validateSystemFields, parseTableData, AllSchemaOperations } from '@wix-velo/velo-external-db-commons' import { Pool as MySqlPool } from 'mysql' import { MySqlQuery } from './types' -import { InputField, ISchemaProvider, ResponseField, SchemaOperations, Table } from '@wix-velo/velo-external-db-types' +import { InputField, ISchemaProvider, ResponseField, SchemaOperations, Table, CollectionCapabilities } from '@wix-velo/velo-external-db-types' +import { CollectionOperations, FieldTypes, ReadOnlyOperations, ReadWriteOperations } from './mysql_capabilities' export default class SchemaProvider implements ISchemaProvider { pool: MySqlPool @@ -23,10 +24,12 @@ export default class SchemaProvider implements ISchemaProvider { const currentDb = this.pool.config.connectionConfig.database const data = await this.query('SELECT TABLE_NAME as table_name, COLUMN_NAME as field, DATA_TYPE as type FROM information_schema.columns WHERE TABLE_SCHEMA = ? ORDER BY TABLE_NAME, ORDINAL_POSITION', currentDb) const tables: {[x:string]: {table_name: string, field: string, type: string}[]} = parseTableData( data ) + return Object.entries(tables) .map(([collectionName, rs]) => ({ id: collectionName, - fields: rs.map( this.translateDbTypes.bind(this) ) + fields: rs.map(this.appendAdditionalRowDetails.bind(this)), + capabilities: this.collectionCapabilities(rs.map(r => r.field)) } )) } @@ -47,35 +50,59 @@ export default class SchemaProvider implements ISchemaProvider { await this.query(`CREATE TABLE IF NOT EXISTS ${escapeTable(collectionName)} (${dbColumnsSql}, PRIMARY KEY (${primaryKeySql}))`, [...(columns || []).map((c: { name: any }) => c.name)]) - .catch( translateErrorCodes ) + .catch( err => translateErrorCodes(err, collectionName) ) } async drop(collectionName: string): Promise { await this.query(`DROP TABLE IF EXISTS ${escapeTable(collectionName)}`) - .catch( translateErrorCodes ) + .catch( err => translateErrorCodes(err, collectionName) ) } async addColumn(collectionName: string, column: InputField): Promise { await validateSystemFields(column.name) await this.query(`ALTER TABLE ${escapeTable(collectionName)} ADD ${escapeId(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) - .catch( translateErrorCodes ) + .catch( err => translateErrorCodes(err, collectionName) ) + } + + async changeColumnType(collectionName: string, column: InputField): Promise { + await validateSystemFields(column.name) + await this.query(`ALTER TABLE ${escapeTable(collectionName)} MODIFY ${escapeId(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`) + .catch( err => translateErrorCodes(err, collectionName) ) } async removeColumn(collectionName: string, columnName: string): Promise { await validateSystemFields(columnName) return await this.query(`ALTER TABLE ${escapeTable(collectionName)} DROP COLUMN ${escapeId(columnName)}`) - .catch( translateErrorCodes ) + .catch( err => translateErrorCodes(err, collectionName) ) } - async describeCollection(collectionName: string): Promise { - const res = await this.query(`DESCRIBE ${escapeTable(collectionName)}`) - .catch( translateErrorCodes ) - return res.map((r: { Field: string; Type: string }) => ({ field: r.Field, type: r.Type })) - .map( this.translateDbTypes.bind(this) ) + async describeCollection(collectionName: string): Promise { + interface describeTableResponse { + Field: string, + Type: string, + } + + const res: describeTableResponse[] = await this.query(`DESCRIBE ${escapeTable(collectionName)}`) + .catch( err => translateErrorCodes(err, collectionName) ) + const fields = res.map(r => ({ field: r.Field, type: r.Type })).map(this.appendAdditionalRowDetails.bind(this)) + return { + id: collectionName, + fields: fields as ResponseField[], + capabilities: this.collectionCapabilities(res.map(f => f.Field)) + } } - translateDbTypes(row: ResponseField): ResponseField { + private appendAdditionalRowDetails(row: ResponseField) { row.type = this.sqlSchemaTranslator.translateType(row.type) + row.capabilities = columnCapabilitiesFor(row.type) return row } + + private collectionCapabilities(fieldNames: string[]): CollectionCapabilities { + return { + dataOperations: fieldNames.includes('_id') ? ReadWriteOperations : ReadOnlyOperations, + fieldTypes: FieldTypes, + collectionOperations: CollectionOperations, + } + } } diff --git a/libs/external-db-mysql/src/mysql_utils.spec.ts b/libs/external-db-mysql/src/mysql_utils.spec.ts index a863d6cee..bf3c69ee0 100644 --- a/libs/external-db-mysql/src/mysql_utils.spec.ts +++ b/libs/external-db-mysql/src/mysql_utils.spec.ts @@ -1,6 +1,7 @@ -import { escapeTable, escapeId } from './mysql_utils' -import { errors } from '@wix-velo/velo-external-db-commons' +import { escapeTable, escapeId, columnCapabilitiesFor } from './mysql_utils' +import { errors, AdapterOperators } from '@wix-velo/velo-external-db-commons' const { InvalidQuery } = errors +const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators describe('Mysql Utils', () => { test('escape collection id will not allow dots', () => { @@ -10,4 +11,52 @@ describe('Mysql Utils', () => { test('escape collection id', () => { expect( escapeTable('some_table_name') ).toEqual(escapeId('some_table_name')) }) + + describe('translate column type to column capabilities object', () => { + test('number column type', () => { + expect(columnCapabilitiesFor('number')).toEqual({ + sortable: true, + columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] + }) + }) + test('text column type', () => { + expect(columnCapabilitiesFor('text')).toEqual({ + sortable: true, + columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] + }) + }) + + test('url column type', () => { + expect(columnCapabilitiesFor('url')).toEqual({ + sortable: true, + columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] + }) + }) + + test('boolean column type', () => { + expect(columnCapabilitiesFor('boolean')).toEqual({ + sortable: true, + columnQueryOperators: [eq] + }) + }) + + test('image column type', () => { + expect(columnCapabilitiesFor('image')).toEqual({ + sortable: false, + columnQueryOperators: [] + }) + }) + + test('datetime column type', () => { + expect(columnCapabilitiesFor('datetime')).toEqual({ + sortable: true, + columnQueryOperators: [eq, ne, gt, gte, lt, lte] + }) + }) + + test('unsupported field type will throw', () => { + expect(() => columnCapabilitiesFor('unsupported-type')).toThrowError() + }) + }) + }) diff --git a/libs/external-db-mysql/src/mysql_utils.ts b/libs/external-db-mysql/src/mysql_utils.ts index 60d17aa40..18906427a 100644 --- a/libs/external-db-mysql/src/mysql_utils.ts +++ b/libs/external-db-mysql/src/mysql_utils.ts @@ -1,6 +1,7 @@ import { escapeId } from 'mysql' -import { errors, patchDateTime } from '@wix-velo/velo-external-db-commons' -import { Item } from '@wix-velo/velo-external-db-types' +import { errors, patchDateTime, AdapterOperators } from '@wix-velo/velo-external-db-commons' +import { Item, ColumnCapabilities } from '@wix-velo/velo-external-db-types' +const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators export const wildCardWith = (n: number, char: string) => Array(n).fill(char, 0, n).join(', ') @@ -27,3 +28,42 @@ export const patchItem = (item: Item) => { } export { escapeIdField as escapeId } + +export const columnCapabilitiesFor = (columnType: string): ColumnCapabilities => { + switch (columnType) { + case 'text': + case 'url': + return { + sortable: true, + columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] + } + case 'number': + return { + sortable: true, + columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] + } + case 'boolean': + return { + sortable: true, + columnQueryOperators: [eq] + } + case 'image': + return { + sortable: false, + columnQueryOperators: [] + } + case 'object': + return { + sortable: true, + columnQueryOperators: [eq, ne] + } + case 'datetime': + return { + sortable: true, + columnQueryOperators: [eq, ne, gt, gte, lt, lte] + } + + default: + throw new Error(`${columnType} - Unsupported field type`) + } +} diff --git a/libs/external-db-mysql/src/sql_exception_translator.ts b/libs/external-db-mysql/src/sql_exception_translator.ts index 7bd1b63f4..fa50b4eb4 100644 --- a/libs/external-db-mysql/src/sql_exception_translator.ts +++ b/libs/external-db-mysql/src/sql_exception_translator.ts @@ -1,14 +1,28 @@ import { errors } from '@wix-velo/velo-external-db-commons' const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist, DbConnectionError, ItemAlreadyExists, UnrecognizedError } = errors -export const notThrowingTranslateErrorCodes = (err: any) => { +const extractDuplicatedColumnName = (error: any) => extractValueFromErrorMessage(error.sqlMessage, /Duplicate column name '(.*)'/) +const extractDuplicatedItem = (error: any) => extractValueFromErrorMessage(error.sqlMessage, /Duplicate entry '(.*)' for key .*/) +const extractUnknownColumn = (error: any) => extractValueFromErrorMessage(error.sqlMessage, /Unknown column '(.*)' in 'field list'/) + +const extractValueFromErrorMessage = (msg: string, regex: RegExp) => { + try { + const match = msg.match(regex) + const value = (match && match[1]) + return value || '' + } catch(e) { + return '' + } +} + +export const notThrowingTranslateErrorCodes = (err: any, collectionName: string) => { switch (err.code) { case 'ER_CANT_DROP_FIELD_OR_KEY': - return new FieldDoesNotExist('Collection does not contain a field with this name') + return new FieldDoesNotExist('Collection does not contain a field with this name', collectionName, extractUnknownColumn(err)) case 'ER_DUP_FIELDNAME': - return new FieldAlreadyExists('Collection already has a field with the same name') + return new FieldAlreadyExists('Collection already has a field with the same name', collectionName, extractDuplicatedColumnName(err)) case 'ER_NO_SUCH_TABLE': - return new CollectionDoesNotExists('Collection does not exists') + return new CollectionDoesNotExists('Collection does not exists', collectionName) case 'ER_DBACCESS_DENIED_ERROR': case 'ER_BAD_DB_ERROR': return new DbConnectionError(`Database does not exists or you don't have access to it, sql message: ${err.sqlMessage}`) @@ -18,13 +32,13 @@ export const notThrowingTranslateErrorCodes = (err: any) => { case 'ENOTFOUND': return new DbConnectionError(`Access to database denied - host is unavailable, sql message: ${err.sqlMessage} `) case 'ER_DUP_ENTRY': - return new ItemAlreadyExists(`Item already exists: ${err.sqlMessage}`) + return new ItemAlreadyExists(`Item already exists: ${err.sqlMessage}`, collectionName, extractDuplicatedItem(err)) default : console.error(err) return new UnrecognizedError(`${err.code} ${err.sqlMessage}`) } } -export const translateErrorCodes = (err: any) => { - throw notThrowingTranslateErrorCodes(err) +export const translateErrorCodes = (err: any, collectionName: string) => { + throw notThrowingTranslateErrorCodes(err, collectionName) } diff --git a/libs/external-db-mysql/src/sql_schema_translator.spec.ts b/libs/external-db-mysql/src/sql_schema_translator.spec.ts index 9182dcb37..20ea2e16b 100644 --- a/libs/external-db-mysql/src/sql_schema_translator.spec.ts +++ b/libs/external-db-mysql/src/sql_schema_translator.spec.ts @@ -19,7 +19,7 @@ describe('Sql Schema Column Translator', () => { }) test('decimal float', () => { - expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'float' }) ).toEqual(`${escapeId(ctx.fieldName)} FLOAT(5,2)`) + expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'float' }) ).toEqual(`${escapeId(ctx.fieldName)} FLOAT(15,2)`) }) test('decimal float with precision', () => { @@ -27,7 +27,7 @@ describe('Sql Schema Column Translator', () => { }) test('decimal double', () => { - expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'double' }) ).toEqual(`${escapeId(ctx.fieldName)} DOUBLE(5,2)`) + expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'double' }) ).toEqual(`${escapeId(ctx.fieldName)} DOUBLE(15,2)`) }) test('decimal double with precision', () => { @@ -35,7 +35,7 @@ describe('Sql Schema Column Translator', () => { }) test('decimal generic', () => { - expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'decimal' }) ).toEqual(`${escapeId(ctx.fieldName)} DECIMAL(5,2)`) + expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'decimal' }) ).toEqual(`${escapeId(ctx.fieldName)} DECIMAL(15,2)`) }) test('decimal generic with precision', () => { diff --git a/libs/external-db-mysql/src/sql_schema_translator.ts b/libs/external-db-mysql/src/sql_schema_translator.ts index 8fe629aff..f9d1dbca8 100644 --- a/libs/external-db-mysql/src/sql_schema_translator.ts +++ b/libs/external-db-mysql/src/sql_schema_translator.ts @@ -125,7 +125,7 @@ export default class SchemaColumnTranslator implements IMySqlSchemaColumnTransla const parsed = precision.split(',').map((s: string) => s.trim()).map((s: string) => parseInt(s)) return `(${parsed.join(',')})` } catch (e) { - return '(5,2)' + return '(15,2)' } } diff --git a/libs/external-db-mysql/tests/e2e-testkit/mysql_resources.ts b/libs/external-db-mysql/tests/e2e-testkit/mysql_resources.ts index bf71529ed..013930f6d 100644 --- a/libs/external-db-mysql/tests/e2e-testkit/mysql_resources.ts +++ b/libs/external-db-mysql/tests/e2e-testkit/mysql_resources.ts @@ -3,6 +3,7 @@ import { waitUntil } from 'async-wait-until' import init from '../../src/connection_provider' export { supportedOperations } from '../../src/supported_operations' +export * as capabilities from '../../src/mysql_capabilities' export const connection = () => { const { connection, schemaProvider, cleanup } = init({ host: 'localhost', user: 'test-user', password: 'password', db: 'test-db' }, { connectionLimit: 1, queueLimit: 0 }) diff --git a/libs/external-db-testkit/src/lib/auth_test_support.ts b/libs/external-db-testkit/src/lib/auth_test_support.ts index d086b93e6..0f350bcbf 100644 --- a/libs/external-db-testkit/src/lib/auth_test_support.ts +++ b/libs/external-db-testkit/src/lib/auth_test_support.ts @@ -1,40 +1,53 @@ import * as Chance from 'chance' +import { AxiosRequestHeaders } from 'axios' +import * as jwt from 'jsonwebtoken' +import { authConfig } from '@wix-velo/test-commons' const chance = Chance() const axios = require('axios').create({ baseURL: 'http://localhost:8080', }) -const secretKey = chance.word() +const allowedMetasite = chance.word() +const externalDatabaseId = chance.word() export const authInit = () => { - process.env['SECRET_KEY'] = secretKey + process.env['ALLOWED_METASITES'] = allowedMetasite + process.env['EXTERNAL_DATABASE_ID'] = externalDatabaseId } -const appendSecretKeyToRequest = (dataRaw: string) => { +const appendRoleToRequest = (role: string) => (dataRaw: string) => { const data = JSON.parse( dataRaw ) - return JSON.stringify({ ...data, ...{ requestContext: { settings: { secretKey: secretKey } } } }) + return JSON.stringify({ ...data, ...{ requestContext: { ...data.requestContext, role: role } } }) } -const appendRoleToRequest = (role: string) => (dataRaw: string) => { +const appendJWTHeaderToRequest = (dataRaw: string, headers: AxiosRequestHeaders) => { + headers['Authorization'] = createJwtHeader() const data = JSON.parse( dataRaw ) - return JSON.stringify({ ...data, ...{ requestContext: { ...data.requestContext, role: role } } }) + return JSON.stringify({ ...data } ) +} + +const TOKEN_ISSUER = 'wix-data.wix.com' + +const createJwtHeader = () => { + const token = jwt.sign({ iss: TOKEN_ISSUER, siteId: allowedMetasite, aud: externalDatabaseId }, authConfig.authPrivateKey, { algorithm: 'ES256', keyid: authConfig.kid }) + return `Bearer ${token}` } export const authAdmin = { transformRequest: axios.defaults .transformRequest - .concat( appendSecretKeyToRequest, appendRoleToRequest('BACKEND_CODE') ) } + .concat( appendJWTHeaderToRequest, appendRoleToRequest('BACKEND_CODE') ) } export const authOwner = { transformRequest: axios.defaults .transformRequest - .concat( appendSecretKeyToRequest, appendRoleToRequest('OWNER' ) ) } + .concat( appendJWTHeaderToRequest, appendRoleToRequest('OWNER' ) ) } export const authVisitor = { transformRequest: axios.defaults .transformRequest - .concat( appendSecretKeyToRequest, appendRoleToRequest('VISITOR' ) ) } + .concat( appendJWTHeaderToRequest, appendRoleToRequest('VISITOR' ) ) } -export const authOwnerWithoutSecretKey = { transformRequest: axios.defaults +export const authOwnerWithoutJwt = { transformRequest: axios.defaults .transformRequest .concat( appendRoleToRequest('OWNER' ) ) } -export const errorResponseWith = (status: any, message: string) => ({ response: { data: { message: expect.stringContaining(message) }, status } }) +export const errorResponseWith = (status: any, message: string) => ({ response: { data: { description: expect.stringContaining(message) }, status } }) diff --git a/libs/test-commons/src/index.ts b/libs/test-commons/src/index.ts index 20290e545..ff771e288 100644 --- a/libs/test-commons/src/index.ts +++ b/libs/test-commons/src/index.ts @@ -1,2 +1,3 @@ export * from './libs/test-commons' export * as gen from './libs/gen' +export { authConfig } from './libs/auth-config.json' diff --git a/libs/test-commons/src/libs/auth-config.json b/libs/test-commons/src/libs/auth-config.json new file mode 100644 index 000000000..c2e77245e --- /dev/null +++ b/libs/test-commons/src/libs/auth-config.json @@ -0,0 +1,8 @@ +{ + "authConfig": { + "kid": "7968bd02-7c7d-446e-83c5-5c993c20a140", + "authPublicKey": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEdnP+fQMJYtljus9pnpEWT02T0uqF\nUacdoxL19vmQdii4DAj+S0pbJ/owcc7HsPvNwhJvIwFtk/4Cm+OYp7fXSQ==\n-----END PUBLIC KEY-----", + "authPrivateKey": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEINoWtnYgw8ZcsZkgWDBxAcJF0ziCI4SOVuK17DrQFCWYoAoGCCqGSM49\nAwEHoUQDQgAEdnP+fQMJYtljus9pnpEWT02T0uqFUacdoxL19vmQdii4DAj+S0pb\nJ/owcc7HsPvNwhJvIwFtk/4Cm+OYp7fXSQ==\n-----END EC PRIVATE KEY-----", + "otherAuthPublicKey": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEC0QSOeblgUZjrKzxsLwJ/gcTFV+/\nTIhuEDxhpNaAnY1AvqFuANfCJ++aCWMjmhp1Fy9BZ6pi/lxVJAF4fpMqtw==\n-----END PUBLIC KEY-----" + } +} \ No newline at end of file diff --git a/libs/test-commons/src/libs/test-commons.ts b/libs/test-commons/src/libs/test-commons.ts index a8ada374f..f36c78c7c 100644 --- a/libs/test-commons/src/libs/test-commons.ts +++ b/libs/test-commons/src/libs/test-commons.ts @@ -19,3 +19,21 @@ export const testSupportedOperations = (supportedOperations: SchemaOperations[], return !isObject(lastItem) || lastItem['neededOperations'].every((i: any) => supportedOperations.includes(i)) }) } + +export const streamToArray = async(stream: any) => { + + return new Promise((resolve, reject) => { + const arr: any[] = [] + + stream.on('data', (data: any) => { + arr.push(JSON.parse(data.toString())) + }) + + stream.on('end', () => { + resolve(arr) + }) + + stream.on('error', (err: Error) => reject(err)) + + }) +} diff --git a/libs/velo-external-db-commons/src/libs/errors.ts b/libs/velo-external-db-commons/src/libs/errors.ts index 2bbf64425..ddfd2f1f3 100644 --- a/libs/velo-external-db-commons/src/libs/errors.ts +++ b/libs/velo-external-db-commons/src/libs/errors.ts @@ -1,90 +1,106 @@ class BaseHttpError extends Error { - status: number - constructor(message: string, status: number) { + constructor(message: string) { super(message) - this.status = status } } export class UnauthorizedError extends BaseHttpError { constructor(message: string) { - super(message, 401) + super(message) } } export class CollectionDoesNotExists extends BaseHttpError { - constructor(message: string) { - super(message, 404) + collectionName: string + constructor(message: string, collectionName?: string) { + super(message) + this.collectionName = collectionName || '' } } export class CollectionAlreadyExists extends BaseHttpError { - constructor(message: string) { - super(message, 400) + collectionName: string + constructor(message: string, collectionName?: string) { + super(message) + this.collectionName = collectionName || '' } } export class FieldAlreadyExists extends BaseHttpError { - constructor(message: string) { - super(message, 400) + collectionName: string + fieldName: string + + constructor(message: string, collectionName?: string, fieldName?: string) { + super(message) + this.collectionName = collectionName || '' + this.fieldName = fieldName || '' } } export class ItemAlreadyExists extends BaseHttpError { - constructor(message: string) { - super(message, 400) + itemId: string + collectionName: string + + constructor(message: string, collectionName?: string, itemId?: string) { + super(message) + this.itemId = itemId || '' + this.collectionName = collectionName || '' } } export class FieldDoesNotExist extends BaseHttpError { - constructor(message: string) { - super(message, 404) + itemId: string + collectionName: string + constructor(message: string, collectionName?: string, itemId?: string) { + super(message) + this.itemId = itemId || '' + this.collectionName = collectionName || '' } } export class CannotModifySystemField extends BaseHttpError { constructor(message: string) { - super(message, 400) + super(message) } } export class InvalidQuery extends BaseHttpError { constructor(message: string) { - super(message, 400) + super(message) } } export class InvalidRequest extends BaseHttpError { constructor(message: string) { - super(message, 400) + super(message) } } export class DbConnectionError extends BaseHttpError { constructor(message: string) { - super(message, 500) + super(message) } } export class ItemNotFound extends BaseHttpError { constructor(message: string) { - super(message, 404) + super(message) } } export class UnsupportedOperation extends BaseHttpError { constructor(message: string) { - super(message, 405) + super(message) } } export class UnsupportedDatabase extends BaseHttpError { constructor(message: string) { - super(message, 405) + super(message) } } export class UnrecognizedError extends BaseHttpError { constructor(message: string) { - super(`Unrecognized Error: ${message}`, 400) + super(`Unrecognized Error: ${message}`) } } diff --git a/libs/velo-external-db-core/src/converters/aggregation_transformer.spec.ts b/libs/velo-external-db-core/src/converters/aggregation_transformer.spec.ts index fbcac5680..653b3e9d1 100644 --- a/libs/velo-external-db-core/src/converters/aggregation_transformer.spec.ts +++ b/libs/velo-external-db-core/src/converters/aggregation_transformer.spec.ts @@ -6,6 +6,7 @@ import { errors } from '@wix-velo/velo-external-db-commons' import AggregationTransformer from './aggregation_transformer' import { EmptyFilter } from './utils' import * as driver from '../../test/drivers/filter_transformer_test_support' +import { Group } from '../spi-model/data_source' const chance = Chance() const { InvalidQuery } = errors @@ -17,127 +18,96 @@ describe('Aggregation Transformer', () => { describe('correctly transform Wix functions to adapter functions', () => { each([ - '$avg', '$max', '$min', '$sum' + 'avg', 'max', 'min', 'sum', 'count' ]) - .test('correctly transform [%s]', (f: string) => { - const AdapterFunction = f.substring(1) - expect(env.AggregationTransformer.wixFunctionToAdapterFunction(f)).toEqual((AdapterFunctions as any)[AdapterFunction]) - }) + .test('correctly transform [%s]', (f: string) => { + const AdapterFunction = f as AdapterFunctions + expect(env.AggregationTransformer.wixFunctionToAdapterFunction(f)).toEqual(AdapterFunctions[AdapterFunction]) + }) test('transform unknown function will throw an exception', () => { - expect( () => env.AggregationTransformer.wixFunctionToAdapterFunction('$wrong')).toThrow(InvalidQuery) + expect(() => env.AggregationTransformer.wixFunctionToAdapterFunction('wrong')).toThrow(InvalidQuery) }) }) test('single id field without function or postFilter', () => { - env.driver.stubEmptyFilterFor(null) - - const processingStep = { _id: `$${ctx.fieldName}` } - const postFilteringStep = null + env.driver.stubEmptyFilterForUndefined() + + const group = { by: [ctx.fieldName], aggregation: [] } as Group - expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({ + expect(env.AggregationTransformer.transform({ group })).toEqual({ projection: [{ name: ctx.fieldName }], postFilter: EmptyFilter }) }) test('multiple id fields without function or postFilter', () => { - env.driver.stubEmptyFilterFor(null) - - const processingStep = { - _id: { - field1: `$${ctx.fieldName}`, - field2: `$${ctx.anotherFieldName}` - } - } - const postFilteringStep = null + env.driver.stubEmptyFilterForUndefined() + + const group = { by: [ctx.fieldName, ctx.anotherFieldName], aggregation: [] } as Group - expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({ + expect(env.AggregationTransformer.transform({ group })).toEqual({ projection: [ - { name: ctx.fieldName }, - { name: ctx.anotherFieldName } - ], + { name: ctx.fieldName }, + { name: ctx.anotherFieldName } + ], postFilter: EmptyFilter }) }) test('single id field with function field and without postFilter', () => { - env.driver.stubEmptyFilterFor(null) - - const processingStep = { - _id: `$${ctx.fieldName}`, - [ctx.fieldAlias]: { - $avg: `$${ctx.anotherFieldName}` - } - } - const postFilteringStep = null + env.driver.stubEmptyFilterForUndefined() - expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({ + const group = { by: [ctx.fieldName], aggregation: [{ name: ctx.fieldAlias, avg: ctx.anotherFieldName }] } as Group + + expect(env.AggregationTransformer.transform({ group })).toEqual({ projection: [ - { name: ctx.fieldName }, - { name: ctx.anotherFieldName, alias: ctx.fieldAlias, function: AdapterFunctions.avg } - ], + { name: ctx.fieldName }, + { name: ctx.anotherFieldName, alias: ctx.fieldAlias, function: AdapterFunctions.avg } + ], postFilter: EmptyFilter }) }) test('single id field with count function and without postFilter', () => { - env.driver.stubEmptyFilterFor(null) + env.driver.stubEmptyFilterForUndefined() - const processingStep = { - _id: `$${ctx.fieldName}`, - [ctx.fieldAlias]: { - $sum: 1 - } - } - const postFilteringStep = null + const group = { by: [ctx.fieldName], aggregation: [{ name: ctx.fieldAlias, count: 1 }] } as Group - expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({ + expect(env.AggregationTransformer.transform({ group })).toEqual({ projection: [ - { name: ctx.fieldName }, - { alias: ctx.fieldAlias, function: AdapterFunctions.count, name: '*' } - ], + { name: ctx.fieldName }, + { alias: ctx.fieldAlias, function: AdapterFunctions.count, name: '*' } + ], postFilter: EmptyFilter }) }) - + test('multiple function fields and without postFilter', () => { - env.driver.stubEmptyFilterFor(null) - - const processingStep = { - _id: `$${ctx.fieldName}`, - [ctx.fieldAlias]: { - $avg: `$${ctx.anotherFieldName}` - }, - [ctx.anotherFieldAlias]: { - $sum: `$${ctx.moreFieldName}` - } - } - const postFilteringStep = null + env.driver.stubEmptyFilterForUndefined() - expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({ + const group = { + by: [ctx.fieldName], + aggregation: [{ name: ctx.fieldAlias, avg: ctx.anotherFieldName }, { name: ctx.anotherFieldAlias, sum: ctx.moreFieldName }] + } as Group + + expect(env.AggregationTransformer.transform({ group })).toEqual({ projection: [ - { name: ctx.fieldName }, - { name: ctx.anotherFieldName, alias: ctx.fieldAlias, function: AdapterFunctions.avg }, - { name: ctx.moreFieldName, alias: ctx.anotherFieldAlias, function: AdapterFunctions.sum } - ], + { name: ctx.fieldName }, + { name: ctx.anotherFieldName, alias: ctx.fieldAlias, function: AdapterFunctions.avg }, + { name: ctx.moreFieldName, alias: ctx.anotherFieldAlias, function: AdapterFunctions.sum } + ], postFilter: EmptyFilter }) }) test('function and postFilter', () => { env.driver.givenFilterByIdWith(ctx.id, ctx.filter) - - const processingStep = { - _id: `$${ctx.fieldName}`, - [ctx.fieldAlias]: { - $avg: `$${ctx.anotherFieldName}` - } - } - const postFilteringStep = ctx.filter + const group = { by: [ctx.fieldName], aggregation: [{ name: ctx.fieldAlias, avg: ctx.anotherFieldName }] } as Group + const finalFilter = ctx.filter - expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({ + expect(env.AggregationTransformer.transform({ group, finalFilter })).toEqual({ projection: [ { name: ctx.fieldName }, { name: ctx.anotherFieldName, alias: ctx.fieldAlias, function: AdapterFunctions.avg } diff --git a/libs/velo-external-db-core/src/converters/aggregation_transformer.ts b/libs/velo-external-db-core/src/converters/aggregation_transformer.ts index bc9de23ac..b38fe0f57 100644 --- a/libs/velo-external-db-core/src/converters/aggregation_transformer.ts +++ b/libs/velo-external-db-core/src/converters/aggregation_transformer.ts @@ -1,14 +1,12 @@ -import { isObject } from '@wix-velo/velo-external-db-commons' -import { AdapterAggregation, AdapterFunctions, FieldProjection, FunctionProjection } from '@wix-velo/velo-external-db-types' +import { AdapterAggregation, AdapterFunctions } from '@wix-velo/velo-external-db-types' import { IFilterTransformer } from './filter_transformer' -import { projectionFieldFor, projectionFunctionFor } from './utils' +import { projectionFunctionFor } from './utils' import { errors } from '@wix-velo/velo-external-db-commons' +import { Aggregation, Group } from '../spi-model/data_source' const { InvalidQuery } = errors interface IAggregationTransformer { - transform(aggregation: any): AdapterAggregation - extractProjectionFunctions(functionsObj: { [x: string]: { [s: string]: string | number } }): FunctionProjection[] - extractProjectionFields(fields: { [fieldName: string]: string } | string): FieldProjection[] + transform(aggregation: { group: Group, finalFilter?: any }): AdapterAggregation wixFunctionToAdapterFunction(wixFunction: string): AdapterFunctions } @@ -18,13 +16,13 @@ export default class AggregationTransformer implements IAggregationTransformer { this.filterTransformer = filterTransformer } - transform({ processingStep, postFilteringStep }: any): AdapterAggregation { - const { _id: fields, ...functions } = processingStep + transform({ group, finalFilter }: { group: Group, finalFilter?: any }): AdapterAggregation { + const { by: fields, aggregation } = group - const projectionFields = this.extractProjectionFields(fields) - const projectionFunctions = this.extractProjectionFunctions(functions) - - const postFilter = this.filterTransformer.transform(postFilteringStep) + const projectionFields = fields.map(f => ({ name: f })) + const projectionFunctions = this.aggregationToProjectionFunctions(aggregation) + + const postFilter = this.filterTransformer.transform(finalFilter) const projection = [...projectionFields, ...projectionFunctions] @@ -34,48 +32,19 @@ export default class AggregationTransformer implements IAggregationTransformer { } } - extractProjectionFunctions(functionsObj: { [x: string]: { [s: string]: string | number } }) { - const projectionFunctions: { name: any; alias: any; function: any }[] = [] - Object.keys(functionsObj) - .forEach(fieldAlias => { - Object.entries(functionsObj[fieldAlias]) - .forEach(([func, field]) => { - projectionFunctions.push(projectionFunctionFor(field, fieldAlias, this.wixFunctionToAdapterFunction(func))) - }) - }) - - return projectionFunctions - } - - extractProjectionFields(fields: { [fieldName: string]: string } | string) { - const projectionFields = [] - - if (isObject(fields)) { - projectionFields.push(...Object.values(fields).map(f => projectionFieldFor(f)) ) - } else { - projectionFields.push(projectionFieldFor(fields)) - } - - return projectionFields + aggregationToProjectionFunctions(aggregations: Aggregation[]) { + return aggregations.map(aggregation => { + const { name: fieldAlias, ...rest } = aggregation + const [func, field] = Object.entries(rest)[0] + return projectionFunctionFor(field, fieldAlias, this.wixFunctionToAdapterFunction(func)) + }) } wixFunctionToAdapterFunction(func: string): AdapterFunctions { - return this.wixFunctionToAdapterFunctionString(func) as AdapterFunctions - } - - private wixFunctionToAdapterFunctionString(func: string): string { - switch (func) { - case '$avg': - return AdapterFunctions.avg - case '$max': - return AdapterFunctions.max - case '$min': - return AdapterFunctions.min - case '$sum': - return AdapterFunctions.sum - - default: - throw new InvalidQuery(`Unrecognized function ${func}`) + if (Object.values(AdapterFunctions).includes(func as any)) { + return AdapterFunctions[func as AdapterFunctions] as AdapterFunctions } + + throw new InvalidQuery(`Unrecognized function ${func}`) } } diff --git a/libs/velo-external-db-core/src/converters/filter_transformer.spec.ts b/libs/velo-external-db-core/src/converters/filter_transformer.spec.ts index a9bdf63f2..253b08732 100644 --- a/libs/velo-external-db-core/src/converters/filter_transformer.spec.ts +++ b/libs/velo-external-db-core/src/converters/filter_transformer.spec.ts @@ -144,6 +144,42 @@ describe('Filter Transformer', () => { value: [env.FilterTransformer.transform(ctx.filter)] }) }) + }), + + describe('transform sort', () => { + test('should handle wrong sort', () => { + expect(env.FilterTransformer.transformSort('')).toEqual([]) + expect(env.FilterTransformer.transformSort(undefined)).toEqual([]) + expect(env.FilterTransformer.transformSort(null)).toEqual([]) + }) + + test('transform empty sort', () => { + expect(env.FilterTransformer.transformSort([])).toEqual([]) + }) + + test('transform sort', () => { + const sort = [ + { fieldName: ctx.fieldName, order: 'ASC' }, + ] + expect(env.FilterTransformer.transformSort(sort)).toEqual([{ + fieldName: ctx.fieldName, + direction: 'asc' + }]) + }) + + test('transform sort with multiple fields', () => { + const sort = [ + { fieldName: ctx.fieldName, order: 'ASC' }, + { fieldName: ctx.anotherFieldName, order: 'DESC' }, + ] + expect(env.FilterTransformer.transformSort(sort)).toEqual([{ + fieldName: ctx.fieldName, + direction: 'asc' + }, { + fieldName: ctx.anotherFieldName, + direction: 'desc' + }]) + }) }) interface Enviorment { @@ -158,6 +194,7 @@ describe('Filter Transformer', () => { filter: Uninitialized, anotherFilter: Uninitialized, fieldName: Uninitialized, + anotherFieldName: Uninitialized, fieldValue: Uninitialized, operator: Uninitialized, fieldListValue: Uninitialized, @@ -168,6 +205,7 @@ describe('Filter Transformer', () => { ctx.filter = gen.randomFilter() ctx.anotherFilter = gen.randomFilter() ctx.fieldName = chance.word() + ctx.anotherFieldName = chance.word() ctx.fieldValue = chance.word() ctx.operator = gen.randomOperator() as WixDataMultiFieldOperators | WixDataSingleFieldOperators ctx.fieldListValue = [chance.word(), chance.word(), chance.word(), chance.word(), chance.word()] diff --git a/libs/velo-external-db-core/src/converters/filter_transformer.ts b/libs/velo-external-db-core/src/converters/filter_transformer.ts index 2d8b1017c..608dbfb49 100644 --- a/libs/velo-external-db-core/src/converters/filter_transformer.ts +++ b/libs/velo-external-db-core/src/converters/filter_transformer.ts @@ -1,7 +1,8 @@ import { AdapterOperators, isObject, patchVeloDateValue } from '@wix-velo/velo-external-db-commons' import { EmptyFilter } from './utils' import { errors } from '@wix-velo/velo-external-db-commons' -import { AdapterFilter, AdapterOperator, WixDataFilter, WixDataMultiFieldOperators, } from '@wix-velo/velo-external-db-types' +import { AdapterFilter, AdapterOperator, Sort, WixDataFilter, WixDataMultiFieldOperators, } from '@wix-velo/velo-external-db-types' +import { Sorting } from '../spi-model/data_source' const { InvalidQuery } = errors export interface IFilterTransformer { @@ -41,6 +42,19 @@ export default class FilterTransformer implements IFilterTransformer { } } + transformSort(sort: any): Sort[] { + if (!this.isSortArray(sort)) { + return [] + } + + return (sort as Sorting[]).map(sorting => { + return { + fieldName: sorting.fieldName, + direction: sorting.order.toLowerCase() as 'asc' | 'desc' + } + }) + } + isMultipleFieldOperator(filter: WixDataFilter) { return (Object).values(WixDataMultiFieldOperators).includes(Object.keys(filter)[0]) } @@ -90,4 +104,17 @@ export default class FilterTransformer implements IFilterTransformer { return (!filter || !isObject(filter) || Object.keys(filter)[0] === undefined) } + isSortArray(sort: any): boolean { + + if (!Array.isArray(sort)) { + return false + } + return sort.every((s: any) => { + return this.isSortObject(s) + }) + } + + isSortObject(sort:any): boolean { + return sort.fieldName && sort.order + } } diff --git a/libs/velo-external-db-core/src/converters/utils.ts b/libs/velo-external-db-core/src/converters/utils.ts index 141313312..18fdf0afa 100644 --- a/libs/velo-external-db-core/src/converters/utils.ts +++ b/libs/velo-external-db-core/src/converters/utils.ts @@ -8,10 +8,8 @@ export const projectionFieldFor = (fieldName: any, fieldAlias?: string) => { } export const projectionFunctionFor = (fieldName: string | number, fieldAlias: any, func: any) => { - if (isCountFunc(func, fieldName)) + if (func === AdapterFunctions.count) return { alias: fieldAlias, function: AdapterFunctions.count, name: '*' } - const name = (fieldName as string).substring(1) - return { name, alias: fieldAlias || name, function: func } + + return { name: fieldName as string, alias: fieldAlias || fieldName as string, function: func } } - -const isCountFunc = (func: any, value: any ) => (func === AdapterFunctions.sum && value === 1) diff --git a/libs/velo-external-db-core/src/data_hooks_utils.spec.ts b/libs/velo-external-db-core/src/data_hooks_utils.spec.ts index ce7641887..3a80fdec1 100644 --- a/libs/velo-external-db-core/src/data_hooks_utils.spec.ts +++ b/libs/velo-external-db-core/src/data_hooks_utils.spec.ts @@ -68,14 +68,18 @@ describe('Hooks Utils', () => { expect(dataPayloadFor(DataOperations.Get, randomBodyWith({ itemId: ctx.itemId, projection: ctx.projection }))).toEqual({ itemId: ctx.itemId, projection: ctx.projection }) }) test('Payload for Aggregate should return Aggregation query', () => { - expect(dataPayloadFor(DataOperations.Aggregate, randomBodyWith({ filter: ctx.filter, processingStep: ctx.processingStep, postFilteringStep: ctx.postFilteringStep }))) - .toEqual( - { - filter: ctx.filter, - processingStep: ctx.processingStep, - postFilteringStep: ctx.postFilteringStep - } - ) + expect(dataPayloadFor(DataOperations.Aggregate, randomBodyWith({ + initialFilter: ctx.filter, group: ctx.group, finalFilter: ctx.finalFilter, distinct: ctx.distinct + , paging: ctx.paging, sort: ctx.sort, projection: ctx.projection + }))).toEqual({ + initialFilter: ctx.filter, + distinct: ctx.distinct, + group: ctx.group, + finalFilter: ctx.finalFilter, + sort: ctx.sort, + paging: ctx.paging, + } + ) }) }) @@ -90,8 +94,10 @@ describe('Hooks Utils', () => { items: Uninitialized, itemId: Uninitialized, itemIds: Uninitialized, - processingStep: Uninitialized, - postFilteringStep: Uninitialized + group: Uninitialized, + finalFilter: Uninitialized, + distinct: Uninitialized, + paging: Uninitialized, } beforeEach(() => { @@ -104,5 +110,9 @@ describe('Hooks Utils', () => { ctx.items = chance.word() ctx.itemId = chance.word() ctx.itemIds = chance.word() + ctx.group = chance.word() + ctx.finalFilter = chance.word() + ctx.distinct = chance.word() + ctx.paging = chance.word() }) }) diff --git a/libs/velo-external-db-core/src/data_hooks_utils.ts b/libs/velo-external-db-core/src/data_hooks_utils.ts index 04969ca48..d01239f67 100644 --- a/libs/velo-external-db-core/src/data_hooks_utils.ts +++ b/libs/velo-external-db-core/src/data_hooks_utils.ts @@ -1,5 +1,6 @@ import { Item, WixDataFilter } from '@wix-velo/velo-external-db-types' -import { AggregationQuery, FindQuery, RequestContext } from './types' +import { AggregateRequest } from './spi-model/data_source' +import { FindQuery, RequestContext } from './types' export const DataHooksForAction: { [key: string]: string[] } = { @@ -92,10 +93,13 @@ export const dataPayloadFor = (operation: DataOperations, body: any) => { return { itemIds: body.itemIds as string[] } case DataOperations.Aggregate: return { - filter: body.filter, - processingStep: body.processingStep, - postFilteringStep: body.postFilteringStep - } as AggregationQuery + initialFilter: body.initialFilter, + distinct: body.distinct, + group: body.group, + finalFilter: body.finalFilter, + sort: body.sort, + paging: body.paging, + } as Partial case DataOperations.Count: return { filter: body.filter as WixDataFilter } } diff --git a/libs/velo-external-db-core/src/index.ts b/libs/velo-external-db-core/src/index.ts index bf90254bb..1b9b9ab86 100644 --- a/libs/velo-external-db-core/src/index.ts +++ b/libs/velo-external-db-core/src/index.ts @@ -16,6 +16,8 @@ import { RoleAuthorizationService } from '@wix-velo/external-db-security' import { ConfigValidator, AuthorizationConfigValidator, CommonConfigValidator } from '@wix-velo/external-db-config' import { ConnectionCleanUp } from '@wix-velo/velo-external-db-types' import { Router } from 'express' +import { CollectionCapability } from './spi-model/capabilities' +import { decodeBase64 } from './utils/base64_utils' export class ExternalDbRouter { connector: DbConnector @@ -36,7 +38,7 @@ export class ExternalDbRouter { constructor({ connector, config, hooks }: { connector: DbConnector, config: ExternalDbRouterConfig, hooks: {schemaHooks?: SchemaHooks, dataHooks?: DataHooks} }) { this.isInitialized(connector) this.connector = connector - this.configValidator = new ConfigValidator(connector.configValidator, new AuthorizationConfigValidator(config.authorization), new CommonConfigValidator({ secretKey: config.secretKey, vendor: config.vendor, type: config.adapterType }, config.commonExtended)) + this.configValidator = new ConfigValidator(connector.configValidator, new AuthorizationConfigValidator(config.authorization), new CommonConfigValidator({ externalDatabaseId: config.externalDatabaseId, allowedMetasites: config.allowedMetasites, vendor: config.vendor, type: config.adapterType }, config.commonExtended)) this.config = config this.operationService = new OperationService(connector.databaseOperations) this.schemaInformation = new CacheableSchemaInformation(connector.schemaProvider) @@ -66,4 +68,6 @@ export class ExternalDbRouter { } } -export { DataService, SchemaService, OperationService, CacheableSchemaInformation, FilterTransformer, AggregationTransformer, QueryValidator, SchemaAwareDataService, ItemTransformer, Hooks, ServiceContext } +export * as dataSpi from './spi-model/data_source' +export * as schemaUtils from '../src/utils/schema_utils' +export { DataService, SchemaService, OperationService, CacheableSchemaInformation, FilterTransformer, AggregationTransformer, QueryValidator, SchemaAwareDataService, ItemTransformer, Hooks, ServiceContext, CollectionCapability, decodeBase64 } diff --git a/libs/velo-external-db-core/src/router.ts b/libs/velo-external-db-core/src/router.ts index 818109c5e..d36cf6556 100644 --- a/libs/velo-external-db-core/src/router.ts +++ b/libs/velo-external-db-core/src/router.ts @@ -1,35 +1,41 @@ import * as path from 'path' import * as BPromise from 'bluebird' import * as express from 'express' +import type { Response } from 'express' import * as compression from 'compression' import { errorMiddleware } from './web/error-middleware' import { appInfoFor } from './health/app_info' import { errors } from '@wix-velo/velo-external-db-commons' import { extractRole } from './web/auth-role-middleware' import { config } from './roles-config.json' -import { secretKeyAuthMiddleware } from './web/auth-middleware' import { authRoleMiddleware } from './web/auth-role-middleware' import { unless, includes } from './web/middleware-support' import { getAppInfoPage } from './utils/router_utils' import { DataHooksForAction, DataOperations, dataPayloadFor, DataActions, requestContextFor } from './data_hooks_utils' -import { SchemaHooksForAction, SchemaOperations, schemaPayloadFor, SchemaActions } from './schema_hooks_utils' +// import { SchemaHooksForAction } from './schema_hooks_utils' import SchemaService from './service/schema' import OperationService from './service/operation' -import { AnyFixMe } from '@wix-velo/velo-external-db-types' +import { AnyFixMe, Item } from '@wix-velo/velo-external-db-types' import SchemaAwareDataService from './service/schema_aware_data' import FilterTransformer from './converters/filter_transformer' import AggregationTransformer from './converters/aggregation_transformer' import { RoleAuthorizationService } from '@wix-velo/external-db-security' import { DataHooks, Hooks, RequestContext, SchemaHooks, ServiceContext } from './types' import { ConfigValidator } from '@wix-velo/external-db-config' +import { JwtAuthenticator } from './web/jwt-auth-middleware' +import * as dataSource from './spi-model/data_source' +import * as capabilities from './spi-model/capabilities' +import { WixDataFacade } from './web/wix_data_facade' -const { InvalidRequest, ItemNotFound } = errors -const { Find: FIND, Insert: INSERT, BulkInsert: BULK_INSERT, Update: UPDATE, BulkUpdate: BULK_UPDATE, Remove: REMOVE, BulkRemove: BULK_REMOVE, Aggregate: AGGREGATE, Count: COUNT, Get: GET } = DataOperations -let schemaService: SchemaService, operationService: OperationService, externalDbConfigClient: ConfigValidator, schemaAwareDataService: SchemaAwareDataService, cfg: { secretKey?: any; type?: any; vendor?: any }, filterTransformer: FilterTransformer, aggregationTransformer: AggregationTransformer, roleAuthorizationService: RoleAuthorizationService, dataHooks: DataHooks, schemaHooks: SchemaHooks +const { InvalidRequest } = errors +// const { Find: FIND, Insert: INSERT, BulkInsert: BULK_INSERT, Update: UPDATE, BulkUpdate: BULK_UPDATE, Remove: REMOVE, BulkRemove: BULK_REMOVE, Aggregate: AGGREGATE, Count: COUNT, Get: GET } = DataOperations +const { Aggregate: AGGREGATE } = DataOperations + +let schemaService: SchemaService, operationService: OperationService, externalDbConfigClient: ConfigValidator, schemaAwareDataService: SchemaAwareDataService, cfg: { externalDatabaseId: string, allowedMetasites: string, type?: any; vendor?: any, wixDataBaseUrl: string }, filterTransformer: FilterTransformer, aggregationTransformer: AggregationTransformer, roleAuthorizationService: RoleAuthorizationService, dataHooks: DataHooks //schemaHooks: SchemaHooks export const initServices = (_schemaAwareDataService: SchemaAwareDataService, _schemaService: SchemaService, _operationService: OperationService, - _externalDbConfigClient: ConfigValidator, _cfg: { secretKey?: string, type?: string, vendor?: string }, + _externalDbConfigClient: ConfigValidator, _cfg: { externalDatabaseId: string, allowedMetasites: string, type?: string, vendor?: string, wixDataBaseUrl: string }, _filterTransformer: FilterTransformer, _aggregationTransformer: AggregationTransformer, _roleAuthorizationService: RoleAuthorizationService, _hooks?: Hooks) => { schemaService = _schemaService @@ -41,7 +47,7 @@ export const initServices = (_schemaAwareDataService: SchemaAwareDataService, _s aggregationTransformer = _aggregationTransformer roleAuthorizationService = _roleAuthorizationService dataHooks = _hooks?.dataHooks || {} - schemaHooks = _hooks?.schemaHooks || {} + // schemaHooks = _hooks?.schemaHooks || {} } const serviceContext = (): ServiceContext => ({ @@ -56,11 +62,11 @@ const executeDataHooksFor = async(action: string, payload: AnyFixMe, requestCont }, payload) } -const executeSchemaHooksFor = async(action: string, payload: any, requestContext: RequestContext, customContext: any) => { - return BPromise.reduce(SchemaHooksForAction[action], async(lastHookResult: any, hookName: string) => { - return await executeHook(schemaHooks, hookName, lastHookResult, requestContext, customContext) - }, payload) -} +// const executeSchemaHooksFor = async(action: string, payload: any, requestContext: RequestContext, customContext: any) => { +// return BPromise.reduce(SchemaHooksForAction[action], async(lastHookResult: any, hookName: string) => { +// return await executeHook(schemaHooks, hookName, lastHookResult, requestContext, customContext) +// }, payload) +// } const executeHook = async(hooks: DataHooks | SchemaHooks, _actionName: string, payload: AnyFixMe, requestContext: RequestContext, customContext: any) => { const actionName = _actionName as keyof typeof hooks @@ -82,10 +88,20 @@ export const createRouter = () => { router.use(express.json()) router.use(compression()) router.use('/assets', express.static(path.join(__dirname, 'assets'))) - router.use(unless(['/', '/provision', '/favicon.ico'], secretKeyAuthMiddleware({ secretKey: cfg.secretKey }))) + const jwtAuthenticator = new JwtAuthenticator(cfg.externalDatabaseId, cfg.allowedMetasites, new WixDataFacade(cfg.wixDataBaseUrl)) + router.use(unless(['/', '/info', '/capabilities', '/favicon.ico'], jwtAuthenticator.authorizeJwt())) config.forEach(({ pathPrefix, roles }) => router.use(includes([pathPrefix], authRoleMiddleware({ roles })))) + const streamCollection = (collection: any[], res: Response) => { + res.contentType('application/x-ndjson') + collection.forEach(item => { + res.write(JSON.stringify(item)) + }) + res.end() + } + + // *************** INFO ********************** router.get('/', async(req, res) => { const appInfo = await appInfoFor(operationService, externalDbConfigClient) @@ -94,117 +110,103 @@ export const createRouter = () => { res.send(appInfoPage) }) + router.get('/capabilities', async(req, res) => { + const capabilitiesResponse = { + capabilities: { + collection: [capabilities.CollectionCapability.CREATE] + } as capabilities.Capabilities + } as capabilities.GetCapabilitiesResponse + + res.json(capabilitiesResponse) + }) + router.post('/provision', async(req, res) => { const { type, vendor } = cfg - res.json({ type, vendor, protocolVersion: 2 }) + res.json({ type, vendor, protocolVersion: 3, adapterVersion: 'v1' }) + }) + + router.get('/info', async(req, res) => { + const { externalDatabaseId } = cfg + res.json({ dataSourceId: externalDatabaseId }) }) // *************** Data API ********************** - router.post('/data/find', async(req, res, next) => { + router.post('/data/query', async(req, res, next) => { try { - const { collectionName } = req.body - const customContext = {} - const { filter, sort, skip, limit, projection } = await executeDataHooksFor(DataActions.BeforeFind, dataPayloadFor(FIND, req.body), requestContextFor(FIND, req.body), customContext) - - await roleAuthorizationService.authorizeRead(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.find(collectionName, filterTransformer.transform(filter), sort, skip, limit, projection) - - const dataAfterAction = await executeDataHooksFor(DataActions.AfterFind, data, requestContextFor(FIND, req.body), customContext) - res.json(dataAfterAction) + const queryRequest: dataSource.QueryRequest = req.body + const query = queryRequest.query + + const offset = query.paging ? query.paging.offset : 0 + const limit = query.paging ? query.paging.limit : 50 + + const data = await schemaAwareDataService.find( + queryRequest.collectionId, + filterTransformer.transform(query.filter), + filterTransformer.transformSort(query.sort), + offset, + limit, + query.fields, + queryRequest.omitTotalCount + ) + + const responseParts = data.items.map(dataSource.QueryResponsePart.item) + const metadata = dataSource.QueryResponsePart.pagingMetadata(responseParts.length, offset, data.totalCount) + + streamCollection([...responseParts, ...[metadata]], res) } catch (e) { next(e) } }) - router.post('/data/aggregate', async(req, res, next) => { + router.post('/data/count', async(req, res, next) => { try { - const { collectionName } = req.body - const customContext = {} - const { filter, processingStep, postFilteringStep } = await executeDataHooksFor(DataActions.BeforeAggregate, dataPayloadFor(AGGREGATE, req.body), requestContextFor(AGGREGATE, req.body), customContext) - await roleAuthorizationService.authorizeRead(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.aggregate(collectionName, filterTransformer.transform(filter), aggregationTransformer.transform({ processingStep, postFilteringStep })) - const dataAfterAction = await executeDataHooksFor(DataActions.AfterAggregate, data, requestContextFor(AGGREGATE, req.body), customContext) - res.json(dataAfterAction) - } catch (e) { - next(e) - } - }) + const countRequest: dataSource.CountRequest = req.body + const data = await schemaAwareDataService.count( + countRequest.collectionId, + filterTransformer.transform(countRequest.filter), + ) - router.post('/data/insert', async(req, res, next) => { - try { - const { collectionName } = req.body - const customContext = {} - const { item } = await executeDataHooksFor(DataActions.BeforeInsert, dataPayloadFor(INSERT, req.body), requestContextFor(INSERT, req.body), customContext) - await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.insert(collectionName, item) + const response = { + totalCount: data.totalCount + } as dataSource.CountResponse - const dataAfterAction = await executeDataHooksFor(DataActions.AfterInsert, data, requestContextFor(INSERT, req.body), customContext) - res.json(dataAfterAction) + res.json(response) } catch (e) { next(e) } }) - router.post('/data/insert/bulk', async(req, res, next) => { + router.post('/data/insert', async(req, res, next) => { try { - const { collectionName } = req.body - const customContext = {} - const { items } = await executeDataHooksFor(DataActions.BeforeBulkInsert, dataPayloadFor(BULK_INSERT, req.body), requestContextFor(BULK_INSERT, req.body), customContext) + const insertRequest: dataSource.InsertRequest = req.body - await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.bulkInsert(collectionName, items) + const collectionName = insertRequest.collectionId - const dataAfterAction = await executeDataHooksFor(DataActions.AfterBulkInsert, data, requestContextFor(BULK_INSERT, req.body), customContext) - res.json(dataAfterAction) - } catch (e) { - next(e) - } - }) + const data = insertRequest.overwriteExisting ? + await schemaAwareDataService.bulkUpsert(collectionName, insertRequest.items) : + await schemaAwareDataService.bulkInsert(collectionName, insertRequest.items) - router.post('/data/get', async(req, res, next) => { - try { - const { collectionName } = req.body - const customContext = {} - const { itemId, projection } = await executeDataHooksFor(DataActions.BeforeGetById, dataPayloadFor(GET, req.body), requestContextFor(GET, req.body), customContext) - await roleAuthorizationService.authorizeRead(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.getById(collectionName, itemId, projection) - - const dataAfterAction = await executeDataHooksFor(DataActions.AfterGetById, data, requestContextFor(GET, req.body), customContext) - if (!dataAfterAction.item) { - throw new ItemNotFound('Item not found') - } - res.json(dataAfterAction) + const responseParts = data.items.map(dataSource.InsertResponsePart.item) + + streamCollection(responseParts, res) } catch (e) { next(e) } }) router.post('/data/update', async(req, res, next) => { + try { - const { collectionName } = req.body - const customContext = {} - const { item } = await executeDataHooksFor(DataActions.BeforeUpdate, dataPayloadFor(UPDATE, req.body), requestContextFor(UPDATE, req.body), customContext) - await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.update(collectionName, item) + const updateRequest: dataSource.UpdateRequest = req.body - const dataAfterAction = await executeDataHooksFor(DataActions.AfterUpdate, data, requestContextFor(UPDATE, req.body), customContext) - res.json(dataAfterAction) - } catch (e) { - next(e) - } - }) + const collectionName = updateRequest.collectionId - router.post('/data/update/bulk', async(req, res, next) => { - try { - const { collectionName } = req.body - const customContext = {} - const { items } = await executeDataHooksFor(DataActions.BeforeBulkUpdate, dataPayloadFor(BULK_UPDATE, req.body), requestContextFor(BULK_UPDATE, req.body), customContext) - await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.bulkUpdate(collectionName, items) + const data = await schemaAwareDataService.bulkUpdate(collectionName, updateRequest.items) - const dataAfterAction = await executeDataHooksFor(DataActions.AfterBulkUpdate, data, requestContextFor(BULK_UPDATE, req.body), customContext) - res.json(dataAfterAction) + const responseParts = data.items.map(dataSource.UpdateResponsePart.item) + + streamCollection(responseParts, res) } catch (e) { next(e) } @@ -212,44 +214,40 @@ export const createRouter = () => { router.post('/data/remove', async(req, res, next) => { try { - const { collectionName } = req.body - const customContext = {} - const { itemId } = await executeDataHooksFor(DataActions.BeforeRemove, dataPayloadFor(REMOVE, req.body), requestContextFor(REMOVE, req.body), customContext) - await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.delete(collectionName, itemId) + const removeRequest: dataSource.RemoveRequest = req.body + const collectionName = removeRequest.collectionId + const idEqExpression = removeRequest.itemIds.map(itemId => ({ _id: { $eq: itemId } })) + const filter = { $or: idEqExpression } - const dataAfterAction = await executeDataHooksFor(DataActions.AfterRemove, data, requestContextFor(REMOVE, req.body), customContext) - res.json(dataAfterAction) - } catch (e) { - next(e) - } - }) + const objectsBeforeRemove = (await schemaAwareDataService.find(collectionName, filterTransformer.transform(filter), undefined, 0, removeRequest.itemIds.length)).items - router.post('/data/remove/bulk', async(req, res, next) => { - try { - const { collectionName } = req.body - const customContext = {} - const { itemIds } = await executeDataHooksFor(DataActions.BeforeBulkRemove, dataPayloadFor(BULK_REMOVE, req.body), requestContextFor(BULK_REMOVE, req.body), customContext) - await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.bulkDelete(collectionName, itemIds) + await schemaAwareDataService.bulkDelete(collectionName, removeRequest.itemIds) + + const responseParts = objectsBeforeRemove.map(dataSource.RemoveResponsePart.item) - const dataAfterAction = await executeDataHooksFor(DataActions.AfterBulkRemove, data, requestContextFor(BULK_REMOVE, req.body), customContext) - res.json(dataAfterAction) + streamCollection(responseParts, res) } catch (e) { next(e) } }) - router.post('/data/count', async(req, res, next) => { + router.post('/data/aggregate', async(req, res, next) => { try { - const { collectionName } = req.body + const aggregationRequest = req.body as dataSource.AggregateRequest + const { collectionId, paging, sort } = aggregationRequest + const offset = paging ? paging.offset : 0 + const limit = paging ? paging.limit : 50 + const customContext = {} - const { filter } = await executeDataHooksFor(DataActions.BeforeCount, dataPayloadFor(COUNT, req.body), requestContextFor(COUNT, req.body), customContext) - await roleAuthorizationService.authorizeRead(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.count(collectionName, filterTransformer.transform(filter)) + const { initialFilter, group, finalFilter } = await executeDataHooksFor(DataActions.BeforeAggregate, dataPayloadFor(AGGREGATE, aggregationRequest), requestContextFor(AGGREGATE, aggregationRequest), customContext) + roleAuthorizationService.authorizeRead(collectionId, extractRole(aggregationRequest)) + const data = await schemaAwareDataService.aggregate(collectionId, filterTransformer.transform(initialFilter), aggregationTransformer.transform({ group, finalFilter }), filterTransformer.transformSort(sort), offset, limit) + const dataAfterAction = await executeDataHooksFor(DataActions.AfterAggregate, data, requestContextFor(AGGREGATE, aggregationRequest), customContext) - const dataAfterAction = await executeDataHooksFor(DataActions.AfterCount, data, requestContextFor(COUNT, req.body), customContext) - res.json(dataAfterAction) + const responseParts = dataAfterAction.items.map(dataSource.AggregateResponsePart.item) + const metadata = dataSource.AggregateResponsePart.pagingMetadata((dataAfterAction.items as Item[]).length, offset, data.totalCount) + + streamCollection([...responseParts, ...[metadata]], res) } catch (e) { next(e) } @@ -257,106 +255,62 @@ export const createRouter = () => { router.post('/data/truncate', async(req, res, next) => { try { - const { collectionName } = req.body - await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body)) - const data = await schemaAwareDataService.truncate(collectionName) - res.json(data) + const trancateRequest = req.body as dataSource.TruncateRequest + await schemaAwareDataService.truncate(trancateRequest.collectionId) + res.json({} as dataSource.TruncateResponse) } catch (e) { next(e) } }) // *********************************************** + // *************** Collections API ********************** - // *************** Schema API ********************** - router.post('/schemas/list', async(req, res, next) => { - try { - const customContext = {} - await executeSchemaHooksFor(SchemaActions.BeforeList, schemaPayloadFor(SchemaOperations.List, req.body), requestContextFor(SchemaOperations.List, req.body), customContext) + router.post('/collections/get', async(req, res, next) => { - const data = await schemaService.list() - - const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterList, data, requestContextFor(SchemaOperations.List, req.body), customContext) - res.json(dataAfterAction) + const { collectionIds } = req.body + try { + const data = await schemaService.list(collectionIds) + streamCollection(data.collection, res) } catch (e) { next(e) } }) - router.post('/schemas/list/headers', async(req, res, next) => { - try { - const customContext = {} - await executeSchemaHooksFor(SchemaActions.BeforeListHeaders, schemaPayloadFor(SchemaOperations.ListHeaders, req.body), requestContextFor(SchemaOperations.ListHeaders, req.body), customContext) - const data = await schemaService.listHeaders() - const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterListHeaders, data, requestContextFor(SchemaOperations.ListHeaders, req.body), customContext) - res.json(dataAfterAction) - } catch (e) { - next(e) - } - }) + router.post('/collections/create', async(req, res, next) => { + const { collection } = req.body - router.post('/schemas/find', async(req, res, next) => { try { - const customContext = {} - const { schemaIds } = await executeSchemaHooksFor(SchemaActions.BeforeFind, schemaPayloadFor(SchemaOperations.Find, req.body), requestContextFor(SchemaOperations.Find, req.body), customContext) - - if (schemaIds && schemaIds.length > 10) { - throw new InvalidRequest('Too many schemas requested') - } - const data = await schemaService.find(schemaIds) - const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterFind, data, requestContextFor(SchemaOperations.Find, req.body), customContext) - res.json(dataAfterAction) + const data = await schemaService.create(collection) + streamCollection([data.collection], res) } catch (e) { next(e) } }) - router.post('/schemas/create', async(req, res, next) => { - try { - const customContext = {} - const { collectionName } = await executeSchemaHooksFor(SchemaActions.BeforeCreate, schemaPayloadFor(SchemaOperations.Create, req.body), requestContextFor(SchemaOperations.Create, req.body), customContext) - const data = await schemaService.create(collectionName) - - const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterCreate, data, requestContextFor(SchemaOperations.Create, req.body), customContext) - - res.json(dataAfterAction) - } catch (e) { - next(e) - } - }) + router.post('/collections/update', async(req, res, next) => { + const { collection } = req.body - router.post('/schemas/column/add', async(req, res, next) => { try { - const { collectionName } = req.body - const customContext = {} - const { column } = await executeSchemaHooksFor(SchemaActions.BeforeColumnAdd, schemaPayloadFor(SchemaOperations.ColumnAdd, req.body), requestContextFor(SchemaOperations.ColumnAdd, req.body), customContext) - - const data = await schemaService.addColumn(collectionName, column) - - const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterColumnAdd, data, requestContextFor(SchemaOperations.ColumnAdd, req.body), customContext) - - res.json(dataAfterAction) + const data = await schemaService.update(collection) + streamCollection([data.collection], res) } catch (e) { next(e) } }) - router.post('/schemas/column/remove', async(req, res, next) => { - try { - const { collectionName } = req.body - const customContext = {} - const { columnName } = await executeSchemaHooksFor(SchemaActions.BeforeColumnRemove, schemaPayloadFor(SchemaOperations.ColumnRemove, req.body), requestContextFor(SchemaOperations.ColumnRemove, req.body), customContext) - - const data = await schemaService.removeColumn(collectionName, columnName) + router.post('/collections/delete', async(req, res, next) => { + const { collectionId } = req.body - const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterColumnRemove, data, requestContextFor(SchemaOperations.ColumnRemove, req.body), customContext) - res.json(dataAfterAction) + try { + const data = await schemaService.delete(collectionId) + streamCollection([data.collection], res) } catch (e) { next(e) } }) - // *********************************************** + router.use(errorMiddleware) diff --git a/libs/velo-external-db-core/src/service/data.spec.ts b/libs/velo-external-db-core/src/service/data.spec.ts index 76f8cf9cf..6dad1589c 100644 --- a/libs/velo-external-db-core/src/service/data.spec.ts +++ b/libs/velo-external-db-core/src/service/data.spec.ts @@ -85,9 +85,10 @@ describe('Data Service', () => { }) test('aggregate api', async() => { - driver.givenAggregateResult(ctx.entities, ctx.collectionName, ctx.filter, ctx.aggregation) + driver.givenAggregateResult(ctx.entities, ctx.collectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit) + driver.givenCountResult(ctx.total, ctx.collectionName, ctx.filter) - return expect(env.dataService.aggregate(ctx.collectionName, ctx.filter, ctx.aggregation)).resolves.toEqual({ items: ctx.entities, totalCount: 0 }) + return expect(env.dataService.aggregate(ctx.collectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit)).resolves.toEqual({ items: ctx.entities, totalCount: ctx.total }) }) diff --git a/libs/velo-external-db-core/src/service/data.ts b/libs/velo-external-db-core/src/service/data.ts index 435ae32d2..17beb7367 100644 --- a/libs/velo-external-db-core/src/service/data.ts +++ b/libs/velo-external-db-core/src/service/data.ts @@ -1,4 +1,4 @@ -import { AdapterAggregation as Aggregation, AdapterFilter as Filter, IDataProvider, Item, ResponseField } from '@wix-velo/velo-external-db-types' +import { AdapterAggregation as Aggregation, AdapterFilter as Filter, IDataProvider, Item, ResponseField, Sort } from '@wix-velo/velo-external-db-types' import { asWixData } from '../converters/data_utils' import { getByIdFilterFor } from '../utils/data_utils' @@ -9,13 +9,14 @@ export default class DataService { this.storage = storage } - async find(collectionName: string, _filter: Filter, sort: any, skip: any, limit: any, projection: any) { + async find(collectionName: string, _filter: Filter, sort: any, skip: any, limit: any, projection: any, omitTotalCount?: boolean): Promise<{items: any[], totalCount?: number}> { const items = this.storage.find(collectionName, _filter, sort, skip, limit, projection) - const totalCount = this.storage.count(collectionName, _filter) + const totalCount = omitTotalCount? undefined : this.storage.count(collectionName, _filter) + return { items: (await items).map(item => asWixData(item, projection)), totalCount: await totalCount - } + } } async getById(collectionName: string, itemId: string, projection: any) { @@ -34,6 +35,11 @@ export default class DataService { return { item: asWixData(resp.items[0]) } } + async bulkUpsert(collectionName: string, items: Item[], fields?: ResponseField[]) { + await this.storage.insert(collectionName, items, fields, true) + return { items: items.map( asWixData ) } + } + async bulkInsert(collectionName: string, items: Item[], fields?: ResponseField[]) { await this.storage.insert(collectionName, items, fields) return { items: items.map( asWixData ) } @@ -63,11 +69,14 @@ export default class DataService { return this.storage.truncate(collectionName) } - async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation) { + + // sort, skip, limit are not really optional, after we'll implement in all the data providers we can remove the ? + async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort?: Sort[], skip?: number, limit?: number) { + const totalCount = this.storage.count(collectionName, filter) return { - items: ((await this.storage.aggregate?.(collectionName, filter, aggregation)) || []) + items: ((await this.storage.aggregate?.(collectionName, filter, aggregation, sort, skip, limit)) || []) .map( asWixData ), - totalCount: 0 + totalCount: await totalCount } } } diff --git a/libs/velo-external-db-core/src/service/schema.spec.ts b/libs/velo-external-db-core/src/service/schema.spec.ts index f0dc7991d..5155d2472 100644 --- a/libs/velo-external-db-core/src/service/schema.spec.ts +++ b/libs/velo-external-db-core/src/service/schema.spec.ts @@ -1,90 +1,212 @@ import * as Chance from 'chance' -import SchemaService from './schema' -import { AllSchemaOperations, errors } from '@wix-velo/velo-external-db-commons' import { Uninitialized } from '@wix-velo/test-commons' +import { errors } from '@wix-velo/velo-external-db-commons' +import SchemaService from './schema' import * as driver from '../../test/drivers/schema_provider_test_support' import * as schema from '../../test/drivers/schema_information_test_support' import * as matchers from '../../test/drivers/schema_matchers' import * as gen from '../../test/gen' -const { schemasListFor, schemaHeadersListFor, schemasWithReadOnlyCapabilitiesFor } = matchers +import { + fieldTypeToWixDataEnum, + compareColumnsInDbAndRequest, + InputFieldsToWixFormatFields, + InputFieldToWixFormatField, +} from '../utils/schema_utils' +import { + Table, + InputField + } from '@wix-velo/velo-external-db-types' + +const { collectionsListFor } = matchers const chance = Chance() describe('Schema Service', () => { + describe('Collection new SPI', () => { + test('retrieve all collections from provider', async() => { + driver.givenAllSchemaOperations() + driver.givenColumnCapabilities() + driver.givenListResult(ctx.dbsWithIdColumn) + + + await expect( env.schemaService.list([]) ).resolves.toEqual(collectionsListFor(ctx.dbsWithIdColumn)) + }) + + test('create new collection without fields', async() => { + driver.givenAllSchemaOperations() + driver.expectCreateOf(ctx.collectionName) + schema.expectSchemaRefresh() + + await expect(env.schemaService.create({ id: ctx.collectionName, fields: [] })).resolves.toEqual({ + collection: { id: ctx.collectionName, fields: [] } + }) + }) + + test('create new collection with fields', async() => { + const fields = [{ + key: ctx.column.name, + type: fieldTypeToWixDataEnum(ctx.column.type), + }] + driver.givenAllSchemaOperations() + schema.expectSchemaRefresh() + driver.expectCreateWithFieldsOf(ctx.collectionName, fields) + + await expect(env.schemaService.create({ id: ctx.collectionName, fields })).resolves.toEqual({ + collection: { id: ctx.collectionName, fields } + }) + }) + + test('update collection - add new columns', async() => { + const newFields = [{ + key: ctx.column.name, + type: fieldTypeToWixDataEnum(ctx.column.type), + }] + + driver.givenAllSchemaOperations() + schema.expectSchemaRefresh() + driver.givenFindResults([ { id: ctx.collectionName, fields: [] } ]) + + await env.schemaService.update({ id: ctx.collectionName, fields: newFields }) + + + expect(driver.schemaProvider.addColumn).toBeCalledTimes(1) + expect(driver.schemaProvider.addColumn).toBeCalledWith(ctx.collectionName, { + name: ctx.column.name, + type: ctx.column.type, + subtype: ctx.column.subtype + }) + expect(driver.schemaProvider.removeColumn).not.toBeCalled() + expect(driver.schemaProvider.changeColumnType).not.toBeCalled() + }) + + test('update collection - add new column to non empty collection', async() => { + const currentFields = [{ + field: ctx.column.name, + type: ctx.column.type + }] + const wantedFields = InputFieldsToWixFormatFields([ ctx.column, ctx.anotherColumn ]) + + driver.givenAllSchemaOperations() + schema.expectSchemaRefresh() + driver.givenFindResults([ { + id: ctx.collectionName, + fields: currentFields + }]) + + await env.schemaService.update({ id: ctx.collectionName, fields: wantedFields }) + + const { columnsToAdd } = compareColumnsInDbAndRequest(currentFields, wantedFields) + + columnsToAdd.forEach(c => expect(driver.schemaProvider.addColumn).toBeCalledWith(ctx.collectionName, c)) + expect(driver.schemaProvider.removeColumn).not.toBeCalled() + expect(driver.schemaProvider.changeColumnType).not.toBeCalled() + }) + + test('update collection - remove column', async() => { + const currentFields = [{ + field: ctx.column.name, + type: ctx.column.type + }] + + driver.givenAllSchemaOperations() + schema.expectSchemaRefresh() + driver.givenFindResults([ { + id: ctx.collectionName, + fields: currentFields + }]) + + const { columnsToRemove } = compareColumnsInDbAndRequest(currentFields, []) + + await env.schemaService.update({ id: ctx.collectionName, fields: [] }) + + columnsToRemove.forEach(c => expect(driver.schemaProvider.removeColumn).toBeCalledWith(ctx.collectionName, c)) + expect(driver.schemaProvider.addColumn).not.toBeCalled() + expect(driver.schemaProvider.changeColumnType).not.toBeCalled() + + }) + + test('update collection - change column type', async() => { + const currentField = { + field: ctx.column.name, + type: 'text' + } + + const changedColumnType = { + key: ctx.column.name, + type: fieldTypeToWixDataEnum('number') + } + + driver.givenAllSchemaOperations() + schema.expectSchemaRefresh() + driver.givenFindResults([ { + id: ctx.collectionName, + fields: [currentField] + }]) + + const { columnsToChangeType } = compareColumnsInDbAndRequest([currentField], [changedColumnType]) + + await env.schemaService.update({ id: ctx.collectionName, fields: [changedColumnType] }) + + columnsToChangeType.forEach(c => expect(driver.schemaProvider.changeColumnType).toBeCalledWith(ctx.collectionName, c)) + expect(driver.schemaProvider.addColumn).not.toBeCalled() + expect(driver.schemaProvider.removeColumn).not.toBeCalled() + + }) + + // TODO: create a test for the case + // test('collections without _id column will have read-only capabilities', async() => {}) + + test('run unsupported operations should throw', async() => { + schema.expectSchemaRefresh() + driver.givenAdapterSupportedOperationsWith(ctx.invalidOperations) + const field = InputFieldToWixFormatField({ + name: ctx.column.name, + type: 'text' + }) + const changedTypeField = InputFieldToWixFormatField({ + name: ctx.column.name, + type: 'number' + }) + + driver.givenFindResults([ { id: ctx.collectionName, fields: [] } ]) - test('retrieve all collections from provider', async() => { - driver.givenAllSchemaOperations() - driver.givenListResult(ctx.dbsWithIdColumn) - - await expect( env.schemaService.list() ).resolves.toEqual( schemasListFor(ctx.dbsWithIdColumn, AllSchemaOperations) ) - }) - - test('retrieve short list of all collections from provider', async() => { - driver.givenListHeadersResult(ctx.collections) - - - await expect( env.schemaService.listHeaders() ).resolves.toEqual( schemaHeadersListFor(ctx.collections) ) - }) - - test('retrieve collections by ids from provider', async() => { - driver.givenAllSchemaOperations() - schema.givenSchemaFieldsResultFor(ctx.dbsWithIdColumn) - - await expect( env.schemaService.find(ctx.dbsWithIdColumn.map((db: { id: any }) => db.id)) ).resolves.toEqual( schemasListFor(ctx.dbsWithIdColumn, AllSchemaOperations) ) - }) - - test('create collection name', async() => { - driver.givenAllSchemaOperations() - driver.expectCreateOf(ctx.collectionName) - schema.expectSchemaRefresh() - - await expect(env.schemaService.create(ctx.collectionName)).resolves.toEqual({}) - }) - - test('add column for collection name', async() => { - driver.givenAllSchemaOperations() - driver.expectCreateColumnOf(ctx.column, ctx.collectionName) - schema.expectSchemaRefresh() - - await expect(env.schemaService.addColumn(ctx.collectionName, ctx.column)).resolves.toEqual({}) - }) - - test('remove column from collection name', async() => { - driver.givenAllSchemaOperations() - driver.expectRemoveColumnOf(ctx.column, ctx.collectionName) - schema.expectSchemaRefresh() + await expect(env.schemaService.update({ id: ctx.collectionName, fields: [field] })).rejects.toThrow(errors.UnsupportedOperation) - await expect(env.schemaService.removeColumn(ctx.collectionName, ctx.column.name)).resolves.toEqual({}) - }) + driver.givenFindResults([ { id: ctx.collectionName, fields: [{ field: ctx.column.name, type: 'text' }] }]) - test('collections without _id column will have read-only capabilities', async() => { - driver.givenAllSchemaOperations() - driver.givenListResult(ctx.dbsWithoutIdColumn) + await expect(env.schemaService.update({ id: ctx.collectionName, fields: [] })).rejects.toThrow(errors.UnsupportedOperation) + await expect(env.schemaService.update({ id: ctx.collectionName, fields: [changedTypeField] })).rejects.toThrow(errors.UnsupportedOperation) - await expect( env.schemaService.list() ).resolves.toEqual( schemasWithReadOnlyCapabilitiesFor(ctx.dbsWithoutIdColumn) ) - }) - test('run unsupported operations should throw', async() => { - driver.givenAdapterSupportedOperationsWith(ctx.invalidOperations) + }) - await expect(env.schemaService.create(ctx.collectionName)).rejects.toThrow(errors.UnsupportedOperation) - await expect(env.schemaService.addColumn(ctx.collectionName, ctx.column)).rejects.toThrow(errors.UnsupportedOperation) - await expect(env.schemaService.removeColumn(ctx.collectionName, ctx.column.name)).rejects.toThrow(errors.UnsupportedOperation) }) - const ctx = { + interface Ctx { + dbsWithoutIdColumn: Table[], + dbsWithIdColumn: Table[], + collections: string[], + collectionName: string, + column: InputField, + anotherColumn: InputField, + invalidOperations: string[], + } + + + const ctx: Ctx = { dbsWithoutIdColumn: Uninitialized, dbsWithIdColumn: Uninitialized, collections: Uninitialized, collectionName: Uninitialized, column: Uninitialized, + anotherColumn: Uninitialized, invalidOperations: Uninitialized, } - interface Enviorment { + interface Environment { schemaService: SchemaService } - const env: Enviorment = { + const env: Environment = { schemaService: Uninitialized, } @@ -98,6 +220,7 @@ describe('Schema Service', () => { ctx.collections = gen.randomCollections() ctx.collectionName = gen.randomCollectionName() ctx.column = gen.randomColumn() + ctx.anotherColumn = gen.randomColumn() ctx.invalidOperations = [chance.word(), chance.word()] diff --git a/libs/velo-external-db-core/src/service/schema.ts b/libs/velo-external-db-core/src/service/schema.ts index 4ee358bbf..cbeb08f93 100644 --- a/libs/velo-external-db-core/src/service/schema.ts +++ b/libs/velo-external-db-core/src/service/schema.ts @@ -1,7 +1,24 @@ -import { asWixSchema, asWixSchemaHeaders, allowedOperationsFor, appendQueryOperatorsTo, errors } from '@wix-velo/velo-external-db-commons' -import { InputField, ISchemaProvider, Table, SchemaOperations } from '@wix-velo/velo-external-db-types' +import { errors } from '@wix-velo/velo-external-db-commons' +import { ISchemaProvider, + SchemaOperations, + ResponseField, + CollectionCapabilities, + Table +} from '@wix-velo/velo-external-db-types' +import * as collectionSpi from '../spi-model/collection' import CacheableSchemaInformation from './schema_information' -const { Create, AddColumn, RemoveColumn } = SchemaOperations +import { + queriesToWixDataQueryOperators, + fieldTypeToWixDataEnum, + WixFormatFieldsToInputFields, + responseFieldToWixFormat, + compareColumnsInDbAndRequest, + dataOperationsToWixDataQueryOperators, + collectionOperationsToWixDataCollectionOperations, +} from '../utils/schema_utils' + + +const { Create, AddColumn, RemoveColumn, ChangeColumnType } = SchemaOperations export default class SchemaService { storage: ISchemaProvider @@ -11,62 +28,105 @@ export default class SchemaService { this.schemaInformation = schemaInformation } - async list() { - const dbs = await this.storage.list() - const dbsWithAllowedOperations = this.appendAllowedOperationsTo(dbs) - - return { schemas: dbsWithAllowedOperations.map( asWixSchema ) } + async list(collectionIds: string[]): Promise { + const collections = (!collectionIds || collectionIds.length === 0) ? + await this.storage.list() : + await Promise.all(collectionIds.map((collectionName: string) => this.schemaInformation.schemaFor(collectionName))) + + return { + collection: collections.map(this.formatCollection.bind(this)) + } } - async listHeaders() { - const collections = await this.storage.listHeaders() - return { schemas: collections.map((collection) => asWixSchemaHeaders(collection)) } + async create(collection: collectionSpi.Collection): Promise { + await this.storage.create(collection.id, WixFormatFieldsToInputFields(collection.fields)) + await this.schemaInformation.refresh() + return { collection } } - async find(collectionNames: string[]) { - const dbs: Table[] = await Promise.all(collectionNames.map(async(collectionName: string) => ({ id: collectionName, fields: await this.schemaInformation.schemaFieldsFor(collectionName) }))) - const dbsWithAllowedOperations = this.appendAllowedOperationsTo(dbs) + async update(collection: collectionSpi.Collection): Promise { + await this.validateOperation(Create) + + // remove in the end of development + if (!this.storage.changeColumnType) { + throw new Error('Your storage does not support the new collection capabilities API') + } - return { schemas: dbsWithAllowedOperations.map( asWixSchema ) } - } + const collectionColumnsInRequest = collection.fields + const { fields: collectionColumnsInDb } = await this.storage.describeCollection(collection.id) as Table + + const { + columnsToAdd, + columnsToRemove, + columnsToChangeType + } = compareColumnsInDbAndRequest(collectionColumnsInDb, collectionColumnsInRequest) - async create(collectionName: string) { - await this.validateOperation(Create) - await this.storage.create(collectionName) - await this.schemaInformation.refresh() - return {} - } + // Adding columns + if (columnsToAdd.length > 0) { + await this.validateOperation(AddColumn) + } + await Promise.all(columnsToAdd.map(async(field) => await this.storage.addColumn(collection.id, field))) + + // Removing columns + if (columnsToRemove.length > 0) { + await this.validateOperation(RemoveColumn) + } + await Promise.all(columnsToRemove.map(async(fieldName) => await this.storage.removeColumn(collection.id, fieldName))) - async addColumn(collectionName: string, column: InputField) { - await this.validateOperation(AddColumn) - await this.storage.addColumn(collectionName, column) - await this.schemaInformation.refresh() - return {} - } + // Changing columns type + if (columnsToChangeType.length > 0) { + await this.validateOperation(ChangeColumnType) + } + await Promise.all(columnsToChangeType.map(async(field) => await this.storage.changeColumnType?.(collection.id, field))) - async removeColumn(collectionName: string, columnName: string) { - await this.validateOperation(RemoveColumn) - await this.storage.removeColumn(collectionName, columnName) await this.schemaInformation.refresh() - return {} + + return { collection } } - appendAllowedOperationsTo(dbs: Table[]) { - const allowedSchemaOperations = this.storage.supportedOperations() - return dbs.map((db: Table) => ({ - ...db, - allowedSchemaOperations, - allowedOperations: allowedOperationsFor(db), - fields: appendQueryOperatorsTo(db.fields) - })) + async delete(collectionId: string): Promise { + const { fields: collectionFields } = await this.storage.describeCollection(collectionId) as Table + await this.storage.drop(collectionId) + this.schemaInformation.refresh() + return { collection: { + id: collectionId, + fields: responseFieldToWixFormat(collectionFields), + } } } - - async validateOperation(operationName: SchemaOperations) { + private async validateOperation(operationName: SchemaOperations) { const allowedSchemaOperations = this.storage.supportedOperations() if (!allowedSchemaOperations.includes(operationName)) throw new errors.UnsupportedOperation(`Your database doesn't support ${operationName} operation`) } + private formatCollection(collection: Table): collectionSpi.Collection { + return { + id: collection.id, + fields: this.formatFields(collection.fields), + capabilities: collection.capabilities? this.formatCollectionCapabilities(collection.capabilities) : undefined + } + } + + private formatFields(fields: ResponseField[]): collectionSpi.Field[] { + return fields.map( field => ({ + key: field.field, + encrypted: false, + type: fieldTypeToWixDataEnum(field.type), + capabilities: { + sortable: field.capabilities? field.capabilities.sortable: undefined, + queryOperators: field.capabilities? queriesToWixDataQueryOperators(field.capabilities.columnQueryOperators): undefined + } + })) + } + + private formatCollectionCapabilities(capabilities: CollectionCapabilities): collectionSpi.CollectionCapabilities { + return { + dataOperations: capabilities.dataOperations.map(dataOperationsToWixDataQueryOperators), + fieldTypes: capabilities.fieldTypes.map(fieldTypeToWixDataEnum), + collectionOperations: capabilities.collectionOperations.map(collectionOperationsToWixDataCollectionOperations), + } + } + } diff --git a/libs/velo-external-db-core/src/service/schema_aware_data.spec.ts b/libs/velo-external-db-core/src/service/schema_aware_data.spec.ts index 1abfdba5c..fa886709c 100644 --- a/libs/velo-external-db-core/src/service/schema_aware_data.spec.ts +++ b/libs/velo-external-db-core/src/service/schema_aware_data.spec.ts @@ -15,10 +15,10 @@ describe ('Schema Aware Data Service', () => { schema.givenDefaultSchemaFor(ctx.collectionName) queryValidator.givenValidFilterForDefaultFieldsOf(ctx.transformedFilter) queryValidator.givenValidProjectionForDefaultFieldsOf(SystemFields) - data.givenListResult(ctx.entities, ctx.totalCount, ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit, ctx.defaultFields) + data.givenListResult(ctx.entities, ctx.totalCount, ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit, ctx.defaultFields, false) patcher.givenPatchedBooleanFieldsWith(ctx.patchedEntities, ctx.entities) - return expect(env.schemaAwareDataService.find(ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit)).resolves.toEqual({ + return expect(env.schemaAwareDataService.find(ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit, undefined, false)).resolves.toEqual({ items: ctx.patchedEntities, totalCount: ctx.totalCount }) @@ -95,9 +95,9 @@ describe ('Schema Aware Data Service', () => { queryValidator.givenValidFilterForDefaultFieldsOf(ctx.filter) queryValidator.givenValidAggregationForDefaultFieldsOf(ctx.aggregation) - data.givenAggregateResult(ctx.entities, ctx.collectionName, ctx.filter, ctx.aggregation) + data.givenAggregateResult(ctx.entities, ctx.collectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit) - return expect(env.schemaAwareDataService.aggregate(ctx.collectionName, ctx.filter, ctx.aggregation)).resolves.toEqual({ items: ctx.entities, totalCount: 0 }) + return expect(env.schemaAwareDataService.aggregate(ctx.collectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit)).resolves.toEqual({ items: ctx.entities, totalCount: 0 }) }) const ctx = { diff --git a/libs/velo-external-db-core/src/service/schema_aware_data.ts b/libs/velo-external-db-core/src/service/schema_aware_data.ts index 86d3daa6b..4895e2dfc 100644 --- a/libs/velo-external-db-core/src/service/schema_aware_data.ts +++ b/libs/velo-external-db-core/src/service/schema_aware_data.ts @@ -1,4 +1,4 @@ -import { AdapterAggregation as Aggregation, AdapterFilter as Filter, AnyFixMe, Item, ItemWithId, ResponseField } from '@wix-velo/velo-external-db-types' +import { AdapterAggregation as Aggregation, AdapterFilter as Filter, AnyFixMe, Item, ItemWithId, ResponseField, Sort } from '@wix-velo/velo-external-db-types' import QueryValidator from '../converters/query_validator' import DataService from './data' import CacheableSchemaInformation from './schema_information' @@ -15,19 +15,21 @@ export default class SchemaAwareDataService { this.itemTransformer = itemTransformer } - async find(collectionName: string, filter: Filter, sort: any, skip: number, limit: number, _projection?: any): Promise<{ items: ItemWithId[], totalCount: number }> { + async find(collectionName: string, filter: Filter, sort: any, skip: number, limit: number, _projection?: any, omitTotalCount?: boolean): Promise<{ items: ItemWithId[], totalCount?: number }> { const fields = await this.schemaInformation.schemaFieldsFor(collectionName) await this.validateFilter(collectionName, filter, fields) const projection = _projection ?? (await this.schemaInformation.schemaFieldsFor(collectionName)).map(f => f.field) await this.validateProjection(collectionName, projection, fields) + await this.dataService.find(collectionName, filter, sort, skip, limit, projection, omitTotalCount) - const { items, totalCount } = await this.dataService.find(collectionName, filter, sort, skip, limit, projection) - return { items: this.itemTransformer.patchItems(items, fields), totalCount } + const { items, totalCount } = await this.dataService.find(collectionName, filter, sort, skip, limit, projection, omitTotalCount) + return { items: this.itemTransformer.patchItems(items, fields), totalCount } } async getById(collectionName: string, itemId: string, _projection?: any) { await this.validateGetById(collectionName, itemId) const projection = _projection ?? (await this.schemaInformation.schemaFieldsFor(collectionName)).map(f => f.field) + this.validateProjection(collectionName, projection) return await this.dataService.getById(collectionName, itemId, projection) @@ -44,6 +46,12 @@ export default class SchemaAwareDataService { return await this.dataService.insert(collectionName, prepared[0], fields) } + async bulkUpsert(collectionName: string, items: Item[]) { + const fields = await this.schemaInformation.schemaFieldsFor(collectionName) + const prepared = await this.prepareItemsForInsert(fields, items) + return await this.dataService.bulkUpsert(collectionName, prepared, fields) + } + async bulkInsert(collectionName: string, items: Item[]) { const fields = await this.schemaInformation.schemaFieldsFor(collectionName) const prepared = await this.prepareItemsForInsert(fields, items) @@ -72,10 +80,11 @@ export default class SchemaAwareDataService { return await this.dataService.truncate(collectionName) } - async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation) { + // sort, skip, limit are not really optional, after we'll implement in all the data providers we can remove the ? + async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort?: Sort[], skip?: number, limit?: number) { await this.validateAggregation(collectionName, aggregation) await this.validateFilter(collectionName, filter) - return await this.dataService.aggregate(collectionName, filter, aggregation) + return await this.dataService.aggregate(collectionName, filter, aggregation, sort, skip, limit) } async validateFilter(collectionName: string, filter: Filter, _fields?: ResponseField[]) { diff --git a/libs/velo-external-db-core/src/service/schema_information.ts b/libs/velo-external-db-core/src/service/schema_information.ts index 5f55848b3..e27a5ccb1 100644 --- a/libs/velo-external-db-core/src/service/schema_information.ts +++ b/libs/velo-external-db-core/src/service/schema_information.ts @@ -1,5 +1,5 @@ import { errors } from '@wix-velo/velo-external-db-commons' -import { ISchemaProvider, ResponseField } from '@wix-velo/velo-external-db-types' +import { ISchemaProvider, ResponseField, Table } from '@wix-velo/velo-external-db-types' const { CollectionDoesNotExists } = errors import * as NodeCache from 'node-cache' @@ -14,12 +14,16 @@ export default class CacheableSchemaInformation { } async schemaFieldsFor(collectionName: string): Promise { + return (await this.schemaFor(collectionName)).fields + } + + async schemaFor(collectionName: string): Promise
{ const schema = this.cache.get(collectionName) if ( !schema ) { await this.update(collectionName) - return this.cache.get(collectionName) as ResponseField[] + return this.cache.get(collectionName) as Table } - return schema as ResponseField[] + return schema as Table } async update(collectionName: string) { @@ -29,10 +33,11 @@ export default class CacheableSchemaInformation { } async refresh() { + await this.clear() const schema = await this.schemaProvider.list() if (schema && schema.length) - schema.forEach((collection: { id: any; fields: any }) => { - this.cache.set(collection.id, collection.fields, FiveMinutes) + schema.forEach((collection: Table) => { + this.cache.set(collection.id, collection, FiveMinutes) }) } diff --git a/libs/velo-external-db-core/src/spi-model/capabilities.ts b/libs/velo-external-db-core/src/spi-model/capabilities.ts new file mode 100644 index 000000000..09adb1e83 --- /dev/null +++ b/libs/velo-external-db-core/src/spi-model/capabilities.ts @@ -0,0 +1,16 @@ +export interface GetCapabilitiesRequest {} + +// Global capabilities that datasource supports. +export interface GetCapabilitiesResponse { + capabilities: Capabilities +} + +export interface Capabilities { + // Defines which collection operations is supported. + collection: CollectionCapability[] +} + +export enum CollectionCapability { + // Supports creating new collections. + CREATE = 'CREATE' +} diff --git a/libs/velo-external-db-core/src/spi-model/collection.ts b/libs/velo-external-db-core/src/spi-model/collection.ts new file mode 100644 index 000000000..6c032febb --- /dev/null +++ b/libs/velo-external-db-core/src/spi-model/collection.ts @@ -0,0 +1,162 @@ +export type listCollections = (req: ListCollectionsRequest) => Promise + +export type createCollection = (req: CreateCollectionRequest) => Promise + +export type updateCollection = (req: UpdateCollectionRequest) => Promise + +export type deleteCollection = (req: DeleteCollectionRequest) => Promise +export abstract class CollectionService { +} +export interface ListCollectionsRequest { + collectionIds: string[]; +} +export interface ListCollectionsResponsePart { + collection: Collection[]; +} +export interface DeleteCollectionRequest { + collectionId: string; +} +export interface DeleteCollectionResponse { + collection: Collection; +} +export interface CreateCollectionRequest { + collection: Collection; +} +export interface CreateCollectionResponse { + collection: Collection; +} +export interface UpdateCollectionRequest { + collection: Collection; +} +export interface UpdateCollectionResponse { + collection: Collection; +} +export interface Collection { + id: string; + fields: Field[]; + capabilities?: CollectionCapabilities; +} + +export interface Field { + // Identifier of the field. + key: string; + // Value is encrypted when `true`. Global data source capabilities define where encryption takes place. + encrypted?: boolean; + // Type of the field. + type: FieldType; + // Defines what kind of operations this field supports. + // Should be set by datasource itself and ignored in request payload. + capabilities?: FieldCapabilities; + // Additional options for specific field types, should be one of the following + singleReferenceOptions?: SingleReferenceOptions; + multiReferenceOptions?: MultiReferenceOptions; +} + +export interface SingleReferenceOptions { + referencedCollectionId?: string; + referencedNamespace?: string; +} +export interface MultiReferenceOptions { + referencedCollectionId?: string; + referencedNamespace?: string; + referencingFieldKey?: string; +} + +export interface FieldCapabilities { + // Indicates if field can be used to sort items in collection. Defaults to false. + sortable?: boolean; + // Query operators (e.g. equals, less than) that can be used for this field. + queryOperators?: QueryOperator[]; + singleReferenceOptions?: SingleReferenceOptions; + multiReferenceOptions?: MultiReferenceOptions; +} + +export enum QueryOperator { + eq = 0, + lt = 1, + gt = 2, + ne = 3, + lte = 4, + gte = 5, + startsWith = 6, + endsWith = 7, + contains = 8, + hasSome = 9, + hasAll = 10, + exists = 11, + urlized = 12, +} +export interface SingleReferenceOptions { + // `true` when datasource supports `include_referenced_items` in query method natively. + includeSupported?: boolean; +} +export interface MultiReferenceOptions { + // `true` when datasource supports `include_referenced_items` in query method natively. + includeSupported?: boolean; +} + +export interface CollectionCapabilities { + // Lists data operations supported by collection. + dataOperations: DataOperation[]; + // Supported field types. + fieldTypes: FieldType[]; + // Describes what kind of reference capabilities is supported. + referenceCapabilities?: ReferenceCapabilities; + // Lists what kind of modifications this collection accept. + collectionOperations: CollectionOperation[]; + // Defines which indexing operations is supported. + indexing?: IndexingCapabilityEnum[]; + // Defines if/how encryption is supported. + encryption?: Encryption; +} + +export enum DataOperation { + query = 0, + count = 1, + queryReferenced = 2, + aggregate = 3, + insert = 4, + update = 5, + remove = 6, + truncate = 7, + insertReferences = 8, + removeReferences = 9, +} + +export interface ReferenceCapabilities { + supportedNamespaces?: string[]; +} +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface CollectionOperationEnum { +} + +export enum CollectionOperation { + update = 0, + remove = 1, +} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface IndexingCapabilityEnum { +} + +export enum IndexingCapability { + list = 0, + create = 1, + remove = 2, +} + +export enum Encryption { + notSupported = 0, + wixDataNative = 1, + dataSourceNative = 2, +} +export enum FieldType { + text = 0, + number = 1, + boolean = 2, + datetime = 3, + object = 4, + longText = 5, + singleReference = 6, + multiReference = 7, +} diff --git a/libs/velo-external-db-core/src/spi-model/data_source.ts b/libs/velo-external-db-core/src/spi-model/data_source.ts new file mode 100644 index 000000000..e3bd971c5 --- /dev/null +++ b/libs/velo-external-db-core/src/spi-model/data_source.ts @@ -0,0 +1,408 @@ + +export interface QueryRequest { + collectionId: string; + namespace?: string; + query: QueryV2; + includeReferencedItems: string[]; + options: Options; + omitTotalCount: boolean; +} + +export interface QueryV2 { + filter: Filter; + sort?: Sorting[]; + fields: string[]; + fieldsets: string[]; + paging?: Paging; + cursorPaging?: CursorPaging; +} + +export type Filter = any; + +export interface Sorting { + fieldName: string; + order: SortOrder; +} + +export interface Paging { + limit: number; + offset: number; +} + +export interface CursorPaging { + limit: number; + cursor?: string; +} + +export interface Options { + consistentRead: boolean; + appOptions: any; +} + +export enum SortOrder { + ASC = 'ASC', + DESC = 'DESC' +} + +export interface QueryResponsePart { + item?: any; + pagingMetadata?: PagingMetadataV2; +} + +export class QueryResponsePart { + static item(item: any): QueryResponsePart { + return { + item: item + } as QueryResponsePart + } + + static pagingMetadata(count?: number, offset?: number, total?: number): QueryResponsePart { + return { + pagingMetadata: { + count, offset, total, tooManyToCount: false + } as PagingMetadataV2 + } + } +} + +export interface PagingMetadataV2 { + count?: number; + // Offset that was requested. + offset?: number; + // Total number of items that match the query. Returned if offset paging is used and the `tooManyToCount` flag is not set. + total?: number; + // Flag that indicates the server failed to calculate the `total` field. + tooManyToCount?: boolean + // Cursors to navigate through the result pages using `next` and `prev`. Returned if cursor paging is used. + cursors?: Cursors + // Indicates if there are more results after the current page. + // If `true`, another page of results can be retrieved. + // If `false`, this is the last page. + has_next?: boolean +} + +export interface Cursors { + next?: string; + // Cursor pointing to previous page in the list of results. + prev?: string; +} + +export interface CountRequest { + // collection name to query + collectionId: string; + // Optional namespace assigned to collection/installation + namespace?: string; + // query filter https://bo.wix.com/wix-docs/rnd/platformization-guidelines/api-query-language + filter?: any; + // request options + options: Options; +} + +export interface CountResponse { + totalCount: number; +} + +export interface QueryReferencedRequest { + // collection name of referencing item + collectionId: string; + // Optional namespace assigned to collection/installation + namespace?: string; + // referencing item IDs + // NOTE if empty reads all referenced items + itemIds: string[]; + // Multi-reference to read referenced items + referencePropertyName: string; + // Paging + paging: Paging; + cursorPaging: CursorPaging; + // Request options + options: Options; + // subset of properties to return + // empty means all, may not be supported + fields: string[] + // Indicates if total count calculation should be omitted. + // Only affects offset pagination, because cursor paging does not return total count. + omitTotalCount: boolean; +} + +// Let's consider "Album" collection containing "songs" property which +// contains references to "Song" collection. +// When making references request to "Album" collection the following names are used: +// - "Album" is called "referencing collection" +// - "Album" items are called "referencing items" +// - "Song" is called "referenced collection" +// - "Song" items are called "referenced items" +export interface ReferencedItem { + // Requested collection item that references returned item + referencingItemId: string; + // Item from referenced collection that is referenced by referencing item + referencedItemId: string; + // may not be present if can't be resolved (not found or item is in draft state) + // if the only requested field is `_id` item will always be present with only field + item?: any; + } + +export interface QueryReferencedResponsePart { + // overall result will contain single paging_metadata + // and zero or more items + item: ReferencedItem; + pagingMetadata: PagingMetadataV2; +} + +export interface AggregateRequest { + // collection name + collectionId: string; + // Optional namespace assigned to collection/installation + namespace?: string; + // filter to apply before aggregation + initialFilter?: any + // group and aggregate + // property name to return unique values of + // may unwind array values or not, depending on implementation + distinct: string; + group: Group; + // filter to apply after aggregation + finalFilter?: any + // sorting + sort?: Sorting[] + // paging + paging?: Paging; + cursorPaging?: CursorPaging; + // request options + options: Options; + // Indicates if total count calculation should be omitted. + // Only affects offset pagination, because cursor paging does not return total count. + omitTotalCount: boolean; +} + +export interface Group { + // properties to group by, if empty single group would be created + by: string[]; + // aggregations, resulted group will contain field with given name and aggregation value + aggregation: Aggregation[]; +} + +export interface Aggregation { + // result property name + name: string; + + //TODO: should be one of the following + // property to calculate average of + avg?: string; + // property to calculate min of + min?: string; + // property to calculate max of + max?: string; + // property to calculate sum of + sum?: string; + // count items, value is always 1 + count?: number; +} + +export interface AggregateResponsePart { + // query response consists of any number of items plus single paging metadata + // Aggregation result item. + // In case of group request, it should contain a field for each `group.by` value + // and a field for each `aggregation.name`. + // For example, grouping + // ``` + // {by: ["foo", "bar"], aggregation: {name: "someCount", calculate: {count: "baz"}}} + // ``` + // could produce an item: + // ``` + // {foo: "xyz", bar: "456", someCount: 123} + // ``` + // When `group.by` and 'aggregation.name' clash, grouping key should be returned. + // + // In case of distinct request, it should contain single field, for example + // ``` + // {distinct: "foo"} + // ``` + // could produce an item: + // ``` + // {foo: "xyz"} + // ``` + item?: any; + pagingMetadata?: PagingMetadataV2; +} + +export class AggregateResponsePart { + static item(item: any) { + return { + item + } as AggregateResponsePart + } + + static pagingMetadata(count?: number, offset?: number, total?: number): QueryResponsePart { + return { + pagingMetadata: { + count, offset, total, tooManyToCount: false + } as PagingMetadataV2 + } + } +} + +export interface InsertRequest { + // collection name + collectionId: string; + // Optional namespace assigned to collection/installation + namespace?: string; + // Items to insert + items: any[]; + // if true items would be overwritten by _id if present + overwriteExisting: boolean + // request options + options: Options; +} + +export interface InsertResponsePart { + item?: any; + // error from [errors list](errors.proto) + error?: ApplicationError; +} + +export class InsertResponsePart { + static item(item: any) { + return { + item + } as InsertResponsePart + } + + static error(error: ApplicationError) { + return { + error + } as InsertResponsePart + } +} + +export interface UpdateRequest { + // collection name + collectionId: string; + // Optional namespace assigned to collection/installation + namespace?: string; + // Items to update, must include _id + items: any[]; + // request options + options: Options; +} + +export interface UpdateResponsePart { + // results in order of request + item?: any; + // error from [errors list](errors.proto) + error?: ApplicationError; +} + +export class UpdateResponsePart { + static item(item: any) { + return { + item + } as UpdateResponsePart + } + + static error(error: ApplicationError) { + return { + error + } as UpdateResponsePart + } +} + +export interface RemoveRequest { + // collection name + collectionId: string; + // Optional namespace assigned to collection/installation + namespace?: string; + // Items to update, must include _id + itemIds: string[]; + // request options + options: Options; +} + +export interface RemoveResponsePart { + // results in order of request + // results in order of request + item?: any; + // error from [errors list](errors.proto) + error?: ApplicationError; +} + +export class RemoveResponsePart { + static item(item: any) { + return { + item + } as RemoveResponsePart + } + + static error(error: ApplicationError) { + return { + error + } as RemoveResponsePart + } +} + +export interface TruncateRequest { + // collection name + collectionId: string; + // Optional namespace assigned to collection/installation + namespace?: string; + // request options + options: Options; +} + +export interface TruncateResponse {} + +export interface InsertReferencesRequest { + // collection name + collectionId: string; + // Optional namespace assigned to collection/installation + namespace?: string; + // multi-reference property to update + referencePropertyName: string; + // references to insert + references: ReferenceId[] + // request options + options: Options; +} + +export interface InsertReferencesResponsePart { + reference: ReferenceId; + // error from [errors list](errors.proto) + error: ApplicationError; + +} + +export interface ReferenceId { + // Id of item in requested collection + referencingItemId: string; + // Id of item in referenced collection + referencedItemId: string; +} + +export interface RemoveReferencesRequest { + collectionId: string; + // Optional namespace assigned to collection/installation + namespace?: string; + // multi-reference property to update + referencePropertyName: string; + // reference masks to delete + referenceMasks: ReferenceMask[]; + // request options + options: Options; + + +} + +export interface ReferenceMask { + // Referencing item ID or any item if empty + referencingItemId?: string; + // Referenced item ID or any item if empty + referencedItemId?: string; +} + +export interface RemoveReferencesResponse {} + +export interface ApplicationError { + code: string; + description: string; + data: any; +} diff --git a/libs/velo-external-db-core/src/spi-model/errors.ts b/libs/velo-external-db-core/src/spi-model/errors.ts new file mode 100644 index 000000000..f0ba4b352 --- /dev/null +++ b/libs/velo-external-db-core/src/spi-model/errors.ts @@ -0,0 +1,456 @@ +export class ErrorMessage { + static unknownError(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0054, + description + } as ErrorMessage, HttpStatusCode.INTERNAL) + } + + static operationTimeLimitExceeded(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0028, + description + } as ErrorMessage, HttpStatusCode.RESOURCE_EXHAUSTED) + } + + static invalidUpdate(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0007, + description + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static operationIsNotSupportedByCollection(collectionName: string, operation: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0119, + description, + data: { + collectionName, + operation + } as UnsupportedByCollectionDetails + } as ErrorMessage, HttpStatusCode.FAILED_PRECONDITION) + } + + static operationIsNotSupportedByDataSource(collectionName: string, operation: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0120, + description, + data: { + collectionName, + operation + } as UnsupportedByCollectionDetails + } as ErrorMessage, HttpStatusCode.FAILED_PRECONDITION) + } + + static itemAlreadyExists(itemId: string, collectionName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0074, + description, + data: { + itemId, + collectionName + } as InvalidItemDetails + } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS) + } + + static uniqIndexConflict(itemId: string, collectionName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0123, + description, + data: { + itemId, + collectionName + } as InvalidItemDetails + } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS) + } + + static documentTooLargeToIndex(itemId: string, collectionName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0133, + description, + data: { + itemId, + collectionName + } as InvalidItemDetails + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static dollarPrefixedFieldNameNotAllowed(itemId: string, collectionName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0134, + description, + data: { + itemId, + collectionName + } as InvalidItemDetails + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static requestPerMinuteQuotaExceeded(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0014, + description + } as ErrorMessage, HttpStatusCode.RESOURCE_EXHAUSTED) + } + + static processingTimeQuotaExceeded(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0122, + description + } as ErrorMessage, HttpStatusCode.RESOURCE_EXHAUSTED) + } + + static storageSpaceQuotaExceeded(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0091, + description + } as ErrorMessage, HttpStatusCode.RESOURCE_EXHAUSTED) + } + + static documentIsTooLarge(itemId: string, collectionName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0009, + description, + data: { + itemId, + collectionName + } as InvalidItemDetails + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static itemNotFound(itemId: string, collectionName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0073, + description, + data: { + itemId, + collectionName + } as InvalidItemDetails + } as ErrorMessage, HttpStatusCode.NOT_FOUND) + } + + static collectionNotFound(collectionName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0025, + description, + data: { + collectionName + } as InvalidCollectionDetails + } as ErrorMessage, HttpStatusCode.NOT_FOUND) + } + + static collectionDeleted(collectionName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0026, + description, + data: { + collectionName + } as InvalidCollectionDetails + } as ErrorMessage, HttpStatusCode.NOT_FOUND) + } + + static propertyDeleted(collectionName: string, propertyName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0024, + description, + data: { + collectionName, + propertyName + } as InvalidPropertyDetails + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static userDoesNotHavePermissionToPerformAction(collectionName: string, operation: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0027, + description, + data: { + collectionName, + operation + } as PermissionDeniedDetails + } as ErrorMessage, HttpStatusCode.PERMISSION_DENIED) + } + + static genericRequestValidationError(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0075, + description + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static notAMultiReferenceProperty(collectionName: string, propertyName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0020, + description, + data: { + collectionName, + propertyName + } as InvalidPropertyDetails + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static datasetIsTooLargeToSort(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0092, + description + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static payloadIsToolarge(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0109, + description + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static sortingByMultipleArrayFieldsIsNotSupported(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0121, + description + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static offsetPagingIsNotSupported(description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0082, + description + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } + + static referenceAlreadyExists(collectionName: string, propertyName: string, referencingItemId: string, referencedItemId: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0029, + description, + data: { + collectionName, + propertyName, + referencingItemId, + referencedItemId + } as InvalidReferenceDetails + } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS) + } + + static unknownErrorWhileBuildingCollectionIndex(collectionName: string, itemId?: string, details?: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0112, + description, + data: { + collectionName, + itemId, + details, + } as IndexingFailureDetails + } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS) + } + + static duplicateKeyErrorWhileBuildingCollectionIndex(collectionName: string, itemId?: string, details?: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0113, + description, + data: { + collectionName, + itemId, + details, + } as IndexingFailureDetails + } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS) + } + + static documentTooLargeWhileBuildingCollectionIndex(collectionName: string, itemId?: string, details?: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0114, + description, + data: { + collectionName, + itemId, + details, + } as IndexingFailureDetails + } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS) + } + + static collectionAlreadyExists(collectionName: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0104, + description, + data: { + collectionName + } as InvalidCollectionDetails + } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS) + } + + static invalidProperty(collectionName: string, propertyName?: string, description?: string) { + return HttpError.create({ + code: ApiErrors.WDE0147, + description, + data: { + collectionName, + propertyName + } as InvalidPropertyDetails + } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT) + } +} + +export interface HttpError { + message: ErrorMessage, + httpCode: HttpStatusCode +} + +export class HttpError { + static create(message: ErrorMessage, httpCode: HttpStatusCode) { + return { + message, + httpCode + } as HttpError + } +} + +export interface ErrorMessage { + code: ApiErrors, + description?: string, + data: object +} + + + + +enum ApiErrors { + // Unknown error + WDE0054='WDE0054', + // Operation time limit exceeded. + WDE0028='WDE0028', + // Invalid update. Updated object must have a string _id property. + WDE0007='WDE0007', + // Operation is not supported by collection + WDE0119='WDE0119', + // Operation is not supported by data source + WDE0120='WDE0120', + // Item already exists + WDE0074='WDE0074', + // Unique index conflict + WDE0123='WDE0123', + // Document too large to index + WDE0133='WDE0133', + // Dollar-prefixed field name not allowed + WDE0134='WDE0134', + // Requests per minute quota exceeded + WDE0014='WDE0014', + // Processing time quota exceeded + WDE0122='WDE0122', + // Storage space quota exceeded + WDE0091='WDE0091', + // Document is too large + WDE0009='WDE0009', + // Item not found + WDE0073='WDE0073', + // Collection not found + WDE0025='WDE0025', + // Collection deleted + WDE0026='WDE0026', + // Property deleted + WDE0024='WDE0024', + // User doesn't have permissions to perform action + WDE0027='WDE0027', + // Generic request validation error + WDE0075='WDE0075', + // Not a multi-reference property + WDE0020='WDE0020', + // Dataset is too large to sort + WDE0092='WDE0092', + // Payload is too large + WDE0109='WDE0109', + // Sorting by multiple array fields is not supported + WDE0121='WDE0121', + // Offset paging is not supported + WDE0082='WDE0082', + // Reference already exists + WDE0029='WDE0029', + // Unknown error while building collection index + WDE0112='WDE0112', + // Duplicate key error while building collection index + WDE0113='WDE0113', + // Document too large while building collection index + WDE0114='WDE0114', + // Collection already exists + WDE0104='WDE0104', + // Invalid property + WDE0147='WDE0147' +} + +enum HttpStatusCode { + OK = 200, + + //Default error codes (applicable to all endpoints) + + // 401 - Identity missing (missing, invalid or expired oAuth token, + // signed instance or cookies) + UNAUTHENTICATED = 401, + + // 403 - Identity does not have the permission needed for this method / resource + PERMISSION_DENIED = 403, + + // 400 - Bad Request. The client sent malformed body + // or one of the arguments was invalid + INVALID_ARGUMENT = 400, + + // 404 - Resource does not exist + NOT_FOUND = 404, + + // 500 - Internal Server Error + INTERNAL = 500, + + // 503 - Come back later, server is currently unavailable + UNAVAILABLE = 503, + + // 429 - The client has sent too many requests + // in a given amount of time (rate limit) + RESOURCE_EXHAUSTED = 429, + + //Custom error codes - need to be documented + + // 499 - Request cancelled by the client + CANCELED = 499, + + // 409 - Can't recreate same resource or concurrency conflict + ALREADY_EXISTS = 409, + + // 428 - request cannot be executed in current system state + // such as deleting a non-empty folder or paying with no funds + FAILED_PRECONDITION = 428 + + //DO NOT USE IN WIX + // ABORTED = 11; // 409 + // OUT_OF_RANGE = 12; // 400 + // DEADLINE_EXEEDED = 13; // 504 + // DATA_LOSS = 14; // 500 + // UNIMPLEMENTED = 15; // 501 + } + + +interface UnsupportedByCollectionDetails { + collectionName: string + operation: string +} +interface InvalidItemDetails { + itemId: string + collectionName: string +} +interface InvalidCollectionDetails { + collectionName: string +} +interface InvalidPropertyDetails { + collectionName: string + propertyName: string +} +interface PermissionDeniedDetails { + collectionName: string + operation: string +} +interface InvalidReferenceDetails { + collectionName: string + propertyName: string + referencingItemId: string + referencedItemId: string +} +interface IndexingFailureDetails { + collectionName: string + itemId?: string + details?: string +} diff --git a/libs/velo-external-db-core/src/spi-model/filter.ts b/libs/velo-external-db-core/src/spi-model/filter.ts new file mode 100644 index 000000000..31667c37b --- /dev/null +++ b/libs/velo-external-db-core/src/spi-model/filter.ts @@ -0,0 +1,42 @@ + +// type PrimitveType = number | string | boolean +// type PrimitveTypeArray = PrimitveType[] + +// interface Filter { +// root: And +// } + + +// interface ToAdapterType { +// toAdapter(): void +// } + +// type Comperator = Eq | Ne | Lt + +// interface FieldComperator { +// [fieldName: string]: Comperator | PrimitveType | PrimitveTypeArray +// } + +// interface Eq { +// $eq: PrimitveType +// } + +// interface Ne { +// $ne: PrimitveType +// } + +// interface Lt { +// $lt: number +// } + +// interface And { +// $and: Array +// } + +// interface Or { +// $or: Array +// } + +// interface Not { +// $not: FieldComperator | And | Or | Not +// } diff --git a/libs/velo-external-db-core/src/types.ts b/libs/velo-external-db-core/src/types.ts index 7e5effcf4..e687c0810 100644 --- a/libs/velo-external-db-core/src/types.ts +++ b/libs/velo-external-db-core/src/types.ts @@ -1,6 +1,7 @@ import { AdapterFilter, InputField, Item, Sort, WixDataFilter, AsWixSchema, AsWixSchemaHeaders, RoleConfig } from '@wix-velo/velo-external-db-types' import SchemaService from './service/schema' import SchemaAwareDataService from './service/schema_aware_data' +import { AggregateRequest, Group, Paging, Sorting } from './spi-model/data_source' export interface FindQuery { @@ -10,20 +11,17 @@ export interface FindQuery { limit?: number; } -export type AggregationQuery = { - filter?: WixDataFilter, - processingStep?: WixDataFilter, - postProcessingStep?: WixDataFilter -} + export interface Payload { filter?: WixDataFilter | AdapterFilter - sort?: Sort; + sort?: Sort[] | Sorting[]; skip?: number; limit?: number; - postProcessingStep?: WixDataFilter | AdapterFilter; - processingStep?: WixDataFilter | AdapterFilter; - postFilteringStep?: WixDataFilter | AdapterFilter; + initialFilter: WixDataFilter | AdapterFilter; + group?: Group; + finalFilter?: WixDataFilter | AdapterFilter; + paging?: Paging; item?: Item; items?: Item[]; itemId?: string; @@ -81,7 +79,7 @@ export interface DataHooks { afterRemove?: Hook<{ itemId: string }> beforeBulkRemove?: Hook<{ itemIds: string[] }> afterBulkRemove?: Hook<{ itemIds: string[] }> - beforeAggregate?: Hook + beforeAggregate?: Hook afterAggregate?: Hook<{ items: Item[] }> beforeCount?: Hook afterCount?: Hook<{ totalCount: number }> @@ -111,11 +109,13 @@ export interface SchemaHooks { } export interface ExternalDbRouterConfig { - secretKey: string + externalDatabaseId: string + allowedMetasites: string authorization?: { roleConfig: RoleConfig } vendor?: string adapterType?: string commonExtended?: boolean + wixDataBaseUrl: string } export type Hooks = { diff --git a/libs/velo-external-db-core/src/utils/base64_utils.ts b/libs/velo-external-db-core/src/utils/base64_utils.ts new file mode 100644 index 000000000..eabebfc0d --- /dev/null +++ b/libs/velo-external-db-core/src/utils/base64_utils.ts @@ -0,0 +1,9 @@ + +export function decodeBase64(data: string): string { + const buff = Buffer.from(data, 'base64') + return buff.toString('ascii') +} +export function encodeBase64(data: string): string { + const buff = Buffer.from(data, 'utf-8') + return buff.toString('base64') +} diff --git a/libs/velo-external-db-core/src/utils/schema_utils.spec.ts b/libs/velo-external-db-core/src/utils/schema_utils.spec.ts new file mode 100644 index 000000000..ae4c282ef --- /dev/null +++ b/libs/velo-external-db-core/src/utils/schema_utils.spec.ts @@ -0,0 +1,179 @@ +import * as Chance from 'chance' +import { InputField } from '@wix-velo/velo-external-db-types' +import { Uninitialized } from '@wix-velo/test-commons' +import { FieldType as VeloFieldTypeEnum } from '../spi-model/collection' +import { + fieldTypeToWixDataEnum, + wixDataEnumToFieldType, + subtypeToFieldType, + compareColumnsInDbAndRequest, + wixFormatFieldToInputFields +} from './schema_utils' +const chance = Chance() + + +describe('Schema utils functions', () => { + describe('translate our field type to velo field type emun', () => { + test('text type', () => { + expect(fieldTypeToWixDataEnum('text')).toBe(VeloFieldTypeEnum.text) + }) + test('number type', () => { + expect(fieldTypeToWixDataEnum('number')).toBe(VeloFieldTypeEnum.number) + }) + test('boolean type', () => { + expect(fieldTypeToWixDataEnum('boolean')).toBe(VeloFieldTypeEnum.boolean) + }) + test('object type', () => { + expect(fieldTypeToWixDataEnum('object')).toBe(VeloFieldTypeEnum.object) + }) + test('datetime type', () => { + expect(fieldTypeToWixDataEnum('datetime')).toBe(VeloFieldTypeEnum.datetime) + }) + + test('unsupported type will throw an error', () => { + expect(() => fieldTypeToWixDataEnum('unsupported-type')).toThrowError() + }) + }) + + describe('translate velo field type emun to our field type', () => { + test('text type', () => { + expect(wixDataEnumToFieldType(VeloFieldTypeEnum.text)).toBe('text') + }) + test('number type', () => { + expect(wixDataEnumToFieldType(VeloFieldTypeEnum.number)).toBe('number') + }) + test('boolean type', () => { + expect(wixDataEnumToFieldType(VeloFieldTypeEnum.boolean)).toBe('boolean') + }) + test('object type', () => { + expect(wixDataEnumToFieldType(VeloFieldTypeEnum.object)).toBe('object') + }) + + test('datetime type', () => { + expect(wixDataEnumToFieldType(VeloFieldTypeEnum.datetime)).toBe('datetime') + }) + + test('unsupported type will throw an error', () => { + expect(() => wixDataEnumToFieldType(100)).toThrowError() + }) + }) + + describe('translate velo field type enum to our sub type', () => { + test('text type', () => { + expect(subtypeToFieldType(VeloFieldTypeEnum.text)).toBe('string') + }) + test('number type', () => { + expect(subtypeToFieldType(VeloFieldTypeEnum.number)).toBe('float') + }) + test('boolean type', () => { + expect(subtypeToFieldType(VeloFieldTypeEnum.boolean)).toBe('') + }) + test('object type', () => { + expect(subtypeToFieldType(VeloFieldTypeEnum.object)).toBe('') + }) + + test('datetime type', () => { + expect(subtypeToFieldType(VeloFieldTypeEnum.datetime)).toBe('datetime') + }) + + test('unsupported type will throw an error', () => { + expect(() => wixDataEnumToFieldType(100)).toThrowError() + }) + }) + + describe('convert wix format fields to our fields', () => { + test('convert velo format fields to our fields', () => { + expect(wixFormatFieldToInputFields({ key: ctx.columnName, type: fieldTypeToWixDataEnum('text') })).toEqual({ + name: ctx.columnName, + type: 'text', + subtype: 'string', + }) + }) + + }) + + describe('compare columns in db and request function', () => { + test('compareColumnsInDbAndRequest function - add columns', async() => { + const columnsInDb = [{ + field: ctx.column.name, + type: ctx.column.type + }] + const columnsInRequest = [{ + key: ctx.column.name, + type: fieldTypeToWixDataEnum(ctx.column.type), + }] + const newColumn = { + key: ctx.anotherColumn.name, + type: fieldTypeToWixDataEnum(ctx.anotherColumn.type) + } + expect(compareColumnsInDbAndRequest([], []).columnsToAdd).toEqual([]) + expect(compareColumnsInDbAndRequest(columnsInDb, columnsInRequest).columnsToAdd).toEqual([]) + expect(compareColumnsInDbAndRequest(columnsInDb, []).columnsToAdd).toEqual([]) + expect(compareColumnsInDbAndRequest([], columnsInRequest).columnsToAdd).toEqual(columnsInRequest.map(wixFormatFieldToInputFields)) + expect(compareColumnsInDbAndRequest(columnsInDb, [...columnsInRequest, newColumn]).columnsToAdd).toEqual([newColumn].map(wixFormatFieldToInputFields)) + }) + + test('compareColumnsInDbAndRequest function - remove columns', async() => { + const columnsInDb = [{ + field: ctx.column.name, + type: ctx.column.type + }] + const columnsInRequest = [{ + key: ctx.column.name, + type: fieldTypeToWixDataEnum(ctx.column.type), + }] + const newColumn = { + key: ctx.anotherColumn.name, + type: fieldTypeToWixDataEnum(ctx.anotherColumn.type) + } + expect(compareColumnsInDbAndRequest([], []).columnsToRemove).toEqual([]) + expect(compareColumnsInDbAndRequest(columnsInDb, columnsInRequest).columnsToRemove).toEqual([]) + expect(compareColumnsInDbAndRequest(columnsInDb, [...columnsInRequest, newColumn]).columnsToRemove).toEqual([]) + expect(compareColumnsInDbAndRequest(columnsInDb, []).columnsToRemove).toEqual(columnsInDb.map(f => f.field)) + expect(compareColumnsInDbAndRequest(columnsInDb, [newColumn]).columnsToRemove).toEqual(columnsInDb.map(f => f.field)) + }) + + test('compareColumnsInDbAndRequest function - change column type', async() => { + const columnsInDb = [{ + field: ctx.column.name, + type: 'text' + }] + + const columnsInRequest = [{ + key: ctx.column.name, + type: fieldTypeToWixDataEnum('text'), + }] + + const changedColumnType = { + key: ctx.column.name, + type: fieldTypeToWixDataEnum('number') + } + + expect(compareColumnsInDbAndRequest([], []).columnsToChangeType).toEqual([]) + expect(compareColumnsInDbAndRequest(columnsInDb, columnsInRequest).columnsToChangeType).toEqual([]) + expect(compareColumnsInDbAndRequest(columnsInDb, [changedColumnType]).columnsToChangeType).toEqual([changedColumnType].map(wixFormatFieldToInputFields)) + }) + }) + + interface Ctx { + collectionName: string, + columnName: string, + column: InputField, + anotherColumn: InputField, + } + + const ctx: Ctx = { + collectionName: Uninitialized, + columnName: Uninitialized, + column: Uninitialized, + anotherColumn: Uninitialized, + } + + beforeEach(() => { + ctx.collectionName = chance.word({ length: 5 }) + ctx.columnName = chance.word({ length: 5 }) + ctx.column = ({ name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false }) + ctx.anotherColumn = ({ name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false }) + }) + +}) diff --git a/libs/velo-external-db-core/src/utils/schema_utils.ts b/libs/velo-external-db-core/src/utils/schema_utils.ts new file mode 100644 index 000000000..a037c4e53 --- /dev/null +++ b/libs/velo-external-db-core/src/utils/schema_utils.ts @@ -0,0 +1,202 @@ +import { AdapterOperators } from '@wix-velo/velo-external-db-commons' +import { InputField, ResponseField, FieldType, DataOperation, CollectionOperation } from '@wix-velo/velo-external-db-types' +import * as collectionSpi from '../spi-model/collection' +const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators + +export const fieldTypeToWixDataEnum = ( fieldType: string ): collectionSpi.FieldType => { + switch (fieldType) { + case FieldType.text: + return collectionSpi.FieldType.text + case FieldType.longText: + return collectionSpi.FieldType.longText + case FieldType.number: + return collectionSpi.FieldType.number + case FieldType.boolean: + return collectionSpi.FieldType.boolean + case FieldType.object: + return collectionSpi.FieldType.object + case FieldType.datetime: + return collectionSpi.FieldType.datetime + case FieldType.singleReference: + return collectionSpi.FieldType.singleReference + case FieldType.multiReference: + return collectionSpi.FieldType.multiReference + + default: + throw new Error(`${fieldType} - Unsupported field type`) + } +} + +export const wixDataEnumToFieldType = (fieldEnum: number): string => { + switch (fieldEnum) { + case collectionSpi.FieldType.text: + case collectionSpi.FieldType.longText: + return FieldType.text + case collectionSpi.FieldType.number: + return FieldType.number + case collectionSpi.FieldType.datetime: + return FieldType.datetime + case collectionSpi.FieldType.boolean: + return FieldType.boolean + case collectionSpi.FieldType.object: + return FieldType.object + + case collectionSpi.FieldType.singleReference: + case collectionSpi.FieldType.multiReference: + default: + // TODO: throw specific error + throw new Error(`Unsupported field type: ${fieldEnum}`) + } +} + +export const subtypeToFieldType = (fieldEnum: number): string => { + switch (fieldEnum) { + case collectionSpi.FieldType.text: + case collectionSpi.FieldType.longText: + return 'string' + case collectionSpi.FieldType.number: + return 'float' + case collectionSpi.FieldType.datetime: + return 'datetime' + case collectionSpi.FieldType.boolean: + return '' + case collectionSpi.FieldType.object: + return '' + + case collectionSpi.FieldType.singleReference: + case collectionSpi.FieldType.multiReference: + default: + // TODO: throw specific error + throw new Error(`There is no subtype for this type: ${fieldEnum}`) + } + +} + +export const queryOperatorsToWixDataQueryOperators = (queryOperator: string): collectionSpi.QueryOperator => { + switch (queryOperator) { + case eq: + return collectionSpi.QueryOperator.eq + case lt: + return collectionSpi.QueryOperator.lt + case gt: + return collectionSpi.QueryOperator.gt + case ne: + return collectionSpi.QueryOperator.ne + case lte: + return collectionSpi.QueryOperator.lte + case gte: + return collectionSpi.QueryOperator.gte + case string_begins: + return collectionSpi.QueryOperator.startsWith + case string_ends: + return collectionSpi.QueryOperator.endsWith + case string_contains: + return collectionSpi.QueryOperator.contains + case include: + return collectionSpi.QueryOperator.hasSome + // case 'hasAll': + // return QueryOperator.hasAll + // case 'exists': + // return QueryOperator.exists + // case 'urlized': + // return QueryOperator.urlized + default: + throw new Error(`${queryOperator} - Unsupported query operator`) + } +} + +export const dataOperationsToWixDataQueryOperators = (dataOperation: DataOperation): collectionSpi.DataOperation => { + switch (dataOperation) { + case DataOperation.query: + return collectionSpi.DataOperation.query + case DataOperation.count: + return collectionSpi.DataOperation.count + case DataOperation.queryReferenced: + return collectionSpi.DataOperation.queryReferenced + case DataOperation.aggregate: + return collectionSpi.DataOperation.aggregate + case DataOperation.insert: + return collectionSpi.DataOperation.insert + case DataOperation.update: + return collectionSpi.DataOperation.update + case DataOperation.remove: + return collectionSpi.DataOperation.remove + case DataOperation.truncate: + return collectionSpi.DataOperation.truncate + case DataOperation.insertReferences: + return collectionSpi.DataOperation.insertReferences + case DataOperation.removeReferences: + return collectionSpi.DataOperation.removeReferences + + default: + throw new Error(`${dataOperation} - Unsupported data operation`) + } +} + +export const collectionOperationsToWixDataCollectionOperations = (collectionOperations: CollectionOperation): collectionSpi.CollectionOperation => { + switch (collectionOperations) { + case CollectionOperation.update: + return collectionSpi.CollectionOperation.update + case CollectionOperation.remove: + return collectionSpi.CollectionOperation.remove + + default: + throw new Error(`${collectionOperations} - Unsupported collection operation`) + } +} + +export const queriesToWixDataQueryOperators = (queryOperators: string[]): collectionSpi.QueryOperator[] => queryOperators.map(queryOperatorsToWixDataQueryOperators) + + +export const responseFieldToWixFormat = (fields: ResponseField[]): collectionSpi.Field[] => { + return fields.map(field => { + return { + key: field.field, + type: fieldTypeToWixDataEnum(field.type) + } + }) +} + +export const wixFormatFieldToInputFields = (field: collectionSpi.Field): InputField => ({ + name: field.key, + type: wixDataEnumToFieldType(field.type), + subtype: subtypeToFieldType(field.type) +}) + +export const InputFieldToWixFormatField = (field: InputField): collectionSpi.Field => ({ + key: field.name, + type: fieldTypeToWixDataEnum(field.type) +}) + +export const WixFormatFieldsToInputFields = (fields: collectionSpi.Field[]): InputField[] => fields.map(wixFormatFieldToInputFields) + +export const InputFieldsToWixFormatFields = (fields: InputField[]): collectionSpi.Field[] => fields.map(InputFieldToWixFormatField) + +export const compareColumnsInDbAndRequest = ( + columnsInDb: ResponseField[], + columnsInRequest: collectionSpi.Field[] +): { + columnsToAdd: InputField[]; + columnsToRemove: string[]; + columnsToChangeType: InputField[]; +} => { + const collectionColumnsNamesInDb = columnsInDb.map((f) => f.field) + const collectionColumnsNamesInRequest = columnsInRequest.map((f) => f.key) + + const columnsToAdd = columnsInRequest.filter((f) => !collectionColumnsNamesInDb.includes(f.key)) + .map(wixFormatFieldToInputFields) + const columnsToRemove = columnsInDb.filter((f) => !collectionColumnsNamesInRequest.includes(f.field)) + .map((f) => f.field) + + const columnsToChangeType = columnsInRequest.filter((f) => { + const fieldInDb = columnsInDb.find((field) => field.field === f.key) + return fieldInDb && fieldInDb.type !== wixDataEnumToFieldType(f.type) + }) + .map(wixFormatFieldToInputFields) + + return { + columnsToAdd, + columnsToRemove, + columnsToChangeType, + } +} diff --git a/libs/velo-external-db-core/src/web/auth-middleware.spec.ts b/libs/velo-external-db-core/src/web/auth-middleware.spec.ts deleted file mode 100644 index 380e431e3..000000000 --- a/libs/velo-external-db-core/src/web/auth-middleware.spec.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { Uninitialized } from '@wix-velo/test-commons' -import { secretKeyAuthMiddleware } from './auth-middleware' -import * as driver from '../../test/drivers/auth_middleware_test_support' //TODO: change driver location -import { errors } from '@wix-velo/velo-external-db-commons' -const { UnauthorizedError } = errors -import * as Chance from 'chance' -const chance = Chance() - -describe('Auth Middleware', () => { - - const ctx = { - secretKey: Uninitialized, - anotherSecretKey: Uninitialized, - next: Uninitialized, - ownerRole: Uninitialized, - dataPath: Uninitialized, - } - - const env = { - auth: Uninitialized, - } - - beforeEach(() => { - ctx.secretKey = chance.word() - ctx.anotherSecretKey = chance.word() - ctx.next = jest.fn().mockName('next') - - env.auth = secretKeyAuthMiddleware({ secretKey: ctx.secretKey }) - }) - - test('should throw when request does not contain auth', () => { - expect( () => env.auth({ body: { } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - expect( () => env.auth({ body: { requestContext: {} } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - expect( () => env.auth({ body: { requestContext: '' } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - expect( () => env.auth({ body: { requestContext: { settings: {} } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - expect( () => env.auth({ body: { requestContext: { settings: '' } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - expect( () => env.auth({ body: { requestContext: [] } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - expect( () => env.auth({ body: { requestContext: { settings: 'x' } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - expect( () => env.auth({ body: { requestContext: { role: '', settings: 'x' } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - expect( () => env.auth({ body: { requestContext: { role: [], settings: 'x' } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - expect( () => env.auth({ body: { requestContext: { role: {}, settings: 'x' } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - }) - - test('should throw when secret key does not match', () => { - expect( () => env.auth(driver.requestBodyWith(ctx.anotherSecretKey, ctx.ownerRole, ctx.dataPath), Uninitialized, ctx.next) ).toThrow(UnauthorizedError) - }) - - test('should call next when secret key matches', () => { - env.auth(driver.requestBodyWith(ctx.secretKey, ctx.ownerRole, ctx.dataPath), Uninitialized, ctx.next) - - expect(ctx.next).toHaveBeenCalled() - }) -}) diff --git a/libs/velo-external-db-core/src/web/auth-middleware.ts b/libs/velo-external-db-core/src/web/auth-middleware.ts deleted file mode 100644 index 7d85f663b..000000000 --- a/libs/velo-external-db-core/src/web/auth-middleware.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { property } from './middleware-support' -import { errors } from '@wix-velo/velo-external-db-commons' -import { Request } from 'express' -const { UnauthorizedError } = errors - - -const extractSecretKey = (body: any) => property('requestContext.settings.secretKey', body) - -const authorizeSecretKey = (req: Request, secretKey: string) => { - if (extractSecretKey(req.body) !== secretKey) { - throw new UnauthorizedError('You are not authorized') - } -} - -export const secretKeyAuthMiddleware = ({ secretKey }: {secretKey: string}) => { - return (req: any, res: any, next: () => void) => { - authorizeSecretKey(req, secretKey) - next() - } -} diff --git a/libs/velo-external-db-core/src/web/auth-role-middleware.spec.ts b/libs/velo-external-db-core/src/web/auth-role-middleware.spec.ts index 518b2f0c6..22de5e14d 100644 --- a/libs/velo-external-db-core/src/web/auth-role-middleware.spec.ts +++ b/libs/velo-external-db-core/src/web/auth-role-middleware.spec.ts @@ -13,8 +13,6 @@ describe('Auth Role Middleware', () => { permittedRole: Uninitialized, notPermittedRole: Uninitialized, next: Uninitialized, - secretKey: Uninitialized, - } const env = { @@ -41,12 +39,12 @@ describe('Auth Role Middleware', () => { }) test('should allow request with permitted role on request', () => { - env.auth(driver.requestBodyWith(ctx.secretKey, ctx.permittedRole), Uninitialized, ctx.next) + env.auth(driver.requestBodyWith(ctx.permittedRole), Uninitialized, ctx.next) expect(ctx.next).toHaveBeenCalled() }) test('should not allow request with permitted role on request', () => { - expect( () => env.auth(driver.requestBodyWith(ctx.secretKey, ctx.notPermittedRole), Uninitialized, ctx.next) ).toThrow(UnauthorizedError) + expect( () => env.auth(driver.requestBodyWith(ctx.notPermittedRole), Uninitialized, ctx.next) ).toThrow(UnauthorizedError) }) }) diff --git a/libs/velo-external-db-core/src/web/domain-to-spi-error-translator.ts b/libs/velo-external-db-core/src/web/domain-to-spi-error-translator.ts new file mode 100644 index 000000000..e59662ee6 --- /dev/null +++ b/libs/velo-external-db-core/src/web/domain-to-spi-error-translator.ts @@ -0,0 +1,25 @@ +import { errors as domainErrors } from '@wix-velo/velo-external-db-commons' +import { ErrorMessage } from '../spi-model/errors' + +export const domainToSpiErrorTranslator = (err: any) => { + switch(err.constructor) { + case domainErrors.ItemAlreadyExists: + const itemAlreadyExists: domainErrors.ItemAlreadyExists = err + return ErrorMessage.itemAlreadyExists(itemAlreadyExists.itemId, itemAlreadyExists.collectionName, itemAlreadyExists.message) + + case domainErrors.CollectionDoesNotExists: + const collectionDoesNotExists: domainErrors.CollectionDoesNotExists = err + return ErrorMessage.collectionNotFound(collectionDoesNotExists.collectionName, collectionDoesNotExists.message) + + case domainErrors.FieldAlreadyExists: + const fieldAlreadyExists: domainErrors.FieldAlreadyExists = err + return ErrorMessage.itemAlreadyExists(fieldAlreadyExists.fieldName, fieldAlreadyExists.collectionName, fieldAlreadyExists.message) + + case domainErrors.FieldDoesNotExist: + const fieldDoesNotExist: domainErrors.FieldDoesNotExist = err + return ErrorMessage.invalidProperty(fieldDoesNotExist.collectionName, fieldDoesNotExist.itemId) + + default: + return ErrorMessage.unknownError(err.message) + } + } diff --git a/libs/velo-external-db-core/src/web/error-middleware.spec.ts b/libs/velo-external-db-core/src/web/error-middleware.spec.ts index 402dd64b7..d4f8c770f 100644 --- a/libs/velo-external-db-core/src/web/error-middleware.spec.ts +++ b/libs/velo-external-db-core/src/web/error-middleware.spec.ts @@ -2,6 +2,8 @@ import * as Chance from 'chance' import { errors } from '@wix-velo/velo-external-db-commons' import { errorMiddleware } from './error-middleware' import { Uninitialized } from '@wix-velo/test-commons' +import { domainToSpiErrorTranslator } from './domain-to-spi-error-translator' + const chance = Chance() describe('Error Middleware', () => { @@ -24,7 +26,7 @@ describe('Error Middleware', () => { errorMiddleware(err, null, ctx.res) expect(ctx.res.status).toHaveBeenCalledWith(500) - expect(ctx.res.send).toHaveBeenCalledWith( { message: err.message } ) + expect(ctx.res.send).toHaveBeenCalledWith( { description: err.message, code: 'WDE0054' } ) }) test('converts exceptions to http error response', () => { @@ -32,9 +34,9 @@ describe('Error Middleware', () => { .forEach(Exception => { const err = new Exception(chance.word()) errorMiddleware(err, null, ctx.res) - - expect(ctx.res.status).toHaveBeenCalledWith(err.status) - expect(ctx.res.send).toHaveBeenCalledWith( { message: err.message } ) + const spiError = domainToSpiErrorTranslator(err) + // expect(ctx.res.status).toHaveBeenCalledWith(err.status) + expect(ctx.res.send).toHaveBeenCalledWith( spiError.message ) ctx.res.status.mockClear() ctx.res.send.mockClear() diff --git a/libs/velo-external-db-core/src/web/error-middleware.ts b/libs/velo-external-db-core/src/web/error-middleware.ts index 8be013790..b9104e450 100644 --- a/libs/velo-external-db-core/src/web/error-middleware.ts +++ b/libs/velo-external-db-core/src/web/error-middleware.ts @@ -1,9 +1,11 @@ import { NextFunction, Response } from 'express' +import { domainToSpiErrorTranslator } from './domain-to-spi-error-translator' export const errorMiddleware = (err: any, _req: any, res: Response, _next?: NextFunction) => { if (process.env['NODE_ENV'] !== 'test') { console.error(err) } - res.status(err.status || 500) - .send({ message: err.message }) + + const errorMsg = domainToSpiErrorTranslator(err) + res.status(errorMsg.httpCode).send(errorMsg.message) } diff --git a/libs/velo-external-db-core/src/web/jwt-auth-middleware.spec.ts b/libs/velo-external-db-core/src/web/jwt-auth-middleware.spec.ts new file mode 100644 index 000000000..497a02c4d --- /dev/null +++ b/libs/velo-external-db-core/src/web/jwt-auth-middleware.spec.ts @@ -0,0 +1,132 @@ +import { sleep, Uninitialized } from '@wix-velo/test-commons' +import * as driver from '../../test/drivers/auth_middleware_test_support' +import { errors } from '@wix-velo/velo-external-db-commons' +const { UnauthorizedError } = errors +import * as Chance from 'chance' +import { JwtAuthenticator, TOKEN_ISSUER } from './jwt-auth-middleware' +import { + signedToken, + WixDataFacadeMock +} from '../../test/drivers/auth_middleware_test_support' +import { authConfig } from '@wix-velo/test-commons' +import { PublicKeyMap } from './wix_data_facade' + +const chance = Chance() + +describe('JWT Auth Middleware', () => { + + test('should authorize when JWT valid', async() => { + const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: ctx.externalDatabaseId }, ctx.keyId) + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + + expectAuthorized() + }) + + test('should authorize when JWT valid, only with second public key', async() => { + const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: ctx.externalDatabaseId }, ctx.keyId) + env.auth = new JwtAuthenticator(ctx.externalDatabaseId, ctx.allowedMetasites, ctx.otherWixDataMock).authorizeJwt() + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + expectAuthorized() + }) + + test('should throw when JWT siteId is not allowed', async() => { + const token = signedToken({ iss: TOKEN_ISSUER, siteId: chance.word(), aud: ctx.externalDatabaseId }, ctx.keyId) + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + + expectUnauthorized() + }) + + test('should throw when JWT has no siteId claim', async() => { + const token = signedToken({ iss: TOKEN_ISSUER, aud: ctx.externalDatabaseId }, ctx.keyId) + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + + expectUnauthorized() + }) + + test('should throw when JWT issuer is not Wix-Data', async() => { + const token = signedToken({ iss: chance.word(), siteId: ctx.metasite, aud: ctx.externalDatabaseId }, ctx.keyId) + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + + expectUnauthorized() + }) + + test('should throw when JWT has no issuer', async() => { + const token = signedToken({ siteId: ctx.metasite, aud: ctx.externalDatabaseId }, ctx.keyId) + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + + expectUnauthorized() + }) + + test('should throw when JWT audience is not externalDatabaseId of adapter', async() => { + const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: chance.word() }, ctx.keyId) + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + + expectUnauthorized() + }) + + test('should throw when JWT has no audience', async() => { + const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite }, ctx.keyId) + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + + expectUnauthorized() + }) + + test('should throw when JWT kid is not found in Wix-Data keys', async() => { + const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: ctx.externalDatabaseId }, chance.word()) + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + + expectUnauthorized() + }) + + test('should throw when JWT kid is absent', async() => { + const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: ctx.externalDatabaseId }) + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + + expectUnauthorized() + }) + + test('should throw when JWT is expired', async() => { + const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: ctx.externalDatabaseId }, ctx.keyId, '10ms') + await sleep(1000) + await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next) + + expectUnauthorized() + }) + + const ctx = { + externalDatabaseId: Uninitialized, + metasite: Uninitialized, + allowedMetasites: Uninitialized, + next: Uninitialized, + keyId: Uninitialized, + otherWixDataMock: Uninitialized + } + + const env = { + auth: Uninitialized, + } + + const expectUnauthorized = () => { + expect(ctx.next).toHaveBeenCalledWith(new UnauthorizedError('You are not authorized')) + } + + const expectAuthorized = () => { + expect(ctx.next).not.toHaveBeenCalledWith(new UnauthorizedError('You are not authorized')) + expect(ctx.next).toHaveBeenCalledWith() + } + + beforeEach(() => { + ctx.externalDatabaseId = chance.word() + ctx.metasite = chance.word() + ctx.allowedMetasites = ctx.metasite + ctx.keyId = chance.word() + const otherKeyId = chance.word() + ctx.next = jest.fn().mockName('next') + const publicKeys: PublicKeyMap = {} + publicKeys[ctx.keyId] = authConfig.authPublicKey + const otherPublicKeys: PublicKeyMap = {} + otherPublicKeys[otherKeyId] = authConfig.otherAuthPublicKey + ctx.otherWixDataMock = new WixDataFacadeMock(otherPublicKeys, publicKeys) + env.auth = new JwtAuthenticator(ctx.externalDatabaseId, ctx.allowedMetasites, new WixDataFacadeMock(publicKeys)).authorizeJwt() + }) +}) diff --git a/libs/velo-external-db-core/src/web/jwt-auth-middleware.ts b/libs/velo-external-db-core/src/web/jwt-auth-middleware.ts new file mode 100644 index 000000000..082d3f4fc --- /dev/null +++ b/libs/velo-external-db-core/src/web/jwt-auth-middleware.ts @@ -0,0 +1,79 @@ +import { errors } from '@wix-velo/velo-external-db-commons' +const { UnauthorizedError } = errors +import { JwtHeader, JwtPayload, SigningKeyCallback, verify } from 'jsonwebtoken' +import * as express from 'express' +import { IWixDataFacade, PublicKeyMap } from './wix_data_facade' + + +export const TOKEN_ISSUER = 'wix-data.wix.com' + +export class JwtAuthenticator { + publicKeys: PublicKeyMap | undefined + externalDatabaseId: string + allowedMetasites: string[] + wixDataFacade: IWixDataFacade + + constructor(externalDatabaseId: string, allowedMetasites: string, wixDataFacade: IWixDataFacade) { + this.externalDatabaseId = externalDatabaseId + this.allowedMetasites = allowedMetasites ? allowedMetasites.split(',') : [] + this.wixDataFacade = wixDataFacade + } + + authorizeJwt() { + return async(req: express.Request, res: express.Response, next: express.NextFunction) => { + try { + const token = this.extractToken(req.header('authorization')) + this.publicKeys = this.publicKeys ?? await this.wixDataFacade.getPublicKeys(this.externalDatabaseId) + await this.verify(token) + } catch (err: any) { + console.error('Authorization failed: ' + err.message) + next(new UnauthorizedError('You are not authorized')) + } + next() + } + } + + getKey(header: JwtHeader, callback: SigningKeyCallback) { + if (header.kid === undefined) { + callback(new UnauthorizedError('No kid set on JWT header')) + return + } + const publicKey = this.publicKeys![header.kid!] + if (publicKey === undefined) { + callback(new UnauthorizedError(`No public key fetched for kid ${header.kid}. Available keys: ${JSON.stringify(this.publicKeys)}`)) + } else { + callback(null, publicKey) + } + } + + verifyJwt(token: string) { + return new Promise((resolve, reject) => + verify(token, this.getKey.bind(this), { audience: this.externalDatabaseId, issuer: TOKEN_ISSUER }, (err, decoded) => + (err) ? reject(err) : resolve(decoded!) + )) + } + + + async verifyWithRetry(token: string): Promise { + try { + return await this.verifyJwt(token) + } catch (err) { + this.publicKeys = await this.wixDataFacade.getPublicKeys(this.externalDatabaseId) + return await this.verifyJwt(token) + } + } + + async verify(token: string) { + const { siteId } = await this.verifyWithRetry(token) as JwtPayload + if (siteId === undefined || !this.allowedMetasites.includes(siteId)) { + throw new UnauthorizedError(`Unauthorized: ${siteId ? `site not allowed ${siteId}` : 'no siteId'}`) + } + } + + private extractToken(header: string | undefined) { + if (header===undefined) { + throw new UnauthorizedError('No Authorization header') + } + return header.replace(/^(Bearer )/, '') + } +} diff --git a/libs/velo-external-db-core/src/web/wix_data_facade.ts b/libs/velo-external-db-core/src/web/wix_data_facade.ts new file mode 100644 index 000000000..2ef8fa942 --- /dev/null +++ b/libs/velo-external-db-core/src/web/wix_data_facade.ts @@ -0,0 +1,40 @@ +import { errors } from '@wix-velo/velo-external-db-commons' +const { UnauthorizedError } = errors +import axios from 'axios' + +type PublicKeyResponse = { + publicKeys: { + id: string, + publicKeyPem: string + }[]; +}; + +export type PublicKeyMap = { [key: string]: string } + +export interface IWixDataFacade { + getPublicKeys(externalDatabaseId: string): Promise +} + +export class WixDataFacade implements IWixDataFacade { + baseUrl: string + + constructor(baseUrl: string) { + this.baseUrl = baseUrl + } + + async getPublicKeys(externalDatabaseId: string): Promise { + const url = `${this.baseUrl}/v1/external-databases/${externalDatabaseId}/public-keys` + const { data, status } = await axios.get(url, { + headers: { + Accept: 'application/json', + }, + }) + if (status !== 200) { + throw new UnauthorizedError(`failed to get public keys: status ${status}`) + } + return data.publicKeys.reduce((m: PublicKeyMap, { id, publicKeyPem }) => { + m[id] = publicKeyPem + return m + }, {}) + } +} diff --git a/libs/velo-external-db-core/test/drivers/auth_middleware_test_support.ts b/libs/velo-external-db-core/test/drivers/auth_middleware_test_support.ts index 9d06a6642..c84f3a225 100644 --- a/libs/velo-external-db-core/test/drivers/auth_middleware_test_support.ts +++ b/libs/velo-external-db-core/test/drivers/auth_middleware_test_support.ts @@ -1,9 +1,38 @@ +import { IWixDataFacade, PublicKeyMap } from '../../src/web/wix_data_facade' +import * as jwt from 'jsonwebtoken' +import { authConfig } from '@wix-velo/test-commons' +import { SignOptions } from 'jsonwebtoken' -export const requestBodyWith = (secretKey: string, role?: string | undefined, path?: string | undefined) => ({ + +export const requestBodyWith = (role?: string | undefined, path?: string | undefined, authHeader?: string | undefined) => ({ path: path || '/', body: { requestContext: { role: role || 'OWNER', settings: { - secretKey: secretKey - } } } } ) + } } }, + header(_name: string) { return authHeader } +} ) + +export const signedToken = (payload: Record, keyid?: string, expiration= '10000ms') => { + const options = keyid ? { algorithm: 'ES256', expiresIn: expiration, keyid: keyid } : { algorithm: 'ES256', expiresIn: expiration } + return jwt.sign(payload, authConfig.authPrivateKey, options as SignOptions) +} + +export class WixDataFacadeMock implements IWixDataFacade { + publicKeys: PublicKeyMap[] + index: number + + constructor(...publicKeys: PublicKeyMap[]) { + this.publicKeys = publicKeys + this.index = 0 + } + + getPublicKeys(_externalDatabaseId: string): Promise { + const publicKeyToReturn = this.publicKeys[this.index] + if (this.index < this.publicKeys.length-1) { + this.index++ + } + return Promise.resolve(publicKeyToReturn) + } +} diff --git a/libs/velo-external-db-core/test/drivers/data_provider_test_support.ts b/libs/velo-external-db-core/test/drivers/data_provider_test_support.ts index 3efb2e43d..8d019e4d8 100644 --- a/libs/velo-external-db-core/test/drivers/data_provider_test_support.ts +++ b/libs/velo-external-db-core/test/drivers/data_provider_test_support.ts @@ -18,8 +18,8 @@ export const givenCountResult = (total: any, forCollectionName: any, filter: any when(dataProvider.count).calledWith(forCollectionName, filter) .mockResolvedValue(total) -export const givenAggregateResult = (total: any, forCollectionName: any, filter: any, andAggregation: any) => - when(dataProvider.aggregate).calledWith(forCollectionName, filter, andAggregation) +export const givenAggregateResult = (total: any, forCollectionName: any, filter: any, andAggregation: any, sort: any, skip: any, limit: any) => + when(dataProvider.aggregate).calledWith(forCollectionName, filter, andAggregation, sort, skip, limit) .mockResolvedValue(total) export const expectInsertFor = (items: string | any[], forCollectionName: any) => diff --git a/libs/velo-external-db-core/test/drivers/data_service_test_support.ts b/libs/velo-external-db-core/test/drivers/data_service_test_support.ts index 23983974a..2871991ac 100644 --- a/libs/velo-external-db-core/test/drivers/data_service_test_support.ts +++ b/libs/velo-external-db-core/test/drivers/data_service_test_support.ts @@ -17,8 +17,8 @@ export const dataService = { const systemFields = SystemFields.map(({ name, type, subtype }) => ({ field: name, type, subtype }) ) -export const givenListResult = (entities: any, totalCount: any, forCollectionName: any, filter: any, sort: any, skip: any, limit: any, projection: any) => - when(dataService.find).calledWith(forCollectionName, filter, sort, skip, limit, projection) +export const givenListResult = (entities: any, totalCount: any, forCollectionName: any, filter: any, sort: any, skip: any, limit: any, projection: any, omitTotalCount?: boolean) => + when(dataService.find).calledWith(forCollectionName, filter, sort, skip, limit, projection, omitTotalCount) .mockResolvedValue( { items: entities, totalCount } ) export const givenCountResult = (totalCount: any, forCollectionName: any, filter: any) => @@ -57,8 +57,8 @@ export const truncateResultTo = (forCollectionName: any) => when(dataService.truncate).calledWith(forCollectionName) .mockResolvedValue(1) -export const givenAggregateResult = (items: any, forCollectionName: any, filter: any, aggregation: any) => - when(dataService.aggregate).calledWith(forCollectionName, filter, aggregation) +export const givenAggregateResult = (items: any, forCollectionName: any, filter: any, aggregation: any, sort: any, skip: any, limit: any) => + when(dataService.aggregate).calledWith(forCollectionName, filter, aggregation, sort, skip, limit) .mockResolvedValue({ items, totalCount: 0 }) export const reset = () => { diff --git a/libs/velo-external-db-core/test/drivers/filter_transformer_test_support.ts b/libs/velo-external-db-core/test/drivers/filter_transformer_test_support.ts index 67680fb5a..f3cc2a8fe 100644 --- a/libs/velo-external-db-core/test/drivers/filter_transformer_test_support.ts +++ b/libs/velo-external-db-core/test/drivers/filter_transformer_test_support.ts @@ -11,6 +11,10 @@ export const stubEmptyFilterFor = (filter: any) => { .mockReturnValue(EmptyFilter) } +export const stubEmptyFilterForUndefined = () => { + stubEmptyFilterFor(undefined) +} + export const givenFilterByIdWith = (id: any, filter: any) => { when(filterTransformer.transform).calledWith(filter) .mockReturnValue({ diff --git a/libs/velo-external-db-core/test/drivers/schema_matchers.ts b/libs/velo-external-db-core/test/drivers/schema_matchers.ts index cff91f1d8..508319743 100644 --- a/libs/velo-external-db-core/test/drivers/schema_matchers.ts +++ b/libs/velo-external-db-core/test/drivers/schema_matchers.ts @@ -1,4 +1,15 @@ +import { + Table, + CollectionCapabilities, + ResponseField +} from '@wix-velo/velo-external-db-types' import { asWixSchema, allowedOperationsFor, appendQueryOperatorsTo, asWixSchemaHeaders, ReadOnlyOperations } from '@wix-velo/velo-external-db-commons' +import { + fieldTypeToWixDataEnum, + queryOperatorsToWixDataQueryOperators, + dataOperationsToWixDataQueryOperators, + collectionOperationsToWixDataCollectionOperations, +} from '../../src/utils/schema_utils' const appendAllowedOperationsToDbs = (dbs: any[], allowedSchemaOperations: any) => { return dbs.map( (db: { fields: any }) => ({ @@ -25,3 +36,36 @@ export const schemaHeadersListFor = (collections: any) => toHaveSchemas(collecti export const schemasWithReadOnlyCapabilitiesFor = (collections: any) => toHaveSchemas(collections, collectionToHaveReadOnlyCapability) +export const fieldCapabilitiesObjectFor = (fieldCapabilities: { sortable: boolean, columnQueryOperators: string[] }) => expect.objectContaining({ + sortable: fieldCapabilities.sortable, + queryOperators: expect.arrayContaining(fieldCapabilities.columnQueryOperators.map(c => queryOperatorsToWixDataQueryOperators(c))) +}) + +export const fieldInWixFormatFor = (field: ResponseField) => expect.objectContaining({ + key: field.field, + type: fieldTypeToWixDataEnum(field.type), + capabilities: field.capabilities? fieldCapabilitiesObjectFor(field.capabilities) : undefined +}) + +export const fieldsToBeInWixFormat = (fields: ResponseField[]) => expect.arrayContaining(fields.map(f => fieldInWixFormatFor(f))) + +export const collectionCapabilitiesObjectFor = (collectionsCapabilities: CollectionCapabilities) => expect.objectContaining({ + dataOperations: expect.arrayContaining(collectionsCapabilities.dataOperations.map(d => dataOperationsToWixDataQueryOperators(d))), + fieldTypes: expect.arrayContaining(collectionsCapabilities.fieldTypes.map(f => fieldTypeToWixDataEnum(f))), + collectionOperations: expect.arrayContaining(collectionsCapabilities.collectionOperations.map(c => collectionOperationsToWixDataCollectionOperations(c))), +}) + +export const collectionsInWixFormatFor = (collection: Table) => { + return expect.objectContaining({ + id: collection.id, + fields: fieldsToBeInWixFormat(collection.fields), + capabilities: collection.capabilities? collectionCapabilitiesObjectFor(collection.capabilities): undefined + }) +} + +export const collectionsListFor = (collections: Table[]) => { + return expect.objectContaining({ + collection: collections.map(collectionsInWixFormatFor) + }) +} + diff --git a/libs/velo-external-db-core/test/drivers/schema_provider_test_support.ts b/libs/velo-external-db-core/test/drivers/schema_provider_test_support.ts index 3966326c2..2811da4d0 100644 --- a/libs/velo-external-db-core/test/drivers/schema_provider_test_support.ts +++ b/libs/velo-external-db-core/test/drivers/schema_provider_test_support.ts @@ -1,5 +1,7 @@ import { when } from 'jest-when' -import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons' +import { AllSchemaOperations, AdapterOperators } from '@wix-velo/velo-external-db-commons' +import { Table } from '@wix-velo/velo-external-db-types' +const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators export const schemaProvider = { list: jest.fn(), @@ -8,7 +10,9 @@ export const schemaProvider = { create: jest.fn(), addColumn: jest.fn(), removeColumn: jest.fn(), - supportedOperations: jest.fn() + supportedOperations: jest.fn(), + columnCapabilitiesFor: jest.fn(), + changeColumnType: jest.fn(), } export const givenListResult = (dbs: any) => @@ -23,13 +27,17 @@ export const givenAdapterSupportedOperationsWith = (operations: any) => export const givenAllSchemaOperations = () => when(schemaProvider.supportedOperations).mockReturnValue(AllSchemaOperations) -export const givenFindResults = (dbs: any[]) => - dbs.forEach((db: { id: any; fields: any }) => when(schemaProvider.describeCollection).calledWith(db.id).mockResolvedValue(db.fields) ) +export const givenFindResults = (tables: Table[]) => + tables.forEach((table) => when(schemaProvider.describeCollection).calledWith(table.id).mockResolvedValue({ id: table.id, fields: table.fields, capabilities: table.capabilities })) export const expectCreateOf = (collectionName: any) => when(schemaProvider.create).calledWith(collectionName) .mockResolvedValue(undefined) +export const expectCreateWithFieldsOf = (collectionName: any, column: any) => + when(schemaProvider.create).calledWith(collectionName, column) + .mockResolvedValue(undefined) + export const expectCreateColumnOf = (column: any, collectionName: any) => when(schemaProvider.addColumn).calledWith(collectionName, column) .mockResolvedValue(undefined) @@ -38,6 +46,24 @@ export const expectRemoveColumnOf = (columnName: any, collectionName: any) => when(schemaProvider.removeColumn).calledWith(collectionName, columnName) .mockResolvedValue(undefined) +export const givenColumnCapabilities = () => { + when(schemaProvider.columnCapabilitiesFor).calledWith('text') + .mockReturnValue({ sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] }) + when(schemaProvider.columnCapabilitiesFor).calledWith('number') + .mockReturnValue({ sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] }) + when(schemaProvider.columnCapabilitiesFor).calledWith('boolean') + .mockReturnValue({ sortable: true, columnQueryOperators: [eq] }) + when(schemaProvider.columnCapabilitiesFor).calledWith('url') + .mockReturnValue({ sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] }) + when(schemaProvider.columnCapabilitiesFor).calledWith('datetime') + .mockReturnValue({ sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] }) + when(schemaProvider.columnCapabilitiesFor).calledWith('image') + .mockReturnValue({ sortable: false, columnQueryOperators: [] }) + when(schemaProvider.columnCapabilitiesFor).calledWith('object') + .mockReturnValue({ sortable: false, columnQueryOperators: [eq, ne] }) +} + + export const reset = () => { schemaProvider.list.mockClear() schemaProvider.listHeaders.mockClear() @@ -46,4 +72,6 @@ export const reset = () => { schemaProvider.addColumn.mockClear() schemaProvider.removeColumn.mockClear() schemaProvider.supportedOperations.mockClear() + schemaProvider.columnCapabilitiesFor.mockClear() + schemaProvider.changeColumnType.mockClear() } diff --git a/libs/velo-external-db-core/test/gen.ts b/libs/velo-external-db-core/test/gen.ts index cfb11c806..7e59531b2 100644 --- a/libs/velo-external-db-core/test/gen.ts +++ b/libs/velo-external-db-core/test/gen.ts @@ -1,6 +1,17 @@ import * as Chance from 'chance' import { AdapterOperators } from '@wix-velo/velo-external-db-commons' import { gen as genCommon } from '@wix-velo/test-commons' +import { + CollectionCapabilities, + CollectionOperation, + InputField, + FieldType, + ResponseField, + DataOperation, + Table, + } from '@wix-velo/velo-external-db-types' + + const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators const chance = Chance() @@ -8,13 +19,17 @@ export const invalidOperatorForType = (validOperators: string | string[]) => ran Object.values(AdapterOperators).filter(x => !validOperators.includes(x)) ) -export const randomObjectFromArray = (array: any[]) => array[chance.integer({ min: 0, max: array.length - 1 })] - -export const randomColumn = () => ( { name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false } ) +export const randomObjectFromArray = (array: any[]): T => array[chance.integer({ min: 0, max: array.length - 1 })] +export const randomColumn = (): InputField => ( { name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false } ) +// TODO: random the wix-type filed from the enum export const randomWixType = () => randomObjectFromArray(['number', 'text', 'boolean', 'url', 'datetime', 'object']) +export const randomFieldType = () => randomObjectFromArray(Object.values(FieldType)) + +export const randomCollectionOperation = () => randomObjectFromArray(Object.values(CollectionOperation)) + export const randomOperator = () => (chance.pickone(['$ne', '$lt', '$lte', '$gt', '$gte', '$hasSome', '$eq', '$contains', '$startsWith', '$endsWith'])) export const randomFilter = () => { @@ -26,7 +41,7 @@ export const randomFilter = () => { } } -export const randomArrayOf = (gen: any) => { +export const randomArrayOf= (gen: any): T[] => { const arr = [] const num = chance.natural({ min: 2, max: 20 }) for (let i = 0; i < num; i++) { @@ -35,21 +50,38 @@ export const randomArrayOf = (gen: any) => { return arr } -export const randomCollectionName = () => chance.word({ length: 5 }) +export const randomAdapterOperators = () => (chance.pickone([eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include])) + +export const randomDataOperations = () => (chance.pickone(Object.values(DataOperation))) + +export const randomColumnCapabilities = () => ({ + sortable: chance.bool(), + columnQueryOperators: [ randomAdapterOperators() ] +}) + + + +export const randomCollectionCapabilities = (): CollectionCapabilities => ({ + dataOperations: [ randomDataOperations() ], + fieldTypes: [ randomFieldType() ], + collectionOperations: [ randomCollectionOperation() ], +}) + +export const randomCollectionName = ():string => chance.word({ length: 5 }) -export const randomCollections = () => randomArrayOf( randomCollectionName ) +export const randomCollections = () => randomArrayOf( randomCollectionName ) -export const randomWixDataType = () => chance.pickone(['number', 'text', 'boolean', 'url', 'datetime', 'image', 'object' ]) +export const randomWixDataType = () => chance.pickone(['number', 'text', 'boolean', 'datetime', 'object' ]) -export const randomDbField = () => ( { field: chance.word(), type: randomWixDataType(), subtype: chance.word(), isPrimary: chance.bool() } ) +export const randomDbField = (): ResponseField => ( { field: chance.word(), type: randomWixDataType(), subtype: chance.word(), isPrimary: chance.bool(), capabilities: randomColumnCapabilities() } ) -export const randomDbFields = () => randomArrayOf( randomDbField ) +export const randomDbFields = () => randomArrayOf( randomDbField ) -export const randomDb = () => ( { id: randomCollectionName(), fields: randomDbFields() }) +export const randomDb = (): Table => ( { id: randomCollectionName(), fields: randomDbFields(), capabilities: randomCollectionCapabilities() }) -export const randomDbs = () => randomArrayOf( randomDb ) +export const randomDbs = (): Table[] => randomArrayOf( randomDb ) -export const randomDbsWithIdColumn = () => randomDbs().map(i => ({ ...i, fields: [ ...i.fields, { field: '_id', type: 'text' }] })) +export const randomDbsWithIdColumn = (): Table[] => randomDbs().map(i => ({ ...i, fields: [ ...i.fields, { field: '_id', type: 'text', capabilities: randomColumnCapabilities() }] })) export const truthyValue = () => chance.pickone(['true', '1', 1, true]) export const falsyValue = () => chance.pickone(['false', '0', 0, false]) diff --git a/libs/velo-external-db-types/src/collection_types.ts b/libs/velo-external-db-types/src/collection_types.ts new file mode 100644 index 000000000..f351128c6 --- /dev/null +++ b/libs/velo-external-db-types/src/collection_types.ts @@ -0,0 +1,112 @@ +export enum DataOperation { + query = 'query', + count = 'count', + queryReferenced = 'queryReferenced', + aggregate = 'aggregate', + insert = 'insert', + update = 'update', + remove = 'remove', + truncate = 'truncate', + insertReferences = 'insertReferences', + removeReferences = 'removeReferences', +} + +export enum FieldType { + text = 'text', + number = 'number', + boolean = 'boolean', + datetime = 'datetime', + object = 'object', + longText = 'longText', + singleReference = 'singleReference', + multiReference = 'multiReference', +} + +export enum CollectionOperation { + update = 'update', + remove = 'remove', +} + +export enum Encryption { + notSupported = 'notSupported', + wixDataNative = 'wixDataNative', + dataSourceNative = 'dataSourceNative', +} + +export type CollectionCapabilities = { + dataOperations: DataOperation[], + fieldTypes: FieldType[], + collectionOperations: CollectionOperation[], + encryption?: Encryption, +} + +export type ColumnCapabilities = { + sortable: boolean, + columnQueryOperators: string[], +} + +export type FieldAttributes = { + type: string, + subtype?: string, + precision?: number | string, + isPrimary?: boolean, +} + +export enum SchemaOperations { + List = 'list', + ListHeaders = 'listHeaders', + Create = 'createCollection', + Drop = 'dropCollection', + AddColumn = 'addColumn', + RemoveColumn = 'removeColumn', + ChangeColumnType = 'changeColumnType', + Describe = 'describeCollection', + FindWithSort = 'findWithSort', + Aggregate = 'aggregate', + BulkDelete = 'bulkDelete', + Truncate = 'truncate', + UpdateImmediately = 'updateImmediately', + DeleteImmediately = 'deleteImmediately', + StartWithCaseSensitive = 'startWithCaseSensitive', + StartWithCaseInsensitive = 'startWithCaseInsensitive', + Projection = 'projection', + FindObject = 'findObject', + Matches = 'matches', + NotOperator = 'not', + IncludeOperator = 'include', + FilterByEveryField = 'filterByEveryField', +} + +export type InputField = FieldAttributes & { name: string } + +export type ResponseField = FieldAttributes & { + field: string + capabilities?: { + sortable: boolean + columnQueryOperators: string[] + } +} + +export type Table = { + id: string, + fields: ResponseField[] + capabilities?: CollectionCapabilities +} +export interface ISchemaProvider { + list(): Promise + listHeaders(): Promise + supportedOperations(): SchemaOperations[] + create(collectionName: string, columns?: InputField[]): Promise + addColumn(collectionName: string, column: InputField): Promise + removeColumn(collectionName: string, columnName: string): Promise + changeColumnType?(collectionName: string, column: InputField): Promise + describeCollection(collectionName: string): Promise | Promise
+ drop(collectionName: string): Promise + translateDbTypes?(column: InputField | ResponseField | string): ResponseField | string + columnCapabilitiesFor?(columnType: string): ColumnCapabilities + capabilities?(): CollectionCapabilities +} + + + + diff --git a/libs/velo-external-db-types/src/index.ts b/libs/velo-external-db-types/src/index.ts index 2fccbcdcf..2dce2a932 100644 --- a/libs/velo-external-db-types/src/index.ts +++ b/libs/velo-external-db-types/src/index.ts @@ -1,3 +1,11 @@ +import { + ResponseField, + SchemaOperations, + ISchemaProvider +} from './collection_types' + +export * from './collection_types' + export enum AdapterOperator { //in velo-external-db-core eq = 'eq', gt = 'gt', @@ -16,30 +24,6 @@ export enum AdapterOperator { //in velo-external-db-core matches = 'matches' } -export enum SchemaOperations { - List = 'list', - ListHeaders = 'listHeaders', - Create = 'createCollection', - Drop = 'dropCollection', - AddColumn = 'addColumn', - RemoveColumn = 'removeColumn', - Describe = 'describeCollection', - FindWithSort = 'findWithSort', - Aggregate = 'aggregate', - BulkDelete = 'bulkDelete', - Truncate = 'truncate', - UpdateImmediately = 'updateImmediately', - DeleteImmediately = 'deleteImmediately', - StartWithCaseSensitive = 'startWithCaseSensitive', - StartWithCaseInsensitive = 'startWithCaseInsensitive', - Projection = 'projection', - FindObject = 'findObject', - Matches = 'matches', - NotOperator = 'not', - IncludeOperator = 'include', - FilterByEveryField = 'filterByEveryField', -} - export type FieldWithQueryOperators = ResponseField & { queryOperators: string[] } export interface AsWixSchemaHeaders { @@ -116,45 +100,15 @@ export type AdapterAggregation = { export interface IDataProvider { find(collectionName: string, filter: AdapterFilter, sort: any, skip: number, limit: number, projection: string[]): Promise; count(collectionName: string, filter: AdapterFilter): Promise; - insert(collectionName: string, items: Item[], fields?: ResponseField[]): Promise; + insert(collectionName: string, items: Item[], fields?: ResponseField[], upsert?: boolean): Promise; update(collectionName: string, items: Item[], fields?: any): Promise; delete(collectionName: string, itemIds: string[]): Promise; truncate(collectionName: string): Promise; - aggregate?(collectionName: string, filter: AdapterFilter, aggregation: AdapterAggregation): Promise; + // sort, skip, limit are not really optional, after we'll implement in all the data providers we can remove the ? + aggregate?(collectionName: string, filter: AdapterFilter, aggregation: AdapterAggregation, sort?: Sort[], skip?: number, limit?: number ): Promise; } -export type TableHeader = { - id: string -} - -export type Table = TableHeader & { fields: ResponseField[] } - -export type FieldAttributes = { - type: string, - subtype?: string, - precision?: number | string, - isPrimary?: boolean, -} - -export type InputField = FieldAttributes & { name: string } - -export type ResponseField = FieldAttributes & { field: string } - -export interface ISchemaProvider { - list(): Promise - listHeaders(): Promise - supportedOperations(): SchemaOperations[] - create(collectionName: string, columns?: InputField[]): Promise - addColumn(collectionName: string, column: InputField): Promise - removeColumn(collectionName: string, columnName: string): Promise - describeCollection(collectionName: string): Promise - drop(collectionName: string): Promise - translateDbTypes?(column: InputField | ResponseField | string): ResponseField | string -} - -export interface IBaseHttpError extends Error { - status: number; -} +export interface IBaseHttpError extends Error {} type ValidConnectionResult = { valid: true } type InvalidConnectionResult = { valid: false, error: IBaseHttpError } @@ -226,15 +180,6 @@ export enum WixDataFunction { $sum = '$sum', } -export type WixDataAggregation = { - processingStep: { - _id: string | { [key: string]: any } - [key: string]: any - // [fieldAlias: string]: {[key in WixDataFunction]: string | number }, - } - postFilteringStep: WixDataFilter -} - export type WixDataRole = 'OWNER' | 'BACKEND_CODE' | 'MEMBER' | 'VISITOR' export type VeloRole = 'Admin' | 'Member' | 'Visitor' diff --git a/package.json b/package.json index e1be6ab4b..164b95ccb 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build:full-image": "nx run velo-external-db:build-image ", "lint": "eslint --cache ./", "lint:fix": "eslint --cache --fix ./", - "test": "npm run test:core; npm run test:postgres; npm run test:spanner; npm run test:mysql; npm run test:mssql; npm run test:firestore; npm run test:mongo; npm run test:airtable; npm run test:dynamodb; npm run test:bigquery", + "test": "npm run test:core; npm run test:mysql;", "test:core": "nx run-many --skip-nx-cache --target=test --projects=@wix-velo/external-db-config,@wix-velo/velo-external-db-core,@wix-velo/external-db-security", "test:postgres": "TEST_ENGINE=postgres nx run-many --skip-nx-cache --target=test --projects=@wix-velo/external-db-postgres,velo-external-db", "test:postgres13": "npm run test:postgres", @@ -49,6 +49,7 @@ "ejs": "^3.1.8", "express": "^4.17.2", "google-spreadsheet": "^3.3.0", + "jsonwebtoken": "^8.5.1", "moment": "^2.29.3", "mongodb": "^4.6.0", "mssql": "^8.1.0", @@ -82,6 +83,7 @@ "@types/google-spreadsheet": "^3.3.0", "@types/jest": "^27.4.1", "@types/jest-when": "^3.5.0", + "@types/jsonwebtoken": "^8.5.9", "@types/mssql": "^8.0.2", "@types/mysql": "^2.15.21", "@types/node": "^16.11.7", diff --git a/workspace.json b/workspace.json index 7a9d98698..0a2f18be0 100644 --- a/workspace.json +++ b/workspace.json @@ -2,16 +2,7 @@ "version": 2, "projects": { "@wix-velo/external-db-config": "libs/external-db-config", - "@wix-velo/external-db-postgres": "libs/external-db-postgres", "@wix-velo/external-db-mysql": "libs/external-db-mysql", - "@wix-velo/external-db-mssql": "libs/external-db-mssql", - "@wix-velo/external-db-spanner": "libs/external-db-spanner", - "@wix-velo/external-db-mongo": "libs/external-db-mongo", - "@wix-velo/external-db-firestore": "libs/external-db-firestore", - "@wix-velo/external-db-airtable": "libs/external-db-airtable", - "@wix-velo/external-db-bigquery": "libs/external-db-bigquery", - "@wix-velo/external-db-dynamodb": "libs/external-db-dynamodb", - "@wix-velo/external-db-google-sheets": "libs/external-db-google-sheets", "@wix-velo/external-db-security": "libs/external-db-security", "@wix-velo/test-commons": "libs/test-commons", "@wix-velo/velo-external-db-commons": "libs/velo-external-db-commons",