Implementation to v3

.eslintrc.json

@@ -61,7 +61,8 @@
             "@typescript-eslint/no-unused-vars": [ "error", { "ignoreRestSiblings": true, "argsIgnorePattern": "^_" } ],
             "@typescript-eslint/no-explicit-any": "off",
             "@typescript-eslint/ban-ts-comment": "warn",
-            "eol-last": [ "error", "always" ]
+            "eol-last": [ "error", "always" ], 
+            "@typescript-eslint/no-empty-interface": "off"
           }
         }
       ]

.github/workflows/main.yml

@@ -38,14 +38,7 @@ jobs:
     strategy:
       matrix:
         database: [
-                   "postgres", "postgres13", "postgres12", "postgres11", "postgres10", "postgres9",
-                   "spanner",
-                   "mysql", "mysql5",
-                   "mssql", "mssql17",
-                   "mongo", "mongo4",
-                   "firestore",
-                   "dynamodb",
-                   "google-sheets"
+                   "mysql", "mysql5"
                    ]
 
     env:

apps/velo-external-db/src/app.ts

@@ -4,10 +4,20 @@ import { ExternalDbRouter, Hooks } from '@wix-velo/velo-external-db-core'
 import { engineConnectorFor } from './storage/factory'
 
 
-const initConnector = async(hooks?: Hooks) => {
-    const { vendor, type: adapterType } = readCommonConfig()
+process.env.CLOUD_VENDOR = 'azure'
+process.env.TYPE = 'mysql'
+process.env.EXTERNAL_DATABASE_ID = ''
+process.env.ALLOWED_METASITES = ''
+process.env['TYPE'] = 'mysql'
+process.env['HOST'] = 'localhost'
+process.env['USER'] = 'test-user'
+process.env['PASSWORD'] = 'password'
+process.env['DB'] = 'test-db'
+
+const initConnector = async(wixDataBaseUrl?: string, hooks?: Hooks) => {
+    const { vendor, type: adapterType, externalDatabaseId, allowedMetasites } = readCommonConfig()
     const configReader = create()
-    const { authorization, secretKey, ...dbConfig } = await configReader.readConfig()
+    const { authorization, ...dbConfig } = await configReader.readConfig()
 
     const { connector: engineConnector, providers, cleanup } = await engineConnectorFor(adapterType, dbConfig)
 
@@ -17,22 +27,24 @@ const initConnector = async(hooks?: Hooks) => {
             authorization: {
                 roleConfig: authorization
             },
-            secretKey,
+            externalDatabaseId,
+            allowedMetasites,
             vendor,
             adapterType,
-            commonExtended: true
+            commonExtended: true,
+            wixDataBaseUrl: wixDataBaseUrl || 'https://www.wixapis.com/wix-data'
         },
         hooks
     })
 
     return { externalDbRouter, cleanup: async() => await cleanup(), schemaProvider: providers.schemaProvider }
 }
 
-export const createApp = async() => {
+export const createApp = async(wixDataBaseUrl?: string) => {
     const app = express()
-    const initConnectorResponse = await initConnector()
+    const initConnectorResponse = await initConnector(wixDataBaseUrl)
     app.use(initConnectorResponse.externalDbRouter.router)
     const server = app.listen(8080, () => console.log('Connector listening on port 8080'))
 
-    return { server, ...initConnectorResponse, reload: () => initConnector() }
+    return { server, ...initConnectorResponse, reload: () => initConnector(wixDataBaseUrl) }
 }

apps/velo-external-db/src/storage/factory.ts

@@ -7,42 +7,42 @@ export const engineConnectorFor = async(_type: string, config: any): Promise<Dat
             const { postgresFactory } = require('@wix-velo/external-db-postgres')
             return await postgresFactory(config)
         }
-        case 'spanner': {
-            const { spannerFactory } = require('@wix-velo/external-db-spanner')
-            return await spannerFactory(config)
-        }
-        case 'firestore': {
-            const { firestoreFactory } = require('@wix-velo/external-db-firestore')
-            return await firestoreFactory(config)
-        }
-        case 'mssql': {
-            const { mssqlFactory } = require('@wix-velo/external-db-mssql')
-            return await mssqlFactory(config)
-        }
+        // case 'spanner': {
+        //     const { spannerFactory } = require('@wix-velo/external-db-spanner')
+        //     return await spannerFactory(config)
+        // }
+        // case 'firestore': {
+        //     const { firestoreFactory } = require('@wix-velo/external-db-firestore')
+        //     return await firestoreFactory(config)
+        // }
+        // case 'mssql': {
+        //     const { mssqlFactory } = require('@wix-velo/external-db-mssql')
+        //     return await mssqlFactory(config)
+        // }
         case 'mysql': {
             const { mySqlFactory } = require('@wix-velo/external-db-mysql')
             return await mySqlFactory(config)
         }
-        case 'mongo': {
-            const { mongoFactory } = require('@wix-velo/external-db-mongo')
-            return await mongoFactory(config)
-        }
-        case 'google-sheet': {
-            const { googleSheetFactory } = require('@wix-velo/external-db-google-sheets')
-            return await googleSheetFactory(config)
-        }
-        case 'airtable': {
-            const { airtableFactory } = require('@wix-velo/external-db-airtable')
-            return await airtableFactory(config)
-        }
-        case 'dynamodb': {
-            const { dynamoDbFactory } = require('@wix-velo/external-db-dynamodb')
-            return await dynamoDbFactory(config)
-        }
-        case 'bigquery': {
-            const { bigqueryFactory } = require('@wix-velo/external-db-bigquery')
-            return await bigqueryFactory(config)
-        }
+        // case 'mongo': {
+        //     const { mongoFactory } = require('@wix-velo/external-db-mongo')
+        //     return await mongoFactory(config)
+        // }
+        // case 'google-sheet': {
+        //     const { googleSheetFactory } = require('@wix-velo/external-db-google-sheets')
+        //     return await googleSheetFactory(config)
+        // }
+        // case 'airtable': {
+        //     const { airtableFactory } = require('@wix-velo/external-db-airtable')
+        //     return await airtableFactory(config)
+        // }
+        // case 'dynamodb': {
+        //     const { dynamoDbFactory } = require('@wix-velo/external-db-dynamodb')
+        //     return await dynamoDbFactory(config)
+        // }
+        // case 'bigquery': {
+        //     const { bigqueryFactory } = require('@wix-velo/external-db-bigquery')
+        //     return await bigqueryFactory(config)
+        // }
         default: {
             const { stubFactory } = require('./stub-db/stub-connector')
             return await stubFactory(type, config)

apps/velo-external-db/test/drivers/data_api_rest_test_support.ts

@@ -1,9 +1,70 @@
+import axios from 'axios'
 import { Item } from '@wix-velo/velo-external-db-types'
+import { dataSpi } from '@wix-velo/velo-external-db-core'
+import { streamToArray } from '@wix-velo/test-commons'
 
-const axios = require('axios').create({
+const axiosInstance = axios.create({
     baseURL: 'http://localhost:8080'
 })
 
-export const givenItems = async(items: Item[], collectionName: string, auth: any) => await axios.post('/data/insert/bulk', { collectionName: collectionName, items: items }, auth)
+export const insertRequest = (collectionName: string, items: Item[], overwriteExisting: boolean): dataSpi.InsertRequest => ({
+    collectionId: collectionName,
+    items: items,
+    overwriteExisting,
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    }
+})
+
+export const updateRequest = (collectionName: string, items: Item[]): dataSpi.UpdateRequest => ({
+    collectionId: collectionName,
+    items: items,
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    }
+})
+
+export const countRequest = (collectionName: string): dataSpi.CountRequest => ({
+    collectionId: collectionName,
+    filter: '',
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    },
+})
+
+export const queryRequest = (collectionName: string, sort: dataSpi.Sorting[], fields: string[], filter?: dataSpi.Filter): dataSpi.QueryRequest => ({
+    collectionId: collectionName,
+    query: {
+        filter: filter ?? '',
+        sort: sort,
+        fields: fields,
+        fieldsets: undefined,
+        paging: {
+            limit: 25,
+            offset: 0,
+        },
+        cursorPaging: null
+    },
+    includeReferencedItems: [],
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    },
+    omitTotalCount: false
+})
+
+
+export const queryCollectionAsArray = (collectionName: string, sort: dataSpi.Sorting[], fields: string[], auth: any, filter?: dataSpi.Filter) =>
+    axiosInstance.post('/data/query',
+        queryRequest(collectionName, sort, fields, filter), { responseType: 'stream', transformRequest: auth.transformRequest })
+        .then(response => streamToArray(response.data))
+
+
+export const pagingMetadata = (total: number, count: number): dataSpi.QueryResponsePart => ({ pagingMetadata: { count: count, offset: 0, total: total, tooManyToCount: false } })
+
 
-export const expectAllDataIn = async(collectionName: string, auth: any) => (await axios.post('/data/find', { collectionName: collectionName, filter: '', sort: '', skip: 0, limit: 25 }, auth)).data
+export const givenItems = async(items: Item[], collectionName: string, auth: any) =>
+    await axiosInstance.post('/data/insert', insertRequest(collectionName, items, false), { responseType: 'stream', transformRequest: auth.transformRequest })

apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts

@@ -1,5 +1,6 @@
 import { SystemFields, asWixSchemaHeaders } from '@wix-velo/velo-external-db-commons'
 import { InputField } from '@wix-velo/velo-external-db-types'
+import { schemaUtils } from '@wix-velo/velo-external-db-core'
 
 export const responseWith = (matcher: any) => expect.objectContaining( { data: matcher } )
 
@@ -40,3 +41,35 @@ const toHaveCollections = (collections: string[]) => expect.objectContaining( {
 const listToHaveCollection = (collectionName: string) => expect.objectContaining( {
     schemas: expect.arrayContaining( [ expect.objectContaining( { id: collectionName } ) ] )
 } )
+
+const collectionCapabilities = (_collectionOperations: any[], _dataOperations: any[], _fieldTypes: any[]) => ({
+    collectionOperations: expect.any(Array),
+    dataOperations: expect.any(Array),
+    fieldTypes: expect.any(Array)
+})
+
+const fieldCapabilitiesMatcher = () => expect.objectContaining({
+    queryOperators: expect.any(Array),
+    sortable: expect.any(Boolean),
+})
+
+const filedMatcher = (field: InputField) => ({
+    key: field.name,
+    capabilities: fieldCapabilitiesMatcher(),
+    encrypted: expect.any(Boolean),
+    type: schemaUtils.fieldTypeToWixDataEnum(field.type)
+})
+
+const fieldsMatcher = (fields: InputField[]) => expect.toIncludeSameMembers(fields.map(filedMatcher))
+
+export const collectionResponsesWith = (collectionName: string, fields: InputField[]) => ({
+    id: collectionName,
+    capabilities: collectionCapabilities([], [], []),
+    fields: fieldsMatcher(fields),
+})
+
+export const createCollectionResponse = (collectionName: string, fields: InputField[]) => ({
+    id: collectionName,
+    capabilities: collectionCapabilities([], [], []),
+    fields: fieldsMatcher(fields),
+})

apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts

@@ -1,14 +1,34 @@
+import axios from 'axios'
 import { InputField } from '@wix-velo/velo-external-db-types'
+import { streamToArray } from '@wix-velo/test-commons'
+import { schemaUtils } from '@wix-velo/velo-external-db-core'
 
-const axios = require('axios').create({
+
+const axiosClient = axios.create({
     baseURL: 'http://localhost:8080'
 })
 
 export const givenCollection = async(name: string, columns: InputField[], auth: any) => {
-    await axios.post('/schemas/create', { collectionName: name }, auth)
-    for (const column of columns) {
-        await axios.post('/schemas/column/add', { collectionName: name, column: column }, auth)
+    const collection = {
+        id: name,
+        fields: columns.map(schemaUtils.InputFieldToWixFormatField)
     }
+    await axiosClient.post('/collections/create', { collection }, { ...auth, responseType: 'stream' })
 }
 
-export const retrieveSchemaFor = async(collectionName: string, auth: any) => axios.post('/schemas/find', { schemaIds: [collectionName] }, auth)
+export const deleteAllCollections = async(auth: any) => {
+    const res = await axiosClient.post('/collections/get', { collectionIds: [] }, { ...auth, responseType: 'stream' })
+    const dataRes = await streamToArray(res.data) as any []
+    const collectionIds = dataRes.map(d => d.id)
+
+    for (const collectionId of collectionIds) {
+        await axiosClient.post('/collections/delete', { collectionId }, { ...auth, responseType: 'stream' })
+    }
+
+}
+
+export const retrieveSchemaFor = async(collectionName: string, auth: any) => {
+    const collectionGetStream = await axiosClient.post('/collections/get', { collectionIds: [collectionName] }, { ...auth, responseType: 'stream' })
+    const [collectionGetRes] = await streamToArray(collectionGetStream.data) as any[]
+    return collectionGetRes
+}

apps/velo-external-db/test/drivers/schema_provider_matchers.ts

@@ -1,6 +1,18 @@
-export const hasSameSchemaFieldsLike = (fields: {field: string, [x: string]: any}[]) => expect.arrayContaining( fields.map((f: any) => expect.objectContaining( f ) ))
+import { SystemFields } from '@wix-velo/velo-external-db-commons'
+import { ResponseField } from '@wix-velo/velo-external-db-types'
 
-export const collectionWithDefaultFields = () => hasSameSchemaFieldsLike([ { field: '_id', type: 'text' },
-                                                                          { field: '_createdDate', type: 'datetime' },
-                                                                          { field: '_updatedDate', type: 'datetime' },
-                                                                          { field: '_owner', type: 'text' } ])
+export const hasSameSchemaFieldsLike = (fields: ResponseField[]) => expect.arrayContaining(fields.map((f) => expect.objectContaining( f )))
+
+export const toContainDefaultFields = () => hasSameSchemaFieldsLike(SystemFields.map(f => ({ field: f.name, type: f.type })))
+
+export const collectionToContainFields = (collectionName: string, fields: ResponseField[], capabilities: any) => ({
+    id: collectionName,
+    fields: hasSameSchemaFieldsLike(fields),
+    capabilities: {
+        collectionOperations: capabilities.CollectionOperations,
+        dataOperations: capabilities.ReadWriteOperations,
+        fieldTypes: capabilities.FieldTypes 
+    }
+})
+
+export const toBeDefaultCollectionWith = (collectionName: string, capabilities: any) => collectionToContainFields(collectionName, SystemFields.map(f => ({ field: f.name, type: f.type })), capabilities)

apps/velo-external-db/test/drivers/wix_data_resources.ts

@@ -0,0 +1,17 @@
+import { Server } from 'http'
+import { app as mockServer } from './wix_data_testkit'
+
+let _server: Server
+const PORT = 9001
+
+export const initWixDataEnv = async() => {
+    _server = mockServer.listen(PORT)
+}
+
+export const shutdownWixDataEnv = async() => {
+    _server.close()
+}
+
+export const wixDataBaseUrl = () => {
+    return `http://localhost:${PORT}`
+}

apps/velo-external-db/test/drivers/wix_data_testkit.ts

@@ -0,0 +1,33 @@
+import { authConfig } from '@wix-velo/test-commons'
+import * as express from 'express'
+
+export const app = express()
+
+app.set('case sensitive routing', true)
+
+app.use(express.json())
+
+app.get('/v1/external-databases/:externalDatabaseId/public-keys', (_req, res) => {
+    res.json({
+        publicKeys: [
+            { id: authConfig.kid, publicKeyPem: authConfig.authPublicKey },
+        ]
+    })
+})
+
+app.use((_req, res) => {
+    res.status(404)
+    res.json({ error: 'NOT_FOUND' })
+})
+
+app.use((err, _req, res, next) => {
+    res.status(err.status)
+    res.json({
+        error: {
+            message: err.message,
+            status: err.status,
+            error: err.error
+        }
+    })
+    next()
+})