diff --git a/.vscode/launch.json b/.vscode/launch.json index dc4db4087af..db831f3dae3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -103,7 +103,8 @@ "UPLOAD_URL": "/files", "AI_BOT_URL": "http://localhost:4010", "STATS_URL": "http://huly.local:4900", - "QUEUE_CONFIG": "localhost:19092" + "QUEUE_CONFIG": "localhost:19092", + "RATE_LIMIT_MAX": "25000" }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeVersion": "20", @@ -120,9 +121,9 @@ "args": ["src/__start.ts"], "env": { "FULLTEXT_URL": "http://localhost:4710", - // "DB_URL": "mongodb://localhost:27018", + "DB_URL": "mongodb://localhost:27018", // "DB_URL": "postgresql://postgres:example@localhost:5432", - "DB_URL": "postgresql://root@huly.local:26258/defaultdb?sslmode=disable", + // "DB_URL": "postgresql://root@huly.local:26258/defaultdb?sslmode=disable", // "GREEN_URL": "http://huly.local:6767?token=secret", "SERVER_PORT": "3335", "METRICS_CONSOLE": "false", @@ -183,7 +184,7 @@ "request": "launch", "args": ["src/__start.ts"], "env": { - "MONGO_URL": "mongodb://localhost:27017", + // "MONGO_URL": "mongodb://localhost:27017", // "DB_URL": "mongodb://localhost:27017", "DB_URL": "postgresql://root@huly.local:26257/defaultdb?sslmode=disable", "SERVER_SECRET": "secret", @@ -217,12 +218,13 @@ "args": ["src/__start.ts"], "env": { // "MONGO_URL": "mongodb://localhost:27018", - // "DB_URL": "mongodb://localhost:27018", + "DB_URL": "mongodb://localhost:27018", // "DB_URL": "postgresql://postgres:example@localhost:5432", - "DB_URL": "postgresql://root@huly.local:26258/defaultdb?sslmode=disable", + // "DB_URL": "postgresql://root@huly.local:26258/defaultdb?sslmode=disable", "SERVER_SECRET": "secret", - "REGION_INFO": "|Mongo;pg|Postgres;cockroach|CockroachDB", - "TRANSACTOR_URL": "ws://transactor:3334;ws://localhost:3334", + "QUEUE_CONFIG": "huly.local:19092", + "REGION_INFO": "|America;europe|Europe", + "TRANSACTOR_URL": "ws://huly.local:3334;ws://huly.local:3334,ws://huly.local:3335;ws://huly.local:3335;europe", "ACCOUNTS_URL": "http://localhost:3003", "ACCOUNT_PORT": "3003", "FRONT_URL": "http://localhost:8083", diff --git a/common/config/rush/command-line.json b/common/config/rush/command-line.json index ea7f52bf2b4..dbd7e740220 100644 --- a/common/config/rush/command-line.json +++ b/common/config/rush/command-line.json @@ -4,7 +4,6 @@ */ { "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json", - "phases": [ { "name": "_phase:build", diff --git a/common/scripts/rush_docker.sh b/common/scripts/rush_docker.sh new file mode 100644 index 00000000000..a3aa33f3f84 --- /dev/null +++ b/common/scripts/rush_docker.sh @@ -0,0 +1,3 @@ +export RUSH_ALLOW_UNSUPPORTED_NODEJS=1 +rush docker:build -p 20 \ + --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext --to @hcengineering/pod-love --to @hcengineering/pod-mail --to @hcengineering/pod-datalake --to @hcengineering/pod-inbound-mail --to @hcengineering/pod-export --to @hcengineering/pod-msg2file diff --git a/dev/doc-import-tool/src/import.ts b/dev/doc-import-tool/src/import.ts index d3c207bae4f..54fc658fc81 100644 --- a/dev/doc-import-tool/src/import.ts +++ b/dev/doc-import-tool/src/import.ts @@ -47,7 +47,7 @@ export default async function importExtractedFile ( try { console.log(`Connected to ${transactorUrl}`) - const txops = new TxOperations(connection, core.account.System) + const txops = new TxOperations(connection, core.account.System, workspaceId) try { const docId = await createDocument(txops, extractedFile, config) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 6105895572f..5c38b631110 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -302,6 +302,7 @@ services: environment: - QUEUE_CONFIG=${QUEUE_CONFIG} - REGION= + - ENDPOINT=ws://huly.local:3333 - SERVER_PORT=3333 - SERVER_SECRET=secret - ENABLE_COMPRESSION=true @@ -338,6 +339,7 @@ services: - QUEUE_CONFIG=${QUEUE_CONFIG} - SERVER_PORT=3332 - REGION=cockroach + - ENDPOINT=ws://huly.local:3332 - SERVER_SECRET=secret - ENABLE_COMPRESSION=true - FULLTEXT_URL=http://huly.local:4702 diff --git a/dev/import-tool/src/index.ts b/dev/import-tool/src/index.ts index d70b360b877..0e1819ef021 100644 --- a/dev/import-tool/src/index.ts +++ b/dev/import-tool/src/index.ts @@ -95,7 +95,7 @@ export function importTool (): void { console.log('Connecting to Transactor URL: ', selectedWs.endpoint) const connection = await createClient(selectedWs.endpoint, selectedWs.token) - const client = new TxOperations(connection, socialId) + const client = new TxOperations(connection, socialId, selectedWs.workspace) const fileUploader = new FrontFileUploader( getFrontUrl(), selectedWs.workspace, diff --git a/dev/prod/src/platform.ts b/dev/prod/src/platform.ts index a0596b1af7d..b488998e1ca 100644 --- a/dev/prod/src/platform.ts +++ b/dev/prod/src/platform.ts @@ -67,7 +67,7 @@ import { uploaderId } from '@hcengineering/uploader' import { mediaId } from '@hcengineering/media' import recorder, { recorderId } from '@hcengineering/recorder' import { viewId } from '@hcengineering/view' -import workbench, { workbenchId } from '@hcengineering/workbench' +import workbench, { workbenchId, workbenchAppsId } from '@hcengineering/workbench' import { mailId } from '@hcengineering/mail' import { chatId } from '@hcengineering/chat' import github, { githubId } from '@hcengineering/github' @@ -147,6 +147,7 @@ import { preferenceId } from '@hcengineering/preference' import { uiId } from '@hcengineering/ui/src/plugin' import { configureAnalytics } from './analytics' + export interface Config { ACCOUNTS_URL: string UPLOAD_URL: string @@ -487,6 +488,7 @@ export async function configurePlatform() { uiPlugin.metadata.Routes, new Map([ [workbenchId, workbench.component.WorkbenchApp], + [workbenchAppsId, workbench.component.WorkbenchApps], [loginId, login.component.LoginApp], [onboardId, onboard.component.OnboardApp], [githubId, github.component.ConnectApp], diff --git a/dev/tool/src/benchmark.ts b/dev/tool/src/benchmark.ts index 8a11b574eb2..296a60671b9 100644 --- a/dev/tool/src/benchmark.ts +++ b/dev/tool/src/benchmark.ts @@ -403,7 +403,7 @@ export function benchmarkWorker (): void { if (msg.options.mode === 'find-all') { const benchmarkPersonId = (core.account.System + '_benchmark') as PersonId - const opt = new TxOperations(connection, benchmarkPersonId) + const opt = new TxOperations(connection, benchmarkPersonId, msg.workspaceId) parentPort?.postMessage({ type: 'operate', workId: msg.workId @@ -544,7 +544,7 @@ export async function stressBenchmark (transactor: string, mode: StressBenchmark export async function testFindAll (endpoint: string, workspace: WorkspaceUuid, account: PersonUuid): Promise { const connection = await connect(endpoint, workspace, account) try { - const client = new TxOperations(connection, core.account.System) + const client = new TxOperations(connection, core.account.System, workspace) const start = platformNow() const res = await client.findAll( recruit.class.Applicant, @@ -569,7 +569,7 @@ export async function generateWorkspaceData ( email: string ): Promise { const connection = await connect(endpoint, workspace) - const client = new TxOperations(connection, core.account.System) + const client = new TxOperations(connection, core.account.System, workspace) try { const emailSocialString = buildSocialIdString({ type: SocialIdType.EMAIL, value: email }) const person = await getPersonBySocialKey(client, emailSocialString) diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts index 90e58aea9b7..81ccfbc7f05 100644 --- a/dev/tool/src/clean.ts +++ b/dev/tool/src/clean.ts @@ -86,11 +86,10 @@ export async function cleanWorkspace ( opt: { recruit: boolean, tracker: boolean, removedTx: boolean } ): Promise { const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup', - model: 'upgrade' + mode: 'backup' })) as unknown as CoreClient & BackupClient try { - const ops = new TxOperations(connection, core.account.System) + const ops = new TxOperations(connection, core.account.System, workspaceId) const hierarchy = ops.getHierarchy() @@ -164,6 +163,7 @@ export async function cleanWorkspace ( client.close() } } catch (err: any) { + // TODO: Add force-close console.trace(err) } finally { await connection.close() @@ -214,6 +214,7 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceUuid, tran objectId: { $in: removedDocs.map((it) => it.objectId) } }) await connection.clean( + workspaceId, DOMAIN_TX, toRemove.map((it) => it._id) ) @@ -232,8 +233,7 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceUuid, tran export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl: string): Promise { const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup', - model: 'upgrade' + mode: 'backup' })) as unknown as CoreClient & BackupClient try { let count = 0 @@ -289,7 +289,7 @@ export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl: } } - await connection.clean(DOMAIN_TX, toRemove) + await connection.clean(workspaceId, DOMAIN_TX, toRemove) count += toRemove.length console.log('processed', count) @@ -298,6 +298,8 @@ export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl: } catch (err: any) { console.trace(err) } finally { + // TODO: Add force-close + await connection.sendForceClose(workspaceId) await connection.close() } } @@ -307,7 +309,7 @@ export async function cleanArchivedSpaces (workspaceId: WorkspaceUuid, transacto })) as unknown as CoreClient & BackupClient try { const count = 0 - const ops = new TxOperations(connection, core.account.System) + const ops = new TxOperations(connection, core.account.System, workspaceId) while (true) { const spaces = await connection.findAll(core.class.Space, { archived: true }, { limit: 1000 }) if (spaces.length === 0) { @@ -370,7 +372,7 @@ export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceUuid, tran // We have found duplicate one, let's rename it. const doc = TxProcessor.createDoc2Doc(c as unknown as TxCreateDoc) if (doc.message !== '' && doc.message.trim() !== '

') { - await connection.clean(DOMAIN_TX, [c._id]) + await connection.clean(workspaceId, DOMAIN_TX, [c._id]) if (oldValue.get(cid) === doc.message.trim()) { console.log('delete tx', cid, doc.message) } else { @@ -379,9 +381,9 @@ export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceUuid, tran // Remove previous transaction. c.objectId = generateId() doc._id = c.objectId as Ref - await connection.upload(DOMAIN_TX, [c]) + await connection.upload(workspaceId, DOMAIN_TX, [c]) // Also we need to create snapsot - await connection.upload(DOMAIN_ACTIVITY, [doc]) + await connection.upload(workspaceId, DOMAIN_ACTIVITY, [doc]) } } } @@ -475,7 +477,7 @@ export async function fixSkills ( // fix skills with + and - if (step === '3') { console.log('STEP 3') - const ops = new TxOperations(connection, core.account.System) + const ops = new TxOperations(connection, core.account.System, workspaceId) const regex = /\S+(?:[-+]\S+)+/g const tagsToClean = (await connection.findAll(tags.class.TagElement, { category: { @@ -532,7 +534,7 @@ export async function fixSkills ( } })) as TagElement[] goodTags = goodTags.sort((a, b) => b.title.length - a.title.length).filter((t) => t.title.length > 2) - const ops = new TxOperations(connection, core.account.System) + const ops = new TxOperations(connection, core.account.System, workspaceId) const tagsToClean = (await connection.findAll(tags.class.TagElement, { category: { $in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref[] @@ -669,8 +671,7 @@ export async function restoreRecruitingTaskTypes ( transactorUrl: string ): Promise { const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup', - model: 'upgrade' + mode: 'backup' })) as unknown as CoreClient & BackupClient const client = getMongoClient(mongoUrl) try { @@ -772,6 +773,7 @@ export async function restoreRecruitingTaskTypes ( statusCategories.sort(compareCategories) const createTxNew: TxCreateDoc = { + _uuid: workspaceId, _id: generateId(), _class: core.class.TxCreateDoc, space: core.space.Tx, @@ -822,7 +824,9 @@ export async function restoreRecruitingTaskTypes ( } catch (err: any) { console.trace(err) } finally { + // TODO: Add force-close client.close() + await connection.sendForceClose(workspaceId) await connection.close() } } @@ -833,8 +837,7 @@ export async function restoreHrTaskTypesFromUpdates ( transactorUrl: string ): Promise { const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup', - model: 'upgrade' + mode: 'backup' })) as unknown as CoreClient & BackupClient const client = getMongoClient(mongoUrl) try { @@ -927,6 +930,7 @@ export async function restoreHrTaskTypesFromUpdates ( const ofClassClass = hierarchy.getClass(recruit.class.Applicant) await db.collection>(DOMAIN_TX).insertOne({ + _uuid: workspaceId, _id: generateId(), _class: core.class.TxCreateDoc, space: core.space.Tx, @@ -946,6 +950,7 @@ export async function restoreHrTaskTypesFromUpdates ( }) createTaskTypeTx = { + _uuid: workspaceId, _id: generateId(), _class: core.class.TxCreateDoc, space: core.space.Tx, @@ -980,6 +985,7 @@ export async function restoreHrTaskTypesFromUpdates ( const ofClassClass = hierarchy.getClass(recruit.class.Vacancy) await db.collection>(DOMAIN_TX).insertOne({ + _uuid: workspaceId, _id: generateId(), _class: core.class.TxCreateDoc, space: core.space.Tx, @@ -999,6 +1005,7 @@ export async function restoreHrTaskTypesFromUpdates ( }) const createProjectTypeTx: TxCreateDoc = { + _uuid: workspaceId, _id: generateId(), _class: core.class.TxCreateDoc, space: core.space.Tx, @@ -1031,6 +1038,7 @@ export async function restoreHrTaskTypesFromUpdates ( console.trace(err) } finally { client.close() + await connection.sendForceClose(workspaceId) await connection.close() } } @@ -1127,7 +1135,7 @@ export async function removeDuplicateIds ( // await updateId(ctx, wsClient, db, storageAdapter, wsDataId, doc) // } // } - // await wsClient.sendForceClose() + // await wsClient.sendForceClose(workspaceId) // await wsClient.close() // await db.collection(DOMAIN_MIGRATION).insertOne({ // _id: generateId(), diff --git a/dev/tool/src/configuration.ts b/dev/tool/src/configuration.ts index c560b8d2f49..db990595cfd 100644 --- a/dev/tool/src/configuration.ts +++ b/dev/tool/src/configuration.ts @@ -41,7 +41,7 @@ export async function changeConfiguration ( } const enable = (cmd.enable ?? '').trim().split(',') console.log('enable', enable) - const ops = new TxFactory(core.account.ConfigUser) + const ops = new TxFactory(core.account.ConfigUser, workspaceId) if (enable.length > 0) { const p = config.filter((it) => enable.includes(it.pluginId) || enable.includes('*')) for (const pp of p) { diff --git a/dev/tool/src/db.ts b/dev/tool/src/db.ts index c7c62fdf989..b8d2d858663 100644 --- a/dev/tool/src/db.ts +++ b/dev/tool/src/db.ts @@ -97,7 +97,7 @@ async function moveWorkspace ( const token = generateToken(systemAccountUuid, wsId, { service: 'tool' }) const endpoint = await getTransactorEndpoint(token, 'external') const connection = (await connect(endpoint, wsId, undefined, { - model: 'upgrade' + mode: 'backup' })) as unknown as Client & BackupClient for (const collection of collections) { const domain = translateDomain(collection.collectionName) @@ -158,7 +158,7 @@ async function moveWorkspace ( } // TODO: FIXME // await updateWorkspace(accountDb, ws, { region }) - await connection.sendForceClose() + await connection.sendForceClose(wsId) await connection.close() } catch (err) { console.log('Error when move workspace', ws.name ?? ws.url, err) diff --git a/dev/tool/src/github.ts b/dev/tool/src/github.ts index ac4156b0d9c..8d9aa79d54d 100644 --- a/dev/tool/src/github.ts +++ b/dev/tool/src/github.ts @@ -94,15 +94,15 @@ export async function performGithubAccountMigrations (db: Db, region: string | n let idx: number | undefined while (true) { - const info = await client.loadChunk(DOMAIN_MODEL_TX, idx) + const info = await client.loadChunk(ws.uuid, DOMAIN_MODEL_TX, idx) idx = info.idx const ids = Array.from(info.docs.map((it) => it.id as Ref)) - const docs = (await client.loadDocs(DOMAIN_MODEL_TX, ids)).filter((it) => + const docs = (await client.loadDocs(ws.uuid, DOMAIN_MODEL_TX, ids)).filter((it) => TxProcessor.isExtendsCUD(it._class) ) as TxCUD[] accountsTxes.push(...docs) if (info.finished && idx !== undefined) { - await client.closeChunk(info.idx) + await client.closeChunk(ws.uuid, info.idx) break } } diff --git a/dev/tool/src/gmail.ts b/dev/tool/src/gmail.ts index ab35f1bb546..cd9443feb3e 100644 --- a/dev/tool/src/gmail.ts +++ b/dev/tool/src/gmail.ts @@ -274,15 +274,15 @@ export async function loadAccounts (ws: WorkspaceInfoWithStatus): Promise<(Doc & let idx: number | undefined while (true) { - const info = await client.loadChunk(DOMAIN_MODEL_TX, idx) + const info = await client.loadChunk(ws.uuid, DOMAIN_MODEL_TX, idx) idx = info.idx const ids = Array.from(info.docs.map((it) => it.id as Ref)) - const docs = (await client.loadDocs(DOMAIN_MODEL_TX, ids)).filter((it) => + const docs = (await client.loadDocs(ws.uuid, DOMAIN_MODEL_TX, ids)).filter((it) => TxProcessor.isExtendsCUD(it._class) ) as TxCUD[] accountsTxes.push(...docs) if (info.finished && idx !== undefined) { - await client.closeChunk(info.idx) + await client.closeChunk(ws.uuid, info.idx) break } } diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 2f88e5939b5..c5e0d9b5734 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -36,6 +36,7 @@ import { } from '@hcengineering/server-backup' import serverClientPlugin, { getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client' import { + createBackupPipeline, registerAdapterFactory, registerDestroyFactory, registerServerPlugins, @@ -88,8 +89,12 @@ import { shutdownPostgres } from '@hcengineering/postgres' import { + createDummyStorageAdapter, QueueTopic, workspaceEvents, + wrapPipeline, + type Pipeline, + type PipelineFactory, type QueueWorkspaceMessage, type StorageAdapter } from '@hcengineering/server-core' @@ -1163,6 +1168,16 @@ export function devTool ( const workspaceStorage: StorageAdapter | undefined = storageConfig !== undefined ? buildStorageFromConfig(storageConfig) : undefined + + const { dbUrl, txes } = prepareTools() + + const pipelineFactory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { + externalStorage: workspaceStorage ?? createDummyStorageAdapter(), + usePassedCtx: true + }) + + let pipeline: Pipeline | undefined + await restore(toolCtx, await getWorkspaceTransactorEndpoint(workspace), wsIds, storage, { date: parseInt(date ?? '-1'), merge: cmd.merge, @@ -1171,7 +1186,13 @@ export function devTool ( include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';')), skip: new Set(cmd.skip.split(';')), storageAdapter: workspaceStorage, - historyFile: cmd.historyFile + historyFile: cmd.historyFile, + getConnection: async () => { + if (pipeline === undefined) { + pipeline = await pipelineFactory(toolCtx, wsIds, () => {}, null, null) + } + return wrapPipeline(toolCtx, pipeline, wsIds) + } }) const queue = getPlatformQueue('tool', ws.region) const wsProducer = queue.getProducer(toolCtx, QueueTopic.Workspace) @@ -1350,14 +1371,15 @@ export function devTool ( program .command('backup-s3-download ') + .option('-s, --skip ', 'A list of ; separated domain names to skip during backup', '') .description('Download a full backup from s3 to local dir') - .action(async (bucketName: string, dirName: string, storeIn: string, cmd) => { + .action(async (bucketName: string, dirName: string, storeIn: string, cmd: { skip: string }) => { const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) const backupIds = { uuid: bucketName as WorkspaceUuid, dataId: bucketName as WorkspaceDataId, url: '' } try { const storage = await createStorageBackupStorage(toolCtx, storageAdapter, backupIds, dirName) - await backupDownload(storage, storeIn) + await backupDownload(storage, storeIn, new Set(cmd.skip.split(';'))) } catch (err: any) { toolCtx.error('failed to size backup', { err }) } diff --git a/dev/tool/src/workspace.ts b/dev/tool/src/workspace.ts index 7d221913084..7a8b9c3ba85 100644 --- a/dev/tool/src/workspace.ts +++ b/dev/tool/src/workspace.ts @@ -111,7 +111,7 @@ export async function updateField ( if (cmd.type === 'boolean') valueToPut = cmd.value === 'true' setByPath(doc, cmd.attribute.split('.'), valueToPut) - await connection.upload(connection.getHierarchy().getDomain(doc?._class), [doc]) + await connection.upload(workspaceId, connection.getHierarchy().getDomain(doc?._class), [doc]) } finally { await connection.close() } diff --git a/models/board/src/migration.ts b/models/board/src/migration.ts index 1716aedac7c..e1efc8fa28a 100644 --- a/models/board/src/migration.ts +++ b/models/board/src/migration.ts @@ -14,7 +14,7 @@ // import { boardId, type Card } from '@hcengineering/board' -import { TxOperations } from '@hcengineering/core' +import { type TxOperations } from '@hcengineering/core' import { createOrUpdate, tryMigrate, @@ -96,8 +96,7 @@ export const boardOperation: MigrateOperation = { { state: 'board0001', func: async (client) => { - const ops = new TxOperations(client, core.account.System) - await createDefaults(ops) + await createDefaults(client) } } ]) diff --git a/models/card/src/migration.ts b/models/card/src/migration.ts index d7b2e90f3cc..7872b3dc830 100644 --- a/models/card/src/migration.ts +++ b/models/card/src/migration.ts @@ -13,19 +13,19 @@ // limitations under the License. // -import { type Card, cardId, DOMAIN_CARD } from '@hcengineering/card' -import core, { type Ref, TxOperations, type Client, type Data, type Doc } from '@hcengineering/core' +import { cardId, DOMAIN_CARD, type Card } from '@hcengineering/card' +import core, { type TxOperations, type Data, type Doc, type Ref } from '@hcengineering/core' import { + createOrUpdate, tryMigrate, tryUpgrade, type MigrateOperation, type MigrationClient, - type MigrationUpgradeClient, - createOrUpdate + type MigrationUpgradeClient } from '@hcengineering/model' +import tags from '@hcengineering/tags' import view from '@hcengineering/view' import card from '.' -import tags from '@hcengineering/tags' export const cardOperation: MigrateOperation = { async migrate (client: MigrationClient, mode): Promise { @@ -60,8 +60,7 @@ export const cardOperation: MigrateOperation = { { state: 'create-defaults', func: async (client) => { - const tx = new TxOperations(client, core.account.System) - await createDefaultProject(tx) + await createDefaultProject(client) } }, { @@ -77,13 +76,12 @@ export const cardOperation: MigrateOperation = { } } -async function fillParentInfo (client: Client): Promise { - const txOp = new TxOperations(client, core.account.System) +async function fillParentInfo (client: TxOperations): Promise { const cards = await client.findAll(card.class.Card, { parentInfo: { $exists: false }, parent: { $ne: null } }) const cache = new Map, Card>() for (const val of cards) { if (val.parent == null) continue - const parent = await getCardParentWithParentInfo(txOp, val.parent, cache) + const parent = await getCardParentWithParentInfo(client, val.parent, cache) if (parent !== undefined) { const parentInfo = [ ...(parent.parentInfo ?? []), @@ -93,7 +91,7 @@ async function fillParentInfo (client: Client): Promise { title: parent.title } ] - await txOp.update(val, { parentInfo }) + await client.update(val, { parentInfo }) val.parentInfo = parentInfo cache.set(val._id, val) } @@ -137,15 +135,14 @@ async function getCardParentWithParentInfo ( return doc } -async function removeVariantViewlets (client: Client): Promise { - const txOp = new TxOperations(client, core.account.System) +async function removeVariantViewlets (client: TxOperations): Promise { const desc = client .getHierarchy() .getDescendants(card.class.Card) .filter((c) => c !== card.class.Card) const viewlets = await client.findAll(view.class.Viewlet, { attachTo: { $in: desc }, variant: { $exists: true } }) for (const viewlet of viewlets) { - await txOp.remove(viewlet) + await client.remove(viewlet) } } @@ -173,8 +170,7 @@ function extractObjectData (doc: T): Data { return data as Data } -async function migrateViewlets (client: Client): Promise { - const txOp = new TxOperations(client, core.account.System) +async function migrateViewlets (client: TxOperations): Promise { const viewlets = await client.findAll(view.class.Viewlet, { attachTo: card.class.Card, variant: { $exists: false } }) const masterTags = await client.findAll(card.class.MasterTag, {}) const currentViewlets = await client.findAll(view.class.Viewlet, { attachTo: { $in: masterTags.map((p) => p._id) } }) @@ -199,13 +195,13 @@ async function migrateViewlets (client: Client): Promise { (p) => p.attachTo === masterTag._id && p.variant === viewlet.variant && p.descriptor === viewlet.descriptor ) if (current === undefined) { - await txOp.createDoc(view.class.Viewlet, core.space.Model, { + await client.createDoc(view.class.Viewlet, core.space.Model, { ...base, config: resConfig, attachTo: masterTag._id }) } else { - await txOp.diffUpdate(current, { + await client.diffUpdate(current, { ...base, config: resConfig, attachTo: masterTag._id @@ -258,10 +254,9 @@ async function migrateSpaces (client: MigrationClient): Promise { await client.update(DOMAIN_CARD, { space: core.space.Workspace }, { space: card.space.Default }) } -async function defaultLabels (client: Client): Promise { - const ops = new TxOperations(client, core.account.System) +async function defaultLabels (client: TxOperations): Promise { await createOrUpdate( - ops, + client, tags.class.TagCategory, core.space.Workspace, { @@ -275,7 +270,7 @@ async function defaultLabels (client: Client): Promise { ) await createOrUpdate( - ops, + client, tags.class.TagElement, core.space.Workspace, { @@ -289,7 +284,7 @@ async function defaultLabels (client: Client): Promise { ) await createOrUpdate( - ops, + client, tags.class.TagElement, core.space.Workspace, { diff --git a/models/chunter/src/migration.ts b/models/chunter/src/migration.ts index abfa2b41395..0b12963073b 100644 --- a/models/chunter/src/migration.ts +++ b/models/chunter/src/migration.ts @@ -13,16 +13,18 @@ // limitations under the License. // +import { type DocUpdateMessage } from '@hcengineering/activity' import { chunterId, type ThreadMessage } from '@hcengineering/chunter' +import contact, { getAllAccounts } from '@hcengineering/contact' import core, { - TxOperations, + DOMAIN_TX, + notEmpty, + type TxOperations, type Class, type Doc, type Domain, type Ref, - type Space, - DOMAIN_TX, - notEmpty + type Space } from '@hcengineering/core' import { tryMigrate, @@ -31,11 +33,9 @@ import { type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model' -import activity, { migrateMessagesSpace, DOMAIN_ACTIVITY } from '@hcengineering/model-activity' -import notification from '@hcengineering/notification' -import contact, { getAllAccounts } from '@hcengineering/contact' +import activity, { DOMAIN_ACTIVITY, migrateMessagesSpace } from '@hcengineering/model-activity' import { DOMAIN_DOC_NOTIFY, DOMAIN_NOTIFICATION } from '@hcengineering/model-notification' -import { type DocUpdateMessage } from '@hcengineering/activity' +import notification from '@hcengineering/notification' import { DOMAIN_CHUNTER } from './index' import chunter from './plugin' @@ -43,8 +43,7 @@ import chunter from './plugin' export const DOMAIN_COMMENT = 'comment' as Domain export async function createDocNotifyContexts ( - client: MigrationUpgradeClient, - tx: TxOperations, + client: TxOperations, objectId: Ref, objectClass: Ref>, objectSpace: Ref @@ -59,7 +58,7 @@ export async function createDocNotifyContexts ( const existingDNCUsers = new Set(docNotifyContexts.map((it) => it.user)) for (const account of accounts.filter((it) => !existingDNCUsers.has(it))) { - await tx.createDoc(notification.class.DocNotifyContext, core.space.Space, { + await client.createDoc(notification.class.DocNotifyContext, core.space.Space, { user: account, objectId, objectClass, @@ -70,22 +69,22 @@ export async function createDocNotifyContexts ( } } -export async function createGeneral (client: MigrationUpgradeClient, tx: TxOperations): Promise { - const current = await tx.findOne(chunter.class.Channel, { _id: chunter.space.General }) +export async function createGeneral (client: MigrationUpgradeClient): Promise { + const current = await client.findOne(chunter.class.Channel, { _id: chunter.space.General }) if (current !== undefined) { if (current.autoJoin === undefined) { - await tx.update(current, { + await client.update(current, { autoJoin: true }) - await joinEmployees(current, tx) + await joinEmployees(current, client) } } else { - const createTx = await tx.findOne(core.class.TxCreateDoc, { + const createTx = await client.findOne(core.class.TxCreateDoc, { objectId: chunter.space.General }) if (createTx === undefined) { - await tx.createDoc( + await client.createDoc( chunter.class.Channel, core.space.Space, { @@ -94,7 +93,7 @@ export async function createGeneral (client: MigrationUpgradeClient, tx: TxOpera topic: 'General Channel', private: false, archived: false, - members: await getAllAccounts(tx), + members: await getAllAccounts(client), autoJoin: true }, chunter.space.General @@ -102,7 +101,7 @@ export async function createGeneral (client: MigrationUpgradeClient, tx: TxOpera } } - await createDocNotifyContexts(client, tx, chunter.space.General, chunter.class.Channel, core.space.Space) + await createDocNotifyContexts(client, chunter.space.General, chunter.class.Channel, core.space.Space) } async function joinEmployees (current: Space, tx: TxOperations): Promise { @@ -120,22 +119,22 @@ async function joinEmployees (current: Space, tx: TxOperations): Promise { }) } -export async function createRandom (client: MigrationUpgradeClient, tx: TxOperations): Promise { - const current = await tx.findOne(chunter.class.Channel, { _id: chunter.space.Random }) +export async function createRandom (client: MigrationUpgradeClient): Promise { + const current = await client.findOne(chunter.class.Channel, { _id: chunter.space.Random }) if (current !== undefined) { if (current.autoJoin === undefined) { - await tx.update(current, { + await client.update(current, { autoJoin: true }) - await joinEmployees(current, tx) + await joinEmployees(current, client) } } else { - const createTx = await tx.findOne(core.class.TxCreateDoc, { + const createTx = await client.findOne(core.class.TxCreateDoc, { objectId: chunter.space.Random }) if (createTx === undefined) { - await tx.createDoc( + await client.createDoc( chunter.class.Channel, core.space.Space, { @@ -144,7 +143,7 @@ export async function createRandom (client: MigrationUpgradeClient, tx: TxOperat topic: 'Random Talks', private: false, archived: false, - members: await getAllAccounts(tx), + members: await getAllAccounts(client), autoJoin: true }, chunter.space.Random @@ -152,7 +151,7 @@ export async function createRandom (client: MigrationUpgradeClient, tx: TxOperat } } - await createDocNotifyContexts(client, tx, chunter.space.Random, chunter.class.Channel, core.space.Space) + await createDocNotifyContexts(client, chunter.space.Random, chunter.class.Channel, core.space.Space) } async function convertCommentsToChatMessages (client: MigrationClient): Promise { @@ -313,9 +312,8 @@ export const chunterOperation: MigrateOperation = { { state: 'create-defaults-v2', func: async (client) => { - const tx = new TxOperations(client, core.account.System) - await createGeneral(client, tx) - await createRandom(client, tx) + await createGeneral(client) + await createRandom(client) } } ]) diff --git a/models/contact/src/migration.ts b/models/contact/src/migration.ts index 82b54a4df2d..6121e7d2d27 100644 --- a/models/contact/src/migration.ts +++ b/models/contact/src/migration.ts @@ -261,6 +261,7 @@ async function createSocialIdentities (client: MigrationClient): Promise { if (socialId == null) continue const socialIdObj: SocialIdentity = { + _uuid: client.wsIds.uuid, _id: socialId as SocialIdentityRef, _class: contact.class.SocialIdentity, space: contact.space.Contacts, @@ -428,6 +429,7 @@ async function createUserProfiles (client: MigrationClient): Promise { const title = d.name != null && d.name !== '' ? formatName(d.name) : 'Profile' const userProfile: UserProfile = { + _uuid: client.wsIds.uuid, _id: generateId(), _class: contact.class.UserProfile, space: contact.space.Contacts, diff --git a/models/controlled-documents/src/migration.ts b/models/controlled-documents/src/migration.ts index 871cbe5d9f9..de71cadadf0 100644 --- a/models/controlled-documents/src/migration.ts +++ b/models/controlled-documents/src/migration.ts @@ -35,7 +35,7 @@ import { type Ref, SortingOrder, toIdMap, - TxOperations + type TxOperations } from '@hcengineering/core' import { createDefaultSpace, @@ -558,14 +558,13 @@ export const documentsOperation: MigrateOperation = { { state: 'init-documents', func: async (client) => { - const tx = new TxOperations(client, core.account.System) await createDefaultSpace(client, documents.space.Documents, { name: 'Documents', description: 'Documents' }) - await createQualityDocumentsSpace(tx) - await createTemplatesSpace(tx) - await createTemplateSequence(tx) - await createTagCategories(tx) - await createDocumentCategories(tx) - await createProductChangeControlTemplate(tx) + await createQualityDocumentsSpace(client) + await createTemplatesSpace(client) + await createTemplateSequence(client) + await createTagCategories(client) + await createDocumentCategories(client) + await createProductChangeControlTemplate(client) } } ]) diff --git a/models/core/src/core.ts b/models/core/src/core.ts index 5e837923886..79fd446afa8 100644 --- a/models/core/src/core.ts +++ b/models/core/src/core.ts @@ -54,7 +54,8 @@ import { type Type, type TypeAny, type Version, - DOMAIN_SEQUENCE + DOMAIN_SEQUENCE, + type WorkspaceUuid } from '@hcengineering/core' import { Hidden, @@ -87,6 +88,8 @@ export class TObj implements Obj { @Model(core.class.Doc, core.class.Obj) @UX(core.string.Object) export class TDoc extends TObj implements Doc { + declare _uuid: WorkspaceUuid + @Prop(TypeRef(core.class.Doc), core.string.Id) @Hidden() // @Index(IndexKind.Indexed) // - automatically indexed by default. diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 1b7eccc7d11..39a44f76bf7 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -86,6 +86,7 @@ async function migrateStatusesToModel (client: MigrationClient, mode: MigrateMod : status.modifiedBy const tx: TxCreateDoc = { + _uuid: client.wsIds.uuid, _id: generateId(), _class: core.class.TxCreateDoc, space: core.space.Tx, @@ -581,6 +582,7 @@ async function migrateAccounts (client: MigrationClient): Promise { accountUuidBySocialKey ) const tx: TxUpdateDoc = { + _uuid: client.wsIds.uuid, _id: generateId(), _class: core.class.TxUpdateDoc, space: core.space.Tx, diff --git a/models/core/src/tx.ts b/models/core/src/tx.ts index f688761ecfd..158a5d9cd4e 100644 --- a/models/core/src/tx.ts +++ b/models/core/src/tx.ts @@ -49,9 +49,6 @@ export class TTx extends TDoc implements Tx { objectSpace!: Ref } -@Model(core.class.TxModelUpgrade, core.class.Tx, DOMAIN_TX) -export class TTxModelUpgrade extends TTx {} - @Model(core.class.TxCUD, core.class.Tx) export class TTxCUD extends TTx implements TxCUD { @Prop(TypeRef(core.class.Doc), core.string.Object) diff --git a/models/drive/src/migration.ts b/models/drive/src/migration.ts index 317028c4178..2d0f6c89ba0 100644 --- a/models/drive/src/migration.ts +++ b/models/drive/src/migration.ts @@ -49,6 +49,7 @@ async function migrateFileVersions (client: MigrationClient): Promise { const fileVersionId: Ref = generateId() await client.create(DOMAIN_DRIVE, { + _uuid: client.wsIds.uuid, _id: fileVersionId, _class: drive.class.FileVersion, attachedTo: file._id, diff --git a/models/hr/src/migration.ts b/models/hr/src/migration.ts index a0fa8fe9e8a..c0f95b4fef0 100644 --- a/models/hr/src/migration.ts +++ b/models/hr/src/migration.ts @@ -15,7 +15,7 @@ import { type Space, - TxOperations, + type TxOperations, type Ref, type Class, type Doc, @@ -128,8 +128,7 @@ export const hrOperation: MigrateOperation = { { state: 'create-defaults-v2', func: async (client) => { - const tx = new TxOperations(client, core.account.System) - await createDepartment(tx) + await createDepartment(client) } } ]) diff --git a/models/lead/src/migration.ts b/models/lead/src/migration.ts index 8033b862b08..134ded16be8 100644 --- a/models/lead/src/migration.ts +++ b/models/lead/src/migration.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { DOMAIN_MODEL_TX, TxOperations, type Ref, type Status } from '@hcengineering/core' +import { DOMAIN_MODEL_TX, type TxOperations, type Ref, type Status } from '@hcengineering/core' import { leadId, type Lead } from '@hcengineering/lead' import { tryMigrate, @@ -193,8 +193,7 @@ export const leadOperation: MigrateOperation = { { state: 'u-default-funnel', func: async (client) => { - const ops = new TxOperations(client, core.account.System) - await createDefaults(ops) + await createDefaults(client) } } ]) diff --git a/models/love/src/migration.ts b/models/love/src/migration.ts index 3d2a634fddd..fc4e710258f 100644 --- a/models/love/src/migration.ts +++ b/models/love/src/migration.ts @@ -14,7 +14,7 @@ // import contact from '@hcengineering/contact' -import { TxOperations, type Ref, type Space } from '@hcengineering/core' +import { type TxOperations, type Ref, type Space } from '@hcengineering/core' import drive from '@hcengineering/drive' import { MeetingStatus, @@ -55,26 +55,24 @@ async function createDefaultFloor (tx: TxOperations): Promise { } async function createRooms (client: MigrationUpgradeClient): Promise { - const tx = new TxOperations(client, core.account.System) const rooms = await client.findAll(love.class.Room, {}) for (const room of rooms) { - await tx.remove(room) + await client.remove(room) } const employees = await client.findAll(contact.mixin.Employee, { active: true }) const data = createDefaultRooms(employees.map((p) => p._id)) for (const room of data) { const _class = isOffice(room) ? love.class.Office : love.class.Room - await tx.createDoc(_class, core.space.Workspace, room, room._id) + await client.createDoc(_class, core.space.Workspace, room, room._id) } } async function createReception (client: MigrationUpgradeClient): Promise { - const tx = new TxOperations(client, core.account.System) - const current = await tx.findOne(love.class.Room, { + const current = await client.findOne(love.class.Room, { _id: love.ids.Reception }) if (current !== undefined) return - await tx.createDoc( + await client.createDoc( love.class.Room, core.space.Workspace, { @@ -182,8 +180,7 @@ export const loveOperation: MigrateOperation = { { state: 'initial-defaults', func: async (client) => { - const tx = new TxOperations(client, core.account.System) - await createDefaultFloor(tx) + await createDefaultFloor(client) } }, { diff --git a/models/process/src/migration.ts b/models/process/src/migration.ts index d26c1a93c7e..dbfc9632a87 100644 --- a/models/process/src/migration.ts +++ b/models/process/src/migration.ts @@ -13,12 +13,12 @@ // limitations under the License. // -import core, { type Ref, TxOperations } from '@hcengineering/core' +import { type Ref } from '@hcengineering/core' import { - tryUpgrade, type MigrateOperation, type MigrationClient, - type MigrationUpgradeClient + type MigrationUpgradeClient, + tryUpgrade } from '@hcengineering/model' import { type Func, parseContext, type ProcessFunction } from '@hcengineering/process' import { processId } from '.' @@ -60,7 +60,6 @@ function getContext (value: string): string | undefined { } async function migrateStateFuncs (client: MigrationUpgradeClient): Promise { - const txOp = new TxOperations(client, core.account.System) const states = await client.findAll(process.class.State, {}) for (const state of states) { let changed = false @@ -76,7 +75,7 @@ async function migrateStateFuncs (client: MigrationUpgradeClient): Promise } } if (changed) { - await txOp.updateDoc(state._class, state.space, state._id, { actions }) + await client.updateDoc(state._class, state.space, state._id, { actions }) } } } diff --git a/models/recruit/src/migration.ts b/models/recruit/src/migration.ts index aaebfabcb3c..4ad38bd147b 100644 --- a/models/recruit/src/migration.ts +++ b/models/recruit/src/migration.ts @@ -17,7 +17,7 @@ import { getCategories } from '@anticrm/skillset' import core, { DOMAIN_MODEL_TX, toIdMap, - TxOperations, + type TxOperations, type Doc, type Ref, type Space, @@ -88,8 +88,7 @@ export const recruitOperation: MigrateOperation = { { state: 'create-defaults-v2', func: async (client) => { - const tx = new TxOperations(client, core.account.System) - await createDefaults(client, tx) + await createDefaults(client, client) } } ]) diff --git a/models/server-activity/src/migration.ts b/models/server-activity/src/migration.ts index f2270f57ef5..27d56855157 100644 --- a/models/server-activity/src/migration.ts +++ b/models/server-activity/src/migration.ts @@ -45,7 +45,7 @@ import { import { generateDocUpdateMessages } from '@hcengineering/server-activity-resources' function getActivityControl (client: MigrationClient): ActivityControl { - const txFactory = new TxFactory(core.account.System, false) + const txFactory = new TxFactory(core.account.System, client.wsIds.uuid, false) return { ctx: new MeasureMetricsContext('migration', {}), @@ -91,6 +91,7 @@ async function generateDocUpdateMessageByTx ( const domain = client.hierarchy.getDomain(createTx.objectClass) await client.create(domain, { + _uuid: client.wsIds.uuid, _id: createTx.objectId, _class: createTx.objectClass, space: createTx.objectSpace, diff --git a/models/task/src/migration.ts b/models/task/src/migration.ts index 3a8c7641553..404e063f35e 100644 --- a/models/task/src/migration.ts +++ b/models/task/src/migration.ts @@ -19,7 +19,7 @@ import { DOMAIN_SEQUENCE, DOMAIN_STATUS, DOMAIN_TX, - TxOperations, + type TxOperations, toIdMap, type Attribute, type Class, @@ -602,10 +602,8 @@ export const taskOperation: MigrateOperation = { { state: 'u-task-001', func: async (client) => { - const tx = new TxOperations(client, core.account.System) - await createOrUpdate( - tx, + client, tags.class.TagCategory, core.space.Workspace, { diff --git a/models/templates/src/migration.ts b/models/templates/src/migration.ts index 01ea9245c84..dea9a59243f 100644 --- a/models/templates/src/migration.ts +++ b/models/templates/src/migration.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import core, { TxOperations } from '@hcengineering/core' +import core from '@hcengineering/core' import { tryUpgrade, type MigrateOperation, @@ -30,12 +30,11 @@ export const templatesOperation: MigrateOperation = { { state: 'create-defaults', func: async (client) => { - const tx = new TxOperations(client, core.account.System) - const current = await tx.findOne(core.class.Space, { + const current = await client.findOne(core.class.Space, { _id: templates.space.Templates }) if (current === undefined) { - await tx.createDoc( + await client.createDoc( templates.class.TemplateCategory, core.space.Space, { @@ -49,9 +48,9 @@ export const templatesOperation: MigrateOperation = { templates.space.Templates ) } else if (current.private) { - await tx.update(current, { private: false }) + await client.update(current, { private: false }) } else if (current.autoJoin !== true) { - await tx.update(current, { autoJoin: true }) + await client.update(current, { autoJoin: true }) } } } diff --git a/models/time/src/migration.ts b/models/time/src/migration.ts index 4c078f7bfaf..74db3d966b4 100644 --- a/models/time/src/migration.ts +++ b/models/time/src/migration.ts @@ -13,15 +13,14 @@ // limitations under the License. // -import { TxOperations } from '@hcengineering/core' import { type MigrateOperation, type MigrationClient, type MigrationUpgradeClient, + createDefaultSpace, createOrUpdate, tryMigrate, - tryUpgrade, - createDefaultSpace + tryUpgrade } from '@hcengineering/model' import core from '@hcengineering/model-core' import tags from '@hcengineering/tags' @@ -66,9 +65,8 @@ export const timeOperation: MigrateOperation = { { state: 'u-time-0001', func: async (client) => { - const tx = new TxOperations(client, core.account.System) await createOrUpdate( - tx, + client, tags.class.TagCategory, core.space.Workspace, { diff --git a/models/tracker/src/migration.ts b/models/tracker/src/migration.ts index ac2f219eaac..b6294744289 100644 --- a/models/tracker/src/migration.ts +++ b/models/tracker/src/migration.ts @@ -20,7 +20,7 @@ import core, { type Ref, type Status, type TxCreateDoc, - TxOperations, + type TxOperations, generateId, toIdMap } from '@hcengineering/core' @@ -260,6 +260,7 @@ async function migrateStatusesToModel (client: MigrationClient): Promise { : status.modifiedBy const tx: TxCreateDoc = { + _uuid: client.wsIds.uuid, _id: generateId(), _class: core.class.TxCreateDoc, space: core.space.Tx, @@ -404,8 +405,7 @@ export const trackerOperation: MigrateOperation = { { state: 'create-defaults', func: async (client) => { - const tx = new TxOperations(client, core.account.System) - await createDefaults(tx) + await createDefaults(client) } } ]) diff --git a/models/training/src/migration.ts b/models/training/src/migration.ts index 507640d29cd..ed86741db8d 100644 --- a/models/training/src/migration.ts +++ b/models/training/src/migration.ts @@ -47,9 +47,8 @@ export const trainingOperation: MigrateOperation = { { state: 'create-defaults', func: async (client) => { - const tx = new TxOperations(client, core.account.System) - await ensureTypedSpace(tx) - await ensureSequence(tx) + await ensureTypedSpace(client) + await ensureSequence(client) } } ]) diff --git a/packages/account-client/src/client.ts b/packages/account-client/src/client.ts index 21dda058f9f..1c716c23a62 100644 --- a/packages/account-client/src/client.ts +++ b/packages/account-client/src/client.ts @@ -59,6 +59,7 @@ export interface AccountClient { getUserWorkspaces: () => Promise selectWorkspace: ( workspaceUrl: string, + singleWorkspace?: boolean, kind?: 'external' | 'internal' | 'byregion', externalRegions?: string[] ) => Promise @@ -283,12 +284,13 @@ class AccountClientImpl implements AccountClient { async selectWorkspace ( workspaceUrl: string, + singleWorkspace?: boolean, kind: 'external' | 'internal' | 'byregion' = 'external', externalRegions: string[] = [] ): Promise { const request = { method: 'selectWorkspace' as const, - params: { workspaceUrl, kind, externalRegions } + params: { workspaceUrl, kind, externalRegions, singleWorkspace: singleWorkspace ?? true } } return await this.rpc(request) diff --git a/packages/account-client/src/types.ts b/packages/account-client/src/types.ts index 42a63a17ced..853c6259186 100644 --- a/packages/account-client/src/types.ts +++ b/packages/account-client/src/types.ts @@ -7,7 +7,8 @@ import { type Timestamp, type SocialId as SocialIdBase, PersonUuid, - type WorkspaceMode + type WorkspaceMode, + type EndpointInfo } from '@hcengineering/core' export interface LoginInfo { @@ -16,12 +17,6 @@ export interface LoginInfo { socialId?: PersonId token?: string } - -export interface EndpointInfo { - internalUrl: string - externalUrl: string - region: string -} export interface WorkspaceVersion { versionMajor: number versionMinor: number @@ -30,6 +25,7 @@ export interface WorkspaceVersion { export interface LoginInfoWorkspace { url: string + name?: string dataId?: WorkspaceDataId mode: WorkspaceMode version: WorkspaceVersion @@ -39,6 +35,7 @@ export interface LoginInfoWorkspace { } export interface LoginInfoWithWorkspaces extends LoginInfo { + personalWorkspace: WorkspaceUuid // Information necessary to handle user <--> transactor connectivity. workspaces: Record socialIds: SocialId[] @@ -48,6 +45,7 @@ export interface LoginInfoWithWorkspaces extends LoginInfo { * @public */ export interface WorkspaceLoginInfo extends LoginInfo { + personalWorkspace: WorkspaceUuid // personal workspace uuid, could be core.workspace.System in case of system account. workspace: WorkspaceUuid // worspace uuid workspaceDataId?: WorkspaceDataId workspaceUrl: string diff --git a/packages/api-client/src/client.ts b/packages/api-client/src/client.ts index db9bf10d44c..70083f95676 100644 --- a/packages/api-client/src/client.ts +++ b/packages/api-client/src/client.ts @@ -105,7 +105,7 @@ class PlatformClientImpl implements PlatformClient { private readonly connection: Client, private readonly user: PersonId ) { - this.client = new TxOperations(connection, user) + this.client = new TxOperations(connection, user, workspace) this.markup = createMarkupOperations(url, workspace, token, config) } diff --git a/packages/api-client/src/rest/rest.ts b/packages/api-client/src/rest/rest.ts index 075fa1c8dc4..86733916ea2 100644 --- a/packages/api-client/src/rest/rest.ts +++ b/packages/api-client/src/rest/rest.ts @@ -34,7 +34,8 @@ import { SocialIdType, type Tx, type TxResult, - type WithLookup + type WithLookup, + type WorkspaceUuid } from '@hcengineering/core' import { PlatformError, type Status, unknownError } from '@hcengineering/platform' @@ -43,7 +44,7 @@ import { getWorkspaceToken } from '../utils' import type { RestClient } from './types' import { extractJson, withRetry } from './utils' -export function createRestClient (endpoint: string, workspaceId: string, token: string): RestClient { +export function createRestClient (endpoint: string, workspaceId: WorkspaceUuid, token: string): RestClient { return new RestClientImpl(endpoint, workspaceId, token) } @@ -68,7 +69,7 @@ export class RestClientImpl implements RestClient { limit: number = 1000 constructor ( endpoint: string, - readonly workspace: string, + readonly workspace: WorkspaceUuid, readonly token: string ) { this.endpoint = endpoint.replace('ws', 'http') @@ -118,7 +119,7 @@ export class RestClientImpl implements RestClient { throw new PlatformError(result.error) } - if (result.lookupMap !== undefined) { + if (result.lookupMap != null) { // We need to extract lookup map to document lookups for (const d of result) { if (d.$lookup !== undefined) { diff --git a/packages/api-client/src/rest/tx.ts b/packages/api-client/src/rest/tx.ts index 46f365c059e..2054147c7ec 100644 --- a/packages/api-client/src/rest/tx.ts +++ b/packages/api-client/src/rest/tx.ts @@ -15,6 +15,7 @@ import { type Account, + type AccountWorkspace, type Class, type Client, type Doc, @@ -31,13 +32,14 @@ import { type Tx, TxOperations, type TxResult, - type WithLookup + type WithLookup, + type WorkspaceUuid } from '@hcengineering/core' import { RestClientImpl } from './rest' export async function createRestTxOperations ( endpoint: string, - workspaceId: string, + workspaceId: WorkspaceUuid, token: string ): Promise { const restClient = new RestClientImpl(endpoint, workspaceId, token) @@ -45,21 +47,36 @@ export async function createRestTxOperations ( const account = await restClient.getAccount() const { hierarchy, model } = await restClient.getModel() - return new TxOperations(new RestTxClient(restClient, hierarchy, model, account), account.socialIds[0]) + return new TxOperations(new RestTxClient(restClient, hierarchy, model, account), account.socialIds[0], workspaceId) } class RestTxClient implements Client { + hierarchy: Hierarchy + model: ModelDb constructor ( readonly client: RestClientImpl, - readonly hierarchy: Hierarchy, - readonly model: ModelDb, + _hierarchy: Hierarchy, + _model: ModelDb, readonly account: Account - ) {} + ) { + this.hierarchy = _hierarchy + this.model = _model + } close (): Promise { return Promise.resolve() } + getAvailableWorkspaces (): WorkspaceUuid[] { + return Array.from(Object.entries(this.account.workspaces)) + .filter(([, v]) => !v.maintenance && v.enabled) + .map(([it]) => it as WorkspaceUuid) + } + + getWorkspaces (): Record { + return this.account.workspaces + } + async findAll( _class: Ref>, query: DocumentQuery, diff --git a/packages/api-client/src/storage/client.ts b/packages/api-client/src/storage/client.ts index e0c068a6cf1..0bb2b82ddaa 100644 --- a/packages/api-client/src/storage/client.ts +++ b/packages/api-client/src/storage/client.ts @@ -75,6 +75,7 @@ export class StorageClientImpl implements StorageClient { const lastModified = Date.parse(headers.get('Last-Modified') ?? '') const size = parseInt(headers.get('Content-Length') ?? '0', 10) return { + _uuid: this.workspace, provider: '', _class: core.class.Blob, _id: objectName as Ref, @@ -121,6 +122,7 @@ export class StorageClientImpl implements StorageClient { if (Object.hasOwn(result[0], 'id')) { const fileResult = result[0] as BlobUploadSuccess return { + _uuid: this.workspace, _class: core.class.Blob, _id: fileResult.id as Ref, space: core.space.Configuration, diff --git a/packages/core/src/__tests__/client.test.ts b/packages/core/src/__tests__/client.test.ts index 1695232430d..fc18f2f13a5 100644 --- a/packages/core/src/__tests__/client.test.ts +++ b/packages/core/src/__tests__/client.test.ts @@ -14,10 +14,10 @@ // limitations under the License. // import { type IntlString, type Plugin } from '@hcengineering/platform' -import { ClientConnectEvent, type DocChunk } from '..' -import type { Class, Data, Doc, Domain, PluginConfiguration, Ref, Space, Timestamp } from '../classes' +import { ClientConnectEvent, type DocChunk, systemAccount, type WorkspaceUuid } from '..' import { ClassifierKind, DOMAIN_MODEL } from '../classes' -import { type ClientConnection, createClient } from '../client' +import type { Account, Class, Data, Doc, PluginConfiguration, Ref, Timestamp, type Space } from '../classes' +import { type ClientConnection, createClient, type SubscribedWorkspaceInfo } from '../client' import core from '../component' import { Hierarchy } from '../hierarchy' import { ModelDb, TxDb } from '../memdb' @@ -41,7 +41,7 @@ function filterPlugin (plugin: Plugin): (txes: Tx[]) => Tx[] { describe('client', () => { it('should create client and spaces', async () => { const klass = core.class.Space - const client = new TxOperations(await createClient(connect), core.account.System) + const client = new TxOperations(await createClient(connect), core.account.System, core.workspace.Model) const result = await client.findAll(klass, {}) expect(result).toHaveLength(2) @@ -70,7 +70,7 @@ describe('client', () => { }) it('should create client with plugins', async () => { - const txFactory = new TxFactory(core.account.System) + const txFactory = new TxFactory(core.account.System, core.workspace.Model) const txes = genMinModel() txes.push( @@ -87,7 +87,7 @@ describe('client', () => { ) ) - async function connectPlugin (handler: (tx: Tx) => void): Promise { + async function connectPlugin (handler: (tx: Tx[]) => void): Promise { const hierarchy = new Hierarchy() for (const tx of txes) hierarchy.tx(tx) @@ -104,21 +104,39 @@ describe('client', () => { } return new (class implements ClientConnection { - handler?: (event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise + handler?: ( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise set onConnect ( - handler: ((event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise) | undefined + handler: + | (( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise) + | undefined ) { this.handler = handler - void this.handler?.(ClientConnectEvent.Connected, '', {}) + void this.handler?.(ClientConnectEvent.Connected, {}, {}) } get onConnect (): - | ((event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise) + | (( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise) | undefined { return this.handler } + async getAccount (): Promise { + return systemAccount + } + isConnected = (): boolean => true findAll = findAll pushHandler = (): void => {} @@ -137,27 +155,33 @@ describe('client', () => { close = async (): Promise => {} - loadChunk = async (domain: Domain, idx?: number): Promise => ({ + loadChunk = async (): Promise => ({ idx: -1, docs: [], finished: true }) - async getDomainHash (domain: Domain): Promise { + async getDomainHash (): Promise { return generateId() } - async closeChunk (idx: number): Promise {} - async loadDocs (domain: Domain, docs: Ref[]): Promise { + async closeChunk (): Promise {} + async loadDocs (): Promise { return [] } - async upload (domain: Domain, docs: Doc[]): Promise {} - async clean (domain: Domain, docs: Ref[]): Promise {} + async upload (): Promise {} + async clean (): Promise {} async loadModel (last: Timestamp): Promise { return txes } + async subscribe (): Promise { + return {} + } + + async unsubscribe (): Promise {} + async sendForceClose (): Promise {} })() } @@ -173,8 +197,9 @@ describe('client', () => { const txCreateDoc1 = txFactory.createTxCreateDoc(core.class.PluginConfiguration, core.space.Model, pluginData1) txes.push(txCreateDoc1) const client1 = new TxOperations( - await createClient(connectPlugin, filterPlugin('testPlugin1' as Plugin)), - core.account.System + await createClient(connectPlugin, { modelFilter: filterPlugin('testPlugin1' as Plugin) }), + core.account.System, + core.workspace.Model ) const result1 = await client1.findAll(core.class.PluginConfiguration, {}) @@ -193,8 +218,9 @@ describe('client', () => { const txCreateDoc2 = txFactory.createTxCreateDoc(core.class.PluginConfiguration, core.space.Model, pluginData2) txes.push(txCreateDoc2) const client2 = new TxOperations( - await createClient(connectPlugin, filterPlugin('testPlugin1' as Plugin)), - core.account.System + await createClient(connectPlugin, { modelFilter: filterPlugin('testPlugin1' as Plugin) }), + core.account.System, + core.workspace.Model ) const result2 = await client2.findAll(core.class.PluginConfiguration, {}) @@ -219,8 +245,9 @@ describe('client', () => { ) txes.push(txUpdateDoc) const client3 = new TxOperations( - await createClient(connectPlugin, filterPlugin('testPlugin2' as Plugin)), - core.account.System + await createClient(connectPlugin, { modelFilter: filterPlugin('testPlugin2' as Plugin) }), + core.account.System, + core.workspace.Model ) const result3 = await client3.findAll(core.class.PluginConfiguration, {}) diff --git a/packages/core/src/__tests__/connection.ts b/packages/core/src/__tests__/connection.ts index 69d704a2333..f8c264c076c 100644 --- a/packages/core/src/__tests__/connection.ts +++ b/packages/core/src/__tests__/connection.ts @@ -13,9 +13,9 @@ // limitations under the License. // -import { ClientConnectEvent, type DocChunk, generateId } from '..' -import type { Class, Doc, Domain, Ref, Timestamp } from '../classes' -import { type ClientConnection } from '../client' +import { ClientConnectEvent, type DocChunk, generateId, systemAccount, type WorkspaceUuid } from '..' +import type { Account, Class, Doc, Domain, Ref, Timestamp } from '../classes' +import { type ClientConnection, type SubscribedWorkspaceInfo } from '../client' import core from '../component' import { Hierarchy } from '../hierarchy' import { ModelDb, TxDb } from '../memdb' @@ -24,7 +24,7 @@ import type { Tx } from '../tx' import { DOMAIN_TX } from '../tx' import { genMinModel } from './minmodel' -export async function connect (handler: (tx: Tx) => void): Promise { +export async function connect (handler: (tx: Tx[]) => void): Promise { const txes = genMinModel() const hierarchy = new Hierarchy() @@ -47,19 +47,45 @@ export async function connect (handler: (tx: Tx) => void): Promise true findAll = findAll - handler?: (event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise + handler?: ( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise + + async subscribe (): Promise { + return {} + } + + async unsubscribe (): Promise {} set onConnect ( - handler: ((event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise) | undefined + handler: + | (( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise) + | undefined ) { this.handler = handler - void this.handler?.(ClientConnectEvent.Connected, '', {}) + void this.handler?.(ClientConnectEvent.Connected, {}, {}) } - get onConnect (): ((event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise) | undefined { + get onConnect (): + | (( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise) + | undefined { return this.handler } + async getAccount (): Promise { + return systemAccount + } + async searchFulltext (query: SearchQuery, options: SearchOptions): Promise { return { docs: [] } } @@ -76,7 +102,7 @@ export async function connect (handler: (tx: Tx) => void): Promise {} - async loadChunk (domain: Domain, idx?: number): Promise { + async loadChunk (workspaceId: WorkspaceUuid, domain: Domain, idx?: number): Promise { return { idx: -1, docs: [], @@ -84,17 +110,17 @@ export async function connect (handler: (tx: Tx) => void): Promise { + async getDomainHash (workspaceId: WorkspaceUuid, domain: Domain): Promise { return generateId() } - async closeChunk (idx: number): Promise {} - async loadDocs (domain: Domain, docs: Ref[]): Promise { + async closeChunk (workspaceId: WorkspaceUuid, idx: number): Promise {} + async loadDocs (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { return [] } - async upload (domain: Domain, docs: Doc[]): Promise {} - async clean (domain: Domain, docs: Ref[]): Promise {} + async upload (workspaceId: WorkspaceUuid, domain: Domain, docs: Doc[]): Promise {} + async clean (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise {} async loadModel (last: Timestamp): Promise { return txes } diff --git a/packages/core/src/__tests__/memdb.test.ts b/packages/core/src/__tests__/memdb.test.ts index 33333d2b130..aca3c23eb84 100644 --- a/packages/core/src/__tests__/memdb.test.ts +++ b/packages/core/src/__tests__/memdb.test.ts @@ -13,8 +13,8 @@ // limitations under the License. // -import { type Client } from '..' -import type { Class, Doc, Obj, Ref } from '../classes' +import { type Client, type WorkspaceUuid } from '..' +import type { AccountWorkspace, Class, Doc, Obj, Ref } from '../classes' import core from '../component' import { Hierarchy } from '../hierarchy' import { ModelDb, TxDb } from '../memdb' @@ -22,11 +22,11 @@ import { TxOperations } from '../operations' import { type DocumentQuery, type FindOptions, - SortingOrder, - type WithLookup, - type SearchQuery, type SearchOptions, - type SearchResult + type SearchQuery, + type SearchResult, + SortingOrder, + type WithLookup } from '../storage' import { type Tx } from '../tx' import { genMinModel, test, type TestMixin } from './minmodel' @@ -34,7 +34,7 @@ import { genMinModel, test, type TestMixin } from './minmodel' const txes = genMinModel() class ClientModel extends ModelDb implements Client { - notify?: ((...tx: Tx[]) => void) | undefined + notify?: ((tx: Tx[]) => void) | undefined getHierarchy (): Hierarchy { return this.hierarchy @@ -44,6 +44,9 @@ class ClientModel extends ModelDb implements Client { return this } + getWorkspaces: () => Record = () => ({}) + getAvailableWorkspaces: () => WorkspaceUuid[] = () => [] + async findOne( _class: Ref>, query: DocumentQuery, @@ -84,7 +87,7 @@ describe('memdb', () => { it('should create space', async () => { const { model } = await createModel() - const client = new TxOperations(model, core.account.System) + const client = new TxOperations(model, core.account.System, core.workspace.Model) const result = await client.findAll(core.class.Space, {}) expect(result).toHaveLength(2) @@ -126,7 +129,7 @@ describe('memdb', () => { it('should create mixin', async () => { const { model } = await createModel() - const ops = new TxOperations(model, core.account.System) + const ops = new TxOperations(model, core.account.System, core.workspace.Model) await ops.createMixin(core.class.Obj, core.class.Class, core.space.Model, test.mixin.TestMixin, { arr: ['hello'] @@ -140,7 +143,7 @@ describe('memdb', () => { const result = await model.findAll(core.class.Space, {}) expect(result.length).toBe(2) - const ops = new TxOperations(model, core.account.System) + const ops = new TxOperations(model, core.account.System, core.workspace.Model) await ops.removeDoc(result[0]._class, result[0].space, result[0]._id) const result2 = await model.findAll(core.class.Space, {}) expect(result2).toHaveLength(1) @@ -249,7 +252,7 @@ describe('memdb', () => { it('limit and sorting', async () => { const hierarchy = new Hierarchy() for (const tx of txes) hierarchy.tx(tx) - const model = new TxOperations(new ClientModel(hierarchy), core.account.System) + const model = new TxOperations(new ClientModel(hierarchy), core.account.System, core.workspace.Model) for (const tx of txes) await model.tx(tx) const without = await model.findAll(core.class.Space, {}) @@ -274,7 +277,7 @@ describe('memdb', () => { it('should add attached document', async () => { const { model } = await createModel() - const client = new TxOperations(model, core.account.System) + const client = new TxOperations(model, core.account.System, core.workspace.Model) const result = await client.findAll(core.class.Space, {}) expect(result).toHaveLength(2) @@ -288,7 +291,7 @@ describe('memdb', () => { it('lookups', async () => { const { model } = await createModel() - const client = new TxOperations(model, core.account.System) + const client = new TxOperations(model, core.account.System, core.workspace.Model) const spaces = await client.findAll(core.class.Space, {}) expect(spaces).toHaveLength(2) @@ -343,7 +346,7 @@ describe('memdb', () => { it('mixin lookups', async () => { const { model } = await createModel() - const client = new TxOperations(model, core.account.System) + const client = new TxOperations(model, core.account.System, core.workspace.Model) const spaces = await client.findAll(core.class.Space, {}) expect(spaces).toHaveLength(2) @@ -379,7 +382,7 @@ describe('memdb', () => { expect.assertions(1) const { model } = await createModel() - const client = new TxOperations(model, core.account.System) + const client = new TxOperations(model, core.account.System, core.workspace.Model) const spaces = await client.findAll(core.class.Space, {}) const task = await client.createDoc(test.class.Task, spaces[0]._id, { name: 'TSK1', diff --git a/packages/core/src/__tests__/minmodel.ts b/packages/core/src/__tests__/minmodel.ts index cf9f4931260..b42966e40a3 100644 --- a/packages/core/src/__tests__/minmodel.ts +++ b/packages/core/src/__tests__/minmodel.ts @@ -21,7 +21,7 @@ import core from '../component' import type { DocumentUpdate, TxCUD, TxCreateDoc, TxRemoveDoc, TxUpdateDoc } from '../tx' import { DOMAIN_TX, TxFactory } from '../tx' -const txFactory = new TxFactory(core.account.System) +const txFactory = new TxFactory(core.account.System, core.workspace.Model) function createClass (_class: Ref>, attributes: Data>): TxCreateDoc { return txFactory.createTxCreateDoc(core.class.Class, core.space.Model, attributes, _class) diff --git a/packages/core/src/backup.ts b/packages/core/src/backup.ts index 348a6606d3f..f9f8d526343 100644 --- a/packages/core/src/backup.ts +++ b/packages/core/src/backup.ts @@ -1,3 +1,4 @@ +import type { WorkspaceUuid } from '.' import { type Doc, type Domain, type Ref } from './classes' import { type DocInfo } from './server' @@ -19,14 +20,14 @@ export interface DocChunk { * @public */ export interface BackupClient { - loadChunk: (domain: Domain, idx?: number) => Promise - closeChunk: (idx: number) => Promise + loadChunk: (workspaceId: WorkspaceUuid, domain: Domain, idx?: number) => Promise + closeChunk: (workspaceId: WorkspaceUuid, idx: number) => Promise - loadDocs: (domain: Domain, docs: Ref[]) => Promise - upload: (domain: Domain, docs: Doc[]) => Promise - clean: (domain: Domain, docs: Ref[]) => Promise + loadDocs: (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]) => Promise + upload: (workspaceId: WorkspaceUuid, domain: Domain, docs: Doc[]) => Promise + clean: (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]) => Promise - getDomainHash: (domain: Domain) => Promise + getDomainHash: (workspaceId: WorkspaceUuid, domain: Domain) => Promise - sendForceClose: () => Promise + sendForceClose: (workspace: WorkspaceUuid) => Promise } diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index 6fca816f2db..2163301253e 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -69,12 +69,24 @@ export interface Obj { _class: Ref> } +export interface AccountWorkspace { + url?: string + name?: string + role: AccountRole + maintenance: boolean + enabled: boolean +} + export interface Account { uuid: AccountUuid - role: AccountRole + role: AccountRole // Role in current workspace + targetWorkspace: WorkspaceUuid // Will point to a selected workspace, if not will point to personal workspace. + personalWorkspace: WorkspaceUuid // Will be filled with personal workspace UUID or will be core.workspace.NoPersonalWorkspace primarySocialId: PersonId socialIds: PersonId[] - fullSocialIds: SocialId[] + fullSocialIds: Map + socialIdsByValue: Map + workspaces: Record } /** @@ -106,6 +118,7 @@ export interface BasePerson { */ export interface Doc extends Obj { _id: Ref + _uuid: WorkspaceUuid space: Ref modifiedOn: Timestamp modifiedBy: PersonId @@ -813,6 +826,7 @@ export interface WorkspaceInfo { createdBy?: PersonUuid // Should always be set for NEW workspaces billingAccount?: PersonUuid // Should always be set for NEW workspaces allowReadOnlyGuest?: boolean // Should always be set for NEW workspaces + personal?: boolean } export interface BackupStatus { @@ -825,6 +839,12 @@ export interface BackupStatus { backups: number } +export interface EndpointInfo { + internalUrl: string + externalUrl: string + region: string +} + export interface WorkspaceInfoWithStatus extends WorkspaceInfo { isDisabled?: boolean versionMajor: number @@ -834,6 +854,8 @@ export interface WorkspaceInfoWithStatus extends WorkspaceInfo { mode: WorkspaceMode processingProgress?: number backupInfo?: BackupStatus + + endpoint: EndpointInfo } export interface WorkspaceMemberInfo { diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts deleted file mode 100644 index 5a1e8236152..00000000000 --- a/packages/core/src/client.ts +++ /dev/null @@ -1,485 +0,0 @@ -// -// Copyright © 2020 Anticrm Platform Contributors. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Analytics } from '@hcengineering/analytics' -import { type BackupClient, type DocChunk } from './backup' -import { type Class, DOMAIN_MODEL, type Doc, type Domain, type Ref, type Timestamp } from './classes' -import core from './component' -import { Hierarchy } from './hierarchy' -import { type MeasureContext, MeasureMetricsContext } from './measurements' -import { ModelDb } from './memdb' -import type { - DocumentQuery, - FindOptions, - FindResult, - FulltextStorage, - SearchOptions, - SearchQuery, - SearchResult, - Storage, - TxResult, - WithLookup -} from './storage' -import { type Tx, type TxCUD, type TxWorkspaceEvent, WorkspaceEvent } from './tx' -import { platformNow, platformNowDiff, toFindResult } from './utils' - -/** - * @public - */ -export type TxHandler = (...tx: Tx[]) => void - -/** - * @public - */ -export interface Client extends Storage, FulltextStorage { - notify?: (...tx: Tx[]) => void - getHierarchy: () => Hierarchy - getModel: () => ModelDb - findOne: ( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ) => Promise | undefined> - close: () => Promise - getConnection?: () => ClientConnection -} - -/** - * @public - */ -export interface LoadModelResponse { - // A diff or a full set of transactions. - transactions: Tx[] - // A current hash chain - hash: string - // If full model is returned, on hash diff for request - full: boolean -} - -/** - * @public - */ -export enum ClientConnectEvent { - Connected, // In case we just connected to server, and receive a full model - Reconnected, // In case we re-connected to server and receive and apply diff. - - // Client could cause back a few more states. - Upgraded, // In case client code receive a full new model and need to be rebuild. - Refresh, // In case we detect query refresh is required - Maintenance // In case workspace are in maintenance mode -} - -export type Handler = (...result: any[]) => void - -/** - * @public - */ -export interface ClientConnection extends Storage, FulltextStorage, BackupClient { - isConnected: () => boolean - - close: () => Promise - onConnect?: (event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise - - // If hash is passed, will return LoadModelResponse - loadModel: (last: Timestamp, hash?: string) => Promise - - getLastHash?: (ctx: MeasureContext) => Promise - pushHandler: (handler: Handler) => void -} - -class ClientImpl implements Client, BackupClient { - notify?: (...tx: Tx[]) => void - hierarchy!: Hierarchy - model!: ModelDb - private readonly appliedModelTransactions = new Set>() - constructor (private readonly conn: ClientConnection) {} - - getConnection (): ClientConnection { - return this.conn - } - - setModel (hierarchy: Hierarchy, model: ModelDb): void { - this.hierarchy = hierarchy - this.model = model - } - - getHierarchy (): Hierarchy { - return this.hierarchy - } - - getModel (): ModelDb { - return this.model - } - - async findAll( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise> { - const domain = this.hierarchy.getDomain(_class) - const data = - domain === DOMAIN_MODEL - ? await this.model.findAll(_class, query, options) - : await this.conn.findAll(_class, query, options) - - // In case of mixin we need to create mixin proxies. - - // Update mixins & lookups - const result = data.map((v) => { - return this.hierarchy.updateLookupMixin(_class, v, options) - }) - return toFindResult(result, data.total) - } - - async searchFulltext (query: SearchQuery, options: SearchOptions): Promise { - return await this.conn.searchFulltext(query, options) - } - - async findOne( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise | undefined> { - return (await this.findAll(_class, query, { ...options, limit: 1 }))[0] - } - - async tx (tx: Tx): Promise { - if (tx.objectSpace === core.space.Model) { - this.hierarchy.tx(tx) - await this.model.tx(tx) - this.appliedModelTransactions.add(tx._id) - } - // We need to handle it on server, before performing local live query updates. - return await this.conn.tx(tx) - } - - async updateFromRemote (...tx: Tx[]): Promise { - for (const t of tx) { - try { - if (t.objectSpace === core.space.Model) { - const hasTx = this.appliedModelTransactions.has(t._id) - if (!hasTx) { - this.hierarchy.tx(t) - await this.model.tx(t) - } else { - this.appliedModelTransactions.delete(t._id) - } - } - } catch (err) { - // console.error('failed to apply model transaction, skipping', t) - continue - } - } - this.notify?.(...tx) - } - - async close (): Promise { - await this.conn.close() - } - - async loadChunk (domain: Domain, idx?: number): Promise { - return await this.conn.loadChunk(domain, idx) - } - - async getDomainHash (domain: Domain): Promise { - return await this.conn.getDomainHash(domain) - } - - async closeChunk (idx: number): Promise { - await this.conn.closeChunk(idx) - } - - async loadDocs (domain: Domain, docs: Ref[]): Promise { - return await this.conn.loadDocs(domain, docs) - } - - async upload (domain: Domain, docs: Doc[]): Promise { - await this.conn.upload(domain, docs) - } - - async clean (domain: Domain, docs: Ref[]): Promise { - await this.conn.clean(domain, docs) - } - - async sendForceClose (): Promise { - await this.conn.sendForceClose() - } -} - -/** - * @public - */ -export interface TxPersistenceStore { - load: () => Promise - store: (model: LoadModelResponse) => Promise -} - -export type ModelFilter = (tx: Tx[]) => Tx[] - -/** - * @public - */ -export async function createClient ( - connect: (txHandler: TxHandler) => Promise, - // If set will build model with only allowed plugins. - modelFilter?: ModelFilter, - txPersistence?: TxPersistenceStore, - _ctx?: MeasureContext -): Promise { - const ctx = _ctx ?? new MeasureMetricsContext('createClient', {}) - let client: ClientImpl | null = null - - // Temporal buffer, while we apply model - let txBuffer: Tx[] | undefined = [] - - let hierarchy = new Hierarchy() - let model = new ModelDb(hierarchy) - - let lastTx: string | undefined - - function txHandler (...tx: Tx[]): void { - if (tx == null || tx.length === 0) { - return - } - if (client === null) { - txBuffer?.push(...tx) - } else { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - client.updateFromRemote(...tx) - } - for (const t of tx) { - if (t._class === core.class.TxWorkspaceEvent && (t as TxWorkspaceEvent).event === WorkspaceEvent.LastTx) { - lastTx = (t as TxWorkspaceEvent).params.lastTx - } - } - } - const conn = await ctx.with('connect', {}, () => connect(txHandler)) - - let { mode, current, addition } = await ctx.with('load-model', {}, (ctx) => loadModel(ctx, conn, txPersistence)) - switch (mode) { - case 'same': - case 'upgrade': - ctx.withSync('build-model', {}, (ctx) => { - buildModel(ctx, current, modelFilter, hierarchy, model) - }) - break - case 'addition': - ctx.withSync('build-model', {}, (ctx) => { - buildModel(ctx, current.concat(addition), modelFilter, hierarchy, model) - }) - } - current = [] - addition = [] - - txBuffer = txBuffer.filter((tx) => tx.space !== core.space.Model) - - client = new ClientImpl(conn) - client.setModel(hierarchy, model) - - txHandler(...txBuffer) - txBuffer = undefined - - const oldOnConnect: - | ((event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise) - | undefined = conn.onConnect - conn.onConnect = async (event, _lastTx, data) => { - console.log('Client: onConnect', event) - if (event === ClientConnectEvent.Maintenance) { - lastTx = _lastTx - await oldOnConnect?.(ClientConnectEvent.Maintenance, _lastTx, data) - return - } - // Find all new transactions and apply - let { mode, current, addition } = await ctx.with('load-model', {}, (ctx) => loadModel(ctx, conn, txPersistence)) - - switch (mode) { - case 'upgrade': - // We have upgrade procedure and need rebuild all stuff. - hierarchy = new Hierarchy() - model = new ModelDb(hierarchy) - client.setModel(hierarchy, model) - - ctx.withSync('build-model', {}, (ctx) => { - buildModel(ctx, current, modelFilter, hierarchy, model) - }) - current = [] - await oldOnConnect?.(ClientConnectEvent.Upgraded, _lastTx, data) - // No need to fetch more stuff since upgrade was happened. - break - case 'addition': - ctx.withSync('build-model', {}, (ctx) => { - buildModel(ctx, current.concat(addition), modelFilter, hierarchy, model) - }) - break - } - current = [] - addition = [] - - if (lastTx === undefined) { - // No need to do anything here since we connected. - await oldOnConnect?.(event, _lastTx, data) - lastTx = _lastTx - return - } - - if (lastTx === _lastTx) { - // Same lastTx, no need to refresh - await oldOnConnect?.(ClientConnectEvent.Reconnected, _lastTx, data) - return - } - lastTx = _lastTx - // We need to trigger full refresh on queries, etc. - await oldOnConnect?.(ClientConnectEvent.Refresh, lastTx, data) - } - - return client -} - -// Ignore Employee accounts. -// We may still have them in transactions in old workspaces even with global accounts. -function isPersonAccount (tx: Tx): boolean { - return ( - (tx._class === core.class.TxCreateDoc || - tx._class === core.class.TxUpdateDoc || - tx._class === core.class.TxRemoveDoc) && - ((tx as TxCUD).objectClass === 'contact:class:PersonAccount' || - (tx as TxCUD).objectClass === 'core:class:Account') - ) -} - -async function loadModel ( - ctx: MeasureContext, - conn: ClientConnection, - persistence?: TxPersistenceStore -): Promise<{ mode: 'same' | 'addition' | 'upgrade', current: Tx[], addition: Tx[] }> { - const t = platformNow() - - const current = (await ctx.with('persistence-load', {}, () => persistence?.load())) ?? { - full: true, - transactions: [], - hash: '' - } - - if (conn.getLastHash !== undefined && (await conn.getLastHash(ctx)) === current.hash) { - // We have same model hash. - return { mode: 'same', current: current.transactions, addition: [] } - } - const lastTxTime = getLastTxTime(current.transactions) - const result = await ctx.with('connection-load-model', { hash: current.hash !== '' }, (ctx) => - conn.loadModel(lastTxTime, current.hash) - ) - - if (Array.isArray(result)) { - // Fallback to old behavior, only for tests - return { - mode: 'same', - current: result, - addition: [] - } - } - - // Save concatenated, if have some more of them. - void ctx - .with('persistence-store', {}, (ctx) => - persistence?.store({ - ...result, - // Store concatinated old + new txes - transactions: result.full ? result.transactions : current.transactions.concat(result.transactions) - }) - ) - .catch((err) => { - Analytics.handleError(err) - }) - - if (typeof window !== 'undefined') { - console.log('find' + (result.full ? 'full model' : 'model diff'), result.transactions.length, platformNowDiff(t)) - } - if (result.full) { - return { mode: 'upgrade', current: result.transactions, addition: [] } - } - return { mode: 'addition', current: current.transactions, addition: result.transactions } -} - -export function buildModel ( - ctx: MeasureContext, - transactions: Tx[], - modelFilter: ModelFilter | undefined, - hierarchy: Hierarchy, - model: ModelDb -): void { - const systemTx: Tx[] = [] - const userTx: Tx[] = [] - - const atxes = transactions - - ctx.withSync('split txes', {}, () => { - atxes.forEach((tx) => - ((tx.modifiedBy === core.account.ConfigUser || tx.modifiedBy === core.account.System) && !isPersonAccount(tx) - ? systemTx - : userTx - ).push(tx) - ) - }) - - userTx.sort(compareTxes) - - let txes = systemTx.concat(userTx) - if (modelFilter !== undefined) { - txes = modelFilter(txes) - } - - ctx.withSync('build hierarchy', {}, () => { - for (const tx of txes) { - try { - hierarchy.tx(tx) - } catch (err: any) { - ctx.warn('failed to apply model transaction, skipping', { - _id: tx._id, - _class: tx._class, - message: err?.message - }) - } - } - }) - ctx.withSync('build model', {}, (ctx) => { - model.addTxes(ctx, txes, false) - }) -} - -function getLastTxTime (txes: Tx[]): number { - let lastTxTime = 0 - for (const tx of txes) { - if (tx.modifiedOn > lastTxTime) { - lastTxTime = tx.modifiedOn - } - } - return lastTxTime -} - -function compareTxes (a: Tx, b: Tx): number { - const result = a.modifiedOn - b.modifiedOn - if (result !== 0) { - return result - } - if (a._class !== b._class) { - if (a._class === core.class.TxCreateDoc) { - return -1 - } - if (b._class === core.class.TxCreateDoc) { - return 1 - } - } - return a._id.localeCompare(b._id) -} diff --git a/packages/core/src/client/client.ts b/packages/core/src/client/client.ts new file mode 100644 index 00000000000..8c5e341cd87 --- /dev/null +++ b/packages/core/src/client/client.ts @@ -0,0 +1,372 @@ +// +// Copyright © 2020 Anticrm Platform Contributors. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Analytics } from '@hcengineering/analytics' +import { deepEqual } from 'fast-equals' +import { BackupClient, DocChunk } from '../backup' +import { + Class, + Doc, + Domain, + DOMAIN_MODEL, + Ref, + type Account, + type AccountUuid, + type AccountWorkspace +} from '../classes' +import core from '../component' +import { Hierarchy } from '../hierarchy' +import { MeasureContext, MeasureMetricsContext } from '../measurements' +import { ModelDb } from '../memdb' +import type { DocumentQuery, FindOptions, FindResult, TxResult, WithLookup } from '../storage' +import { SearchOptions, SearchQuery, SearchResult } from '../storage' +import { Tx, WorkspaceEvent, type TxWorkspaceEvent } from '../tx' +import { platformNow, platformNowDiff, toFindResult, type WorkspaceUuid } from '../utils' +import { + ClientConnectEvent, + type Client, + type ClientConnection, + type ClientConnectOptions, + type ConnectionEvents, + type TxHandler, + type TxPersistenceStore +} from './types' +import { buildModel, getLastTxTime, isModelDomain } from './utils' + +class ClientImpl implements Client, BackupClient { + notify?: (tx: Tx[], workspace?: WorkspaceUuid, target?: string, exclude?: string[]) => void + hierarchy: Hierarchy + model: ModelDb + modelLoaded: boolean = false + + account: Account + + workspaces: Record = {} + availableWorkspaces: WorkspaceUuid[] = [] + + lastTx: Record | undefined = {} + + private readonly appliedModelTransactions = new Set>() + constructor ( + private readonly conn: ClientConnection, + account: Account, + readonly ctx: MeasureContext, + readonly opt?: ClientConnectOptions + ) { + this.account = account + + this.hierarchy = new Hierarchy() + this.model = new ModelDb(this.hierarchy) + } + + onAccount (account: Account): void { + // Do diff and notify about workspace changes. + this.account = account + this.workspaces = account.workspaces + this.availableWorkspaces = Object.entries(this.workspaces) + .filter((it) => !it[1].maintenance && it[1].enabled) + .map((it) => it[0] as WorkspaceUuid) + } + + getConnection (): ClientConnection { + return this.conn + } + + getWorkspaces (): Record { + return this.workspaces + } + + getAvailableWorkspaces (): WorkspaceUuid[] { + return this.availableWorkspaces + } + + getHierarchy (): Hierarchy { + return this.hierarchy + } + + getModel (): ModelDb { + return this.model + } + + async findAll( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + const domain = this.hierarchy.getDomain(_class) + const data = + domain === DOMAIN_MODEL + ? await this.model.findAll(_class, query, options) + : await this.conn.findAll(_class, query, options) + + // In case of mixin we need to create mixin proxies. + + // Update mixins & lookups + const result = data.map((v) => { + return this.hierarchy.updateLookupMixin(_class, v, options) + }) + return toFindResult(result, data.total) + } + + async searchFulltext (query: SearchQuery, options: SearchOptions): Promise { + return await this.conn.searchFulltext(query, options) + } + + async findOne( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise | undefined> { + return (await this.findAll(_class, query, { ...options, limit: 1 }))[0] + } + + async tx (tx: Tx): Promise { + if (isModelDomain(tx, this.hierarchy)) { + this.hierarchy.tx(tx) + await this.model.tx(tx) + this.appliedModelTransactions.add(tx._id) + } + // We need to handle it on server, before performing local live query updates. + return await this.conn.tx(tx) + } + + async updateFromRemote (tx: Tx[], workspace?: WorkspaceUuid, target?: string, exclude?: string[]): Promise { + for (const t of tx) { + try { + if (isModelDomain(t, this.hierarchy)) { + const hasTx = this.appliedModelTransactions.has(t._id) + if (!hasTx) { + this.hierarchy.tx(t) + await this.model.tx(t) + } else { + this.appliedModelTransactions.delete(t._id) + } + } + } catch (err) { + // console.error('failed to apply model transaction, skipping', t) + continue + } + } + this.notify?.(tx, workspace, target, exclude) + } + + async close (): Promise { + await this.conn.close() + } + + async loadChunk (workspaceId: WorkspaceUuid, domain: Domain, idx?: number): Promise { + return await this.conn.loadChunk(workspaceId, domain, idx) + } + + async getDomainHash (workspaceId: WorkspaceUuid, domain: Domain): Promise { + return await this.conn.getDomainHash(workspaceId, domain) + } + + async closeChunk (workspaceId: WorkspaceUuid, idx: number): Promise { + await this.conn.closeChunk(workspaceId, idx) + } + + async loadDocs (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { + return await this.conn.loadDocs(workspaceId, domain, docs) + } + + async upload (workspaceId: WorkspaceUuid, domain: Domain, docs: Doc[]): Promise { + await this.conn.upload(workspaceId, domain, docs) + } + + async clean (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { + await this.conn.clean(workspaceId, domain, docs) + } + + async sendForceClose (workspaceId: WorkspaceUuid): Promise { + await this.conn.sendForceClose(workspaceId) + } + + async loadModelInternal (ctx: MeasureContext): Promise { + let { mode, current, addition } = await ctx.with('load-model', {}, (ctx) => + loadModel(ctx, this.conn, this.opt?.txPersistence) + ) + + if (mode === 'same' && this.modelLoaded) { + // We have same model hash. + return + } + if (mode === 'same' || mode === 'upgrade') { + ctx.withSync('build-model', {}, (ctx) => { + buildModel(ctx, current, this.opt?.modelFilter, this.hierarchy, this.model) + }) + } else if (mode === 'addition') { + ctx.withSync('build-model', {}, (ctx) => { + buildModel(ctx, current.concat(addition), this.opt?.modelFilter, this.hierarchy, this.model) + }) + } + current = [] + addition = [] + } + + async onConnect ( + ctx: MeasureContext, + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ): Promise { + this.lastTx = lastTx + if (this.modelLoaded) { + await this.loadModelInternal(ctx) + + if (this.lastTx === undefined) { + // No need to do anything here since we connected. + await this.opt?.onConnect?.(event, this.lastTx, data) + this.lastTx = lastTx + return + } + + if (deepEqual(this.lastTx, lastTx)) { + // Same lastTx, no need to refresh + await this.opt?.onConnect?.(ClientConnectEvent.Reconnected, lastTx, data) + return + } + this.lastTx = lastTx + // We need to trigger full refresh on queries, etc. + await this.opt?.onConnect?.(ClientConnectEvent.Refresh, lastTx, data) + } + } +} + +/** + * @public + */ +export async function createClient ( + connect: (txHandler: TxHandler, events?: ConnectionEvents) => Promise, + // If set will build model with only allowed plugins. + opt?: ClientConnectOptions +): Promise { + const ctx = opt?._ctx ?? new MeasureMetricsContext('createClient', {}) + let client: ClientImpl | null = null + + // Temporal buffer, while we apply model + let txBuffer: { buff: Tx[], workspace?: WorkspaceUuid, target?: AccountUuid, exclude?: AccountUuid[] }[] | undefined = + [] + + function txHandler (tx: Tx[], workspace?: WorkspaceUuid, target?: AccountUuid, exclude?: AccountUuid[]): void { + if (tx == null || tx.length === 0) { + return + } + if (client === null) { + txBuffer?.push({ buff: tx, workspace, target, exclude }) + } else { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + client.updateFromRemote(tx, workspace, target, exclude) + + for (const t of tx) { + if (t._class === core.class.TxWorkspaceEvent && (t as TxWorkspaceEvent).event === WorkspaceEvent.LastTx) { + client.lastTx = (t as TxWorkspaceEvent).params.lastTx + } + } + } + } + let account: Account | undefined + const conn = await ctx.with('connect', {}, () => + connect(txHandler, { + ...opt, + onAccount: (a) => { + account = a + client?.onAccount(account) + opt?.onAccount?.(a) + }, + onConnect: async (event, _lastTx, data) => { + console.log('Client: onConnect', event) + if (event === ClientConnectEvent.Maintenance) { + await opt?.onConnect?.(event, _lastTx, data) + } + await client?.onConnect(ctx, event, _lastTx, data) + } + }) + ) + + if (account === undefined) { + account = await conn.getAccount() + opt?.onAccount?.(account) + } + + client = new ClientImpl(conn, account, ctx, opt) + client.onAccount(account) + + await client.loadModelInternal(ctx) + + for (const { buff, workspace, target, exclude } of txBuffer) { + txHandler(buff, workspace, target, exclude) + } + txBuffer = undefined + + return client +} + +async function loadModel ( + ctx: MeasureContext, + conn: ClientConnection, + persistence?: TxPersistenceStore +): Promise<{ mode: 'same' | 'addition' | 'upgrade', current: Tx[], addition: Tx[] }> { + const t = platformNow() + + const current = (await ctx.with('persistence-load', {}, () => persistence?.load())) ?? { + full: true, + transactions: [], + hash: '' + } + + if (conn.getLastHash !== undefined) { + const account = await conn.getAccount() + const lastHash = await conn.getLastHash(ctx) + if (lastHash[account.targetWorkspace] === current.hash) { + // We have same model hash. + return { mode: 'same', current: current.transactions, addition: [] } + } + } + const lastTxTime = getLastTxTime(current.transactions) + const result = await ctx.with('connection-load-model', { hash: current.hash !== '' }, (ctx) => + conn.loadModel(lastTxTime, current.hash) + ) + + if (Array.isArray(result)) { + // Fallback to old behavior, only for tests + return { + mode: 'same', + current: result, + addition: [] + } + } + + // Save concatenated, if have some more of them. + void ctx + .with('persistence-store', {}, (ctx) => + persistence?.store({ + ...result, + // Store concatinated old + new txes + transactions: result.full ? result.transactions : current.transactions.concat(result.transactions) + }) + ) + .catch((err) => { + Analytics.handleError(err) + }) + + if (typeof window !== 'undefined') { + console.log('find' + (result.full ? 'full model' : 'model diff'), result.transactions.length, platformNowDiff(t)) + } + if (result.full) { + return { mode: 'upgrade', current: result.transactions, addition: [] } + } + return { mode: 'addition', current: current.transactions, addition: result.transactions } +} diff --git a/packages/core/src/client/index.ts b/packages/core/src/client/index.ts new file mode 100644 index 00000000000..8d02cc151dd --- /dev/null +++ b/packages/core/src/client/index.ts @@ -0,0 +1,3 @@ +export * from './types' +export * from './utils' +export * from './client' diff --git a/packages/core/src/client/types.ts b/packages/core/src/client/types.ts new file mode 100644 index 00000000000..2626c276099 --- /dev/null +++ b/packages/core/src/client/types.ts @@ -0,0 +1,161 @@ +// +// Copyright © 2020 Anticrm Platform Contributors. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { BackupClient } from '../backup' +import { Class, Doc, Ref, Timestamp, type Account, type AccountUuid, type AccountWorkspace } from '../classes' +import { Hierarchy } from '../hierarchy' +import { MeasureContext } from '../measurements' +import { ModelDb } from '../memdb' +import type { + DocumentQuery, + FindOptions, + FindResult, + FulltextStorage, + SearchOptions, + SearchQuery, + SearchResult, + Storage, + TxResult, + WithLookup +} from '../storage' +import { Tx } from '../tx' +import { type WorkspaceUuid } from '../utils' + +/** + * @public + */ +export type TxHandler = (tx: Tx[], workspace?: WorkspaceUuid, target?: AccountUuid, exclude?: AccountUuid[]) => void + +/** + * @public + */ +export interface Client extends Storage, FulltextStorage { + notify?: (tx: Tx[], workspace?: WorkspaceUuid, target?: string, exclude?: string[]) => void + getHierarchy: () => Hierarchy + getModel: () => ModelDb + + // If not called, will cause model and hierarchy to be empty. + + findOne: ( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise | undefined> + close: () => Promise + getConnection?: () => ClientConnection + + // Get a list of active workspaces, not in maintenance mode and enabled. + getAvailableWorkspaces: () => WorkspaceUuid[] + + // A list of active workspaces, workspace could be disabled or in maintenance mode, in this case it will not be here. + getWorkspaces: () => Record +} + +/** + * @public + */ +export interface LoadModelResponse { + // A diff or a full set of transactions. + transactions: Tx[] + // A current hash chain + hash: string + // If full model is returned, on hash diff for request + full: boolean +} + +/** + * @public + */ +export enum ClientConnectEvent { + Connected, // In case we just connected to server, and receive a full model + Reconnected, // In case we re-connected to server and receive and apply diff. + + // Client could cause back a few more states. + Upgraded, // In case client code receive a full new model and need to be rebuild. + Refresh, // In case we detect query refresh is required + Maintenance // In case workspace are in maintenance mode +} + +export type Handler = (...result: any[]) => void + +export interface ConnectionEvents { + onHello?: (serverVersion?: string) => boolean + onUnauthorized?: () => void + onConnect?: ( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise + onDialTimeout?: () => void | Promise + onAccount?: (account: Account) => void +} + +export type SubscribedWorkspaceInfo = Record< +WorkspaceUuid, +{ lastHash: string | undefined, lastTx: string | undefined } +> + +export interface TxOptions { + user?: AccountUuid // For system account only, to akt on behalf of another user +} + +/** + * @public + */ +export interface ClientConnection extends BackupClient { + isConnected: () => boolean + close: () => Promise + + // If hash is passed, will return LoadModelResponse + loadModel: (last: Timestamp, hash?: string, workspace?: WorkspaceUuid) => Promise + getLastHash?: (ctx: MeasureContext) => Promise> + pushHandler: (handler: Handler) => void + getAccount: () => Promise + + // For system account we could subscribe to extra workspaces. + subscribe: (subscription: { + accounts?: AccountUuid[] + workspaces?: WorkspaceUuid[] + }) => Promise + unsubscribe: (subscription: { accounts?: AccountUuid[], workspaces?: WorkspaceUuid[] }) => Promise + + // Operations + + findAll: ( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise> + + tx: (tx: Tx, options?: TxOptions) => Promise + + searchFulltext: (query: SearchQuery, options: SearchOptions) => Promise +} + +/** + * @public + */ +export interface TxPersistenceStore { + load: () => Promise + store: (model: LoadModelResponse) => Promise +} + +export type ModelFilter = (tx: Tx[]) => Tx[] + +export interface ClientConnectOptions extends ConnectionEvents { + modelFilter?: ModelFilter + txPersistence?: TxPersistenceStore + _ctx?: MeasureContext +} diff --git a/packages/core/src/client/utils.ts b/packages/core/src/client/utils.ts new file mode 100644 index 00000000000..44c97e44250 --- /dev/null +++ b/packages/core/src/client/utils.ts @@ -0,0 +1,169 @@ +// +// Copyright © 2025 Anticrm Platform Contributors. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Analytics } from '@hcengineering/analytics' +import { DOMAIN_MODEL, Doc } from '../classes' +import core from '../component' +import { Hierarchy } from '../hierarchy' +import { MeasureContext } from '../measurements' +import { ModelDb } from '../memdb' +import { Tx, TxCUD, TxProcessor } from '../tx' +import { platformNow, platformNowDiff } from '../utils' +import type { ClientConnection, ModelFilter, TxPersistenceStore } from './types' + +export function isModelDomain (tx: Tx, h: Hierarchy): boolean { + return TxProcessor.isExtendsCUD(tx._class) ? h.findDomain((tx as TxCUD).objectClass) === DOMAIN_MODEL : false +} + +// Ignore Employee accounts. +// We may still have them in transactions in old workspaces even with global accounts. +export function isPersonAccount (tx: Tx): boolean { + return ( + (tx._class === core.class.TxCreateDoc || + tx._class === core.class.TxUpdateDoc || + tx._class === core.class.TxRemoveDoc) && + ((tx as TxCUD).objectClass === 'contact:class:PersonAccount' || + (tx as TxCUD).objectClass === 'core:class:Account') + ) +} + +export async function loadModel ( + ctx: MeasureContext, + conn: ClientConnection, + persistence?: TxPersistenceStore +): Promise<{ mode: 'same' | 'addition' | 'upgrade', current: Tx[], addition: Tx[] }> { + const t = platformNow() + + const current = (await ctx.with('persistence-load', {}, () => persistence?.load())) ?? { + full: true, + transactions: [], + hash: '' + } + + if (conn.getLastHash !== undefined) { + const account = await conn.getAccount() + const lastHash = await conn.getLastHash(ctx) + if (lastHash[account.targetWorkspace] === current.hash) { + // We have same model hash. + return { mode: 'same', current: current.transactions, addition: [] } + } + } + const lastTxTime = getLastTxTime(current.transactions) + const result = await ctx.with('connection-load-model', { hash: current.hash !== '' }, (ctx) => + conn.loadModel(lastTxTime, current.hash) + ) + + if (Array.isArray(result)) { + // Fallback to old behavior, only for tests + return { + mode: 'same', + current: result, + addition: [] + } + } + + // Save concatenated, if have some more of them. + void ctx + .with('persistence-store', {}, (ctx) => + persistence?.store({ + ...result, + // Store concatinated old + new txes + transactions: result.full ? result.transactions : current.transactions.concat(result.transactions) + }) + ) + .catch((err) => { + Analytics.handleError(err) + }) + + if (typeof window !== 'undefined') { + console.log('find' + (result.full ? 'full model' : 'model diff'), result.transactions.length, platformNowDiff(t)) + } + if (result.full) { + return { mode: 'upgrade', current: result.transactions, addition: [] } + } + return { mode: 'addition', current: current.transactions, addition: result.transactions } +} + +export function buildModel ( + ctx: MeasureContext, + transactions: Tx[], + modelFilter: ModelFilter | undefined, + hierarchy: Hierarchy, + model: ModelDb +): void { + const systemTx: Tx[] = [] + const userTx: Tx[] = [] + + const atxes = transactions + + ctx.withSync('split txes', {}, () => { + atxes.forEach((tx) => + ((tx.modifiedBy === core.account.ConfigUser || tx.modifiedBy === core.account.System) && !isPersonAccount(tx) + ? systemTx + : userTx + ).push(tx) + ) + }) + + userTx.sort(compareTxes) + + let txes = systemTx.concat(userTx) + if (modelFilter !== undefined) { + txes = modelFilter(txes) + } + + ctx.withSync('build hierarchy', {}, () => { + for (const tx of txes) { + try { + hierarchy.tx(tx) + } catch (err: any) { + ctx.warn('failed to apply model transaction, skipping', { + _id: tx._id, + _class: tx._class, + message: err?.message + }) + } + } + }) + ctx.withSync('build model', {}, (ctx) => { + model.addTxes(ctx, txes, false) + }) +} + +export function getLastTxTime (txes: Tx[]): number { + let lastTxTime = 0 + for (const tx of txes) { + if (tx.modifiedOn > lastTxTime) { + lastTxTime = tx.modifiedOn + } + } + return lastTxTime +} + +function compareTxes (a: Tx, b: Tx): number { + const result = a.modifiedOn - b.modifiedOn + if (result !== 0) { + return result + } + if (a._class !== b._class) { + if (a._class === core.class.TxCreateDoc) { + return -1 + } + if (b._class === core.class.TxCreateDoc) { + return 1 + } + } + return a._id.localeCompare(b._id) +} diff --git a/packages/core/src/component.ts b/packages/core/src/component.ts index 28a6361df39..1ca8d1ed792 100644 --- a/packages/core/src/component.ts +++ b/packages/core/src/component.ts @@ -15,9 +15,9 @@ import type { Asset, IntlString, Metadata, Plugin, StatusCode } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' import type { BenchmarkDoc } from './benchmark' -import { AccountRole } from './classes' import type { Account, + AccountUuid, AnyAttribute, ArrOf, Association, @@ -59,21 +59,12 @@ import type { TypeAny, TypedSpace, UserStatus, - Version, - AccountUuid + Version } from './classes' +import { AccountRole } from './classes' import { type Status, type StatusCategory } from './status' -import type { - Tx, - TxApplyIf, - TxCUD, - TxCreateDoc, - TxMixin, - TxModelUpgrade, - TxRemoveDoc, - TxUpdateDoc, - TxWorkspaceEvent -} from './tx' +import type { Tx, TxApplyIf, TxCUD, TxCreateDoc, TxMixin, TxRemoveDoc, TxUpdateDoc, TxWorkspaceEvent } from './tx' +import type { WorkspaceUuid } from './utils' /** * @public @@ -89,9 +80,13 @@ export const systemAccountUuid = '1749089e-22e6-48de-af4e-165e18fbd2f9' as Accou export const systemAccount: Account = { uuid: systemAccountUuid, role: AccountRole.Owner, + targetWorkspace: systemAccountUuid as any as WorkspaceUuid, + personalWorkspace: systemAccountUuid as any as WorkspaceUuid, primarySocialId: '' as PersonId, socialIds: [], - fullSocialIds: [] + fullSocialIds: new Map(), + socialIdsByValue: new Map(), + workspaces: {} } export const configUserAccountUuid = '0d94731c-0787-4bcd-aefe-304efc3706b1' as AccountUuid @@ -107,7 +102,6 @@ export default plugin(coreId, { Interface: '' as Ref>>, Attribute: '' as Ref>, Tx: '' as Ref>, - TxModelUpgrade: '' as Ref>, TxWorkspaceEvent: '' as Ref>, TxApplyIf: '' as Ref>, TxCUD: '' as Ref>>, @@ -116,6 +110,7 @@ export default plugin(coreId, { TxUpdateDoc: '' as Ref>>, TxRemoveDoc: '' as Ref>>, Space: '' as Ref>, + Workspace: '' as WorkspaceUuid, SystemSpace: '' as Ref>, TypedSpace: '' as Ref>, SpaceTypeDescriptor: '' as Ref>, @@ -192,6 +187,11 @@ export default plugin(coreId, { Configuration: '' as Ref, Workspace: '' as Ref }, + workspace: { + Personal: '#personal' as WorkspaceUuid, // Special identifier to map into users' personal workspace + Any: '#any' as WorkspaceUuid, // Special identifier to map into any workspace, mostly for tests + Model: '#model' as WorkspaceUuid // Special identifier for model system transactions., + }, account: { System: '' as PersonId, ConfigUser: '' as PersonId diff --git a/packages/core/src/operations.ts b/packages/core/src/operations.ts index 467b0049004..ab82e63d73d 100644 --- a/packages/core/src/operations.ts +++ b/packages/core/src/operations.ts @@ -8,10 +8,11 @@ import { type MixinUpdate, type ModelDb, platformNow, - toFindResult + toFindResult, + type WorkspaceUuid } from '.' import type { - PersonId, + AccountWorkspace, AnyAttribute, AttachedData, AttachedDoc, @@ -19,6 +20,7 @@ import type { Data, Doc, Mixin, + PersonId, Ref, Space, Timestamp @@ -50,9 +52,22 @@ export class TxOperations implements Omit { constructor ( readonly client: Client, readonly user: PersonId, + private readonly _workspaceUuid: WorkspaceUuid | (() => WorkspaceUuid), readonly isDerived: boolean = false ) { - this.txFactory = new TxFactory(user, isDerived) + this.txFactory = new TxFactory(user, this._workspaceUuid, isDerived) + } + + get workspaceUuid (): WorkspaceUuid { + return this.txFactory.workspaceUuid + } + + getAvailableWorkspaces (): WorkspaceUuid[] { + return this.client.getAvailableWorkspaces() + } + + getWorkspaces (): Record { + return this.client.getWorkspaces() } getHierarchy (): Hierarchy { @@ -473,9 +488,11 @@ export class ApplyOperations extends TxOperations { this.txes.push(tx as TxCUD) } return {} - } + }, + getAvailableWorkspaces: () => ops.client.getAvailableWorkspaces(), + getWorkspaces: () => ops.client.getWorkspaces() } - super(txClient, ops.user, isDerived ?? false) + super(txClient, ops.user, ops.workspaceUuid, isDerived ?? false) } match(_class: Ref>, query: DocumentQuery): ApplyOperations { @@ -564,9 +581,11 @@ export class TxBuilder extends TxOperations { this.txes.push(tx as TxCUD) } return {} - } + }, + getAvailableWorkspaces: () => [], + getWorkspaces: () => ({}) } - super(txClient, user) + super(txClient, user, core.workspace.Model) } } diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index cbded9e7917..145804d0dcb 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -36,7 +36,7 @@ export interface StorageIterator { close: (ctx: MeasureContext) => Promise } -export type BroadcastTargets = Record string[] | undefined> +export type BroadcastTargets = Record AccountUuid[] | undefined> export interface SessionData { broadcast: { diff --git a/packages/core/src/storage.ts b/packages/core/src/storage.ts index a1959df782a..154be425b27 100644 --- a/packages/core/src/storage.ts +++ b/packages/core/src/storage.ts @@ -16,8 +16,9 @@ import type { Asset, Resource } from '@hcengineering/platform' import type { KeysByType } from 'simplytyped' -import type { Association, AttachedDoc, Class, Doc, Ref, Space } from './classes' +import type { AccountUuid, Association, AttachedDoc, Class, Doc, Ref, Space } from './classes' import type { Tx } from './tx' +import type { WorkspaceUuid } from '.' /** * @public @@ -120,6 +121,9 @@ export type AssociationQuery = [Ref, 1 | -1] */ // eslint-disable-next-line @typescript-eslint/consistent-type-definitions export type FindOptions = { + workspace?: WorkspaceUuid | { $in: WorkspaceUuid[] } | { $nin: WorkspaceUuid[] } + user?: AccountUuid // For system account only, to akt on behalf of another user + limit?: number sort?: SortingQuery lookup?: Lookup @@ -224,6 +228,8 @@ export interface SearchQuery { * @public */ export interface SearchOptions { + workspace?: WorkspaceUuid + user?: AccountUuid // For system account only, to akt on behalf of another user limit?: number } diff --git a/packages/core/src/tx.ts b/packages/core/src/tx.ts index 3be9c812bad..24620aab39b 100644 --- a/packages/core/src/tx.ts +++ b/packages/core/src/tx.ts @@ -34,7 +34,7 @@ import { setObjectValue } from './objvalue' import { _getOperator } from './operator' import { _toDoc } from './proxy' import type { DocumentQuery, TxResult } from './storage' -import { generateId } from './utils' +import { generateId, type WorkspaceUuid } from './utils' /** * @public @@ -52,7 +52,9 @@ export enum WorkspaceEvent { SecurityChange, MaintenanceNotification, BulkUpdate, - LastTx + LastTx, + WorkpaceActive, + WorkspaceMaintenance = 7 } /** @@ -62,6 +64,7 @@ export enum WorkspaceEvent { export interface TxWorkspaceEvent extends Tx { event: WorkspaceEvent params: T + workspace?: WorkspaceUuid } /** @@ -78,11 +81,6 @@ export interface BulkUpdateEvent { _class: Ref>[] } -/** - * @public - */ -export interface TxModelUpgrade extends Tx {} - /** * @public */ @@ -459,11 +457,16 @@ export class TxFactory { private readonly txSpace: Ref constructor ( readonly account: PersonId, + readonly _workspaceUuid: WorkspaceUuid | (() => WorkspaceUuid), readonly isDerived: boolean = false ) { this.txSpace = isDerived ? core.space.DerivedTx : core.space.Tx } + get workspaceUuid (): WorkspaceUuid { + return typeof this._workspaceUuid === 'function' ? this._workspaceUuid() : this._workspaceUuid + } + createTxCreateDoc( _class: Ref>, space: Ref, @@ -473,6 +476,7 @@ export class TxFactory { modifiedBy?: PersonId ): TxCreateDoc { return { + _uuid: this.workspaceUuid, _id: generateId(), _class: core.class.TxCreateDoc, space: this.txSpace, @@ -515,6 +519,7 @@ export class TxFactory { modifiedBy?: PersonId ): TxUpdateDoc { return { + _uuid: this.workspaceUuid, _id: generateId(), _class: core.class.TxUpdateDoc, space: this.txSpace, @@ -536,6 +541,7 @@ export class TxFactory { modifiedBy?: PersonId ): TxRemoveDoc { return { + _uuid: this.workspaceUuid, _id: generateId(), _class: core.class.TxRemoveDoc, space: this.txSpace, @@ -557,6 +563,7 @@ export class TxFactory { modifiedBy?: PersonId ): TxMixin { return { + _uuid: this.workspaceUuid, _id: generateId(), _class: core.class.TxMixin, space: this.txSpace, @@ -583,6 +590,7 @@ export class TxFactory { modifiedBy?: PersonId ): TxApplyIf { return { + _uuid: this.workspaceUuid, _id: generateId(), _class: core.class.TxApplyIf, space: this.txSpace, diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index fb7bf05df2d..e66300e3e7c 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -762,7 +762,13 @@ export function isOwnerOrMaintainer (): boolean { } export function hasAccountRole (acc: Account, targerRole: AccountRole): boolean { - return roleOrder[acc.role] >= roleOrder[targerRole] + if (acc.targetWorkspace == null) { + throw new Error('Account has no target workspace') + } + if (acc.targetWorkspace === '') { + return true + } + return roleOrder[acc.workspaces[acc.targetWorkspace].role] >= roleOrder[targerRole] } export function getBranding (brandings: BrandingMap, key: string | undefined): Branding | null { diff --git a/packages/model/src/dsl.ts b/packages/model/src/dsl.ts index 11a86ef8108..de83556d8fb 100644 --- a/packages/model/src/dsl.ts +++ b/packages/model/src/dsl.ts @@ -128,6 +128,7 @@ export function Prop (type: Type, label: IntlString, extra: Partia return function (target: any, propertyKey: string): void { const txes = getTxes(target) const tx: TxCreateDoc> = { + _uuid: core.workspace.Model, _id: generateId(), _class: core.class.TxCreateDoc, space: core.space.Tx, @@ -257,7 +258,7 @@ function generateIds (objectId: Ref, txes: TxCreateDoc ( ...defaults, ...props } - const tx = new TxOperations(client, core.account.System) - const current = await tx.findOne(core.class.Space, { + const current = await client.findOne(core.class.Space, { _id }) if (current === undefined || current._class !== _class) { if (current !== undefined && current._class !== _class) { - await tx.remove(current) + await client.remove(current) } - await tx.createDoc(_class, core.space.Space, data, _id) + await client.createDoc(_class, core.space.Space, data, _id) } } diff --git a/packages/presentation/src/communication.ts b/packages/presentation/src/communication.ts index 5e5167b2669..529c2907d44 100644 --- a/packages/presentation/src/communication.ts +++ b/packages/presentation/src/communication.ts @@ -367,7 +367,9 @@ class Client { private getSocialId (): SocialID { const me = getCurrentAccount() - const hulySocialId = me.fullSocialIds.find((it) => it.type === SocialIdType.HULY && it.verifiedOn !== undefined) + const hulySocialId = Array.from(me.fullSocialIds.values()).find( + (it) => it.type === SocialIdType.HULY && it.verifiedOn !== undefined + ) const id = hulySocialId?._id ?? me.primarySocialId if (id == null || id === '') { throw new Error('Social id not found') diff --git a/packages/presentation/src/pipeline.ts b/packages/presentation/src/pipeline.ts index b1c17c9069d..3578e0b366a 100644 --- a/packages/presentation/src/pipeline.ts +++ b/packages/presentation/src/pipeline.ts @@ -1,5 +1,6 @@ import { toFindResult, + type AccountWorkspace, type Class, type Client, type Doc, @@ -15,7 +16,8 @@ import { type SearchResult, type Tx, type TxResult, - type WithLookup + type WithLookup, + type WorkspaceUuid } from '@hcengineering/core' import platform, { PlatformError, setPlatformStatus, unknownError, type Resource } from '@hcengineering/platform' @@ -87,6 +89,14 @@ export class PresentationPipelineImpl implements PresentationPipeline { await this.head?.notifyTx(...tx) } + getAvailableWorkspaces (): WorkspaceUuid[] { + return this.client.getAvailableWorkspaces() + } + + getWorkspaces (): Record { + return this.client.getWorkspaces() + } + static create (client: Client, constructors: PresentationMiddlewareCreator[]): PresentationPipeline { const pipeline = new PresentationPipelineImpl(client) pipeline.head = pipeline.buildChain(constructors) diff --git a/packages/presentation/src/plugin.ts b/packages/presentation/src/plugin.ts index a3f191c19b2..bb787f005ee 100644 --- a/packages/presentation/src/plugin.ts +++ b/packages/presentation/src/plugin.ts @@ -148,8 +148,14 @@ export default plugin(presentationId, { CollaboratorUrl: '' as Metadata, Token: '' as Metadata, Endpoint: '' as Metadata, + + // Identifier of personal workspace + PersonalWorkspaceUuid: '' as Metadata, + + // Identifier of target workspace + data WorkspaceUuid: '' as Metadata, WorkspaceDataId: '' as Metadata, + FrontUrl: '' as Asset, LinkPreviewUrl: '' as Metadata, UploadConfig: '' as Metadata, diff --git a/packages/presentation/src/utils.ts b/packages/presentation/src/utils.ts index 58ed9083555..556abfa388d 100644 --- a/packages/presentation/src/utils.ts +++ b/packages/presentation/src/utils.ts @@ -17,7 +17,7 @@ import { Analytics } from '@hcengineering/analytics' import core, { type Account, - AccountRole, + type AccountWorkspace, type ArrOf, type AttachedDoc, type Class, @@ -100,14 +100,23 @@ export const uiContext = new MeasureMetricsContext('client-ui', {}) export const pendingCreatedDocs = writable, boolean>>({}) class UIClient extends TxOperations implements Client { + hook = getMetadata(plugin.metadata.ClientHook) + pendingTxes = new Set>() constructor ( client: Client, - private readonly liveQuery: Client + private readonly liveQuery: Client, + workspace: () => WorkspaceUuid ) { - super(client, getCurrentAccount().primarySocialId) + super(client, getCurrentAccount().primarySocialId, workspace) } - protected pendingTxes = new Set>() + getWorkspaces (): Record { + return this.client.getWorkspaces() + } + + getAvailableWorkspaces (): WorkspaceUuid[] { + return this.client.getAvailableWorkspaces() + } async doNotify (...tx: Tx[]): Promise { const pending = get(pendingCreatedDocs) @@ -245,10 +254,22 @@ const clientProxy = new Proxy( /** * @public */ -export function getClient (): TxOperations & Client { +export function getClient (): TxOperations { return clientProxy } +let targetWorkspace: WorkspaceUuid | undefined + +export function setTargetWorkspace (workspace: WorkspaceUuid): void { + targetWorkspace = workspace +} +export function getTargetWorkspace (): WorkspaceUuid { + if (targetWorkspace === undefined) { + throw new Error('Target workspace is not set') + } + return targetWorkspace +} + export type OnClientListener = (client: Client, account: Account) => void | Promise const onClientListeners: OnClientListener[] = [] @@ -271,71 +292,10 @@ export function addRefreshListener (r: RefreshListener): void { refreshListeners.add(r) } -class ClientHookImpl implements Client { - constructor ( - private readonly client: Client, - private readonly hook: ClientHook - ) {} - - set notify (op: (...tx: Tx[]) => void) { - this.client.notify = op - } - - get notify (): ((...tx: Tx[]) => void) | undefined { - return this.client.notify - } - - getHierarchy (): Hierarchy { - return this.client.getHierarchy() - } - - getModel (): ModelDb { - return this.client.getModel() - } - - async findOne( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise | undefined> { - if (this.hook !== undefined) { - return await this.hook.findOne(this.client, _class, query, options) - } - return await this.client.findOne(_class, query, options) - } - - get getConnection (): (() => ClientConnection) | undefined { - return this.client.getConnection - } - - async close (): Promise { - await this.client.close() - } - - async findAll( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise> { - if (this.hook !== undefined) { - return await this.hook.findAll(this.client, _class, query, options) - } - return await this.client.findAll(_class, query, options) - } +export const singleWorkspace = writable(false) - async tx (tx: Tx): Promise { - if (this.hook !== undefined) { - return await this.hook.tx(this.client, tx) - } - return await this.client.tx(tx) - } - - async searchFulltext (query: SearchQuery, options: SearchOptions): Promise { - if (this.hook !== undefined) { - return await this.hook.searchFulltext(this.client, query, options) - } - return await this.client.searchFulltext(query, options) - } +export function setSingleWorkspace (value: boolean): void { + singleWorkspace.set(value) } /** @@ -370,7 +330,8 @@ export async function setClient (_client: Client): Promise { liveQuery = new LQ(pipeline) - const uiClient = new UIClient(pipeline, liveQuery) + // Select first workspace if not set + const uiClient = new UIClient(pipeline, liveQuery, () => getTargetWorkspace()) client = uiClient @@ -380,7 +341,7 @@ export async function setClient (_client: Client): Promise { await uiClient.doNotify(...t) }) - _client.notify = (...tx: Tx[]) => { + _client.notify = (tx: Tx[]) => { txQueue.push(...tx) void notifyCaller() } @@ -436,9 +397,12 @@ export class LiveQuery { unsubscribe: () => void = () => {} clientRecreated = false + destroyed = false + constructor (noDestroy: boolean = false) { if (!noDestroy) { onDestroy(() => { + this.destroyed = true this.unsubscribe() }) } else { @@ -452,6 +416,9 @@ export class LiveQuery { callback: (result: FindResult) => void | Promise, options?: FindOptions ): boolean { + if (this.destroyed) { + return false + } if (!this.needUpdate(_class, query, callback, options) && !this.clientRecreated) { return false } diff --git a/packages/query/src/__tests__/connection.ts b/packages/query/src/__tests__/connection.ts index b09e3894d93..aa0cfb11bb6 100644 --- a/packages/query/src/__tests__/connection.ts +++ b/packages/query/src/__tests__/connection.ts @@ -14,9 +14,7 @@ // import core, { - BackupClient, Class, - Client, ClientConnectEvent, ClientConnection, Doc, @@ -26,32 +24,26 @@ import core, { DOMAIN_TX, FindOptions, FindResult, - FulltextStorage, generateId, - Handler, Hierarchy, - LoadModelResponse, ModelDb, Ref, SearchOptions, SearchQuery, SearchResult, + systemAccount, Timestamp, Tx, TxDb, - TxResult + TxResult, + type Account, + type AccountWorkspace, + type SubscribedWorkspaceInfo, + type WorkspaceUuid } from '@hcengineering/core' import { genMinModel } from './minmodel' -export async function connect (handler: (tx: Tx) => void): Promise< -Client & -BackupClient & -FulltextStorage & { - isConnected: () => boolean - loadModel: (last: Timestamp, hash?: string) => Promise - pushHandler: (handler: Handler) => void -} -> { +export async function connect (handler: (tx: Tx[]) => void): Promise { const txes = genMinModel() const hierarchy = new Hierarchy() @@ -79,6 +71,18 @@ FulltextStorage & { return true } + getAvailableWorkspaces (): WorkspaceUuid[] { + return [] + } + + async subscribe (): Promise { + return {} + } + + async unsubscribe (): Promise {} + + getWorkspaces: () => Record = () => ({}) + pushHandler (): void {} async findAll( @@ -112,13 +116,13 @@ FulltextStorage & { this.hierarchy.tx(tx) } await Promise.all([this.model.tx(tx), this.transactions.tx(tx)]) - handler(tx) + handler([tx]) return {} } async close (): Promise {} - async loadChunk (domain: Domain, idx?: number): Promise { + async loadChunk (workspaceId: WorkspaceUuid, domain: Domain, idx?: number): Promise { return { idx: -1, docs: [], @@ -126,7 +130,7 @@ FulltextStorage & { } } - async getDomainHash (domain: Domain): Promise { + async getDomainHash (workspaceId: WorkspaceUuid, domain: Domain): Promise { return generateId() } @@ -134,15 +138,15 @@ FulltextStorage & { return txes } - async closeChunk (idx: number): Promise {} + async closeChunk (workspaceId: WorkspaceUuid, idx: number): Promise {} - async loadDocs (domain: Domain, docs: Ref[]): Promise { + async loadDocs (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { return [] } - async upload (domain: Domain, docs: Doc[]): Promise {} + async upload (workspaceId: WorkspaceUuid, domain: Domain, docs: Doc[]): Promise {} - async clean (domain: Domain, docs: Ref[]): Promise {} + async clean (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise {} async searchFulltext (query: SearchQuery, options: SearchOptions): Promise { return { docs: [] } @@ -150,18 +154,38 @@ FulltextStorage & { async sendForceClose (): Promise {} - handler?: (event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise + handler?: ( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise set onConnect ( - handler: ((event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise) | undefined + handler: + | (( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise) + | undefined ) { this.handler = handler - void this.handler?.(ClientConnectEvent.Connected, '', {}) + void this.handler?.(ClientConnectEvent.Connected, {}, {}) } - get onConnect (): ((event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise) | undefined { + get onConnect (): + | (( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise) + | undefined { return this.handler } + + async getAccount (): Promise { + return systemAccount + } } return new TestConnection(hierarchy, model, transactions) diff --git a/packages/query/src/__tests__/minmodel.ts b/packages/query/src/__tests__/minmodel.ts index 6cb584cc6e7..8a9905edc38 100644 --- a/packages/query/src/__tests__/minmodel.ts +++ b/packages/query/src/__tests__/minmodel.ts @@ -32,7 +32,7 @@ import core, { AttachedDoc, ClassifierKind, DOMAIN_MODEL, DOMAIN_TX, TxFactory } import type { IntlString, Plugin } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' -const txFactory = new TxFactory(core.account.System) +const txFactory = new TxFactory(core.account.System, core.workspace.Model) function createClass (_class: Ref>, attributes: Data>): TxCreateDoc { return txFactory.createTxCreateDoc(core.class.Class, core.space.Model, attributes, _class) diff --git a/packages/query/src/__tests__/query.test.ts b/packages/query/src/__tests__/query.test.ts index bd6ea9526ce..81e746af0f3 100644 --- a/packages/query/src/__tests__/query.test.ts +++ b/packages/query/src/__tests__/query.test.ts @@ -38,12 +38,12 @@ interface Channel extends Space { async function getClient (): Promise<{ liveQuery: LiveQuery, factory: TxOperations }> { const storage = await createClient(connect) const liveQuery = new LiveQuery(storage) - storage.notify = (...tx: Tx[]) => { + storage.notify = (tx: Tx[]) => { liveQuery.tx(...tx).catch((err) => { console.log(err) }) } - return { liveQuery, factory: new TxOperations(storage, core.account.System) } + return { liveQuery, factory: new TxOperations(storage, core.account.System, core.workspace.Model) } } describe('query', () => { @@ -390,6 +390,7 @@ describe('query', () => { it('lookup query add doc', async () => { const { liveQuery, factory } = await getClient() const futureSpace: Space = { + _uuid: core.workspace.Model, _id: generateId(), _class: core.class.Space, private: false, @@ -446,6 +447,7 @@ describe('query', () => { it('lookup nested query add doc', async () => { const { liveQuery, factory } = await getClient() const futureSpace: Space = { + _uuid: core.workspace.Model, _id: generateId(), _class: core.class.Space, private: false, diff --git a/packages/query/src/index.ts b/packages/query/src/index.ts index 0836e996c12..cf55557048f 100644 --- a/packages/query/src/index.ts +++ b/packages/query/src/index.ts @@ -58,7 +58,9 @@ import core, { platformNow, reduceCalls, shouldShowArchived, - toFindResult + toFindResult, + type AccountWorkspace, + type WorkspaceUuid } from '@hcengineering/core' import { PlatformError } from '@hcengineering/platform' import { deepEqual } from 'fast-equals' @@ -72,7 +74,7 @@ const CACHE_SIZE = 125 * @public */ export class LiveQuery implements WithTx, Client { - private readonly client: Client + protected readonly client: Client private readonly queries = new Map>, Map>() private readonly queue = new Map() private queryCounter: number = 0 @@ -86,6 +88,14 @@ export class LiveQuery implements WithTx, Client { this.client = client } + getWorkspaces (): Record { + return this.client.getWorkspaces() + } + + getAvailableWorkspaces (): WorkspaceUuid[] { + return this.client.getAvailableWorkspaces() + } + public isClosed (): boolean { return this.closed } @@ -884,6 +894,7 @@ export class LiveQuery implements WithTx, Client { // Check if query is partially matched. private async matchQuery (q: Query, tx: TxUpdateDoc, docCache: Map): Promise { const doc: Doc = { + _uuid: tx._uuid, _id: tx.objectId, _class: tx.objectClass, modifiedBy: tx.modifiedBy, diff --git a/packages/ui-next/src/components/message/MessageInput.svelte b/packages/ui-next/src/components/message/MessageInput.svelte new file mode 100644 index 00000000000..cc2e7e17dfe --- /dev/null +++ b/packages/ui-next/src/components/message/MessageInput.svelte @@ -0,0 +1,353 @@ + + + + + + +
{}} + on:dragleave={() => {}} + on:drop|preventDefault|stopPropagation={fileDrop} +> + + +
+ {#if files.length > 0} +
+ {#each files as file (file.blobId)} +
+ { + if (result !== undefined) { + files = files.filter((it) => it.blobId !== file.blobId) + if (!message?.files?.some((it) => it.blobId === file.blobId)) { + void deleteFile(file.blobId) + } + } + }} + /> +
+ {/each} +
+ {/if} +
+ +
+ +{#if message === undefined} + +{/if} + + diff --git a/plugins/attachment-resources/src/components/AttachmentRefInput.svelte b/plugins/attachment-resources/src/components/AttachmentRefInput.svelte index bad49a09471..e6ccac9048d 100644 --- a/plugins/attachment-resources/src/components/AttachmentRefInput.svelte +++ b/plugins/attachment-resources/src/components/AttachmentRefInput.svelte @@ -205,6 +205,7 @@ const _id: Ref = generateId() attachments.set(_id, { + _uuid: client.workspaceUuid, _id, _class: attachment.class.Attachment, collection: 'attachments', diff --git a/plugins/attachment-resources/src/components/AttachmentStyleBoxCollabEditor.svelte b/plugins/attachment-resources/src/components/AttachmentStyleBoxCollabEditor.svelte index 7fc8547f403..b26d17b5727 100644 --- a/plugins/attachment-resources/src/components/AttachmentStyleBoxCollabEditor.svelte +++ b/plugins/attachment-resources/src/components/AttachmentStyleBoxCollabEditor.svelte @@ -177,6 +177,7 @@ : object.space const attachmentDoc: Attachment = { + _uuid: client.workspaceUuid, _id, _class: attachment.class.Attachment, collection: 'attachments', diff --git a/plugins/attachment-resources/src/components/AttachmentStyledBox.svelte b/plugins/attachment-resources/src/components/AttachmentStyledBox.svelte index c909dcc19e9..ee4d0625413 100644 --- a/plugins/attachment-resources/src/components/AttachmentStyledBox.svelte +++ b/plugins/attachment-resources/src/components/AttachmentStyledBox.svelte @@ -179,6 +179,7 @@ const _id: Ref = generateId() attachments.set(_id, { + _uuid: client.workspaceUuid, _id, _class: attachment.class.Attachment, collection: 'attachments', diff --git a/plugins/bitrix/src/hr.ts b/plugins/bitrix/src/hr.ts index 97673adf2b0..8fccf00c695 100644 --- a/plugins/bitrix/src/hr.ts +++ b/plugins/bitrix/src/hr.ts @@ -1,26 +1,16 @@ import { Organization } from '@hcengineering/contact' -import core, { - PersonId, - Client, - Data, - Doc, - Ref, - SortingOrder, - Status, - TxOperations, - generateId -} from '@hcengineering/core' +import core, { Data, Doc, PersonId, Ref, SortingOrder, Status, TxOperations, generateId } from '@hcengineering/core' import recruit, { Applicant, Vacancy } from '@hcengineering/recruit' import task, { ProjectType, makeRank } from '@hcengineering/task' export async function createVacancy ( - rawClient: Client, + rawClient: TxOperations, name: string, typeId: Ref, account: PersonId, company?: Ref ): Promise> { - const client = new TxOperations(rawClient, account) + const client = new TxOperations(rawClient, account, rawClient.workspaceUuid) const type = await client.findOne(task.class.ProjectType, { _id: typeId }) if (type === undefined) { throw Error(`Failed to find target project type: ${typeId}`) diff --git a/plugins/bitrix/src/sync.ts b/plugins/bitrix/src/sync.ts index 9d11261677e..0de1e80dd01 100644 --- a/plugins/bitrix/src/sync.ts +++ b/plugins/bitrix/src/sync.ts @@ -788,6 +788,7 @@ async function downloadComments ( }) for (const it of commentsData.result) { const c: ChatMessage & BitrixSyncDoc = { + _uuid: ops.client.workspaceUuid, _id: generateId(), _class: chunter.class.ChatMessage, message: processComment(it.COMMENT as string), @@ -808,6 +809,7 @@ async function downloadComments ( c.attachments = (c.attachments ?? 0) + 1 res.blobs.push([ { + _uuid: ops.client.workspaceUuid, _id: generateId(), _class: attachment.class.Attachment, attachedTo: c._id, @@ -863,6 +865,7 @@ async function downloadComments ( const parser = new DOMParser() const c: Message & BitrixSyncDoc = { + _uuid: ops.client.workspaceUuid, _id: generateId(), _class: gmail.class.Message, content: comm.DESCRIPTION, diff --git a/plugins/bitrix/src/utils.ts b/plugins/bitrix/src/utils.ts index ecf2fa00acf..ce54b47e498 100644 --- a/plugins/bitrix/src/utils.ts +++ b/plugins/bitrix/src/utils.ts @@ -97,7 +97,7 @@ export interface ConvertResult { * @public */ export async function convert ( - client: Client, + client: TxOperations, entity: BitrixEntityMapping, space: Ref, fields: BitrixFieldMapping[], @@ -111,6 +111,7 @@ export async function convert ( const hierarchy = client.getHierarchy() const bitrixId = `${rawDocument.ID as string}` const document: BitrixSyncDoc = { + _uuid: client.workspaceUuid, _id: generateId(), type: entity.type, bitrixId, @@ -305,6 +306,7 @@ export async function convert ( .find((it) => it.value === svalue) if (existingC === undefined) { const c: Channel & BitrixSyncDoc = { + _uuid: client.workspaceUuid, _id: generateId(), _class: contact.class.Channel, attachedTo: document._id, @@ -353,6 +355,7 @@ export async function convert ( let tag: TagElement | undefined = allTagElements.find((it) => it.title === vv) if (tag === undefined) { tag = { + _uuid: client.workspaceUuid, _id: generateId(), _class: tags.class.TagElement, category: defaultCategory._id, @@ -367,6 +370,7 @@ export async function convert ( newExtraDocs.push(tag) } const ref: TagReference & BitrixSyncDoc = { + _uuid: client.workspaceUuid, _id: generateId(), attachedTo: existingId ?? document._id, attachedToClass: attr.attributeOf, @@ -461,7 +465,7 @@ export async function convert ( ) const candidate = doc.mixins[recruit.mixin.Candidate] as Data - const ops = new TxOperations(client, document.modifiedBy) + const ops = new TxOperations(client, document.modifiedBy, client.workspaceUuid) let statusName = sourceStatusName let mapping: BitrixStateMapping | undefined for (const t of operation.stateMapping ?? []) { @@ -559,6 +563,7 @@ export async function convert ( const blobRefs: { file: string, id: string }[] = await getDownloadValue(attr, f.operation) for (const blobRef of blobRefs) { const attachDoc: Attachment & BitrixSyncDoc = { + _uuid: client.workspaceUuid, _id: generateId(), bitrixId: `${blobRef.id}`, file: '' as Ref, // Empty since not uploaded yet. diff --git a/plugins/client-resources/src/connection.ts b/plugins/client-resources/src/connection.ts index 0e834fea5cf..71546136293 100644 --- a/plugins/client-resources/src/connection.ts +++ b/plugins/client-resources/src/connection.ts @@ -22,13 +22,27 @@ import client, { pingConst, pongConst } from '@hcengineering/client' +import { EventResult } from '@hcengineering/communication-sdk-types' +import { + Collaborator, + FindCollaboratorsParams, + FindLabelsParams, + FindMessagesGroupsParams, + FindMessagesParams, + FindNotificationContextParams, + FindNotificationsParams, + Label, + Message, + MessagesGroup, + NotificationContext +} from '@hcengineering/communication-types' import core, { Account, + type AccountUuid, Class, ClientConnectEvent, ClientConnection, clone, - Handler, Doc, DocChunk, DocumentQuery, @@ -36,6 +50,7 @@ import core, { FindOptions, FindResult, generateId, + Handler, LoadModelResponse, type MeasureContext, MeasureMetricsContext, @@ -49,9 +64,13 @@ import core, { Tx, TxApplyIf, TxHandler, + type TxOptions, TxResult, + type TxWorkspaceEvent, + WorkspaceEvent, type WorkspaceUuid } from '@hcengineering/core' +import type { SubscribedWorkspaceInfo } from '@hcengineering/core/src/client/types' import platform, { broadcastEvent, getMetadata, @@ -60,22 +79,8 @@ import platform, { Status, UNAUTHORIZED } from '@hcengineering/platform' +import { HelloRequest, HelloResponse, type RateLimitInfo, ReqId, type Response, RPCHandler } from '@hcengineering/rpc' import { uncompress } from 'snappyjs' -import { HelloRequest, HelloResponse, ReqId, type Response, RPCHandler, type RateLimitInfo } from '@hcengineering/rpc' -import { EventResult } from '@hcengineering/communication-sdk-types' -import { - FindLabelsParams, - FindMessagesGroupsParams, - FindMessagesParams, - FindNotificationContextParams, - FindNotificationsParams, - FindCollaboratorsParams, - Label, - Message, - MessagesGroup, - NotificationContext, - Collaborator -} from '@hcengineering/communication-types' const SECOND = 1000 const pingTimeout = 10 * SECOND @@ -128,19 +133,15 @@ class Connection implements ClientConnection { private readonly sessionId: string | undefined private closed = false - private upgrading: boolean = false - private pingResponse: number = Date.now() private helloReceived: boolean = false private account: Account | undefined - onConnect?: (event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise - rpcHandler: RPCHandler - lastHash?: string + lastHash?: Record handlers: Handler[] = [] @@ -148,7 +149,6 @@ class Connection implements ClientConnection { private readonly ctx: MeasureContext, private readonly url: string, handler: TxHandler, - readonly workspace: WorkspaceUuid, readonly user: PersonUuid, readonly opt?: ClientFactoryOptions ) { @@ -172,7 +172,6 @@ class Connection implements ClientConnection { } this.rpcHandler = opt?.useGlobalRPCHandler === true ? globalRPCHandler : new RPCHandler() this.pushHandler(handler) - this.onConnect = opt?.onConnect this.scheduleOpen(this.ctx, false) } @@ -181,9 +180,9 @@ class Connection implements ClientConnection { this.handlers.push(handler) } - async getLastHash (ctx: MeasureContext): Promise { + async getLastHash (ctx: MeasureContext): Promise> { await this.waitOpenConnection(ctx) - return this.lastHash + return this.lastHash ?? {} } private schedulePing (socketId: number): void { @@ -195,11 +194,11 @@ class Connection implements ClientConnection { clearInterval(interval) return } - if (!this.upgrading && this.pingResponse !== 0 && Date.now() - this.pingResponse > hangTimeout) { + if (this.pingResponse !== 0 && Date.now() - this.pingResponse > hangTimeout) { // No ping response from server. if (this.websocket !== null) { - console.log('no ping response from server. Closing socket.', socketId, this.workspace, this.user) + console.log('no ping response from server. Closing socket.', socketId, this.user) clearInterval(this.interval) this.websocket.close(1000) return @@ -332,12 +331,6 @@ class Connection implements ClientConnection { if (resp.error?.code === UNAUTHORIZED.code) { this.opt?.onUnauthorized?.() } - if (resp.error?.code === platform.status.WorkspaceArchived) { - this.opt?.onArchived?.() - } - if (resp.error?.code === platform.status.WorkspaceMigration) { - this.opt?.onMigration?.() - } } if (resp.id !== undefined) { @@ -366,12 +359,6 @@ class Connection implements ClientConnection { if (resp.id === -1) { this.delay = 0 - if (resp.result?.state === 'upgrading') { - void this.onConnect?.(ClientConnectEvent.Maintenance, undefined, resp.result.stats) - this.upgrading = true - this.delay = 3 - return - } if (resp.result === 'hello') { const helloResp = resp as HelloResponse this.binaryMode = helloResp.binary @@ -393,13 +380,9 @@ class Connection implements ClientConnection { return } this.account = helloResp.account - this.helloReceived = true - if (this.upgrading) { - // We need to call upgrade since connection is upgraded - this.opt?.onUpgrade?.() - } + this.opt?.onAccount?.(this.account) - this.upgrading = false + this.helloReceived = true // Notify all waiting connection listeners const handlers = this.onConnectHandlers.splice(0, this.onConnectHandlers.length) for (const h of handlers) { @@ -410,13 +393,15 @@ class Connection implements ClientConnection { v.reconnect?.() } - void this.onConnect?.( - helloResp.reconnect === true ? ClientConnectEvent.Reconnected : ClientConnectEvent.Connected, - helloResp.lastTx, - this.sessionId - )?.catch((err) => { - this.ctx.error('failed to call onConnect', { err }) - }) + void this.opt + ?.onConnect?.( + helloResp.reconnect === true ? ClientConnectEvent.Reconnected : ClientConnectEvent.Connected, + helloResp.lastTx, + this.sessionId + ) + ?.catch((err) => { + this.ctx.error('failed to call onConnect', { err }) + }) this.schedulePing(socketId) return } else { @@ -435,7 +420,7 @@ class Connection implements ClientConnection { if (promise === undefined) { console.error( - new Error(`unknown response id: ${resp.id as string} ${this.workspace} ${this.user}`), + new Error(`unknown response id: ${resp.id as string} ${this.user}`), JSON.stringify(this.requests) ) return @@ -460,7 +445,7 @@ class Connection implements ClientConnection { if (c.data.total !== 0) { total = c.data.total } - if (c.data.lookupMap !== undefined) { + if (c.data.lookupMap != null) { lookupMap = c.data.lookupMap } result = result.concat(c.data) @@ -493,7 +478,6 @@ class Connection implements ClientConnection { resp.error, 'result: ', resp.result, - this.workspace, this.user ) promise.reject(new PlatformError(resp.error)) @@ -518,14 +502,25 @@ class Connection implements ClientConnection { const txArr = Array.isArray(resp.result) ? (resp.result as Tx[]) : [resp.result as Tx] for (const tx of txArr) { - if (tx?._class === core.class.TxModelUpgrade) { - console.log('Processing upgrade', this.workspace, this.user) - this.opt?.onUpgrade?.() + if (tx?._class === core.class.TxWorkspaceEvent) { + const event = tx as TxWorkspaceEvent + // TODO: Check + if (event.event === WorkspaceEvent.WorkspaceMaintenance || event.event === WorkspaceEvent.WorkpaceActive) { + console.log('Processing upgrade', event.workspace, this.user) + + if (this.account !== undefined) { + const ws = this.account.workspaces[event._uuid] + if (ws != null) { + ws.maintenance = event.event === WorkspaceEvent.WorkspaceMaintenance + this.opt?.onAccount?.(this.account) + } + } + } return } } this.handlers.forEach((handler) => { - handler(...txArr) + handler(txArr) }) clearTimeout(this.incomingTimer) @@ -707,7 +702,7 @@ class Connection implements ClientConnection { this.delay += 1 } if (opened) { - console.error('client websocket error:', socketId, this.url, this.workspace, this.user) + console.error('client websocket error:', socketId, this.url, this.user) } void broadcastEvent(client.event.NetworkRequests, -1).catch((err) => { this.ctx.error('failed to broadcast', { err }) @@ -841,7 +836,7 @@ class Connection implements ClientConnection { } } }) - if (result.lookupMap !== undefined) { + if (result.lookupMap != null) { // We need to extract lookup map to document lookups for (const d of result) { if (d.$lookup !== undefined) { @@ -875,10 +870,10 @@ class Connection implements ClientConnection { return result } - tx (tx: Tx): Promise { + tx (tx: Tx, options?: TxOptions): Promise { return this.sendRequest({ method: 'tx', - params: [tx], + params: [tx, options], retry: async () => { if (tx._class === core.class.TxApplyIf) { return (await this.findAll(core.class.Tx, { _id: (tx as TxApplyIf).txes[0]._id }, { limit: 1 })).length === 0 @@ -888,36 +883,42 @@ class Connection implements ClientConnection { }) } - loadChunk (domain: Domain, idx?: number): Promise { - return this.sendRequest({ method: 'loadChunk', params: [domain, idx] }) + loadChunk (workspaceId: WorkspaceUuid, domain: Domain, idx?: number): Promise { + return this.sendRequest({ method: 'loadChunk', params: [workspaceId, domain, idx] }) } - async getDomainHash (domain: Domain): Promise { - return await this.sendRequest({ method: 'getDomainHash', params: [domain] }) + async getDomainHash (workspaceId: WorkspaceUuid, domain: Domain): Promise { + return await this.sendRequest({ method: 'getDomainHash', params: [workspaceId, domain] }) } - closeChunk (idx: number): Promise { - return this.sendRequest({ method: 'closeChunk', params: [idx] }) + closeChunk (workspaceId: WorkspaceUuid, idx: number): Promise { + return this.sendRequest({ method: 'closeChunk', params: [workspaceId, idx] }) } - loadDocs (domain: Domain, docs: Ref[]): Promise { - return this.sendRequest({ method: 'loadDocs', params: [domain, docs] }) + loadDocs (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { + return this.sendRequest({ method: 'loadDocs', params: [workspaceId, domain, docs] }) } - upload (domain: Domain, docs: Doc[]): Promise { - return this.sendRequest({ method: 'upload', params: [domain, docs] }) + upload (workspaceId: WorkspaceUuid, domain: Domain, docs: Doc[]): Promise { + return this.sendRequest({ method: 'upload', params: [workspaceId, domain, docs] }) } - clean (domain: Domain, docs: Ref[]): Promise { - return this.sendRequest({ method: 'clean', params: [domain, docs] }) + clean (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { + return this.sendRequest({ method: 'clean', params: [workspaceId, domain, docs] }) } searchFulltext (query: SearchQuery, options: SearchOptions): Promise { return this.sendRequest({ method: 'searchFulltext', params: [query, options] }) } - sendForceClose (): Promise { - return this.sendRequest({ method: 'forceClose', params: [], allowReconnect: false, overrideId: -2, once: true }) + sendForceClose (workspace: WorkspaceUuid): Promise { + return this.sendRequest({ + method: 'forceClose', + params: [workspace], + allowReconnect: false, + overrideId: -2, + once: true + }) } async sendEvent (event: Event): Promise { @@ -954,6 +955,17 @@ class Connection implements ClientConnection { async unsubscribeQuery (id: number): Promise { await this.sendRequest({ method: 'unsubscribeQuery', params: [id] }) } + + async subscribe (subscription: { + accounts?: AccountUuid[] + workspaces?: WorkspaceUuid[] + }): Promise { + return await this.sendRequest({ method: 'subscribe', params: [subscription] }) + } + + async unsubscribe (subscription: { accounts?: AccountUuid[], workspaces?: WorkspaceUuid[] }): Promise { + await this.sendRequest({ method: 'unsubscribe', params: [subscription] }) + } } /** @@ -962,7 +974,6 @@ class Connection implements ClientConnection { export function connect ( url: string, handler: TxHandler, - workspace: WorkspaceUuid, user: PersonUuid, opt?: ClientFactoryOptions ): ClientConnection { @@ -970,7 +981,6 @@ export function connect ( opt?.ctx?.newChild?.('connection', {}) ?? new MeasureMetricsContext('connection', {}), url, handler, - workspace, user, opt ) diff --git a/plugins/client-resources/src/index.ts b/plugins/client-resources/src/index.ts index 0c88df268f4..86d28895983 100644 --- a/plugins/client-resources/src/index.ts +++ b/plugins/client-resources/src/index.ts @@ -16,10 +16,18 @@ import clientPlugin from '@hcengineering/client' import type { ClientFactoryOptions } from '@hcengineering/client/src' import core, { - Client, + type Class, + type Client, + type ClientConnection, + type ConnectionEvents, + type Doc, LoadModelResponse, + type ModelFilter, type PersonUuid, + type PluginConfiguration, + type Ref, Tx, + type TxCUD, TxHandler, TxPersistenceStore, TxWorkspaceEvent, @@ -28,15 +36,8 @@ import core, { concatLink, createClient, fillConfiguration, - pluginFilterTx, - type Class, - type ClientConnection, - type Doc, - type ModelFilter, - type PluginConfiguration, - type Ref, - type TxCUD, - platformNow + platformNow, + pluginFilterTx } from '@hcengineering/core' import platform, { Severity, Status, getMetadata, getPlugins, setPlatformStatus } from '@hcengineering/platform' import { connect } from './connection' @@ -98,15 +99,11 @@ export default async () => { GetClient: async (token: string, endpoint: string, opt?: ClientFactoryOptions): Promise => { const filterModel = getMetadata(clientPlugin.metadata.FilterModel) ?? 'none' - const handler = async (handler: TxHandler): Promise => { + const handler = async (handler: TxHandler, events?: ConnectionEvents): Promise => { const url = concatLink(endpoint, `/${token}`) - const upgradeHandler: TxHandler = (...txes: Tx[]) => { + const upgradeHandler: TxHandler = (txes: Tx[]) => { for (const tx of txes) { - if (tx?._class === core.class.TxModelUpgrade) { - opt?.onUpgrade?.() - return - } if (tx?._class === core.class.TxWorkspaceEvent) { const event = tx as TxWorkspaceEvent if (event.event === WorkspaceEvent.MaintenanceNotification) { @@ -118,35 +115,35 @@ export default async () => { } } } - handler(...txes) + handler(txes) } const tokenPayload = decodeTokenPayload(token) if (tokenPayload.workspace === undefined || tokenPayload.account === undefined) { throw new Error('Workspace or account not found in token') } - const newOpt = { ...opt } + const newOpt = { ...events } const connectTimeout = opt?.connectionTimeout ?? getMetadata(clientPlugin.metadata.ConnectionTimeout) let connectPromise: Promise | undefined + if ((connectTimeout ?? 0) > 0) { connectPromise = new Promise((resolve, reject) => { const connectTO = setTimeout(() => { if (!clientConnection.isConnected()) { - newOpt.onConnect = undefined + newOpt.onConnect = events?.onConnect void clientConnection?.close() - void opt?.onDialTimeout?.() reject(new Error(`Connection timeout, and no connection established to ${endpoint}`)) } }, connectTimeout) newOpt.onConnect = async (event, lastTx, data) => { // Any event is fine, it means server is alive. clearTimeout(connectTO) - await opt?.onConnect?.(event, lastTx, data) resolve() } }) } - const clientConnection = connect(url, upgradeHandler, tokenPayload.workspace, tokenPayload.account, newOpt) + const clientConnection = connect(url, upgradeHandler, tokenPayload.account, newOpt) + if (connectPromise !== undefined) { await connectPromise } @@ -163,8 +160,12 @@ export default async () => { return txes } - const client = createClient(handler, modelFilter, createModelPersistence(getWSFromToken(token)), opt?.ctx) - return await client + return await createClient(handler, { + ...opt, + modelFilter, + txPersistence: createModelPersistence(getWSFromToken(token)), + _ctx: opt?.ctx + }) } } } diff --git a/plugins/client/src/index.ts b/plugins/client/src/index.ts index 077284f21b3..419ab6b24b9 100644 --- a/plugins/client/src/index.ts +++ b/plugins/client/src/index.ts @@ -13,8 +13,8 @@ // limitations under the License. // -import type { Client, ClientConnectEvent, MeasureContext, TxPersistenceStore } from '@hcengineering/core' -import { type Plugin, type Resource, type Metadata, plugin } from '@hcengineering/platform' +import type { Client, ConnectionEvents, MeasureContext, TxPersistenceStore } from '@hcengineering/core' +import { type Metadata, type Plugin, plugin, type Resource } from '@hcengineering/platform' /** * @public @@ -54,20 +54,12 @@ export enum ClientSocketReadyState { CLOSED = 3 } -export interface ClientFactoryOptions { +export interface ClientFactoryOptions extends ConnectionEvents { socketFactory?: ClientSocketFactory useBinaryProtocol?: boolean useProtocolCompression?: boolean connectionTimeout?: number - onHello?: (serverVersion?: string) => boolean - onUpgrade?: () => void - onUnauthorized?: () => void - onArchived?: () => void - onMigration?: () => void - onConnect?: (event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise ctx?: MeasureContext - onDialTimeout?: () => void | Promise - useGlobalRPCHandler?: boolean } diff --git a/plugins/communication-resources/src/activity.ts b/plugins/communication-resources/src/activity.ts index 1337324758b..a150c62b6d6 100644 --- a/plugins/communication-resources/src/activity.ts +++ b/plugins/communication-resources/src/activity.ts @@ -58,7 +58,7 @@ export async function getAttributeModel ( const model = await getAttributePresenterSafe( client, - update.mixin ?? _class, + update.mixin ?? (_class as any), attrKey, view.mixin.ActivityAttributePresenter ) @@ -67,7 +67,7 @@ export async function getAttributeModel ( return model } - return await getAttributePresenterSafe(client, update.mixin ?? _class, attrKey) + return await getAttributePresenterSafe(client, update.mixin ?? (_class as any), attrKey) } export async function getAttributeValues ( diff --git a/plugins/communication-resources/src/components/TypingPresenter.svelte b/plugins/communication-resources/src/components/TypingPresenter.svelte index 66331dfce19..90f6b52e856 100644 --- a/plugins/communication-resources/src/components/TypingPresenter.svelte +++ b/plugins/communication-resources/src/components/TypingPresenter.svelte @@ -34,7 +34,7 @@ let moreCount: number = 0 let typing: PresenceTyping[] = [] - $: presence = $presenceByObjectId.get(cardId) ?? [] + $: presence = $presenceByObjectId.get(cardId as any) ?? [] $: typing = presence.map((p) => p.presence.typing).filter(notEmpty) $: void updateTypingPersons(typing) diff --git a/plugins/communication-resources/src/components/message/MessageFooter.svelte b/plugins/communication-resources/src/components/message/MessageFooter.svelte index ece63d22721..c2f132e1c79 100644 --- a/plugins/communication-resources/src/components/message/MessageFooter.svelte +++ b/plugins/communication-resources/src/components/message/MessageFooter.svelte @@ -67,7 +67,7 @@
{#each message.files as file (file.blobId)} {/each} diff --git a/plugins/communication-resources/src/components/message/activity/ActivityUpdateTagViewer.svelte b/plugins/communication-resources/src/components/message/activity/ActivityUpdateTagViewer.svelte index 2918436fd2e..a1520da721b 100644 --- a/plugins/communication-resources/src/components/message/activity/ActivityUpdateTagViewer.svelte +++ b/plugins/communication-resources/src/components/message/activity/ActivityUpdateTagViewer.svelte @@ -26,7 +26,7 @@ const client = getClient() const hierarchy = client.getHierarchy() - $: mixin = hierarchy.hasClass(update.tag) ? hierarchy.getClass(update.tag) : undefined + $: mixin = hierarchy.hasClass(update.tag as any) ? hierarchy.getClass(update.tag as any) : undefined {#if mixin !== undefined} diff --git a/plugins/contact/src/utils.ts b/plugins/contact/src/utils.ts index 91c0a51e805..7e923847b5a 100644 --- a/plugins/contact/src/utils.ts +++ b/plugins/contact/src/utils.ts @@ -33,7 +33,8 @@ import { Ref, SocialId, toIdMap, - TxFactory + TxFactory, + type WorkspaceUuid } from '@hcengineering/core' import { getMetadata } from '@hcengineering/platform' import { ColorDefinition } from '@hcengineering/ui' @@ -396,11 +397,12 @@ export async function ensureEmployee ( ctx: MeasureContext, me: Account, client: Pick, + workspace: WorkspaceUuid, socialIds: SocialId[], getGlobalPerson: () => Promise ): Promise | null> { const globalPerson = await getGlobalPerson() - return await ensureEmployeeForPerson(ctx, me, me, client, socialIds, globalPerson) + return await ensureEmployeeForPerson(ctx, me, me, client, socialIds, workspace, globalPerson) } export async function ensureEmployeeForPerson ( @@ -409,15 +411,15 @@ export async function ensureEmployeeForPerson ( person: Account, client: Pick, socialIds: SocialId[], + workspace: WorkspaceUuid, globalPerson?: GlobalPerson ): Promise | null> { - const txFactory = new TxFactory(me.primarySocialId) - const personByUuid = await client.findOne(contact.class.Person, { personUuid: person.uuid }) + const txFactory = new TxFactory(me.primarySocialId, workspace) + const personByUuid = await client.findOne(contact.class.Person, { personUuid: person.uuid }, { workspace }) let personRef: Ref | undefined = personByUuid?._id if (personRef === undefined) { - const socialIdentity = await client.findOne(contact.class.SocialIdentity, { - _id: { $in: person.socialIds as SocialIdentityRef[] } - }) + const socialIdentity = await client.findOne(contact.class.SocialIdentity, { _id: { $in: person.socialIds as SocialIdentityRef[] } }, { workspace } + ) // This social id is confirmed globally as we only have ids of confirmed social identities in socialIds array personRef = socialIdentity?.attachedTo @@ -449,7 +451,7 @@ export async function ensureEmployeeForPerson ( } const existingIdentifiers = toIdMap( - await client.findAll(contact.class.SocialIdentity, { _id: { $in: person.socialIds as SocialIdentityRef[] } }) + await client.findAll(contact.class.SocialIdentity, { _id: { $in: person.socialIds as SocialIdentityRef[] } }, { workspace }) ) for (const socialId of socialIds) { diff --git a/plugins/guest-resources/src/connect.ts b/plugins/guest-resources/src/connect.ts index ced8e950d9a..31c322ae171 100644 --- a/plugins/guest-resources/src/connect.ts +++ b/plugins/guest-resources/src/connect.ts @@ -115,9 +115,6 @@ export async function connect (title: string): Promise { return true }, - onUpgrade: () => { - location.reload() - }, onUnauthorized: () => { void logOut().then(() => { invalidError.set(true) @@ -186,9 +183,13 @@ export async function connect (title: string): Promise { const me: Account = { uuid: account, role: workspaceLoginInfo.role, + targetWorkspace: workspaceLoginInfo.workspace, primarySocialId: '' as PersonId, socialIds: [], - fullSocialIds: [] + fullSocialIds: new Map(), + personalWorkspace: core.workspace.Personal, + socialIdsByValue: new Map(), + workspaces: {} } if (me !== undefined) { diff --git a/plugins/login-resources/src/utils.ts b/plugins/login-resources/src/utils.ts index 662b63d8bf3..53e1cf15acd 100644 --- a/plugins/login-resources/src/utils.ts +++ b/plugins/login-resources/src/utils.ts @@ -386,12 +386,13 @@ export async function getRegionInfo (doNavigate: boolean = true): Promise { const actualToken = token ?? getMetadata(presentation.metadata.Token) ?? undefined try { - const loginInfo = await getAccountClient(actualToken).selectWorkspace(workspaceUrl) + const loginInfo = await getAccountClient(actualToken).selectWorkspace(workspaceUrl, singleWorkspace) return [OK, loginInfo] } catch (err: any) { diff --git a/plugins/login/src/index.ts b/plugins/login/src/index.ts index 965926c2106..e0a9ad9d609 100644 --- a/plugins/login/src/index.ts +++ b/plugins/login/src/index.ts @@ -96,7 +96,11 @@ export default plugin(loginId, { LeaveWorkspace: '' as Resource<(account: string) => Promise>, ChangePassword: '' as Resource<(oldPassword: string, password: string) => Promise>, SelectWorkspace: '' as Resource< - (workspace: string, token: string | null | undefined) => Promise<[Status, WorkspaceLoginInfo | undefined]> + ( + workspace: string, + token: string | null | undefined, + singleWorkspace?: boolean + ) => Promise<[Status, WorkspaceLoginInfo | undefined]> >, ExchangeGuestToken: '' as Resource<(token: string) => Promise>, FetchWorkspace: '' as Resource<() => Promise<[Status, WorkspaceInfoWithStatus | undefined]>>, diff --git a/plugins/love-resources/src/utils.ts b/plugins/love-resources/src/utils.ts index 521fc7208c6..38fb376e729 100644 --- a/plugins/love-resources/src/utils.ts +++ b/plugins/love-resources/src/utils.ts @@ -49,6 +49,7 @@ import presentation, { createQuery, type DocCreatePhase, getClient, + getTargetWorkspace, type ObjectSearchResult } from '@hcengineering/presentation' import { @@ -797,6 +798,7 @@ async function initMeetingMinutes (room: Room): Promise { .replace(',', ' at') const _id = generateId() const newDoc: MeetingMinutes = { + _uuid: getTargetWorkspace(), _id, _class: love.class.MeetingMinutes, attachedTo: room._id, diff --git a/plugins/process-resources/src/middleware.ts b/plugins/process-resources/src/middleware.ts index 3f9b3db94c9..7d10fcd2f38 100644 --- a/plugins/process-resources/src/middleware.ts +++ b/plugins/process-resources/src/middleware.ts @@ -123,7 +123,7 @@ export class ProcessMiddleware extends BasePresentationMiddleware implements Pre const nextState = transition.to if (nextState == null) return const context = await getNextStateUserInput(execution, nextState, execution.context) - const txop = new TxOperations(this.client, getCurrentAccount().primarySocialId) + const txop = new TxOperations(this.client, getCurrentAccount().primarySocialId, etx._uuid) await txop.update(execution, { context }) diff --git a/plugins/setting-resources/src/components/Profile.svelte b/plugins/setting-resources/src/components/Profile.svelte index df5b3ffe143..f0dce4d0afb 100644 --- a/plugins/setting-resources/src/components/Profile.svelte +++ b/plugins/setting-resources/src/components/Profile.svelte @@ -35,7 +35,7 @@ const client = getClient() const account = getCurrentAccount() - const email = account.fullSocialIds.find((si) => si.type === SocialIdType.EMAIL)?.value ?? '' + const email = Array.from(account.fullSocialIds.values()).find((si) => si.type === SocialIdType.EMAIL)?.value ?? '' let firstName = $myEmployeeStore !== undefined ? getFirstName($myEmployeeStore.name) : '' let lastName = $myEmployeeStore !== undefined ? getLastName($myEmployeeStore.name) : '' diff --git a/plugins/tracker-resources/src/components/CreateIssue.svelte b/plugins/tracker-resources/src/components/CreateIssue.svelte index 73d0f0c5e25..96f67bb388a 100644 --- a/plugins/tracker-resources/src/components/CreateIssue.svelte +++ b/plugins/tracker-resources/src/components/CreateIssue.svelte @@ -272,6 +272,7 @@ function tagAsRef (tag: TagElement): TagReference { return { + _uuid: tag._uuid, _class: tags.class.TagReference, _id: generateId(), attachedTo: '' as Ref, diff --git a/plugins/tracker-resources/src/components/projects/ProjectPresenter.svelte b/plugins/tracker-resources/src/components/projects/ProjectPresenter.svelte index cffdc0810a0..8e2319c8626 100644 --- a/plugins/tracker-resources/src/components/projects/ProjectPresenter.svelte +++ b/plugins/tracker-resources/src/components/projects/ProjectPresenter.svelte @@ -13,19 +13,21 @@ // limitations under the License. --> {#if value} @@ -56,5 +58,11 @@
{/if} diff --git a/plugins/view-resources/src/components/SpacePresenter.svelte b/plugins/view-resources/src/components/SpacePresenter.svelte index 414a5c0840f..525b7938120 100644 --- a/plugins/view-resources/src/components/SpacePresenter.svelte +++ b/plugins/view-resources/src/components/SpacePresenter.svelte @@ -15,6 +15,7 @@ --> -{#if $location.path[0] === workbenchId || $location.path[0] === workbenchRes.component.WorkbenchApp} +{#if $location.path[0] === workbenchId || $location.path[0] === workbenchRes.component.WorkbenchApp || !singleWorkspace} {#if $deviceOptionsStore.isMobile && mobileAllowed !== true}
@@ -54,8 +56,8 @@
{:else} - {#key $location.path[1]} - {#await connect(getMetadata(workbenchRes.metadata.PlatformTitle) ?? 'Platform')} + {#key `${$location.path[1]}${singleWorkspace}`} + {#await connect(getMetadata(workbenchRes.metadata.PlatformTitle) ?? 'Platform', singleWorkspace)} {#if ($workspaceCreating ?? -1) >= 0}
diff --git a/plugins/workbench-resources/src/components/WorkbenchApps.svelte b/plugins/workbench-resources/src/components/WorkbenchApps.svelte new file mode 100644 index 00000000000..ba148bf5ea9 --- /dev/null +++ b/plugins/workbench-resources/src/components/WorkbenchApps.svelte @@ -0,0 +1,19 @@ + + + + diff --git a/plugins/workbench-resources/src/connect.ts b/plugins/workbench-resources/src/connect.ts index 5e307d4d0c6..58df7e7fe32 100644 --- a/plugins/workbench-resources/src/connect.ts +++ b/plugins/workbench-resources/src/connect.ts @@ -1,4 +1,3 @@ -import { getClient as getAccountClient } from '@hcengineering/account-client' import { Analytics } from '@hcengineering/analytics' import client from '@hcengineering/client' import { ensureEmployee, setCurrentEmployee } from '@hcengineering/contact' @@ -8,28 +7,16 @@ import core, { type Client, ClientConnectEvent, concatLink, - isWorkspaceCreating, + type Person as GlobalPerson, type MeasureMetricsContext, metricsToString, - type Person as GlobalPerson, - pickPrimarySocialId, setCurrentAccount, - type SocialId, type Version, - versionToString + versionToString, + type WorkspaceUuid } from '@hcengineering/core' -import login, { loginId, type Pages } from '@hcengineering/login' -import platform, { - PlatformEvent, - broadcastEvent, - getMetadata, - getResource, - OK, - setMetadata, - Severity, - Status, - translateCB -} from '@hcengineering/platform' +import login, { loginId } from '@hcengineering/login' +import platform, { broadcastEvent, getMetadata, getResource, OK, PlatformEvent, setMetadata, Severity, Status, translateCB } from '@hcengineering/platform' import presentation, { loadServerConfig, purgeClient, @@ -37,6 +24,8 @@ import presentation, { setClient, setCommunicationClient, setPresentationCookie, + setSingleWorkspace, + setTargetWorkspace, uiContext, upgradeDownloadProgress } from '@hcengineering/presentation' @@ -51,7 +40,7 @@ import { import { get, writable } from 'svelte/store' import plugin from './plugin' -import { logOut, workspaceCreating } from './utils' +import { logOut } from './utils' export const versionError = writable(undefined) const versionStorageKey = 'last_server_version' @@ -68,11 +57,11 @@ export async function disconnect (): Promise { } } -export async function connect (title: string): Promise { +export async function connect (title: string, singleWorkspace: boolean = true): Promise { const ctx = uiContext.newChild('connect', {}) const loc = getCurrentLocation() const wsUrl = loc.path[1] - if (wsUrl === undefined) { + if (singleWorkspace && wsUrl === undefined) { const lastLoc = localStorage.getItem(locationStorageKeyId) if (lastLoc !== null) { const lastLocObj = JSON.parse(lastLoc) @@ -89,7 +78,11 @@ export async function connect (title: string): Promise { } const selectWorkspace = await getResource(login.function.SelectWorkspace) - const [, workspaceLoginInfo] = await ctx.with('select-workspace', {}, async () => await selectWorkspace(wsUrl, null)) + const [, workspaceLoginInfo] = await ctx.with( + 'select-workspace', + {}, + async () => await selectWorkspace(wsUrl, null, singleWorkspace) + ) if (workspaceLoginInfo == null) { console.error( @@ -104,55 +97,21 @@ export async function connect (title: string): Promise { const token = workspaceLoginInfo.token setMetadata(presentation.metadata.Token, workspaceLoginInfo.token) - setMetadata(presentation.metadata.WorkspaceUuid, workspaceLoginInfo.workspace) - setMetadata(presentation.metadata.WorkspaceDataId, workspaceLoginInfo.workspaceDataId) - setMetadata(presentation.metadata.Endpoint, workspaceLoginInfo.endpoint) - - const fetchWorkspace = await getResource(login.function.FetchWorkspace) - let workspace = await ctx.with('fetch-workspace', {}, async () => (await fetchWorkspace())[1]) - - if (workspace == null) { - // something went wrong, workspace not exist, redirect to login - console.error( - `Error fetching workspace ${wsUrl}. It might no longer exist or be inaccessible. Please try to log in again.` - ) - navigate({ - path: [loginId] - }) - return - } - - if (isWorkspaceCreating(workspace.mode)) { - while (true) { - if (wsUrl !== getCurrentLocation().path[1]) return - - workspaceCreating.set(workspace.processingProgress ?? 0) - workspace = await ctx.with('fetch-workspace', {}, async () => (await fetchWorkspace())[1]) - - if (workspace == null) { - // something went wrong, workspace not exist, redirect to login - navigate({ - path: [loginId] - }) - return - } - workspaceCreating.set(workspace.processingProgress ?? 0) + setMetadata(presentation.metadata.Endpoint, workspaceLoginInfo.endpoint) - if (!isWorkspaceCreating(workspace.mode)) { - workspaceCreating.set(-1) - break - } + setMetadata(presentation.metadata.PersonalWorkspaceUuid, workspaceLoginInfo.personalWorkspace) - await new Promise((resolve) => setTimeout(resolve, 1000)) - } - } + // Set for target workspace for now + setMetadata(presentation.metadata.WorkspaceUuid, workspaceLoginInfo.workspace) + setMetadata(presentation.metadata.WorkspaceDataId, workspaceLoginInfo.workspaceDataId) + setMetadata(presentation.metadata.PersonalWorkspaceUuid, core.workspace.Personal) // No personal workspace set in single workspace mode setPresentationCookie(token, workspaceLoginInfo.workspace) setMetadataLocalStorage(login.metadata.LoginEndpoint, workspaceLoginInfo?.endpoint) const endpoint = getMetadata(login.metadata.TransactorOverride) ?? workspaceLoginInfo?.endpoint - const account = workspaceLoginInfo?.account + const account = workspaceLoginInfo.account if (token == null || endpoint == null || account == null) { console.error('Something of the vital auth info is missing. Please try to log in again.') const navigateUrl = encodeURIComponent(JSON.stringify(loc)) @@ -183,11 +142,15 @@ export async function connect (title: string): Promise { const clientFactory = await getResource(client.function.GetClient) let version: Version | undefined + let accountRef: Account | undefined const newClient = await ctx.with( 'create-client', {}, async (ctx) => await clientFactory(token, endpoint, { + onAccount: (a) => { + accountRef = a + }, onHello: (serverVersion?: string) => { const frontVersion = getMetadata(presentation.metadata.FrontVersion) if ( @@ -227,9 +190,6 @@ export async function connect (title: string): Promise { return true }, - onUpgrade: () => { - location.reload() - }, onUnauthorized: () => { void logOut().then(() => { navigate({ @@ -238,24 +198,6 @@ export async function connect (title: string): Promise { }) }) }, - onArchived: () => { - translateCB(plugin.string.WorkspaceIsArchived, {}, get(themeStore).language, (r) => { - const selectWorkspace: Pages = 'selectWorkspace' - navigate({ - path: [loginId, selectWorkspace], - query: {} - }) - }) - }, - onMigration: () => { - // TODO: Rework maitenance mode as well - translateCB(plugin.string.WorkspaceIsMigrating, {}, get(themeStore).language, (r) => { - versionError.set(r) - setTimeout(() => { - location.reload() - }, 5000) - }) - }, // We need to refresh all active live queries and clear old queries. onConnect: async (event: ClientConnectEvent, data: any): Promise => { console.log('WorkbenchClient: onConnect', event) @@ -339,7 +281,7 @@ export async function connect (title: string): Promise { const newLoginInfo = await ctx.with( 'select-workspace', {}, - async () => (await selectWorkspace(wsUrl, token))[1] + async () => (await selectWorkspace(wsUrl, token, singleWorkspace))[1] ) if (newLoginInfo?.endpoint !== endpoint) { console.log('endpoint changed, reloading') @@ -349,23 +291,41 @@ export async function connect (title: string): Promise { }) ) - _client = newClient - - // TODO: should we take the function from some resource like fetchWorkspace/selectWorkspace - // to remove account client dependency? - const accountsUrl = getMetadata(login.metadata.AccountsUrl) - const socialIds: SocialId[] = await getAccountClient(accountsUrl, token).getSocialIds() - - const me: Account = { - uuid: account, - role: workspaceLoginInfo.role, - primarySocialId: pickPrimarySocialId(socialIds)._id, - socialIds: socialIds.map((si) => si._id), - fullSocialIds: socialIds + if (accountRef == null) { + accountRef = await newClient.getConnection?.()?.getAccount() + } + if (accountRef == null) { + throw new Error('Failed to get account') } + _client = newClient + // Ensure employee and social identifiers - const employee = await ensureEmployee(ctx, me, newClient, socialIds, getGlobalPerson) + const employee = await ensureEmployee( + ctx, + accountRef, + newClient, + workspaceLoginInfo.workspace, + Array.from(accountRef.fullSocialIds.values()), + getGlobalPerson + ) + + if (!singleWorkspace) { + // We need to be sure all other employee instances are created. + for (const [ws, info] of Object.entries(accountRef.workspaces)) { + if (ws === workspaceLoginInfo.workspace || info.maintenance || !info.enabled) { + continue + } + await ensureEmployee( + ctx, + accountRef, + newClient, + ws as WorkspaceUuid, + Array.from(accountRef.fullSocialIds.values()), + getGlobalPerson + ) + } + } if (employee == null) { console.log('Failed to ensure employee') @@ -378,11 +338,14 @@ export async function connect (title: string): Promise { Analytics.setUser(account) Analytics.setTag('workspace', wsUrl) - console.log('Logged in with account: ', me) - setCurrentAccount(me) + console.log('Logged in with account: ', accountRef) + setCurrentAccount(accountRef) setCurrentEmployee(employee) + if (workspaceLoginInfo.workspace !== undefined) { + setTargetWorkspace(workspaceLoginInfo.workspace) + } - if (me.role === AccountRole.ReadOnlyGuest) { + if (accountRef.role === AccountRole.ReadOnlyGuest) { await broadcastEvent(PlatformEvent, new Status(Severity.INFO, platform.status.ReadOnlyAccount, {})) } @@ -421,11 +384,13 @@ export async function connect (title: string): Promise { document.title = [wsUrl, title].filter((it) => it).join(' - ') _clientSet = true await ctx.with('set-client', {}, async () => { + setSingleWorkspace(singleWorkspace) await setClient(newClient) + await setCommunicationClient(newClient) }) await ctx.with('broadcast-connected', {}, async () => { - await broadcastEvent(plugin.event.NotifyConnection, me) + await broadcastEvent(plugin.event.NotifyConnection, accountRef) }) console.log(metricsToString((ctx as MeasureMetricsContext).metrics, 'connect', 50)) return newClient diff --git a/plugins/workbench-resources/src/index.ts b/plugins/workbench-resources/src/index.ts index a9b31815bef..5d72882ace9 100644 --- a/plugins/workbench-resources/src/index.ts +++ b/plugins/workbench-resources/src/index.ts @@ -20,6 +20,7 @@ import Archive from './components/Archive.svelte' import SpacePanel from './components/navigator/SpacePanel.svelte' import SpecialView from './components/SpecialView.svelte' import WorkbenchApp from './components/WorkbenchApp.svelte' +import WorkbenchApps from './components/WorkbenchApps.svelte' import { doNavigate, logIn, logOut } from './utils' import Workbench from './components/Workbench.svelte' import ServerManager from './components/ServerManager.svelte' @@ -49,6 +50,7 @@ export * from './sidebar' export default async (): Promise => ({ component: { WorkbenchApp, + WorkbenchApps, ApplicationPresenter, Archive, SpacePanel, diff --git a/plugins/workbench-resources/src/workbench.ts b/plugins/workbench-resources/src/workbench.ts index db145d44520..e9f173f06e6 100644 --- a/plugins/workbench-resources/src/workbench.ts +++ b/plugins/workbench-resources/src/workbench.ts @@ -99,6 +99,7 @@ const syncTabLoc = reduceCalls(async (): Promise => { const me = getCurrentAccount() const newTab: WorkbenchTab = { + _uuid: getClient().workspaceUuid, _id: generateId(), _class: workbench.class.WorkbenchTab, space: core.space.Workspace, diff --git a/plugins/workbench/src/index.ts b/plugins/workbench/src/index.ts index 735411078c7..9c64bdb954a 100644 --- a/plugins/workbench/src/index.ts +++ b/plugins/workbench/src/index.ts @@ -13,12 +13,12 @@ // limitations under the License. // -import { workbenchId, workbenchPlugin } from './plugin' +import { workbenchId, workbenchPlugin, workbenchAppsId } from './plugin' export * from './analytics' export * from './types' export * from './utils' -export { workbenchId } +export { workbenchId, workbenchAppsId } export default workbenchPlugin diff --git a/plugins/workbench/src/plugin.ts b/plugins/workbench/src/plugin.ts index 3f587ee124f..55a324d55d2 100644 --- a/plugins/workbench/src/plugin.ts +++ b/plugins/workbench/src/plugin.ts @@ -34,6 +34,8 @@ import type { /** @public */ export const workbenchId = 'workbench' as Plugin +export const workbenchAppsId = 'apps' as Plugin + /** @public */ export const workbenchPlugin = plugin(workbenchId, { class: { @@ -49,6 +51,7 @@ export const workbenchPlugin = plugin(workbenchId, { }, component: { WorkbenchApp: '' as AnyComponent, + WorkbenchApps: '' as AnyComponent, // Multi workspace version of workbench InviteLink: '' as AnyComponent, Archive: '' as AnyComponent, SpecialView: '' as AnyComponent diff --git a/pods/fulltext/src/__tests__/indexing.spec.ts b/pods/fulltext/src/__tests__/indexing.spec.ts index 7b9b99d7f76..1ce0c17b0d9 100644 --- a/pods/fulltext/src/__tests__/indexing.spec.ts +++ b/pods/fulltext/src/__tests__/indexing.spec.ts @@ -32,8 +32,6 @@ import { dbConfig, dbUrl, elasticIndexName, model, prepare, preparePipeline } fr prepare() -jest.setTimeout(500000) - class TestWorkspaceManager extends WorkspaceManager { public async getWorkspaceInfo (token?: string): Promise { const decodedToken = decodeToken(token ?? '') @@ -52,6 +50,11 @@ class TestWorkspaceManager extends WorkspaceManager { lastBackup: 0, backups: 0 }, + endpoint: { + internalUrl: 'http://localhost:3003', + externalUrl: 'http://localhost:3003', + region: 'test' + }, versionMajor: 0, versionMinor: 0, versionPatch: 0, @@ -159,7 +162,7 @@ describe('full-text-indexing', () => { const dataId = generateId() - const ops = new TxOperations(pipelineClient, core.account.System) + const ops = new TxOperations(pipelineClient, core.account.System, core.workspace.Model) let id: Ref await queue.expectIndexingDoc(dataId, async () => { @@ -190,7 +193,7 @@ describe('full-text-indexing', () => { const dataId = generateId() - const ops = new TxOperations(pipelineClient, core.account.System) + const ops = new TxOperations(pipelineClient, core.account.System, core.workspace.Model) for (let i = 0; i < 1000; i++) { await ops.createDoc(test.class.TestDocument, core.space.Workspace, { diff --git a/pods/fulltext/src/__tests__/minmodel.ts b/pods/fulltext/src/__tests__/minmodel.ts index 75f8cf9910e..c01782473b3 100644 --- a/pods/fulltext/src/__tests__/minmodel.ts +++ b/pods/fulltext/src/__tests__/minmodel.ts @@ -38,7 +38,7 @@ import { plugin } from '@hcengineering/platform' import buildModel from '@hcengineering/model-all' -const txFactory = new TxFactory(core.account.System) +const txFactory = new TxFactory(core.account.System, core.workspace.Model) function createClass (_class: Ref>, attributes: Data>): TxCreateDoc { return txFactory.createTxCreateDoc(core.class.Class, core.space.Model, attributes, _class) diff --git a/pods/fulltext/src/workspace.ts b/pods/fulltext/src/workspace.ts index dbf67292e2d..c301d24f4dd 100644 --- a/pods/fulltext/src/workspace.ts +++ b/pods/fulltext/src/workspace.ts @@ -113,6 +113,7 @@ export class WorkspaceIndexer { _class: classes } const tx: TxWorkspaceEvent = { + _uuid: workspace.uuid, _class: core.class.TxWorkspaceEvent, _id: generateId(), event: WorkspaceEvent.IndexingUpdate, diff --git a/pods/server/package.json b/pods/server/package.json index 4cca2646132..6a7a946f88c 100644 --- a/pods/server/package.json +++ b/pods/server/package.json @@ -82,6 +82,7 @@ "@hcengineering/server-storage": "^0.6.0", "@hcengineering/server-telegram": "^0.6.0", "@hcengineering/server-token": "^0.6.11", + "@hcengineering/client-resources": "^0.6.27", "utf-8-validate": "^6.0.4", "bufferutil": "^4.0.8", "msgpackr": "^1.11.2", diff --git a/pods/server/src/__start.ts b/pods/server/src/__start.ts index 507a70f579a..adb4916ed02 100644 --- a/pods/server/src/__start.ts +++ b/pods/server/src/__start.ts @@ -25,6 +25,8 @@ import serverToken from '@hcengineering/server-token' import { join } from 'path' import { start } from '.' import { profileStart, profileStop } from './inspector' +import client from '@hcengineering/client' +import { WebSocket } from 'ws' configureAnalytics(process.env.SENTRY_DSN, {}) Analytics.setTag('application', 'transactor') @@ -84,6 +86,13 @@ setMetadata(serverNotification.metadata.WebPushUrl, config.webPushUrl) setMetadata(serverAiBot.metadata.EndpointURL, process.env.AI_BOT_URL) setMetadata(serverCalendar.metadata.EndpointURL, process.env.CALENDAR_URL) +const region = process.env.REGION ?? '' +const endpointName = process.env.ENDPOINT_NAME ?? `ws://huly.local:${config.serverPort}` + +setMetadata(client.metadata.ClientSocketFactory, (url) => { + return new WebSocket(url) as any +}) + const { shutdown, sessionManager } = start(metricsContext, config.dbUrl, { fulltextUrl: config.fulltextUrl, storageConfig, @@ -97,7 +106,9 @@ const { shutdown, sessionManager } = start(metricsContext, config.dbUrl, { stop: profileStop }, mongoUrl: config.mongoUrl, - queue + queue, + region, + endpointName }) getStats = (): WorkspaceStatistics[] => { diff --git a/pods/server/src/__tests__/minmodel.ts b/pods/server/src/__tests__/minmodel.ts index 625e0d06d6c..d13db61b5e5 100644 --- a/pods/server/src/__tests__/minmodel.ts +++ b/pods/server/src/__tests__/minmodel.ts @@ -34,7 +34,7 @@ import core, { import type { IntlString, Plugin } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' -export const txFactory = new TxFactory(core.account.System) +export const txFactory = new TxFactory(core.account.System, core.workspace.Model) export function createClass (_class: Ref>, attributes: Data>): TxCreateDoc { return txFactory.createTxCreateDoc(core.class.Class, core.space.Model, attributes, _class) diff --git a/pods/server/src/__tests__/server.test.ts b/pods/server/src/__tests__/server.test.ts index 4271a1a3758..3a45333f667 100644 --- a/pods/server/src/__tests__/server.test.ts +++ b/pods/server/src/__tests__/server.test.ts @@ -19,7 +19,7 @@ import { RPCHandler, type Response } from '@hcengineering/rpc' import { generateToken } from '@hcengineering/server-token' import WebSocket from 'ws' -import { +import core, { Hierarchy, MeasureMetricsContext, ModelDb, @@ -102,7 +102,10 @@ describe('server', () => { }, brandingMap: {}, accountsUrl: '', - queue: createDummyQueue() + queue: createDummyQueue(), + region: '', + endpointName: '', + endpointFactory: () => { return {} as any } } const sessionMgr = startSessionManager(toolCtx, opt) @@ -182,6 +185,7 @@ describe('server', () => { options?: FindOptions ): Promise> => { const d: Doc & { sessionId: string } = { + _uuid: core.workspace.Any, _class: 'result' as Ref>, _id: '1' as Ref, space: '' as Ref, @@ -215,9 +219,14 @@ describe('server', () => { communicationApiFactory: async () => { return {} as any }, + endpointFactory: () => { + throw new Error('Method not implemented.') + }, brandingMap: {}, accountsUrl: '', - queue: createDummyQueue() + queue: createDummyQueue(), + region: '', + endpointName: '' } const cancelOp = startSessionManager(new MeasureMetricsContext('test', {}), opt) const serverShutdown = startHttpServer(toolCtx, sessionMgr, port + 1, opt.accountsUrl, createDummyStorageAdapter()) diff --git a/pods/server/src/rpc.ts b/pods/server/src/rpc.ts index f6907fd912a..ecdc5bf2431 100644 --- a/pods/server/src/rpc.ts +++ b/pods/server/src/rpc.ts @@ -1,31 +1,32 @@ +import { getClient as getAccountClientRaw, type AccountClient } from '@hcengineering/account-client' +import contact, { + AvatarType, + combineName, + type Person, + type SocialIdentity, + type SocialIdentityRef +} from '@hcengineering/contact' import core, { buildSocialIdString, generateId, - systemAccountUuid, pickPrimarySocialId, + systemAccountUuid, TxFactory, TxProcessor, type AttachedData, - type Data, type Class, + type Data, type Doc, type MeasureContext, type Ref, type SearchOptions, type SearchQuery, - type TxCUD + type TxCUD, + type WorkspaceUuid } from '@hcengineering/core' -import type { ClientSessionCtx, ConnectionSocket, Session, SessionManager } from '@hcengineering/server-core' -import { decodeToken } from '@hcengineering/server-token' import { rpcJSONReplacer, type RateLimitInfo } from '@hcengineering/rpc' -import contact, { - AvatarType, - combineName, - type SocialIdentity, - type Person, - type SocialIdentityRef -} from '@hcengineering/contact' -import { type AccountClient, getClient as getAccountClientRaw } from '@hcengineering/account-client' +import type { ConnectionSocket } from '@hcengineering/server-core' +import { decodeToken } from '@hcengineering/server-token' import { createHash } from 'crypto' import { type Express, type Response as ExpressResponse, type Request } from 'express' @@ -36,6 +37,7 @@ import { gzip } from 'zlib' import { retrieveJson } from './utils' import { unknownError } from '@hcengineering/platform' +import type { ClientSessionCtx, Session, SessionManager } from '@hcengineering/server' interface RPCClientInfo { client: ConnectionSocket session: Session @@ -139,7 +141,8 @@ export function registerRPC (app: Express, sessions: SessionManager, ctx: Measur ctx: ClientSessionCtx, session: Session, rateLimit: RateLimitInfo | undefined, - token: string + token: string, + workspaceId: WorkspaceUuid ) => Promise ): Promise { try { @@ -153,34 +156,27 @@ export function registerRPC (app: Express, sessions: SessionManager, ctx: Measur sendError(res, 401, { message: 'Missing Authorization header' }) return } - const workspaceId = decodeURIComponent(req.params.workspaceId) + const workspaceId = decodeURIComponent(req.params.workspaceId) as WorkspaceUuid token = token.split(' ')[1] const decodedToken = decodeToken(token) - if (workspaceId !== decodedToken.workspace) { - sendError(res, 401, { message: 'Invalid workspace', workspace: decodedToken.workspace }) - return - } + // if (workspaceId !== decodedToken.workspace) { + // sendError(res, 401, { message: 'Invalid workspace', workspace: decodedToken.workspace }) + // return + // } let transactorRpc = rpcSessions.get(token) if (transactorRpc === undefined) { const cs: ConnectionSocket = createClosingSocket(token, rpcSessions) - const s = await sessions.addSession(ctx, cs, decodedToken, token, token) - if (!('session' in s)) { - sendError(res, 401, { - message: 'Failed to create session', - mode: 'specialError' in s ? s.specialError ?? '' : 'upgrading' - }) - return - } - transactorRpc = { session: s.session, client: cs, workspaceId: s.workspaceId, context: s.context } + const session = await sessions.addSession(ctx, cs, decodedToken, token, token) + transactorRpc = { session, client: cs, workspaceId, context: s.context } rpcSessions.set(token, transactorRpc) } const rpc = transactorRpc - const rateLimit = await sessions.handleRPC(rpc.context, rpc.session, rpc.client, async (ctx, rateLimit) => { - await operation(ctx, rpc.session, rateLimit, token) + const rateLimit = await sessions.handleRPC(ctx, workspaceId, rpc.session, rpc.client, async (ctx, rateLimit) => { + await operation(ctx, rpc.session, rateLimit, token, workspaceId) }) if (rateLimit !== undefined) { const { remaining, limit, reset, retryAfter } = rateLimit @@ -208,15 +204,15 @@ export function registerRPC (app: Express, sessions: SessionManager, ctx: Measur } app.get('/api/v1/ping/:workspaceId', (req, res) => { - void withSession(req, res, async (ctx, session, rateLimit) => { + void withSession(req, res, async (ctx, session, rateLimit, token, workspaceId) => { await session.ping(ctx) + await sendJson( req, res, { pong: true, - lastTx: ctx.pipeline.context.lastTx, - lastHash: ctx.pipeline.context.lastHash + ...(await sessions.getLastTxHash(workspaceId)) }, rateLimitToHeaders(rateLimit) ) @@ -275,14 +271,17 @@ export function registerRPC (app: Express, sessions: SessionManager, ctx: Measur core.class.Space, core.class.Tx ] - const h = ctx.pipeline.context.hierarchy - const filtered = txes.filter( - (it) => - TxProcessor.isExtendsCUD(it._class) && - allowedClasess.some((cl) => h.isDerived((it as TxCUD).objectClass, cl)) - ) + const workspace = ctx.workspaces[0] + await workspace.with(async (pipeline) => { + const h = pipeline.context.hierarchy + const filtered = txes.filter( + (it) => + TxProcessor.isExtendsCUD(it._class) && + allowedClasess.some((cl) => h.isDerived((it as TxCUD).objectClass, cl)) + ) - await sendJson(req, res, filtered, rateLimitToHeaders(rateLimit)) + await sendJson(req, res, filtered, rateLimitToHeaders(rateLimit)) + }) }) }) @@ -327,14 +326,14 @@ export function registerRPC (app: Express, sessions: SessionManager, ctx: Measur }) app.post('/api/v1/ensure-person/:workspaceId', (req, res) => { - void withSession(req, res, async (ctx, session, rateLimit, token) => { + void withSession(req, res, async (ctx, session, rateLimit, token, workspaceId) => { const { socialType, socialValue, firstName, lastName } = (await retrieveJson(req)) ?? {} const accountClient = getAccountClient(token) const { uuid, socialId } = await accountClient.ensurePerson(socialType, socialValue, firstName, lastName) const primaryPersonId = session.getUser() === systemAccountUuid ? core.account.System : pickPrimarySocialId(session.getSocialIds())._id - const txFactory: TxFactory = new TxFactory(primaryPersonId) + const txFactory: TxFactory = new TxFactory(primaryPersonId, workspaceId) const [person] = await session.findAllRaw(ctx, contact.class.Person, { personUuid: uuid }, { limit: 1 }) let personRef: Ref = person?._id diff --git a/pods/server/src/server.ts b/pods/server/src/server.ts index 3fb68c94ff0..e182aaf05ae 100644 --- a/pods/server/src/server.ts +++ b/pods/server/src/server.ts @@ -14,16 +14,11 @@ // limitations under the License. // -import { type BrandingMap, type MeasureContext, type Tx } from '@hcengineering/core' +import { systemAccount, systemAccountUuid, type BrandingMap, type MeasureContext, type Tx } from '@hcengineering/core' import { buildStorageFromConfig } from '@hcengineering/server-storage' -import { startSessionManager } from '@hcengineering/server' -import { - type CommunicationApiFactory, - type PlatformQueue, - type SessionManager, - type StorageConfiguration -} from '@hcengineering/server-core' +import { startSessionManager, type SessionManager } from '@hcengineering/server' +import { type CommunicationApiFactory, type EndpointConnectionFactory, type PlatformQueue, type StorageConfiguration } from '@hcengineering/server-core' import { Api as CommunicationApi } from '@hcengineering/communication-server' import { @@ -53,6 +48,7 @@ import { } from '@hcengineering/postgres' import { readFileSync } from 'node:fs' import { startHttpServer } from './server_http' +import { connect } from '@hcengineering/client-resources' const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[] registerStringLoaders() @@ -90,6 +86,9 @@ export function start ( } mongoUrl?: string + + region: string + endpointName: string } ): { shutdown: () => Promise, sessionManager: SessionManager } { registerTxAdapterFactory('mongodb', createMongoTxAdapter) @@ -144,14 +143,21 @@ export function start ( ) } + const endpointFactory: EndpointConnectionFactory = (ctx, endpointUrl, handler, opt) => { + return connect(endpointUrl, handler, systemAccountUuid, opt) + } + const sessionManager = startSessionManager(metrics, { pipelineFactory, + endpointFactory, communicationApiFactory, brandingMap: opt.brandingMap, enableCompression: opt.enableCompression, accountsUrl: opt.accountsUrl, profiling: opt.profiling, - queue: opt.queue + queue: opt.queue, + region: opt.region, + endpointName: opt.endpointName }) const shutdown = startHttpServer(metrics, sessionManager, opt.port, opt.accountsUrl, externalStorage) return { diff --git a/pods/server/src/server_http.ts b/pods/server/src/server_http.ts index 4a092bf8301..40868675e4d 100644 --- a/pods/server/src/server_http.ts +++ b/pods/server/src/server_http.ts @@ -28,7 +28,7 @@ import { type WorkspaceIds, type WorkspaceUuid } from '@hcengineering/core' -import platform, { Severity, Status, UNAUTHORIZED, unknownStatus } from '@hcengineering/platform' +import { UNAUTHORIZED } from '@hcengineering/platform' import { RPCHandler, type Response } from '@hcengineering/rpc' import { doSessionOp, @@ -38,6 +38,7 @@ import { processRequest, wipeStatistics, type BlobResponse, + type SessionManager, type WebsocketData } from '@hcengineering/server' import { @@ -45,7 +46,6 @@ import { pingConst, pongConst, type ConnectionSocket, - type SessionManager, type StorageAdapter } from '@hcengineering/server-core' import { decodeToken, type Token } from '@hcengineering/server-token' @@ -261,7 +261,7 @@ export function startHttpServer ( } case 'force-close': { const wsId = req.query.wsId as WorkspaceUuid - void sessions.forceClose(wsId ?? payload.workspace) + void sessions.forceCloseWorkspace(ctx, wsId ?? payload.workspace) res.writeHead(200) res.end() return @@ -454,69 +454,20 @@ export function startHttpServer ( } const cs: ConnectionSocket = createWebsocketClientSocket(ws, data) + const session = sessions.addSession(ctx, cs, token, rawToken, sessionId) + + void session.catch(() => { + // Ignore err + ws.close() + }) const webSocketData: WebsocketData = { connectionSocket: cs, payload: token, token: rawToken, - session: sessions.addSession(ctx, cs, token, rawToken, sessionId), + session, url: '' } - if (webSocketData.session instanceof Promise) { - void webSocketData.session.then((s) => { - if ('error' in s) { - if (s.specialError === 'archived') { - void cs.send( - ctx, - { - id: -1, - error: new Status(Severity.ERROR, platform.status.WorkspaceArchived, { - workspaceUuid: token.workspace - }), - terminate: s.terminate - }, - false, - false - ) - } else if (s.specialError === 'migration') { - void cs.send( - ctx, - { - id: -1, - error: new Status(Severity.ERROR, platform.status.WorkspaceMigration, { - workspaceUuid: token.workspace - }), - terminate: s.terminate - }, - false, - false - ) - } else { - void cs.send( - ctx, - { id: -1, error: unknownStatus(s.error.message ?? 'Unknown error'), terminate: s.terminate }, - false, - false - ) - } - // No connection to account service, retry from client. - setTimeout(() => { - cs.close() - }, 1000) - } - if ('upgrade' in s) { - void cs - .send(ctx, { id: -1, result: { state: 'upgrading', stats: (s as any).upgradeInfo } }, false, false) - .then(() => { - cs.close() - }) - } - }) - void webSocketData.session.catch((err) => { - ctx.error('unexpected error in websocket', { err }) - }) - } - // eslint-disable-next-line @typescript-eslint/no-misused-promises ws.on('message', (msg: RawData) => { try { @@ -530,8 +481,7 @@ export function startHttpServer ( doSessionOp( webSocketData, (s, buff) => { - s.context.measure('receive-data', buff?.length ?? 0) - processRequest(s.session, cs, s.context, s.workspaceId, buff, sessions) + processRequest(ctx, s, cs, buff, sessions) }, buff ) @@ -543,32 +493,30 @@ export function startHttpServer ( } } }) - // eslint-disable-next-line @typescript-eslint/no-misused-promises - ws.on('close', (code: number, reason: Buffer) => { + + const handleClose = (err: Error | null): void => { doSessionOp( webSocketData, (s) => { - if (!(s.session.workspaceClosed ?? false)) { - // remove session after 1seconds, give a time to reconnect. - void sessions.close(ctx, cs, token.workspace) + if (err !== null) { + ctx.error('error', { err, user: s.getUser() }) } + // remove session after 1seconds, give a time to reconnect. + void sessions.close(ctx, s).catch((err) => { + ctx.error('failed to close session', { err }) + }) }, Buffer.from('') ) + } + + // eslint-disable-next-line @typescript-eslint/no-misused-promises + ws.on('close', (code: number, reason: Buffer) => { + handleClose(null) }) ws.on('error', (err) => { - doSessionOp( - webSocketData, - (s) => { - ctx.error('error', { err, user: s.session.getUser() }) - if (!(s.session.workspaceClosed ?? false)) { - // remove session after 1seconds, give a time to reconnect. - void sessions.close(ctx, cs, token.workspace) - } - }, - Buffer.from('') - ) + handleClose(err) }) } wss.on('connection', handleConnection as any) diff --git a/rush.json b/rush.json index dec889e98ce..9fb674d2933 100644 --- a/rush.json +++ b/rush.json @@ -16,7 +16,7 @@ * path segment in the "$schema" field for all your Rush config files. This will ensure * correct error-underlining and tab-completion for editors such as VS Code. */ - "rushVersion": "5.151.0", + "rushVersion": "5.153.2", /** * The next field selects which package manager should be installed and determines its version. @@ -137,7 +137,7 @@ * pre-LTS versions in preparation for supporting the first LTS version, you can use this setting * to disable Rush's warning. */ - // "suppressNodeLtsWarning": false, + "suppressNodeLtsWarning": true, /** * If you would like the version specifiers for your dependencies to be consistent, then @@ -150,7 +150,7 @@ * version. In those cases, you will need to add an entry to the "allowedAlternativeVersions" * section of the common-versions.json. */ - // "ensureConsistentVersions": true, + "ensureConsistentVersions": true, /** * Large monorepos can become intimidating for newcomers if project folder paths don't follow diff --git a/server-plugins/activity-resources/src/newActivity.ts b/server-plugins/activity-resources/src/newActivity.ts index 06af1e86cbb..9032adaaac4 100644 --- a/server-plugins/activity-resources/src/newActivity.ts +++ b/server-plugins/activity-resources/src/newActivity.ts @@ -156,7 +156,7 @@ async function getUpdateText (update: ActivityUpdate, card: Card, hierarchy: Hie if (isUnset) { return await translate(activity.string.UnsetObject, { object: attrName }) } else { - const values = await getAttributeValues(set, attrClass) + const values = await getAttributeValues(set, attrClass as any) if (values !== undefined) { return await translate(activity.string.AttributeSetTo, { name: capitalizeFirstLetter(attrName), @@ -169,7 +169,7 @@ async function getUpdateText (update: ActivityUpdate, card: Card, hierarchy: Hie } if (update.type === ActivityUpdateType.Tag) { - const clazz = hierarchy.getClass(update.tag) + const clazz = hierarchy.getClass(update.tag as any) if (update.action === 'add') { const tagName = await translate(clazz.label, {}) return await translate(activity.string.AddedTag, { title: tagName }) diff --git a/server-plugins/activity-resources/src/references.ts b/server-plugins/activity-resources/src/references.ts index 143d7fa4b18..55048c4f0fd 100644 --- a/server-plugins/activity-resources/src/references.ts +++ b/server-plugins/activity-resources/src/references.ts @@ -592,7 +592,7 @@ async function ActivityReferenceCreate (tx: TxCUD, control: TriggerControl) if (control.hierarchy.isDerived(ctx.objectClass, notification.class.InboxNotification)) return [] if (control.hierarchy.isDerived(ctx.objectClass, activity.class.ActivityReference)) return [] - const txFactory = new TxFactory(control.txFactory.account) + const txFactory = new TxFactory(control.txFactory.account, control.workspace.uuid) const doc = TxProcessor.createDoc2Doc(ctx) const target = guessReferenceObj(control.hierarchy, tx) @@ -641,7 +641,7 @@ async function ActivityReferenceUpdate (tx: TxCUD, control: TriggerControl) return [] } - const txFactory = new TxFactory(control.txFactory.account) + const txFactory = new TxFactory(control.txFactory.account, control.workspace.uuid) const doc = TxProcessor.updateDoc2Doc(rawDoc, ctx) const target = guessReferenceObj(control.hierarchy, tx) @@ -678,7 +678,7 @@ async function ActivityReferenceRemove (tx: TxCUD, control: TriggerControl) } if (hasMarkdown) { - const txFactory = new TxFactory(control.txFactory.account) + const txFactory = new TxFactory(control.txFactory.account, control.workspace.uuid) const txes: Tx[] = await getRemoveActivityReferenceTxes(control, txFactory, ctx.objectId) if (txes.length !== 0) { diff --git a/server-plugins/controlled-documents-resources/src/index.ts b/server-plugins/controlled-documents-resources/src/index.ts index 0677cacaf05..d31b6c426e3 100644 --- a/server-plugins/controlled-documents-resources/src/index.ts +++ b/server-plugins/controlled-documents-resources/src/index.ts @@ -331,7 +331,7 @@ export async function OnDocPlannedEffectiveDateChanged ( // make doc effective immediately if required if (tx.operations.plannedEffectiveDate === 0 && doc.controlledState === ControlledDocumentState.Approved) { // Create with not derived tx factory in order for notifications to work - const factory = new TxFactory(control.txFactory.account) + const factory = new TxFactory(control.txFactory.account, control.workspace.uuid) await control.apply(control.ctx, makeDocEffective(doc, factory)) } } @@ -356,7 +356,7 @@ export async function OnDocApprovalRequestApproved ( } // Create with not derived tx factory in order for notifications to work - const factory = new TxFactory(control.txFactory.account) + const factory = new TxFactory(control.txFactory.account, control.workspace.uuid) await control.apply(control.ctx, makeDocEffective(doc, factory)) // make doc effective immediately } diff --git a/server-plugins/time-resources/src/index.ts b/server-plugins/time-resources/src/index.ts index a7dd25ba1cb..9fb4a67aed5 100644 --- a/server-plugins/time-resources/src/index.ts +++ b/server-plugins/time-resources/src/index.ts @@ -145,7 +145,7 @@ export async function OnWorkSlotCreate (txes: Tx[], control: TriggerControl): Pr } const nextStatus = typeStatuses.find((p) => p.category === task.statusCategory.Active) if (nextStatus !== undefined) { - const factory = new TxFactory(control.txFactory.account) + const factory = new TxFactory(control.txFactory.account, control.workspace.uuid) const innerTx = factory.createTxUpdateDoc(issue._class, issue.space, issue._id, { status: nextStatus._id }) @@ -194,7 +194,7 @@ export async function OnToDoRemove (txes: Tx[], control: TriggerControl): Promis if (project !== undefined) { const type = (await control.modelDb.findAll(task.class.ProjectType, { _id: project.type }))[0] if (type !== undefined && type.classic) { - const factory = new TxFactory(control.txFactory.account) + const factory = new TxFactory(control.txFactory.account, control.workspace.uuid) const taskType = (await control.modelDb.findAll(task.class.TaskType, { _id: issue.kind }))[0] if (taskType !== undefined) { const statuses = await control.modelDb.findAll(core.class.Status, { _id: { $in: taskType.statuses } }) @@ -553,7 +553,7 @@ export async function IssueToDoDone ( for (const workslot of workslots) { total += (workslot.dueDate - workslot.date) / 1000 / 60 } - const factory = new TxFactory(control.txFactory.account) + const factory = new TxFactory(control.txFactory.account, control.workspace.uuid) const issue = ( await control.findAll(control.ctx, todo.attachedToClass, { _id: todo.attachedTo as Ref }) )[0] diff --git a/server/account-service/src/index.ts b/server/account-service/src/index.ts index 1839dba90da..4b73c9d82ce 100644 --- a/server/account-service/src/index.ts +++ b/server/account-service/src/index.ts @@ -111,10 +111,12 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap const dbNs = process.env.DB_NS const accountsDb = getAccountDB(dbUrl, dbNs) + let migrationsApplied = false const migrations = accountsDb.then(async ([db]) => { if (oldAccsUrl !== undefined) { await migrateFromOldAccounts(oldAccsUrl, db, oldAccsNs) console.log('Migrations verified/done') + migrationsApplied = true } }) @@ -336,6 +338,7 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap const meta = getRequestMeta(ctx.request.headers) const request = ctx.request.body as any + const method = methods[request.method as AccountMethods] if (method === undefined) { const response = { @@ -350,7 +353,9 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap } const [db] = await accountsDb - await migrations + if (!migrationsApplied) { + await migrations + } let host: string | undefined const origin = ctx.request.headers.origin ?? ctx.request.headers.referer diff --git a/server/account/src/collections/mongo.ts b/server/account/src/collections/mongo.ts index e7648f0f758..a4396134e7a 100644 --- a/server/account/src/collections/mongo.ts +++ b/server/account/src/collections/mongo.ts @@ -578,7 +578,10 @@ export class MongoAccountDB implements AccountDB { }) } - async createWorkspace (data: WorkspaceData, status: WorkspaceStatusData): Promise { + async createWorkspace ( + data: WorkspaceData & { uuid?: WorkspaceUuid }, + status: WorkspaceStatusData + ): Promise { const res = await this.workspace.insertOne(data) await this.workspaceStatus.insertOne({ diff --git a/server/account/src/collections/postgres/postgres.ts b/server/account/src/collections/postgres/postgres.ts index 959c3325e15..de58d14bcc2 100644 --- a/server/account/src/collections/postgres/postgres.ts +++ b/server/account/src/collections/postgres/postgres.ts @@ -744,7 +744,7 @@ export class PostgresAccountDB implements AccountDB { ) } - async createWorkspace (data: WorkspaceData, status: WorkspaceStatusData): Promise { + async createWorkspace (data: WorkspaceData & { uuid?: WorkspaceUuid }, status: WorkspaceStatusData): Promise { return await this.withRetry(async (rTx) => { const workspaceUuid = await this.workspace.insertOne(data, rTx) await this.workspaceStatus.insertOne({ ...status, workspaceUuid }, rTx) diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index fa0fab45f4d..cd76c3d1505 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -13,7 +13,7 @@ // limitations under the License. // import { Analytics } from '@hcengineering/analytics' -import { +import core, { type AccountInfo, AccountRole, type AccountUuid, @@ -27,6 +27,7 @@ import { type PersonId, type PersonUuid, SocialIdType, + systemAccount, systemAccountUuid, type WorkspaceMemberInfo, type WorkspaceUuid @@ -61,6 +62,7 @@ import { createAccount, createWorkspaceRecord, doJoinByInvite, + doReleaseSocialId, EndpointKind, generatePassword, getAccount, @@ -70,6 +72,7 @@ import { getFrontUrl, getInviteEmail, getMailUrl, + getPersonalWorkspace, getPersonName, getRegions, getRolePower, @@ -84,7 +87,6 @@ import { isEmail, isOtpValid, normalizeValue, - doReleaseSocialId, selectWorkspace, sendEmail, sendEmailConfirmation, @@ -1257,6 +1259,10 @@ export async function getWorkspaceInfo ( const isGuest = extra?.guest === 'true' const skipAssignmentCheck = isGuest || account === systemAccountUuid + if (workspaceUuid === '' || workspaceUuid == null) { + throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspaceUuid })) + } + if (!skipAssignmentCheck) { const role = await db.getWorkspaceRole(account, workspaceUuid) @@ -1347,6 +1353,12 @@ export async function getLoginInfoByToken ( throw new PlatformError(new Status(Severity.ERROR, platform.status.InternalServerError, {})) } + const account = await db.account.findOne({ uuid: accountUuid }) + + if (account == null) { + throw new PlatformError(new Status(Severity.ERROR, platform.status.InternalServerError, {})) + } + const loginInfo = { account: accountUuid, name: getPersonName(person), @@ -1431,7 +1443,8 @@ export async function getLoginWithWorkspaceInfo ( return { account: accountUuid, workspaces: {}, - socialIds: [] + socialIds: [], + personalWorkspace: core.workspace.Personal } } } @@ -1457,6 +1470,24 @@ export async function getLoginWithWorkspaceInfo ( throw new PlatformError(new Status(Severity.ERROR, platform.status.InternalServerError, {})) } + const account = (accountUuid === systemAccountUuid) ? systemAccount : await db.account.findOne({ uuid: accountUuid }) + + if (account == null) { + throw new PlatformError(new Status(Severity.ERROR, platform.status.InternalServerError, {})) + } + + if (accountUuid === systemAccountUuid) { + return { + account: accountUuid, + name: 'System', + personalWorkspace: core.workspace.Any, + workspaces: {}, + socialIds: [] + } + } + + const personalWorkspace = await getPersonalWorkspace(db, account) + const userWorkspaces = (await db.getAccountWorkspaces(accountUuid)).filter((it) => isActiveMode(it.status.mode)) const roles: Map = await getWorkspaceRoles(db, accountUuid) @@ -1466,6 +1497,7 @@ export async function getLoginWithWorkspaceInfo ( name: getPersonName(person), socialId: socialIds[0]?._id, token, + personalWorkspace: personalWorkspace.uuid, workspaces: Object.fromEntries( isSystem || isDocGuest ? [] @@ -1474,6 +1506,7 @@ export async function getLoginWithWorkspaceInfo ( { url: it.url, dataId: it.dataId, + name: it.name, mode: it.status.mode, endpoint: getWorkspaceEndpoint(info, it.uuid, it.region), role: roles.get(it.uuid) ?? null, diff --git a/server/account/src/types.ts b/server/account/src/types.ts index 16f9ca96e9b..ebe165b58da 100644 --- a/server/account/src/types.ts +++ b/server/account/src/types.ts @@ -28,9 +28,9 @@ import { type PersonUuid, type SocialId as SocialIdBase, type WorkspaceDataId, - type WorkspaceUuid + type WorkspaceUuid, + type EndpointInfo } from '@hcengineering/core' -import type { EndpointInfo } from './utils' /* ========= D A T A B A S E E N T I T I E S ========= */ export enum Location { @@ -56,6 +56,7 @@ export interface SocialId extends SocialIdBase { export interface Account { uuid: AccountUuid + accountWorkspace?: WorkspaceUuid automatic?: boolean timezone?: string locale?: string @@ -114,6 +115,7 @@ export interface Workspace { createdBy?: PersonUuid billingAccount?: PersonUuid createdOn?: Timestamp + personal?: boolean } export interface OTP { @@ -173,6 +175,7 @@ export type IntegrationSecretKey = Omit export interface WorkspaceInfoWithStatus extends Workspace { status: WorkspaceStatus + endpoint: EndpointInfo } export type WorkspaceData = Omit @@ -200,7 +203,7 @@ export interface AccountDB { integrationSecret: DbCollection init: () => Promise - createWorkspace: (data: WorkspaceData, status: WorkspaceStatusData) => Promise + createWorkspace: (data: WorkspaceData & { uuid?: WorkspaceUuid }, status: WorkspaceStatusData) => Promise updateAllowReadOnlyGuests: (workspaceId: WorkspaceUuid, readOnlyGuestsAllowed: boolean) => Promise assignWorkspace: (accountId: AccountUuid, workspaceId: WorkspaceUuid, role: AccountRole) => Promise batchAssignWorkspace: (data: [AccountUuid, WorkspaceUuid, AccountRole][]) => Promise @@ -293,6 +296,7 @@ export interface LoginInfo { export interface LoginInfoWorkspace { url: string + name?: string dataId?: WorkspaceDataId mode: WorkspaceMode version: WorkspaceVersion @@ -303,13 +307,14 @@ export interface LoginInfoWorkspace { } export interface LoginInfoWithWorkspaces extends LoginInfo { + personalWorkspace: WorkspaceUuid // Information necessary to handle user <--> transactor connectivity. workspaces: Record socialIds: SocialId[] } export interface WorkspaceLoginInfo extends LoginInfo { - workspace: WorkspaceUuid + workspace: WorkspaceUuid // In case of multi workspace mode, it will be personal workspace workspaceUrl: string workspaceDataId?: WorkspaceDataId endpoint: string diff --git a/server/account/src/utils.ts b/server/account/src/utils.ts index d749727f37c..84cb15f0bfe 100644 --- a/server/account/src/utils.ts +++ b/server/account/src/utils.ts @@ -17,6 +17,7 @@ import { type AccountUuid, type Branding, concatLink, + type EndpointInfo, generateId, groupByArray, isActiveMode, @@ -41,6 +42,7 @@ import otpGenerator from 'otp-generator' import { Analytics } from '@hcengineering/analytics' import { sharedPipelineContextVars } from '@hcengineering/server-pipeline' import { decodeTokenVerbose, generateToken, TokenError } from '@hcengineering/server-token' +import { isAdminEmail } from './admin' import { MongoAccountDB } from './collections/mongo' import { PostgresAccountDB } from './collections/postgres/postgres' import { accountPlugin } from './plugin' @@ -61,7 +63,6 @@ import { type WorkspaceLoginInfo, type WorkspaceStatus } from './types' -import { isAdminEmail } from './admin' export const GUEST_ACCOUNT = 'b6996120-416f-49cd-841e-e4a5d2e49c9b' export const READONLY_GUEST_ACCOUNT = '83bbed9a-0867-4851-be32-31d49d1d42ce' @@ -255,14 +256,14 @@ export const _getRegions = (): RegionInfo[] => { return _regionInfo } -export interface EndpointInfo { - internalUrl: string - externalUrl: string - region: string -} +let _endpointInfo: Map | undefined export function getEndpointInfo (): Map { - return groupByArray(getEndpoints().map(toTransactor), (it) => it.region) + if (_endpointInfo !== undefined) { + return _endpointInfo + } + _endpointInfo = groupByArray(getEndpoints().map(toTransactor), (it) => it.region) + return _endpointInfo } export const selectKind = (kind: EndpointKind, it: EndpointInfo): string => { @@ -544,6 +545,7 @@ export async function selectWorkspace ( token: string | undefined, params: { workspaceUrl: string + singleWorkspace?: boolean kind: 'external' | 'internal' | 'byregion' externalRegions?: string[] }, @@ -558,11 +560,13 @@ export async function selectWorkspace ( let accountUuid: AccountUuid let extra: Record | undefined + let tokenWorkspaceUuid: WorkspaceUuid try { const decodedToken = decodeTokenVerbose(ctx, token ?? '') accountUuid = decodedToken.account + tokenWorkspaceUuid = decodedToken.workspace if (workspace == null) { - workspace = await getWorkspaceById(db, decodedToken.workspace) + workspace = await getWorkspaceById(db, tokenWorkspaceUuid) } extra = decodedToken.extra } catch (e) { @@ -573,11 +577,7 @@ export async function selectWorkspace ( } } - if (workspace == null) { - ctx.error('Workspace not found in selectWorkspace', { workspaceUrl, kind, accountUuid, extra }) - throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspaceUrl })) - } - + const singleWorkspace = params.singleWorkspace ?? true const getKind = (region: string | undefined): EndpointKind => { switch (kind) { case 'external': @@ -610,6 +610,46 @@ export async function selectWorkspace ( } } + let account = await db.account.findOne({ uuid: accountUuid }) + + if (accountUuid !== systemAccountUuid && account == null) { + throw new PlatformError(new Status(Severity.ERROR, platform.status.AccountNotFound, {})) + } + + if (!singleWorkspace && account != null) { + const workspace = await getWorkspaceByUrl(db, workspaceUrl) + if (workspace == null && workspaceUrl !== '') { + ctx.error('Workspace not found in selectWorkspace', { workspaceUrl, kind, accountUuid, extra }) + throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspaceUrl })) + } + + // We need to create a personal workspace for the account. + const personalWorkspace = await getPersonalWorkspace(db, account) + + // Do not check for workspace if we are in multi workspace mode. + return { + account: accountUuid, + // We should use a personal workspace as endpoint for person. + endpoint: getEndpoint(personalWorkspace.uuid, personalWorkspace.region, getKind(personalWorkspace.region)), + workspace: workspace?.uuid ?? personalWorkspace.uuid, + workspaceUrl: workspace?.url ?? personalWorkspace.url, + workspaceDataId: workspace?.dataId ?? personalWorkspace.dataId, + role: + workspace != null + ? (await db.getWorkspaceRole(accountUuid, workspace.uuid)) ?? AccountRole.User + : AccountRole.Owner, // Think about more correct role + token: generateToken(accountUuid, '' as WorkspaceUuid, extra) // Generate multi workspace token + } + } + + if (workspace == null) { + throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, { workspaceUrl })) + } + + if (accountUuid !== systemAccountUuid && meta !== undefined) { + void setTimezoneIfNotDefined(ctx, db, accountUuid, account, meta) + } + if (accountUuid === systemAccountUuid || extra?.admin === 'true') { return { account: accountUuid, @@ -622,7 +662,6 @@ export async function selectWorkspace ( } let role = await db.getWorkspaceRole(accountUuid, workspace.uuid) - let account = await db.account.findOne({ uuid: accountUuid }) if ((role == null || account == null) && workspace.allowReadOnlyGuest) { accountUuid = READONLY_GUEST_ACCOUNT as AccountUuid @@ -726,6 +765,45 @@ export async function updateAllowReadOnlyGuests ( return { guestPerson, guestSocialIds: guestSocialIds.filter((si) => si.isDeleted !== true) } } +export async function getPersonalWorkspace (db: AccountDB, account: Account): Promise { + let personalWorkspace = await getWorkspaceById(db, account.uuid as any as WorkspaceUuid) + if (personalWorkspace == null) { + // We need to create a personal workspace for the account. + await db.createWorkspace( + { + uuid: account.uuid as any as WorkspaceUuid, + name: 'Personal', + url: '', + branding: 'personal', + personal: true, + allowReadOnlyGuest: false, + region: + getRegions() + .filter((it) => it.name !== '') + .shift()?.name ?? '' + }, + { + mode: 'active', + versionMajor: 0, + versionMinor: 0, + versionPatch: 0, + isDisabled: false + } + ) + personalWorkspace = await getWorkspaceById(db, account.uuid as any as WorkspaceUuid) + } + if (personalWorkspace == null) { + throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, {})) + } + + const currentRole = await db.getWorkspaceRole(account.uuid, personalWorkspace.uuid) + + if (currentRole == null) { + await db.assignWorkspace(account.uuid, personalWorkspace.uuid, AccountRole.Owner) + } + return personalWorkspace +} + export async function updateWorkspaceRole ( ctx: MeasureContext, db: AccountDB, @@ -1051,7 +1129,8 @@ export async function getWorkspaceInfoWithStatusById ( return { ...ws, - status + status, + endpoint: getWorkspaceEndpoint(getEndpointInfo(), uuid, ws.region) } } @@ -1065,10 +1144,11 @@ export async function getWorkspacesInfoWithStatusByIds ( return sm }, {}) const workspaces = await db.workspace.find({ uuid: { $in: uuids } }) - + const info = getEndpointInfo() return workspaces.map((it) => ({ ...it, - status: statusesMap[it.uuid] + status: statusesMap[it.uuid], + endpoint: getWorkspaceEndpoint(info, it.uuid, it.region) })) } @@ -1358,7 +1438,8 @@ export async function getWorkspaces ( return workspaces.map((it) => ({ ...it, - status: statusesMap[it.uuid] + status: statusesMap[it.uuid], + endpoint: getWorkspaceEndpoint(getEndpointInfo(), it.uuid, it.region) })) } diff --git a/server/backup/src/backup.ts b/server/backup/src/backup.ts index 607de53d785..4c233d11672 100644 --- a/server/backup/src/backup.ts +++ b/server/backup/src/backup.ts @@ -40,7 +40,7 @@ import core, { type WorkspaceIds, type WorkspaceUuid } from '@hcengineering/core' -import { BlobClient, createClient, getTransactorEndpoint } from '@hcengineering/server-client' +import { BlobClient, createClient } from '@hcengineering/server-client' import { estimateDocSize, type StorageAdapter } from '@hcengineering/server-core' import { generateToken } from '@hcengineering/server-token' import { deepEqual } from 'fast-equals' @@ -622,7 +622,7 @@ export async function cloneWorkspace ( // await sourceConnection.close() // }) // await ctx.with('close-target', {}, async (ctx) => { - // await targetConnection.sendForceClose() + // await targetConnection.sendForceClose(workspaceId) // await targetConnection.close() // }) // } @@ -949,7 +949,7 @@ export async function backup ( } while (true) { try { - const currentChunk = await ctx.with('loadChunk', {}, () => connection.loadChunk(domain, idx)) + const currentChunk = await ctx.with('loadChunk', {}, () => connection.loadChunk(workspaceId, domain, idx)) if (domain === DOMAIN_BLOB) { result.blobsSize += currentChunk.size ?? 0 } else { @@ -1022,7 +1022,7 @@ export async function backup ( workspace: workspaceId }) await ctx.with('closeChunk', {}, async () => { - await connection.closeChunk(idx as number) + await connection.closeChunk(workspaceId, idx as number) }) break } @@ -1030,7 +1030,7 @@ export async function backup ( ctx.error('failed to load chunks', { error: err }) if (idx !== undefined) { await ctx.with('closeChunk', {}, async () => { - await connection.closeChunk(idx as number) + await connection.closeChunk(workspaceId, idx as number) }) } // Try again @@ -1069,7 +1069,7 @@ export async function backup ( removed: 0 } - const dHash = await connection.getDomainHash(domain) + const dHash = await connection.getDomainHash(workspaceId, domain) if (backupInfo.domainHashes[domain] === dHash && !fullCheck) { ctx.info('no changes in domain', { domain }) return @@ -1111,6 +1111,7 @@ export async function backup ( same, async (docs) => { const serverDocs = await connection.loadDocs( + workspaceId, domain, docs.map((it) => it._id) ) @@ -1193,7 +1194,7 @@ export async function backup ( }) let docs: Doc[] = [] try { - docs = await ctx.with('<<<< load-docs', {}, async () => await connection.loadDocs(domain, needRetrieve)) + docs = await ctx.with('<<<< load-docs', {}, async () => await connection.loadDocs(workspaceId, domain, needRetrieve)) lastSize = docs.reduce((p, it) => p + estimateDocSize(it), 0) if (docs.length !== needRetrieve.length) { ctx.error('failed to retrieve all documents', { @@ -1644,7 +1645,7 @@ export async function backupSize (storage: BackupStorage): Promise { /** * @public */ -export async function backupDownload (storage: BackupStorage, storeIn: string): Promise { +export async function backupDownload (storage: BackupStorage, storeIn: string, skipDomains: Set): Promise { const infoFile = 'backup.json.gz' const sizeFile = 'backup.size.gz' @@ -1654,15 +1655,16 @@ export async function backupDownload (storage: BackupStorage, storeIn: string): let size = 0 const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString()) - console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) + console.log('Downloading workspace:', backupInfo.workspace ?? '', backupInfo.version, backupInfo.snapshots.length) let sizeInfo: Record = {} if (await storage.exists(sizeFile)) { + console.log('Parse size file') sizeInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(sizeFile))).toString()) } - console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version) - const addFileSize = async (file: string | undefined | null, force: boolean = false): Promise => { + const downloadFile = async (file: string | undefined | null, force: boolean = false): Promise => { + console.log('Download file', file) if (file != null) { const target = join(storeIn, file) const dir = dirname(target) @@ -1699,17 +1701,21 @@ export async function backupDownload (storage: BackupStorage, storeIn: string): // Let's calculate data size for backup for (const sn of backupInfo.snapshots) { - for (const [, d] of Object.entries(sn.domains)) { - await addFileSize(d.snapshot) + console.log('processing', sn.date) + for (const [k, d] of Object.entries(sn.domains)) { + if (skipDomains.has(k)) { + continue + } + await downloadFile(d.snapshot) for (const snp of d.snapshots ?? []) { - await addFileSize(snp) + await downloadFile(snp) } for (const snp of d.storage ?? []) { - await addFileSize(snp) + await downloadFile(snp) } } } - await addFileSize(infoFile, true) + await downloadFile(infoFile, true) console.log('Backup size', size / (1024 * 1024), 'Mb') } @@ -1828,7 +1834,7 @@ export async function restore ( recheck?: boolean include?: Set skip?: Set - getConnection?: () => Promise + getConnection: () => Promise storageAdapter?: StorageAdapter token?: string progress?: (progress: number) => Promise @@ -1878,26 +1884,10 @@ export async function restore ( opt.token ?? generateToken(systemAccountUuid, workspaceId, { service: 'backup', - mode: 'backup', - model: 'upgrade' + mode: 'backup' }) - const connection = - opt.getConnection !== undefined - ? await opt.getConnection() - : ((await createClient(transactorUrl, token)) as CoreClient & BackupClient) - - if (opt.getConnection === undefined) { - try { - let serverEndpoint = await getTransactorEndpoint(token, 'external') - serverEndpoint = serverEndpoint.replaceAll('wss://', 'https://').replace('ws://', 'http://') - await fetch(serverEndpoint + `/api/v1/manage?token=${token}&operation=force-close`, { - method: 'PUT' - }) - } catch (err: any) { - // Ignore - } - } + const connection = await opt.getConnection() const blobClient = new BlobClient(transactorUrl, token, wsIds, { storageAdapter: opt.storageAdapter }) console.log('connected') @@ -1935,7 +1925,7 @@ export async function restore ( } async function processDomain (c: Domain): Promise { - const dHash = await connection.getDomainHash(c) + const dHash = await connection.getDomainHash(workspaceId, c) if (backupInfo.domainHashes[c] === dHash) { ctx.info('no changes in domain', { domain: c }) return @@ -1964,7 +1954,7 @@ export async function restore ( await opt.progress?.(domainProgress) } const st = Date.now() - const it = await connection.loadChunk(c, idx) + const it = await connection.loadChunk(workspaceId, c, idx) dataSize += it.size ?? 0 chunks++ @@ -1987,7 +1977,7 @@ export async function restore ( } } finally { if (idx !== undefined) { - await connection.closeChunk(idx) + await connection.closeChunk(workspaceId, idx) } } ctx.info('loaded', { @@ -2056,6 +2046,7 @@ export async function restore ( // We need to download all documents and compare them. const serverDocs = toIdMap( await connection.loadDocs( + workspaceId, c, docs.map((it) => it._id) ) @@ -2072,7 +2063,7 @@ export async function restore ( }) } try { - await connection.upload(c, docsToSend) + await connection.upload(workspaceId, c, docsToSend) } catch (err: any) { ctx.error('error during upload', { err, docs: JSON.stringify(docs) }) } @@ -2264,7 +2255,7 @@ export async function restore ( while (docsToRemove.length > 0) { const part = docsToRemove.splice(0, 10000) try { - await connection.clean(c, part) + await connection.clean(workspaceId, c, part) } catch (err: any) { ctx.error('failed to clean, will retry', { error: err, workspaceId }) docsToRemove.push(...part) @@ -2324,7 +2315,7 @@ export async function restore ( return false } finally { if (opt.getConnection === undefined && connection !== undefined) { - await connection.sendForceClose() + await connection.sendForceClose(workspaceId) await connection.close() } } diff --git a/server/client/src/account.ts b/server/client/src/account.ts index 5c921c8ac82..bed1d6cd2b1 100644 --- a/server/client/src/account.ts +++ b/server/client/src/account.ts @@ -54,7 +54,7 @@ export async function getTransactorEndpoint ( const st = Date.now() while (true) { try { - const workspaceInfo = await accountClient.selectWorkspace('', kind, externalRegions) + const workspaceInfo = await accountClient.selectWorkspace('', true, kind, externalRegions) if (workspaceInfo === undefined) { throw new Error('Workspace not found') } diff --git a/server/collaborator/src/platform.ts b/server/collaborator/src/platform.ts index 6401aa8b436..508f170b9aa 100644 --- a/server/collaborator/src/platform.ts +++ b/server/collaborator/src/platform.ts @@ -44,7 +44,7 @@ async function getTxOperations (client: Client, token: Token, isDerived: boolean primarySocialString = pickPrimarySocialId(socialIds)._id } - return new TxOperations(client, primarySocialString, isDerived) + return new TxOperations(client, primarySocialString, token.workspace, isDerived) } /** diff --git a/server/core/src/benchmark/index.ts b/server/core/src/benchmark/index.ts index 44f95a68aad..e43fc6846ba 100644 --- a/server/core/src/benchmark/index.ts +++ b/server/core/src/benchmark/index.ts @@ -79,6 +79,7 @@ class BenchmarkDbAdapter extends DummyDbAdapter { const dataSize = typeof request.size === 'number' ? request.size : request.size.from + Math.random() * request.size.to result.push({ + _uuid: core.workspace.Any, _class: core.class.BenchmarkDoc, _id: generateId(), modifiedBy: core.account.System, diff --git a/server/core/src/storage.ts b/server/core/src/storage.ts index 7f6abdb014a..2954622c096 100644 --- a/server/core/src/storage.ts +++ b/server/core/src/storage.ts @@ -30,6 +30,12 @@ export interface ChunkInfo { iterator: StorageIterator } +export interface LoadChunkResponse { + idx: number + docs: DocInfo[] + finished: boolean +} + /** * @public */ @@ -39,15 +45,7 @@ export class BackupClientOps { idIndex = 0 chunkInfo = new Map() - loadChunk ( - ctx: MeasureContext, - domain: Domain, - idx?: number - ): Promise<{ - idx: number - docs: DocInfo[] - finished: boolean - }> { + loadChunk (ctx: MeasureContext, domain: Domain, idx?: number): Promise { return ctx.with('load-chunk', {}, async (ctx) => { idx = idx ?? this.idIndex++ let chunk: ChunkInfo | undefined = this.chunkInfo.get(idx) diff --git a/server/core/src/triggers.ts b/server/core/src/triggers.ts index 311f55cd7a9..6baafda1cbb 100644 --- a/server/core/src/triggers.ts +++ b/server/core/src/triggers.ts @@ -109,7 +109,7 @@ export class Triggers { trigger.op = await trigger.op } for (const [k, v] of group.entries()) { - tctrl.txFactory = new TxFactory(k, true) + tctrl.txFactory = new TxFactory(k, ctrl.workspace.uuid, true) try { const tresult = await trigger.op(v, tctrl) result.push(...tresult) diff --git a/server/core/src/types.ts b/server/core/src/types.ts index 6a18512ce49..f5129d35b72 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -13,22 +13,14 @@ // limitations under the License. // -import { - type ServerApi as CommunicationApi, - type RequestEvent as CommunicationEvent, - type EventResult -} from '@hcengineering/communication-sdk-types' -import { - type FindMessagesGroupsParams, - type FindMessagesParams, - type Message, - type MessagesGroup -} from '@hcengineering/communication-types' +import { type ServerApi as CommunicationApi, type EventResult } from '@hcengineering/communication-sdk-types' import { type Account, type AccountUuid, type Branding, type Class, + type ClientConnection, + type ClientConnectOptions, type Doc, type DocumentQuery, type Domain, @@ -51,6 +43,7 @@ import { type Timestamp, type Tx, type TxFactory, + type TxHandler, type TxResult, type UserStatus, type WorkspaceIds, @@ -58,14 +51,14 @@ import { } from '@hcengineering/core' import type { Asset, Resource } from '@hcengineering/platform' import type { LiveQuery } from '@hcengineering/query' -import type { RateLimitInfo, ReqId, Request, Response } from '@hcengineering/rpc' -import type { Token } from '@hcengineering/server-token' +import type { ReqId, Request, Response } from '@hcengineering/rpc' import { type Readable } from 'stream' import type { DbAdapter, DomainHelper } from './adapter' -import type { StatisticsElement, WorkspaceStatistics } from './stats' +import { type PlatformQueue, type PlatformQueueProducer, type QueueTopic } from './queue' import { type StorageAdapter } from './storage' -import { type PlatformQueueProducer, type QueueTopic, type PlatformQueue } from './queue' +import type { Token } from '@hcengineering/server-token' +import type { StatisticsElement } from './stats' export interface ServerFindOptions extends FindOptions { domain?: Domain // Allow to find for Doc's in specified domain only. @@ -134,8 +127,8 @@ export interface Middleware { export type BroadcastFunc = ( ctx: MeasureContext, tx: Tx[], - targets?: string | string[], - exclude?: string[] + targets?: AccountUuid | AccountUuid[], + exclude?: AccountUuid[] ) => void export type BroadcastSessionsFunc = (ctx: MeasureContext, sessionIds: string[], result: any) => void @@ -247,6 +240,16 @@ export type PipelineFactory = ( communicationApi: CommunicationApi | null ) => Promise +/** + * @public + */ +export type EndpointConnectionFactory = ( + ctx: MeasureContext, + endpoint: string, + handler: TxHandler, + opt: ClientConnectOptions +) => ClientConnection + export type CommunicationApiFactory = ( ctx: MeasureContext, ws: WorkspaceIds, @@ -543,6 +546,7 @@ export interface SessionRequest { id: string params: any start: number + workspaceId?: WorkspaceUuid } export interface ClientSessionCtx { @@ -671,76 +675,5 @@ export function disableLogging (): void { LOGGING_ENABLED = false } -export interface AddSessionActive { - session: Session - context: MeasureContext - workspaceId: WorkspaceUuid -} - -export type GetWorkspaceResponse = - | { upgrade: true, progress?: number } - | { error: any, terminate?: boolean, specialError?: 'archived' | 'migration' } - -export type AddSessionResponse = AddSessionActive | GetWorkspaceResponse - -/** - * @public - */ -export interface SessionManager { - // workspaces: Map - sessions: Map - - addSession: ( - ctx: MeasureContext, - ws: ConnectionSocket, - token: Token, - rawToken: string, - sessionId: string | undefined - ) => Promise - - broadcastAll: (workspace: WorkspaceUuid, tx: Tx[], targets?: string[]) => void - - close: (ctx: MeasureContext, ws: ConnectionSocket, workspaceId: WorkspaceUuid) => Promise - - forceClose: (wsId: WorkspaceUuid, ignoreSocket?: ConnectionSocket) => Promise - - closeWorkspaces: (ctx: MeasureContext) => Promise - - scheduleMaintenance: (timeMinutes: number) => void - - profiling?: { - start: () => void - stop: () => Promise - } - - handleRequest: ( - requestCtx: MeasureContext, - service: S, - ws: ConnectionSocket, - request: Request, - workspace: WorkspaceUuid - ) => Promise - - handleRPC: ( - requestCtx: MeasureContext, - service: S, - ws: ConnectionSocket, - operation: (ctx: ClientSessionCtx, rateLimit?: RateLimitInfo) => Promise - ) => Promise - - createOpContext: ( - ctx: MeasureContext, - sendCtx: MeasureContext, - pipeline: Pipeline, - communicationApi: CommunicationApi, - requestId: Request['id'], - service: Session, - ws: ConnectionSocket, - rateLimit?: RateLimitInfo - ) => ClientSessionCtx - - getStatistics: () => WorkspaceStatistics[] -} - export const pingConst = 'ping' export const pongConst = 'pong!' diff --git a/server/core/src/utils.ts b/server/core/src/utils.ts index 84a213dd488..3f320e54e45 100644 --- a/server/core/src/utils.ts +++ b/server/core/src/utils.ts @@ -24,10 +24,12 @@ import core, { type Ref, type SearchResult, type SessionData, + type SubscribedWorkspaceInfo, type Tx, type TxResult, type TxWorkspaceEvent, - type WorkspaceIds + type WorkspaceIds, + type WorkspaceUuid } from '@hcengineering/core' import { PlatformError, unknownError } from '@hcengineering/platform' import { createHash, type Hash } from 'crypto' @@ -202,8 +204,12 @@ export class SessionDataImpl implements SessionData { } } -export function createBroadcastEvent (classes: Ref>[]): TxWorkspaceEvent { +export function createBroadcastEvent ( + workspace: WorkspaceUuid, + classes: Ref>[] +): TxWorkspaceEvent { return { + _uuid: workspace, _class: core.class.TxWorkspaceEvent, _id: generateId(), event: WorkspaceEvent.BulkUpdate, @@ -260,17 +266,19 @@ export function wrapPipeline ( findAll: (_class, query, options) => pipeline.findAll(ctx, _class, query, options), findOne: async (_class, query, options) => (await pipeline.findAll(ctx, _class, query, { ...options, limit: 1 })).shift(), - clean: (domain, docs) => backupOps.clean(ctx, domain, docs), + clean: (workspaceId, domain, docs) => backupOps.clean(ctx, domain, docs), close: () => pipeline.close(), - closeChunk: (idx) => backupOps.closeChunk(ctx, idx), + closeChunk: (workspaceId, idx) => backupOps.closeChunk(ctx, idx), getHierarchy: () => pipeline.context.hierarchy, getModel: () => pipeline.context.modelDb, - loadChunk: (domain, idx) => backupOps.loadChunk(ctx, domain, idx), - getDomainHash: (domain) => backupOps.getDomainHash(ctx, domain), - loadDocs: (domain, docs) => backupOps.loadDocs(ctx, domain, docs), - upload: (domain, docs) => backupOps.upload(ctx, domain, docs), + loadChunk: (workspaceId, domain, idx) => backupOps.loadChunk(ctx, domain, idx), + getDomainHash: (workspaceid, domain) => backupOps.getDomainHash(ctx, domain), + loadDocs: (workspaceId, domain, docs) => backupOps.loadDocs(ctx, domain, docs), + upload: (workspaceId, domain, docs) => backupOps.upload(ctx, domain, docs), searchFulltext: async (query, options) => ({ docs: [], total: 0 }), sendForceClose: async () => {}, + getAvailableWorkspaces: () => [], + getWorkspaces: () => ({}), tx: async (tx) => { const result = await pipeline.tx(ctx, [tx]) if (doBroadcast) { @@ -286,19 +294,45 @@ export function wrapAdapterToClient (ctx: MeasureContext, storageAdapter: DbAdap class TestClientConnection implements ClientConnection { isConnected = (): boolean => true - handler?: (event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise + handler?: ( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise + + async subscribe (): Promise { + return {} + } + + async unsubscribe (): Promise {} set onConnect ( - handler: ((event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise) | undefined + handler: + | (( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise) + | undefined ) { this.handler = handler - void this.handler?.(ClientConnectEvent.Connected, '', {}) + void this.handler?.(ClientConnectEvent.Connected, {}, {}) } - get onConnect (): ((event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise) | undefined { + get onConnect (): + | (( + event: ClientConnectEvent, + lastTx: Record | undefined, + data: any + ) => Promise) + | undefined { return this.handler } + async getAccount (): Promise { + return systemAccount + } + pushHandler (): void {} async findAll( @@ -319,23 +353,23 @@ export function wrapAdapterToClient (ctx: MeasureContext, storageAdapter: DbAdap async close (): Promise {} - async loadChunk (domain: Domain): Promise { + async loadChunk (workspaceId: WorkspaceUuid, domain: Domain): Promise { throw new Error('unsupported') } - async getDomainHash (domain: Domain): Promise { + async getDomainHash (workspaceId: WorkspaceUuid, domain: Domain): Promise { return await storageAdapter.getDomainHash(ctx, domain) } - async closeChunk (idx: number): Promise {} + async closeChunk (workspaceId: WorkspaceUuid, idx: number): Promise {} - async loadDocs (domain: Domain, docs: Ref[]): Promise { + async loadDocs (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { return [] } - async upload (domain: Domain, docs: Doc[]): Promise {} + async upload (workspaceId: WorkspaceUuid, domain: Domain, docs: Doc[]): Promise {} - async clean (domain: Domain, docs: Ref[]): Promise {} + async clean (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise {} async loadModel (): Promise { return txes diff --git a/server/datalake/src/index.ts b/server/datalake/src/index.ts index bc5b440be9a..6700e022e84 100644 --- a/server/datalake/src/index.ts +++ b/server/datalake/src/index.ts @@ -128,6 +128,7 @@ export class DatalakeService implements StorageAdapter { for (const blob of res.blobs) { buffer.push({ + _uuid: wsIds.uuid, _id: blob.name as Ref, _class: core.class.Blob, etag: blob.etag, @@ -153,6 +154,7 @@ export class DatalakeService implements StorageAdapter { const result = await this.retry(ctx, () => this.client.statObject(ctx, wsIds.uuid, objectName)) if (result !== undefined) { return { + _uuid: wsIds.uuid, provider: '', _class: core.class.Blob, _id: objectName as Ref, diff --git a/server/middleware/src/broadcast.ts b/server/middleware/src/broadcast.ts index 9e6f5f1c327..b7f39a72103 100644 --- a/server/middleware/src/broadcast.ts +++ b/server/middleware/src/broadcast.ts @@ -14,7 +14,9 @@ // import { + groupByArray, TxProcessor, + type AccountUuid, type BroadcastTargets, type Class, type Doc, @@ -70,9 +72,9 @@ export class BroadcastMiddleware extends BaseMiddleware implements Middleware { // Combine targets by sender - const toSendTarget = new Map() + const toSendTarget = new Map() - const getTxes = (key: string): Tx[] => { + const getTxes = (key: AccountUuid | ''): Tx[] => { let txes = toSendTarget.get(key) if (txes === undefined) { txes = [...(toSendTarget.get('') ?? [])] // We also need to add all from to all @@ -83,7 +85,7 @@ export class BroadcastMiddleware extends BaseMiddleware implements Middleware { // Put current user as send target for (const txd of tx) { - let target: string[] | undefined + let target: AccountUuid[] | undefined for (const tt of Object.values(targets ?? {})) { target = tt(txd) if (target !== undefined) { @@ -107,8 +109,8 @@ export class BroadcastMiddleware extends BaseMiddleware implements Middleware { const handleSend = async ( ctx: MeasureContext, derived: Tx[], - target?: string, - exclude?: string[] + target?: AccountUuid, + exclude?: AccountUuid[] ): Promise => { if (derived.length === 0) { return @@ -127,29 +129,32 @@ export class BroadcastMiddleware extends BaseMiddleware implements Middleware { // Then send targeted and all other for (const [k, v] of toSendTarget.entries()) { - void handleSend(ctx, v, k) + void handleSend(ctx, v, k as AccountUuid) } // Send all other except us. - await handleSend(ctx, toSendAll, undefined, Array.from(toSendTarget.keys())) + await handleSend(ctx, toSendAll, undefined, Array.from(toSendTarget.keys()) as AccountUuid[]) } private async sendWithPart ( derived: Tx[], ctx: MeasureContext, - target: string | undefined, - exclude: string[] | undefined + target: AccountUuid | undefined, + exclude: AccountUuid[] | undefined ): Promise { - const classes = new Set>>() - for (const dtx of derived) { - if (TxProcessor.isExtendsCUD(dtx._class)) { - classes.add((dtx as TxCUD).objectClass) - const attachedToClass = (dtx as TxCUD).attachedToClass - if (attachedToClass !== undefined) { - classes.add(attachedToClass) + const byWorkspace = groupByArray(derived, (it) => it._uuid) + for (const [uuid, dtxs] of byWorkspace) { + const classes = new Set>>() + for (const dtx of dtxs) { + if (TxProcessor.isExtendsCUD(dtx._class)) { + classes.add((dtx as TxCUD).objectClass) + const attachedToClass = (dtx as TxCUD).attachedToClass + if (attachedToClass !== undefined) { + classes.add(attachedToClass) + } } } + const bevent = createBroadcastEvent(uuid, Array.from(classes)) + this.broadcast(ctx, [bevent], target, exclude) } - const bevent = createBroadcastEvent(Array.from(classes)) - this.broadcast(ctx, [bevent], target, exclude) } } diff --git a/server/middleware/src/identity.ts b/server/middleware/src/identity.ts index e94cde9bc83..6d07ef24670 100644 --- a/server/middleware/src/identity.ts +++ b/server/middleware/src/identity.ts @@ -47,19 +47,18 @@ export class IdentityMiddleware extends BaseMiddleware implements Middleware { tx (ctx: MeasureContext, txes: Tx[]): Promise { const account = ctx.contextData.account - if (account.uuid === systemAccountUuid || account.fullSocialIds.some((it) => it.value === aiBotAccountEmail)) { + if (account.uuid === systemAccountUuid || account.socialIdsByValue.has(aiBotAccountEmail)) { // TODO: We need to enhance allowed list in case of user service, on behalf of user activities. // We pass for system accounts and services. return this.provideTx(ctx, txes) } function checkTx (tx: Tx): void { - const mxAccount = ctx.contextData.socialStringsToUsers.get(tx.modifiedBy) - if (mxAccount === undefined || mxAccount !== account.uuid) { + if (!ctx.contextData.account.socialIds.includes(tx.modifiedBy)) { throw new PlatformError( new Status(Severity.ERROR, platform.status.AccountMismatch, { account: account.uuid, - requiredAccount: mxAccount + requiredAccount: ctx.contextData.account.uuid }) ) } diff --git a/server/middleware/src/index.ts b/server/middleware/src/index.ts index d937345d303..bbbd27591a2 100644 --- a/server/middleware/src/index.ts +++ b/server/middleware/src/index.ts @@ -24,7 +24,6 @@ export * from './domainFind' export * from './domainTx' export * from './fulltext' export * from './liveQuery' -export * from './lookup' export * from './lowLevel' export * from './model' export * from './modified' diff --git a/server/middleware/src/liveQuery.ts b/server/middleware/src/liveQuery.ts index 8d570a39443..11c719af0c0 100644 --- a/server/middleware/src/liveQuery.ts +++ b/server/middleware/src/liveQuery.ts @@ -79,7 +79,9 @@ export class LiveQueryMiddleware extends BaseMiddleware implements Middleware { searchFulltext: async (query: SearchQuery, options: SearchOptions) => { // Cast client doesn't support fulltext search return { docs: [] } - } + }, + getAvailableWorkspaces: () => [], + getWorkspaces: () => ({}) } } diff --git a/server/middleware/src/lookup.ts b/server/middleware/src/lookup.ts deleted file mode 100644 index 7b5f1da4adc..00000000000 --- a/server/middleware/src/lookup.ts +++ /dev/null @@ -1,118 +0,0 @@ -// -// Copyright © 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { - type Class, - type Doc, - type DocumentQuery, - type FindOptions, - type FindResult, - type MeasureContext, - type Ref, - clone, - toFindResult -} from '@hcengineering/core' -import { BaseMiddleware, type Middleware, type PipelineContext } from '@hcengineering/server-core' -/** - * @public - */ -export class LookupMiddleware extends BaseMiddleware implements Middleware { - private constructor (context: PipelineContext, next?: Middleware) { - super(context, next) - } - - static async create ( - ctx: MeasureContext, - context: PipelineContext, - next: Middleware | undefined - ): Promise { - return new LookupMiddleware(context, next) - } - - override async findAll( - ctx: MeasureContext, - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise> { - const result = await this.provideFindAll(ctx, _class, query, options) - // Fill lookup map to make more compact representation - - if (options?.lookup !== undefined) { - const newResult: T[] = [] - let counter = 0 - const idClassMap: Record = {} - - function mapDoc (doc: Doc): number { - const key = doc._class + '@' + doc._id - let docRef = idClassMap[key] - if (docRef === undefined) { - docRef = { id: ++counter, doc, count: -1 } - idClassMap[key] = docRef - } - docRef.count++ - return docRef.id - } - - for (const d of result) { - const newDoc: any = { ...d } - if (d.$lookup !== undefined) { - newDoc.$lookup = clone(d.$lookup) - newResult.push(newDoc) - for (const [k, v] of Object.entries(d.$lookup)) { - if (!Array.isArray(v)) { - newDoc.$lookup[k] = v != null ? mapDoc(v) : v - } else { - newDoc.$lookup[k] = v.map((it) => (it != null ? mapDoc(it) : it)) - } - } - } else { - newResult.push(newDoc) - } - } - const lookupMap = Object.fromEntries(Array.from(Object.values(idClassMap)).map((it) => [it.id, it.doc])) - return this.cleanQuery(toFindResult(newResult, result.total, lookupMap), query, lookupMap) - } - - // We need to get rid of simple query parameters matched in documents - return this.cleanQuery(result, query) - } - - private cleanQuery( - result: FindResult, - query: DocumentQuery, - lookupMap?: Record - ): FindResult { - const newResult: T[] = [] - for (const doc of result) { - let _doc = doc - let cloned = false - for (const [k, v] of Object.entries(query)) { - if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') { - if ((_doc as any)[k] === v) { - if (!cloned) { - _doc = { ...doc } as any - cloned = true - } - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete (_doc as any)[k] - } - } - } - newResult.push(_doc) - } - return toFindResult(newResult, result.total, lookupMap) - } -} diff --git a/server/middleware/src/pluginConfig.ts b/server/middleware/src/pluginConfig.ts index 9faa4bc2648..1e6199918db 100644 --- a/server/middleware/src/pluginConfig.ts +++ b/server/middleware/src/pluginConfig.ts @@ -50,7 +50,7 @@ export class PluginConfigurationMiddleware extends BaseMiddleware implements Mid tx (ctx: MeasureContext, txes: Tx[]): Promise { const account = ctx.contextData.account - if (account.uuid === systemAccountUuid || account.fullSocialIds.some((it) => it.value === aiBotAccountEmail)) { + if (account.uuid === systemAccountUuid || account.socialIdsByValue.has(aiBotAccountEmail)) { // We pass for system accounts and services. return this.provideTx(ctx, txes) } diff --git a/server/middleware/src/private.ts b/server/middleware/src/private.ts index f8e4b82e253..685020bce71 100644 --- a/server/middleware/src/private.ts +++ b/server/middleware/src/private.ts @@ -29,7 +29,8 @@ import core, { type TxCUD, TxProcessor, systemAccountUuid, - type SessionData + type SessionData, + type AccountUuid } from '@hcengineering/core' import platform, { PlatformError, Severity, Status } from '@hcengineering/platform' import { @@ -69,7 +70,7 @@ export class PrivateMiddleware extends BaseMiddleware implements Middleware { tx (ctx: MeasureContext, txes: Tx[]): Promise { for (const tx of txes) { - let target: PersonUuid[] | undefined + let target: AccountUuid[] | undefined if (this.isTargetDomain(tx)) { const account = ctx.contextData.account if (!account.socialIds.includes(tx.modifiedBy) && account.uuid !== systemAccountUuid) { diff --git a/server/middleware/src/spaceSecurity.ts b/server/middleware/src/spaceSecurity.ts index 2351ea6c014..b79b4ebb127 100644 --- a/server/middleware/src/spaceSecurity.ts +++ b/server/middleware/src/spaceSecurity.ts @@ -191,7 +191,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar if (createTx.objectClass === core.class.SystemSpace) { this.systemSpaces.add(createTx.objectId) } else { - const res = TxProcessor.createDoc2Doc(createTx) + const res = TxProcessor.createDoc2Doc(createTx) this.addSpace(res) } } @@ -253,9 +253,10 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar } } - private brodcastEvent (ctx: MeasureContext, users: AccountUuid[], space?: Ref): void { - const targets = this.getTargets(users) + private brodcastEvent (ctx: MeasureContext, users: AccountUuid[] | undefined, space?: Ref): void { + const targets = users != null ? this.getTargets(users) : [] const tx: TxWorkspaceEvent = { + _uuid: this.context.workspace.uuid, _class: core.class.TxWorkspaceEvent, _id: generateId(), event: WorkspaceEvent.SecurityChange, @@ -266,10 +267,13 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar params: null } ctx.contextData.broadcast.txes.push(tx) - ctx.contextData.broadcast.targets['security' + tx._id] = (it) => { - // TODO: I'm not sure it is called - if (it._id === tx._id) { - return targets + + if (targets !== null) { + ctx.contextData.broadcast.targets['security' + tx._id] = (it) => { + // TODO: I'm not sure it is called + if (it._id === tx._id) { + return targets + } } } } @@ -281,10 +285,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar } private broadcastAll (ctx: MeasureContext, space: SpaceWithMembers): void { - const { socialStringsToUsers } = ctx.contextData - const accounts = Array.from(new Set(socialStringsToUsers.values())) - - this.brodcastEvent(ctx, accounts, space._id) + this.brodcastEvent(ctx, undefined, space._id) } private async handleUpdate (ctx: MeasureContext, tx: TxCUD): Promise { @@ -343,7 +344,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar } } - getTargets (accounts: AccountUuid[]): string[] { + getTargets (accounts: AccountUuid[]): AccountUuid[] { const res = Array.from(new Set(accounts)) // We need to add system account for targets for integrations to work properly res.push(systemAccountUuid) diff --git a/server/middleware/src/triggers.ts b/server/middleware/src/triggers.ts index fb2cfcf97ec..7758c4e2616 100644 --- a/server/middleware/src/triggers.ts +++ b/server/middleware/src/triggers.ts @@ -270,7 +270,7 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware { attachedTo: Pick, update: DocumentUpdate ): Tx { - const txFactory = new TxFactory(modifiedBy, true) + const txFactory = new TxFactory(modifiedBy, this.context.workspace.uuid, true) const baseClass = this.context.hierarchy.getBaseClass(_class) if (baseClass !== _class) { // Mixin operation is required. @@ -462,7 +462,7 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware { private deleteObject (ctx: MeasureContext, object: Doc, removedMap: Map, Doc>): Tx[] { const result: Tx[] = [] - const factory = new TxFactory(object.modifiedBy, true) + const factory = new TxFactory(object.modifiedBy, this.context.workspace.uuid, true) if (this.context.hierarchy.isDerived(object._class, core.class.AttachedDoc)) { const adoc = object as AttachedDoc const nestedTx = factory.createTxRemoveDoc(adoc._class, adoc.space, adoc._id) @@ -515,7 +515,7 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware { if (rtx.operations.space === undefined || rtx.operations.space === rtx.objectSpace) { continue } - const factory = new TxFactory(tx.modifiedBy, true) + const factory = new TxFactory(tx.modifiedBy, this.context.workspace.uuid, true) for (const [, attribute] of this.context.hierarchy.getAllAttributes(rtx.objectClass)) { if (!this.context.hierarchy.isDerived(attribute.type._class, core.class.Collection)) { continue diff --git a/server/middleware/src/txPush.ts b/server/middleware/src/txPush.ts index c626d17667d..cecbec69951 100644 --- a/server/middleware/src/txPush.ts +++ b/server/middleware/src/txPush.ts @@ -78,6 +78,7 @@ export class TxMiddleware extends BaseMiddleware implements Middleware { this.context.lastTx = txToStore[txToStore.length - 1]._id // We need to deliver information to all clients so far. const evt: TxWorkspaceEvent = { + _uuid: this.context.workspace.uuid, _class: core.class.TxWorkspaceEvent, _id: generateId(), event: WorkspaceEvent.LastTx, diff --git a/server/minio/src/index.ts b/server/minio/src/index.ts index 0174c5c88b6..cf2d54afb7e 100644 --- a/server/minio/src/index.ts +++ b/server/minio/src/index.ts @@ -229,6 +229,7 @@ export class MinioService implements StorageAdapter { if (data.name !== undefined) { const _id = this.stripPrefix(rootPrefix, data.name) buffer.push({ + _uuid: wsIds.uuid, _id: _id as Ref, _class: core.class.Blob, etag: data.etag, @@ -283,6 +284,7 @@ export class MinioService implements StorageAdapter { const result = await this.client.statObject(this.getBucketId(wsIds), this.getDocumentKey(wsIds, objectName)) const rootPrefix = this.rootPrefix(wsIds) return { + _uuid: wsIds.uuid, provider: '', _class: core.class.Blob, _id: this.stripPrefix(rootPrefix, objectName) as Ref, diff --git a/server/mongo/src/__tests__/minmodel.ts b/server/mongo/src/__tests__/minmodel.ts index be5c1b76553..9fab675bfa8 100644 --- a/server/mongo/src/__tests__/minmodel.ts +++ b/server/mongo/src/__tests__/minmodel.ts @@ -36,7 +36,7 @@ import type { IntlString, Plugin } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' import { taskPlugin } from './tasks' -export const txFactory = new TxFactory(core.account.System) +export const txFactory = new TxFactory(core.account.System, core.workspace.Model) export function createClass (_class: Ref>, attributes: Data>): TxCreateDoc { return txFactory.createTxCreateDoc(core.class.Class, core.space.Model, attributes, _class) diff --git a/server/mongo/src/__tests__/storage.test.ts b/server/mongo/src/__tests__/storage.test.ts index 0464657fd0e..79528eaf22f 100644 --- a/server/mongo/src/__tests__/storage.test.ts +++ b/server/mongo/src/__tests__/storage.test.ts @@ -115,7 +115,7 @@ describe('mongo operations', () => { return wrapAdapterToClient(ctx, serverStorage, txes) }) - operations = new TxOperations(client, core.account.System) + operations = new TxOperations(client, core.account.System, core.workspace.Model) } it('check add', async () => { diff --git a/server/mongo/src/__tests__/tasks.ts b/server/mongo/src/__tests__/tasks.ts index 12997282f44..a4161cb54f8 100644 --- a/server/mongo/src/__tests__/tasks.ts +++ b/server/mongo/src/__tests__/tasks.ts @@ -1,4 +1,4 @@ -import { +import core, { type AttachedDoc, type Class, ClassifierKind, @@ -79,6 +79,7 @@ export function createTask (name: string, rate: number, description: string): Da } export const doc1: Task = { + _uuid: core.workspace.Any, _id: 'd1' as Ref, _class: taskPlugin.class.Task, name: 'my-space', diff --git a/server/postgres/src/__tests__/conversion.spec.ts b/server/postgres/src/__tests__/conversion.spec.ts index b8ccaf07bff..cae6283915b 100644 --- a/server/postgres/src/__tests__/conversion.spec.ts +++ b/server/postgres/src/__tests__/conversion.spec.ts @@ -71,7 +71,7 @@ describe('array decoding', () => { }) }) -const factory = new TxFactory('email:test' as PersonId) +const factory = new TxFactory('email:test' as PersonId, core.workspace.Model) function upd (id: string, partial: DocumentUpdate): Tx { return factory.createTxUpdateDoc( test.class.ComplexClass, diff --git a/server/postgres/src/__tests__/minmodel.ts b/server/postgres/src/__tests__/minmodel.ts index 2089d852ae9..a06ef3130b7 100644 --- a/server/postgres/src/__tests__/minmodel.ts +++ b/server/postgres/src/__tests__/minmodel.ts @@ -37,7 +37,7 @@ import type { IntlString, Plugin } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' import { taskPlugin } from './tasks' -export const txFactory = new TxFactory(core.account.System) +export const txFactory = new TxFactory(core.account.System, core.workspace.Model) export function createClass (_class: Ref>, attributes: Data>): TxCreateDoc { return txFactory.createTxCreateDoc(core.class.Class, core.space.Model, attributes, _class) diff --git a/server/postgres/src/__tests__/storage.test.ts b/server/postgres/src/__tests__/storage.test.ts index 88ab3b367d7..7e92e49fb4b 100644 --- a/server/postgres/src/__tests__/storage.test.ts +++ b/server/postgres/src/__tests__/storage.test.ts @@ -124,7 +124,7 @@ describe('postgres operations', () => { return wrapAdapterToClient(ctx, serverStorage, txes) }) - operations = new TxOperations(client, core.account.System) + operations = new TxOperations(client, core.account.System, core.workspace.Model) } it('check add', async () => { diff --git a/server/postgres/src/__tests__/tasks.ts b/server/postgres/src/__tests__/tasks.ts index 773b9c05217..943719a36b5 100644 --- a/server/postgres/src/__tests__/tasks.ts +++ b/server/postgres/src/__tests__/tasks.ts @@ -1,4 +1,4 @@ -import { +import core, { type AttachedDoc, type Class, ClassifierKind, @@ -80,6 +80,7 @@ export function createTask (name: string, rate: number, description: string): Da } export const doc1: Task = { + _uuid: core.workspace.Any, _id: 'd1' as Ref, _class: taskPlugin.class.Task, name: 'my-space', diff --git a/server/postgres/src/utils.ts b/server/postgres/src/utils.ts index 6284b01e734..0c5c5c5afdf 100644 --- a/server/postgres/src/utils.ts +++ b/server/postgres/src/utils.ts @@ -349,6 +349,7 @@ export function convertDoc ( schemaAndFields?: SchemaAndFields ): DBDoc { const extractedFields: Doc & Record = { + _uuid: workspaceId, _id: doc._id, space: doc.space, createdBy: doc.createdBy, @@ -605,6 +606,7 @@ export function parseDocWithProjection ( resultData = filterProjection(data, projection) } const res = { + _uuid: workspaceId, ...resultData, ...rest } as any as T @@ -636,7 +638,7 @@ export function parseDoc (doc: DBDoc, schema: Schema): T { return res } -export interface DBDoc extends Doc { +export interface DBDoc extends Omit { workspaceId: WorkspaceUuid data: Record [key: string]: any diff --git a/server/rpc/src/rpc.ts b/server/rpc/src/rpc.ts index 010bfcd0ff1..7bf6badeacd 100644 --- a/server/rpc/src/rpc.ts +++ b/server/rpc/src/rpc.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import type { Account } from '@hcengineering/core' +import type { Account, WorkspaceUuid } from '@hcengineering/core' import platform, { PlatformError, Severity, Status } from '@hcengineering/platform' import { Packr } from 'msgpackr' @@ -47,14 +47,14 @@ export interface HelloResponse extends Response { binary: boolean reconnect?: boolean serverVersion: string - lastTx?: string - lastHash?: string // Last model hash + lastTx?: Record + lastHash?: Record // Last model hash account: Account useCompression?: boolean } export function rpcJSONReplacer (key: string, value: any): any { - if (Array.isArray(value) && ((value as any).total !== undefined || (value as any).lookupMap !== undefined)) { + if (Array.isArray(value) && ((value as any).total !== undefined || (value as any).lookupMap != null)) { return { dataType: 'TotalArray', total: (value as any).total, @@ -104,6 +104,10 @@ export interface Response { time?: number // Server time to perform operation bfst?: number // Server time to perform operation queue?: number + + // Special for system account clients. + target?: string + exclude?: string[] } export class RPCHandler { diff --git a/server/s3/src/index.ts b/server/s3/src/index.ts index 49e33144aa5..2f6ab0c086b 100644 --- a/server/s3/src/index.ts +++ b/server/s3/src/index.ts @@ -282,6 +282,7 @@ export class S3Service implements StorageAdapter { for (const data of res.Contents ?? []) { const _id = this.stripPrefix(rootPrefix, data.Key ?? '') buffer.push({ + _uuid: wsIds.uuid, _id: _id as Ref, _class: core.class.Blob, etag: data.ETag ?? '', @@ -311,6 +312,7 @@ export class S3Service implements StorageAdapter { }) const rootPrefix = this.rootPrefix(wsIds) return { + _uuid: wsIds.uuid, provider: '', _class: core.class.Blob, _id: this.stripPrefix(rootPrefix, objectName) as Ref, diff --git a/server/server-pipeline/src/pipeline.ts b/server/server-pipeline/src/pipeline.ts index cdd30289c7e..476efdf99bb 100644 --- a/server/server-pipeline/src/pipeline.ts +++ b/server/server-pipeline/src/pipeline.ts @@ -25,7 +25,6 @@ import { FullTextMiddleware, IdentityMiddleware, LiveQueryMiddleware, - LookupMiddleware, LowLevelMiddleware, MarkDerivedEntryMiddleware, ModelMiddleware, @@ -117,7 +116,6 @@ export function createServerPipeline ( const conf = getConfig(metrics, dbUrl, wsMetrics, opt, extensions) const middlewares: MiddlewareCreator[] = [ - LookupMiddleware.create, IdentityMiddleware.create, ModifiedMiddleware.create, PluginConfigurationMiddleware.create, diff --git a/server/server-storage/src/tests/memAdapters.ts b/server/server-storage/src/tests/memAdapters.ts index 68671cea88e..bc379725a64 100644 --- a/server/server-storage/src/tests/memAdapters.ts +++ b/server/server-storage/src/tests/memAdapters.ts @@ -103,6 +103,7 @@ export class MemStorageAdapter implements StorageAdapter { const data = Buffer.concat(buffer as any) const dataId = getDataId(wsIds) const dta = { + _uuid: wsIds.uuid, _class: core.class.Blob, _id: objectName as any, contentType, diff --git a/server/server/src/__test__/account.ts b/server/server/src/__test__/account.ts new file mode 100644 index 00000000000..4ad3fc104fd --- /dev/null +++ b/server/server/src/__test__/account.ts @@ -0,0 +1,106 @@ +import type { LoginInfoWithWorkspaces, LoginInfoWorkspace } from '@hcengineering/account-client' +import core, { + AccountRole, + SocialIdType, + systemAccountUuid, + type AccountUuid, + type PersonId, + type WorkspaceInfoWithStatus, + type WorkspaceUuid +} from '@hcengineering/core' +import { decodeToken } from '@hcengineering/server-token' +import type { TSessionManager } from '../sessionManager' + +export const workspaces: Record = { + ['test1' as WorkspaceUuid]: { + url: 'test', + mode: 'active', + version: { versionMajor: 1, versionMinor: 0, versionPatch: 0 }, + name: 'test', + role: AccountRole.Owner, + endpoint: { + internalUrl: 'endpoint', + externalUrl: 'endpoint', + region: 'region' + } + }, + ['test2' as WorkspaceUuid]: { + url: 'test2', + mode: 'active', + version: { versionMajor: 1, versionMinor: 0, versionPatch: 0 }, + name: 'test2', + role: AccountRole.Owner, + endpoint: { + internalUrl: 'endpoint2', + externalUrl: 'endpoint2', + region: 'region2' + } + }, + ['test3' as WorkspaceUuid]: { + url: 'test3', + mode: 'active', + version: { versionMajor: 1, versionMinor: 0, versionPatch: 0 }, + name: 'test3', + role: AccountRole.Owner, + endpoint: { + internalUrl: 'endpoint2', + externalUrl: 'endpoint2', + region: 'region2' + } + } +} + +export const workspaceRef = { + test1: { ...workspaces['test1' as WorkspaceUuid], uuid: 'test1' as WorkspaceUuid }, + test2: { ...workspaces['test2' as WorkspaceUuid], uuid: 'test2' as WorkspaceUuid }, + test3: { ...workspaces['test3' as WorkspaceUuid], uuid: 'test3' as WorkspaceUuid } +} + +export function hookSessionManagerAccount (sessionManager: TSessionManager): void { + ;(sessionManager as any).getWorkspaceInfo = async ( + token: string, + updateLastVisit: boolean = false + ): Promise => { + const tok = decodeToken(token) + const info = workspaces[tok.workspace] + return { + uuid: tok.workspace, + url: info.url, + region: info.endpoint.region, + versionMajor: info.version.versionMajor, + versionMinor: info.version.versionMinor, + versionPatch: info.version.versionPatch, + lastVisit: Date.now(), + mode: info.mode, + processingProgress: info.progress ?? 0, + name: info.name ?? '', + createdOn: new Date('2022-01-01').getTime(), + endpoint: info.endpoint + } + } + sessionManager.getLoginWithWorkspaceInfo = async (token: string): Promise => { + const tok = decodeToken(token) + if (tok.account === systemAccountUuid) { + return { + account: tok.account, + personalWorkspace: core.workspace.Any, + socialIds: [], + workspaces: {} + } + } + return { + account: tok.account, + personalWorkspace: tok.workspace, + workspaces, + socialIds: [ + { + _id: ('sid' + tok.account) as PersonId, + type: SocialIdType.EMAIL, + value: tok.account + '@test.com', + key: tok.account + '@test.com', + personUuid: tok.account + } + ] + } + } +} diff --git a/server/server/src/__test__/collectQueue.ts b/server/server/src/__test__/collectQueue.ts new file mode 100644 index 00000000000..02a6f90b437 --- /dev/null +++ b/server/server/src/__test__/collectQueue.ts @@ -0,0 +1,78 @@ +import type { MeasureContext, WorkspaceUuid } from '@hcengineering/core' +import type { ConsumerHandle, PlatformQueue, PlatformQueueProducer, QueueTopic } from '@hcengineering/server-core' + +/** + * A dummy implementation of PlatformQueueProducer for testing and development + */ +class CollectQueueProducer implements PlatformQueueProducer { + constructor (readonly topic: QueueTopic | string) {} + entries: { id: WorkspaceUuid | string, value: T }[] = [] + async send (id: WorkspaceUuid | string, msgs: T[]): Promise { + this.entries.push(...msgs.map((it) => ({ id, value: it }))) + } + + async close (): Promise { + await Promise.resolve() + } + + getQueue (): PlatformQueue { + return new CollectQueue() + } +} + +/** + * A dummy implementation of PlatformQueue for testing and development + */ +export class CollectQueue implements PlatformQueue { + producers: PlatformQueueProducer[] = [] + getProducer(ctx: MeasureContext, topic: QueueTopic | string): PlatformQueueProducer { + const p = new CollectQueueProducer(topic) + this.producers.push(p) + return p + } + + getClientId (): string { + return 'collect' + } + + async shutdown (): Promise {} + + createConsumer( + ctx: MeasureContext, + topic: QueueTopic | string, + groupId: string, + onMessage: ( + msg: { id: WorkspaceUuid | string, value: T }[], + queue: { + pause: () => void + heartbeat: () => Promise + } + ) => Promise, + options?: { + bulkSize?: number + fromBegining?: boolean + } + ): ConsumerHandle { + return { + close: async (): Promise => { + await Promise.resolve() + }, + isConnected: (): boolean => { + return false + } + } + } + + async createTopics (tx: number): Promise { + await Promise.resolve() + } + + async deleteTopics (topics?: (QueueTopic | string)[]): Promise {} +} + +/** + * Creates a new instance of the dummy queue implementation + */ +export function createCollectQueue (): PlatformQueue { + return new CollectQueue() +} diff --git a/server/server/src/__test__/connection.ts b/server/server/src/__test__/connection.ts new file mode 100644 index 00000000000..aae066f3730 --- /dev/null +++ b/server/server/src/__test__/connection.ts @@ -0,0 +1,255 @@ +import { + ClientConnectEvent, + generateId, + type Account, + type AccountUuid, + type Class, + type ClientConnection, + type ClientConnectOptions, + type Doc, + type DocChunk, + type DocumentQuery, + type Domain, + type FindOptions, + type FindResult, + type Handler, + type LoadModelResponse, + type MeasureContext, + type Ref, + type SearchOptions, + type SearchQuery, + type SearchResult, + type Storage, + type SubscribedWorkspaceInfo, + type Timestamp, + type Tx, + type TxHandler, + type TxOptions, + type TxResult, + type WorkspaceUuid +} from '@hcengineering/core' +import type { Request, Response } from '@hcengineering/rpc' +import type { ConnectionSocket } from '@hcengineering/server-core' +import type { ClientSession } from '../client' +import type { TSessionManager } from '../sessionManager' +import type { ClientSessionCtx } from '../types' +import type { Workspace } from '../workspace' + +export class CollectConnectionSocket implements ConnectionSocket { + id: string = generateId() + private _isClosed: boolean = false + private readonly _data: Record = {} + private backpressurePromise: Promise | undefined + + isClosed: boolean = false + + messagesTo: Response[] = [] + + close: () => void = () => { + this._isClosed = true + this.isClosed = true + } + + send: (ctx: MeasureContext, msg: Response, binary: boolean, compression: boolean) => Promise = async ( + ctx, + msg, + binary, + compression + ) => { + if (this._isClosed) { + throw new Error('Connection is closed') + } + + this.messagesTo.push(msg) + } + + sendPong: () => void = () => { + // Simulate pong response + } + + data: () => Record = () => { + return { ...this._data } + } + + readRequest: (buffer: Buffer, binary: boolean) => Request = (buffer, binary) => { + // Mock request parsing from buffer + return JSON.parse(buffer.toString()) + } + + isBackpressure: () => boolean = () => { + return this.backpressurePromise !== undefined + } + + backpressure: (ctx: MeasureContext) => Promise = async (ctx) => { + if (this.backpressurePromise === undefined) { + this.backpressurePromise = new Promise((resolve) => { + setTimeout(() => { + this.backpressurePromise = undefined + resolve() + }, 100) + }) + } + await this.backpressurePromise + } + + checkState: () => boolean = () => { + return !this._isClosed + } +} + +export class SMClientConnection implements ClientConnection { + reqId: number = 0 + + constructor ( + readonly ctx: MeasureContext, + readonly endpoint: string, + readonly account: Account, + readonly sessionManager: TSessionManager, + readonly session: ClientSession, + + readonly handler: TxHandler, + readonly options?: ClientConnectOptions + ) { + if (options?.onConnect !== undefined) { + void options.onConnect(ClientConnectEvent.Connected, {}, {}) + } + if (options?.onAccount !== undefined) { + options.onAccount(this.account) + } + sessionManager.addBroadcastHandler(this.handler) + } + + isConnected: () => boolean = () => true + + close: () => Promise = async () => {} + + async loadModel (last: Timestamp, hash?: string, workspace?: WorkspaceUuid): Promise { + return await this.session.loadModelRaw(this.toContext(this.ctx, workspace), last, hash) + } + + getLastHash?: ((ctx: MeasureContext) => Promise>) | undefined = + async () => ({}) + + pushHandler: (handler: Handler) => void = () => {} + + private getWorkspace (workspace: WorkspaceUuid | undefined): Workspace { + if (workspace === undefined) { + throw new Error('Workspace not specified') + } + + const workspaceRef = this.sessionManager.workspaces.get(workspace) + if (workspaceRef === undefined) { + throw new Error('Workspace not found') + } + return workspaceRef + } + + async getAccount (): Promise { + return this.account + } + + subscribe (subscription: { + accounts?: AccountUuid[] + workspaces?: WorkspaceUuid[] + }): Promise { + return this.sessionManager.handleSubcribe(this.ctx, this.session, subscription, true) + } + + unsubscribe (subscription: { accounts?: AccountUuid[], workspaces?: WorkspaceUuid[] }): Promise { + return this.sessionManager.handleSubcribe(this.ctx, this.session, subscription, false).then() + } + + toContext (ctx: MeasureContext, workspace: WorkspaceUuid | undefined): ClientSessionCtx { + const workspaceRef = this.getWorkspace(workspace) + return this.sessionManager.createOpContext( + this.ctx, + this.ctx, + [workspaceRef], + this.reqId++, + this.session, + new CollectConnectionSocket(), + undefined + ) + } + + findAll( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + return this.session.findAllRaw(this.toContext(this.ctx, options?.workspace as WorkspaceUuid), _class, query) + } + + tx (tx: Tx, options?: TxOptions): Promise { + return this.session.txRaw(this.toContext(this.ctx, tx._uuid), tx, options?.user) + } + + searchFulltext (query: SearchQuery, options: SearchOptions): Promise { + return this.session.searchFulltextRaw(this.toContext(this.ctx, options.workspace), query, options) + } + + loadChunk (workspaceId: WorkspaceUuid, domain: Domain, idx?: number): Promise { + return this.session.loadChunkRaw(this.toContext(this.ctx, workspaceId), workspaceId, domain, idx) + } + + closeChunk (workspaceId: WorkspaceUuid, idx: number): Promise { + return this.session.closeChunkRaw(this.toContext(this.ctx, workspaceId), workspaceId, idx) + } + + loadDocs (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { + return this.session.loadDocsRaw(this.toContext(this.ctx, workspaceId), workspaceId, domain, docs) + } + + upload (workspaceId: WorkspaceUuid, domain: Domain, docs: Doc[]): Promise { + return this.session.uploadRaw(this.toContext(this.ctx, workspaceId), workspaceId, domain, docs) + } + + clean (workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { + return this.session.cleanRaw(this.toContext(this.ctx, workspaceId), workspaceId, domain, docs) + } + + async getDomainHash (workspaceId: WorkspaceUuid, domain: Domain): Promise { + return await this.session.getDomainHashRaw(this.toContext(this.ctx, workspaceId), workspaceId, domain) + } + + async sendForceClose (workspace: WorkspaceUuid): Promise { + // TODO: Check if required for tests + } +} + +export class ClientConnectionWrapper implements Storage { + constructor ( + readonly sessionManager: TSessionManager, + readonly session: ClientSession + ) {} + + findAll: ( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise> = async (_class, query, options) => { + const ctx = this.sessionManager.createOpContext( + this.sessionManager.ctx, + this.sessionManager.ctx, + Array.from(this.sessionManager.workspaces.values()), + 0, + this.session, + new CollectConnectionSocket(), + undefined + ) + return await this.session.findAllRaw(ctx, _class, query, options) + } + + tx: (tx: Tx) => Promise = async (tx) => { + const ctx = this.sessionManager.createOpContext( + this.sessionManager.ctx, + this.sessionManager.ctx, + Array.from(this.sessionManager.workspaces.values()), + 0, + this.session, + new CollectConnectionSocket(), + undefined + ) + return await this.session.txRaw(ctx, tx) + } +} diff --git a/server/server/src/__test__/sessions.test.ts b/server/server/src/__test__/sessions.test.ts new file mode 100644 index 00000000000..495672f451c --- /dev/null +++ b/server/server/src/__test__/sessions.test.ts @@ -0,0 +1,48 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +import { MeasureMetricsContext, type AccountUuid } from '@hcengineering/core' +import { prepapre2SM1EP } from './util2SM1EP' +import core from '@hcengineering/core' +import { ClientConnectionWrapper } from './connection' +import { workspaceRef } from './account' + +describe('sessions', () => { + it('check add session', async () => { + const ctx = new MeasureMetricsContext('test', {}) + + const { newSession, endpointMgr, endpoint2Mgr } = await prepapre2SM1EP(ctx) + + const session = await newSession('user1' as AccountUuid) + + expect(session.sessionId).toBe('s1') + expect(session.account.uuid).toBe('user1' as AccountUuid) + + expect(endpointMgr.accounts.size).toBe(1) + expect(endpointMgr.workspaces.size).toBe(3) + + expect(endpointMgr.endpointClients.size).toBe(1) + + expect(endpoint2Mgr.accounts.size).toBe(2) + expect(endpoint2Mgr.workspaces.size).toBe(2) + + expect(endpointMgr.sessions.size).toBe(1) + expect(endpoint2Mgr.sessions.size).toBe(1) + + const cc = new ClientConnectionWrapper(endpointMgr, session) + + const r1 = await cc.findAll(core.class.BenchmarkDoc, {}) + expect(r1.length).toBe(3) + + const r2 = await cc.findAll(core.class.BenchmarkDoc, {}, { workspace: workspaceRef.test1.uuid }) + expect(r2.length).toBe(1) + + const r3 = await cc.findAll(core.class.BenchmarkDoc, {}, { workspace: workspaceRef.test2.uuid }) + expect(r3.length).toBe(1) + + const r4 = await cc.findAll( + core.class.BenchmarkDoc, + {}, + { workspace: { $in: [workspaceRef.test1.uuid, workspaceRef.test3.uuid] } } + ) + expect(r4.length).toBe(2) + }) +}) diff --git a/server/server/src/__test__/test-middleware.ts b/server/server/src/__test__/test-middleware.ts new file mode 100644 index 00000000000..3fdff169d77 --- /dev/null +++ b/server/server/src/__test__/test-middleware.ts @@ -0,0 +1,60 @@ +import core, { + generateId, + toFindResult, + type Class, + type Doc, + type DocumentQuery, + type FindOptions, + type FindResult, + type MeasureContext, + type Ref, + type SessionData, + type Tx +} from '@hcengineering/core' +import { + BaseMiddleware, + type Middleware, + type MiddlewareCreator, + type PipelineContext, + type TxMiddlewareResult +} from '@hcengineering/server-core' + +export class CollectMiddleware extends BaseMiddleware { + constructor ( + ctx: MeasureContext, + readonly context: PipelineContext, + protected readonly next: Middleware | undefined + ) { + super(context, next) + } + + static create (): MiddlewareCreator { + return async (ctx, context, next) => { + return new CollectMiddleware(ctx, context, next) + } + } + + async tx (ctx: MeasureContext, tx: Tx[]): Promise { + return {} + } + + async findAll( + ctx: MeasureContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + return toFindResult([ + { + _class, + _id: generateId(), + _uuid: this.context.workspace.uuid, + modifiedBy: core.account.System, + modifiedOn: Date.now(), + space: core.space.DerivedTx, + query, + options + } as any + ]) + } +} diff --git a/server/server/src/__test__/util2SM1EP.ts b/server/server/src/__test__/util2SM1EP.ts new file mode 100644 index 00000000000..5c1c0a56b20 --- /dev/null +++ b/server/server/src/__test__/util2SM1EP.ts @@ -0,0 +1,144 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +import type { ServerApi } from '@hcengineering/communication-sdk-types' +import { + Hierarchy, + type MeasureMetricsContext, + ModelDb, + systemAccount, + systemAccountUuid, + type AccountUuid, + type Branding, + type MeasureContext, + type WorkspaceIds, + type WorkspaceUuid +} from '@hcengineering/core' +import { + createPipeline, + type BroadcastFunc, + type CommunicationApiFactory, + type EndpointConnectionFactory, + type PipelineFactory +} from '@hcengineering/server-core' +import { generateToken } from '@hcengineering/server-token' +import type { ClientSession } from '../client' +import { createSessionManager, type TSessionManager } from '../sessionManager' +import { hookSessionManagerAccount } from './account' +import { createCollectQueue } from './collectQueue' +import { CollectConnectionSocket, SMClientConnection } from './connection' +import { CollectMiddleware } from './test-middleware' +export async function prepapre2SM1EP ( + ctx: MeasureMetricsContext +): Promise<{ + newSession: (user: AccountUuid, target?: WorkspaceUuid) => Promise + endpointMgr: TSessionManager + endpoint2Mgr: TSessionManager + }> { + const queue = createCollectQueue() + const hierarchy = new Hierarchy() + const modelDb = new ModelDb(hierarchy) + + const pipelineFactory: PipelineFactory = async ( + ctx: MeasureContext, + ws: WorkspaceIds, + broadcast: BroadcastFunc, + branding: Branding | null, + communicationApi: ServerApi | null + ) => { + return await createPipeline(ctx, [CollectMiddleware.create()], { + hierarchy, + modelDb, + queue, + workspace: ws, + branding, + contextVars: {}, + communicationApi + }) + } + + const communicationApiFactory: CommunicationApiFactory = async (ctx, ws, br) => { + return jest.mocked({ ctx, ws, br } as any) + } + + const endpoint2Mgr: TSessionManager = createSessionManager( + ctx, + 'region', + 'endpoint2', + {}, + { + pingTimeout: 10000, + reconnectTimeout: 30 // seconds to reconnect + }, + { + start: () => {}, + stop: async () => '' + }, + 'account', + false, + queue, + pipelineFactory, + () => { + throw new Error('Endpoint not found') + }, + communicationApiFactory, + false + ) as TSessionManager + + hookSessionManagerAccount(endpoint2Mgr) + const sysSocket = new CollectConnectionSocket() + const sysSession = await endpoint2Mgr.addSession( + ctx, + sysSocket, + { + account: systemAccountUuid, + workspace: '' as WorkspaceUuid + }, + generateToken(systemAccountUuid), + 'root-s1' + ) + + const endpointFactory: EndpointConnectionFactory = (ctx, ws, handler, opt) => { + if (ws === 'endpoint2') { + return new SMClientConnection(ctx, ws, systemAccount, endpoint2Mgr, sysSession as ClientSession, handler, opt) + } + throw new Error('Endpoint not found') + } + + const endpointMgr: TSessionManager = createSessionManager( + ctx, + 'region', + 'endpoint', + {}, + { + pingTimeout: 10000, + reconnectTimeout: 30 // seconds to reconnect + }, + { + start: () => {}, + stop: async () => '' + }, + 'account', + false, + queue, + pipelineFactory, + endpointFactory, + communicationApiFactory, + false + ) as TSessionManager + + hookSessionManagerAccount(endpointMgr) + + const newSession = async (user: AccountUuid, target?: WorkspaceUuid): Promise => { + const socket = new CollectConnectionSocket() + return (await endpointMgr.addSession( + ctx, + socket, + { + account: 'user1' as AccountUuid, + workspace: '' as WorkspaceUuid + }, + generateToken('user1' as AccountUuid), + 's1' + )) as ClientSession + } + return { endpointMgr, endpoint2Mgr, newSession } +} diff --git a/server/server/src/client.ts b/server/server/src/client.ts index 2e3620c1377..61de06354f4 100644 --- a/server/server/src/client.ts +++ b/server/server/src/client.ts @@ -30,10 +30,12 @@ import { type MessagesGroup } from '@hcengineering/communication-types' import { - type AccountUuid, generateId, + groupByArray, + toFindResult, TxProcessor, type Account, + type AccountUuid, type Class, type Doc, type DocumentQuery, @@ -52,23 +54,27 @@ import { type Timestamp, type Tx, type TxCUD, + type TxOptions, type TxResult, type WorkspaceDataId, - type WorkspaceIds + type WorkspaceIds, + type WorkspaceUuid } from '@hcengineering/core' import { PlatformError, unknownError } from '@hcengineering/platform' import { BackupClientOps, createBroadcastEvent, SessionDataImpl, - type ClientSessionCtx, type ConnectionSocket, + type LoadChunkResponse, type Pipeline, - type Session, type SessionRequest, type StatisticsElement } from '@hcengineering/server-core' + import { type Token } from '@hcengineering/server-token' +import { mapLookup } from './lookup' +import { type ClientSessionCtx, type Session } from './types' const useReserveContext = (process.env.USE_RESERVE_CTX ?? 'true') === 'true' @@ -93,21 +99,31 @@ export class ClientSession implements Session { ops: BackupClientOps | undefined opsPipeline: Pipeline | undefined isAdmin: boolean + workspaceClosed = false + + subscribedUsers = new Set() constructor ( - readonly token: Token, - readonly workspace: WorkspaceIds, + readonly ctx: MeasureContext, + protected readonly token: Token, + readonly socket: ConnectionSocket, + readonly workspaces: Set, readonly account: Account, readonly info: LoginInfoWithWorkspaces, readonly allowUpload: boolean ) { this.isAdmin = this.token.extra?.admin === 'true' + this.subscribedUsers.add(account.uuid) } getUser (): AccountUuid { return this.token.account } + async getAccount (ctx: ClientSessionCtx): Promise { + await ctx.sendResponse(ctx.requestId, this.account) + } + getUserSocialIds (): PersonId[] { return this.account.socialIds } @@ -134,43 +150,51 @@ export class ClientSession implements Session { } async loadModel (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string): Promise { - try { - this.includeSessionContext(ctx) - const result = await ctx.ctx.with('load-model', {}, () => ctx.pipeline.loadModel(ctx.ctx, lastModelTx, hash)) - await ctx.sendResponse(ctx.requestId, result) - } catch (err) { - await ctx.sendError(ctx.requestId, 'Failed to loadModel', unknownError(err)) - ctx.ctx.error('failed to loadModel', { err }) - } + // TODO: Model is from first workspace for now. + const workspace = ctx.workspaces[0] + await workspace.with(async (pipeline, communicationApi) => { + try { + this.includeSessionContext(ctx, pipeline, this.account) + const result = await ctx.ctx.with('load-model', {}, () => pipeline.loadModel(ctx.ctx, lastModelTx, hash)) + await ctx.sendResponse(ctx.requestId, result) + } catch (err) { + await ctx.sendError(ctx.requestId, 'Failed to loadModel', unknownError(err)) + ctx.ctx.error('failed to loadModel', { err }) + } + }) } async loadModelRaw (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string): Promise { - this.includeSessionContext(ctx) - return await ctx.ctx.with('load-model', {}, (_ctx) => ctx.pipeline.loadModel(_ctx, lastModelTx, hash)) + // TODO: Model is from first workspace for now. + const workspace = ctx.workspaces[0] + return await workspace.with(async (pipeline, communicationApi) => { + this.includeSessionContext(ctx, pipeline, this.account) + return await ctx.ctx.with('load-model', {}, (_ctx) => pipeline.loadModel(_ctx, lastModelTx, hash)) + }) } - includeSessionContext (ctx: ClientSessionCtx): void { - const dataId = this.workspace.dataId ?? (this.workspace.uuid as unknown as WorkspaceDataId) + includeSessionContext (ctx: ClientSessionCtx, pipeline: Pipeline, account: Account): void { + const dataId = pipeline.context.workspace.dataId ?? (pipeline.context.workspace.uuid as unknown as WorkspaceDataId) const contextData = new SessionDataImpl( - this.account, + account, this.sessionId, this.isAdmin, undefined, { - ...this.workspace, + ...pipeline.context.workspace, dataId }, false, undefined, undefined, - ctx.pipeline.context.modelDb, + pipeline.context.modelDb, ctx.socialStringsToUsers, this.token.extra?.service ?? '🤦‍♂️user' ) ctx.ctx.contextData = contextData } - findAllRaw( + async findAllRaw( ctx: ClientSessionCtx, _class: Ref>, query: DocumentQuery, @@ -179,8 +203,39 @@ export class ClientSession implements Session { this.lastRequest = Date.now() this.total.find++ this.current.find++ - this.includeSessionContext(ctx) - return ctx.pipeline.findAll(ctx.ctx, _class, query, options) + + const result: FindResult = toFindResult([], -1) + + let workspaces = ctx.workspaces + + if (options?.workspace !== undefined) { + if (typeof options.workspace === 'string') { + workspaces = workspaces.filter((it) => it.wsId.uuid === options.workspace) + } else if ('$in' in options.workspace && options.workspace.$in !== undefined) { + const $in = options.workspace.$in + workspaces = workspaces.filter((it) => $in.includes(it.wsId.uuid)) + } else if ('$nin' in options.workspace && options.workspace.$nin !== undefined) { + const $nin = options.workspace.$nin + workspaces = workspaces.filter((it) => !$nin.includes(it.wsId.uuid)) + } + } + + const useUser = options?.user !== undefined ? ctx.getAccount(options?.user) : this.account + + for (const workspace of workspaces) { + await workspace.with(async (pipeline, communicationApi) => { + this.includeSessionContext(ctx, pipeline, useUser) + const part = await pipeline.findAll(ctx.ctx, _class, query, options) + result.push(...part) + if (part.total !== -1) { + if (result.total === -1) { + result.total = 0 + } + result.total += part.total + } + }) + } + return result } async findAll( @@ -190,7 +245,8 @@ export class ClientSession implements Session { options?: FindOptions ): Promise { try { - await ctx.sendResponse(ctx.requestId, await this.findAllRaw(ctx, _class, query, options)) + const result = await this.findAllRaw(ctx, _class, query, options) + await ctx.sendResponse(ctx.requestId, mapLookup(query, result, options)) } catch (err) { await ctx.sendError(ctx.requestId, 'Failed to findAll', unknownError(err)) ctx.ctx.error('failed to findAll', { err }) @@ -200,8 +256,12 @@ export class ClientSession implements Session { async searchFulltext (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions): Promise { try { this.lastRequest = Date.now() - this.includeSessionContext(ctx) - await ctx.sendResponse(ctx.requestId, await ctx.pipeline.searchFulltext(ctx.ctx, query, options)) + const workspace = ctx.workspaces[0] + const useUser = options?.user !== undefined ? ctx.getAccount(options?.user) : this.account + await workspace.with(async (pipeline, communicationApi) => { + this.includeSessionContext(ctx, pipeline, useUser) + await ctx.sendResponse(ctx.requestId, await pipeline.searchFulltext(ctx.ctx, query, options)) + }) } catch (err) { await ctx.sendError(ctx.requestId, 'Failed to searchFulltext', unknownError(err)) ctx.ctx.error('failed to searchFulltext', { err }) @@ -210,13 +270,18 @@ export class ClientSession implements Session { async searchFulltextRaw (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions): Promise { this.lastRequest = Date.now() - this.includeSessionContext(ctx) - return await ctx.pipeline.searchFulltext(ctx.ctx, query, options) + const workspace = ctx.workspaces[0] + const useUser = options?.user !== undefined ? ctx.getAccount(options?.user) : this.account + return await workspace.with(async (pipeline, communicationApi) => { + this.includeSessionContext(ctx, pipeline, useUser) + return await pipeline.searchFulltext(ctx.ctx, query, options) + }) } async txRaw ( ctx: ClientSessionCtx, - tx: Tx + tx: Tx, + accountOverride?: AccountUuid ): Promise<{ result: TxResult broadcastPromise: Promise @@ -225,48 +290,58 @@ export class ClientSession implements Session { this.lastRequest = Date.now() this.total.tx++ this.current.tx++ - this.includeSessionContext(ctx) - - let cid = 'client_' + generateId() - ctx.ctx.id = cid - let onEnd = useReserveContext ? ctx.pipeline.context.adapterManager?.reserveContext?.(cid) : undefined - let result: TxResult - try { - result = await ctx.pipeline.tx(ctx.ctx, [tx]) - } finally { - onEnd?.() + const workspace = ctx.workspaces.find((it) => it.wsId.uuid === tx._uuid) + if (workspace === undefined) { + throw new Error('Workspace not found') } - // Send result immideately - await ctx.sendResponse(ctx.requestId, result) + const useUser = accountOverride !== undefined ? ctx.getAccount(accountOverride) : this.account - // We need to broadcast all collected transactions - const broadcastPromise = ctx.pipeline.handleBroadcast(ctx.ctx) + return await workspace.with(async (pipeline, communicationApi) => { + this.includeSessionContext(ctx, pipeline, useUser) - // ok we could perform async requests if any - const asyncs = (ctx.ctx.contextData as SessionData).asyncRequests ?? [] - let asyncsPromise: Promise | undefined - if (asyncs.length > 0) { - cid = 'client_async_' + generateId() + let cid = 'client_' + generateId() ctx.ctx.id = cid - onEnd = useReserveContext ? ctx.pipeline.context.adapterManager?.reserveContext?.(cid) : undefined - const handleAyncs = async (): Promise => { - try { - for (const r of asyncs) { - await r(ctx.ctx) + + let onEnd = useReserveContext ? pipeline.context.adapterManager?.reserveContext?.(cid) : undefined + let result: TxResult + try { + result = await pipeline.tx(ctx.ctx, [tx]) + } finally { + onEnd?.() + } + // Send result immideately + await ctx.sendResponse(ctx.requestId, result) + + // We need to broadcast all collected transactions + const broadcastPromise = pipeline.handleBroadcast(ctx.ctx) + + // ok we could perform async requests if any + const asyncs = (ctx.ctx.contextData as SessionData).asyncRequests ?? [] + let asyncsPromise: Promise | undefined + if (asyncs.length > 0) { + cid = 'client_async_' + generateId() + ctx.ctx.id = cid + onEnd = useReserveContext ? pipeline.context.adapterManager?.reserveContext?.(cid) : undefined + + const handleAyncs = async (): Promise => { + try { + for (const r of (ctx.ctx.contextData as SessionData).asyncRequests ?? []) { + await r(ctx.ctx) + } + } finally { + onEnd?.() } - } finally { - onEnd?.() } + asyncsPromise = handleAyncs() } - asyncsPromise = handleAyncs() - } - return { result, broadcastPromise, asyncsPromise } + return { result, broadcastPromise, asyncsPromise } + }) } - async tx (ctx: ClientSessionCtx, tx: Tx): Promise { + async tx (ctx: ClientSessionCtx, tx: Tx, options?: TxOptions): Promise { try { - const { broadcastPromise, asyncsPromise } = await this.txRaw(ctx, tx) + const { broadcastPromise, asyncsPromise } = await this.txRaw(ctx, tx, options?.user) await broadcastPromise if (asyncsPromise !== undefined) { await asyncsPromise @@ -277,29 +352,34 @@ export class ClientSession implements Session { } } - broadcast (ctx: MeasureContext, socket: ConnectionSocket, tx: Tx[]): void { - if (this.tx.length > 10000) { - const classes = new Set>>() - for (const dtx of tx) { - if (TxProcessor.isExtendsCUD(dtx._class)) { - classes.add((dtx as TxCUD).objectClass) - const attachedToClass = (dtx as TxCUD).attachedToClass - if (attachedToClass !== undefined) { - classes.add(attachedToClass) + broadcast (ctx: MeasureContext, socket: ConnectionSocket, tx: Tx[], target?: string, exclude?: string[]): void { + if (tx.length > 10000) { + const byWorkspace = groupByArray(tx, (it) => it._uuid) + for (const [uuid, tx] of byWorkspace) { + const classes = new Set>>() + for (const dtx of tx) { + if (TxProcessor.isExtendsCUD(dtx._class)) { + classes.add((dtx as TxCUD).objectClass) + const attachedToClass = (dtx as TxCUD).attachedToClass + if (attachedToClass !== undefined) { + classes.add(attachedToClass) + } } } + const bevent = createBroadcastEvent(uuid, Array.from(classes)) + void socket.send( + ctx, + { + result: [bevent], + target, + exclude + }, + this.binaryMode, + this.useCompression + ) } - const bevent = createBroadcastEvent(Array.from(classes)) - void socket.send( - ctx, - { - result: [bevent] - }, - this.binaryMode, - this.useCompression - ) } else { - void socket.send(ctx, { result: tx }, this.binaryMode, this.useCompression) + void socket.send(ctx, { result: tx, target, exclude }, this.binaryMode, this.useCompression) } } @@ -314,32 +394,65 @@ export class ClientSession implements Session { return this.ops } - async loadChunk (ctx: ClientSessionCtx, domain: Domain, idx?: number): Promise { + async loadChunkRaw ( + ctx: ClientSessionCtx, + workspaceId: WorkspaceUuid, + domain: Domain, + idx?: number + ): Promise { this.lastRequest = Date.now() + const workspace = ctx.workspaces.find((it) => it.wsId.uuid === workspaceId) + if (workspace === undefined) { + throw new Error('Workspace not found') + } + return await workspace.with(async (pipeline, communicationApi) => { + return await this.getOps(pipeline).loadChunk(ctx.ctx, domain, idx) + }) + } + + async loadChunk (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain, idx?: number): Promise { try { - const result = await this.getOps(ctx.pipeline).loadChunk(ctx.ctx, domain, idx) - await ctx.sendResponse(ctx.requestId, result) + await ctx.sendResponse(ctx.requestId, this.loadChunkRaw(ctx, workspaceId, domain)) } catch (err: any) { await ctx.sendError(ctx.requestId, 'Failed to upload', unknownError(err)) ctx.ctx.error('failed to loadChunk', { domain, err }) } } - async getDomainHash (ctx: ClientSessionCtx, domain: Domain): Promise { + async getDomainHashRaw (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain): Promise { this.lastRequest = Date.now() + const workspace = ctx.workspaces.find((it) => it.wsId.uuid === workspaceId) + if (workspace === undefined) { + throw new Error('Workspace not found') + } + return await workspace.with(async (pipeline, communicationApi) => { + return await this.getOps(pipeline).getDomainHash(ctx.ctx, domain) + }) + } + + async getDomainHash (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain): Promise { try { - const result = await this.getOps(ctx.pipeline).getDomainHash(ctx.ctx, domain) - await ctx.sendResponse(ctx.requestId, result) + await ctx.sendResponse(ctx.requestId, this.getDomainHashRaw(ctx, workspaceId, domain)) } catch (err: any) { await ctx.sendError(ctx.requestId, 'Failed to upload', unknownError(err)) ctx.ctx.error('failed to getDomainHash', { domain, err }) } } - async closeChunk (ctx: ClientSessionCtx, idx: number): Promise { + async closeChunkRaw (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, idx: number): Promise { + this.lastRequest = Date.now() + const workspace = ctx.workspaces.find((it) => it.wsId.uuid === workspaceId) + if (workspace === undefined) { + throw new Error('Workspace not found') + } + await workspace.with(async (pipeline, communicationApi) => { + await this.getOps(pipeline).closeChunk(ctx.ctx, idx) + }) + } + + async closeChunk (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, idx: number): Promise { try { - this.lastRequest = Date.now() - await this.getOps(ctx.pipeline).closeChunk(ctx.ctx, idx) + await this.closeChunkRaw(ctx, workspaceId, idx) await ctx.sendResponse(ctx.requestId, {}) } catch (err: any) { await ctx.sendError(ctx.requestId, 'Failed to closeChunk', unknownError(err)) @@ -347,50 +460,97 @@ export class ClientSession implements Session { } } - async loadDocs (ctx: ClientSessionCtx, domain: Domain, docs: Ref[]): Promise { + async loadDocsRaw ( + ctx: ClientSessionCtx, + workspaceId: WorkspaceUuid, + domain: Domain, + docs: Ref[] + ): Promise { this.lastRequest = Date.now() + + const workspace = ctx.workspaces.find((it) => it.wsId.uuid === workspaceId) + if (workspace === undefined) { + throw new Error('Workspace not found') + } + return await workspace.with(async (pipeline, communicationApi) => { + return await this.getOps(pipeline).loadDocs(ctx.ctx, domain, docs) + }) + } + + async loadDocs (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { try { - const result = await this.getOps(ctx.pipeline).loadDocs(ctx.ctx, domain, docs) - await ctx.sendResponse(ctx.requestId, result) + await ctx.sendResponse(ctx.requestId, await this.loadDocsRaw(ctx, workspaceId, domain, docs)) } catch (err: any) { await ctx.sendError(ctx.requestId, 'Failed to loadDocs', unknownError(err)) ctx.ctx.error('failed to loadDocs', { domain, err }) } } - async upload (ctx: ClientSessionCtx, domain: Domain, docs: Doc[]): Promise { + async uploadRaw (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain, docs: Doc[]): Promise { + this.lastRequest = Date.now() if (!this.allowUpload) { - await ctx.sendResponse(ctx.requestId, { error: 'Upload not allowed' }) + return } + const workspace = ctx.workspaces.find((it) => it.wsId.uuid === workspaceId) + if (workspace === undefined) { + throw new Error('Workspace not found') + } + await workspace.with(async (pipeline, communicationApi) => { + await this.getOps(pipeline).upload(ctx.ctx, domain, docs) + }) + } + + async upload (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain, docs: Doc[]): Promise { this.lastRequest = Date.now() + if (!this.allowUpload) { + await ctx.sendResponse(ctx.requestId, { error: 'Upload not allowed' }) + return + } try { - await this.getOps(ctx.pipeline).upload(ctx.ctx, domain, docs) + await this.uploadRaw(ctx, workspaceId, domain, docs) + await ctx.sendResponse(ctx.requestId, {}) } catch (err: any) { await ctx.sendError(ctx.requestId, 'Failed to upload', unknownError(err)) ctx.ctx.error('failed to loadDocs', { domain, err }) - return } - await ctx.sendResponse(ctx.requestId, {}) } - async clean (ctx: ClientSessionCtx, domain: Domain, docs: Ref[]): Promise { + async cleanRaw (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { + this.lastRequest = Date.now() if (!this.allowUpload) { await ctx.sendResponse(ctx.requestId, { error: 'Clean not allowed' }) + return + } + const workspace = ctx.workspaces.find((it) => it.wsId.uuid === workspaceId) + if (workspace === undefined) { + throw new Error('Workspace not found') } + await workspace.with(async (pipeline, communicationApi) => { + await this.getOps(pipeline).clean(ctx.ctx, domain, docs) + }) + } + + async clean (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]): Promise { this.lastRequest = Date.now() + if (!this.allowUpload) { + await ctx.sendResponse(ctx.requestId, { error: 'Clean not allowed' }) + return + } try { - await this.getOps(ctx.pipeline).clean(ctx.ctx, domain, docs) + await this.cleanRaw(ctx, workspaceId, domain, docs) + await ctx.sendResponse(ctx.requestId, {}) } catch (err: any) { await ctx.sendError(ctx.requestId, 'Failed to clean', unknownError(err)) ctx.ctx.error('failed to clean', { domain, err }) - return } - await ctx.sendResponse(ctx.requestId, {}) } async eventRaw (ctx: ClientSessionCtx, event: CommunicationEvent): Promise { this.lastRequest = Date.now() - return await ctx.communicationApi.event(this.getCommunicationCtx(), event) + const workspace = ctx.workspaces[0] + return await workspace.with(async (pipeline, communicationApi) => { + return await communicationApi.event(this.getCommunicationCtx(workspace.wsId), event) + }) } async event (ctx: ClientSessionCtx, event: CommunicationEvent): Promise { @@ -400,7 +560,10 @@ export class ClientSession implements Session { async findMessagesRaw (ctx: ClientSessionCtx, params: FindMessagesParams, queryId?: number): Promise { this.lastRequest = Date.now() - return await ctx.communicationApi.findMessages(this.getCommunicationCtx(), params, queryId) + const workspace = ctx.workspaces[0] + return await workspace.with(async (pipeline, communicationApi) => { + return await communicationApi.findMessages(this.getCommunicationCtx(workspace.wsId), params, queryId) + }) } async findMessages (ctx: ClientSessionCtx, params: FindMessagesParams, queryId?: number): Promise { @@ -410,7 +573,10 @@ export class ClientSession implements Session { async findMessagesGroupsRaw (ctx: ClientSessionCtx, params: FindMessagesGroupsParams): Promise { this.lastRequest = Date.now() - return await ctx.communicationApi.findMessagesGroups(this.getCommunicationCtx(), params) + const workspace = ctx.workspaces[0] + return await workspace.with(async (pipeline, communicationApi) => { + return await communicationApi.findMessagesGroups(this.getCommunicationCtx(workspace.wsId), params) + }) } async findMessagesGroups (ctx: ClientSessionCtx, params: FindMessagesGroupsParams): Promise { @@ -419,8 +585,11 @@ export class ClientSession implements Session { } async findNotifications (ctx: ClientSessionCtx, params: FindNotificationsParams): Promise { - const result = await ctx.communicationApi.findNotifications(this.getCommunicationCtx(), params) - await ctx.sendResponse(ctx.requestId, result) + const workspace = ctx.workspaces[0] + await workspace.with(async (pipeline, communicationApi) => { + const result = await communicationApi.findNotifications(this.getCommunicationCtx(workspace.wsId), params) + await ctx.sendResponse(ctx.requestId, result) + }) } async findNotificationContexts ( @@ -428,31 +597,51 @@ export class ClientSession implements Session { params: FindNotificationContextParams, queryId?: number ): Promise { - const result = await ctx.communicationApi.findNotificationContexts(this.getCommunicationCtx(), params, queryId) - await ctx.sendResponse(ctx.requestId, result) + const workspace = ctx.workspaces[0] + await workspace.with(async (pipeline, communicationApi) => { + const result = await communicationApi.findNotificationContexts( + this.getCommunicationCtx(workspace.wsId), + params, + queryId + ) + await ctx.sendResponse(ctx.requestId, result) + }) } async findLabels (ctx: ClientSessionCtx, params: FindLabelsParams): Promise { - const result = await ctx.communicationApi.findLabels(this.getCommunicationCtx(), params) - await ctx.sendResponse(ctx.requestId, result) + const workspace = ctx.workspaces[0] + await workspace.with(async (pipeline, communicationApi) => { + const result = await communicationApi.findLabels(this.getCommunicationCtx(workspace.wsId), params) + await ctx.sendResponse(ctx.requestId, result) + }) } async findCollaborators (ctx: ClientSessionCtx, params: FindCollaboratorsParams): Promise { - const result = await ctx.communicationApi.findCollaborators(this.getCommunicationCtx(), params) - await ctx.sendResponse(ctx.requestId, result) + const workspace = ctx.workspaces[0] + await workspace.with(async (pipeline, communicationApi) => { + const result = await communicationApi.findCollaborators(this.getCommunicationCtx(workspace.wsId), params) + await ctx.sendResponse(ctx.requestId, result) + }) } async unsubscribeQuery (ctx: ClientSessionCtx, id: number): Promise { this.lastRequest = Date.now() - await ctx.communicationApi.unsubscribeQuery(this.getCommunicationCtx(), id) - await ctx.sendResponse(ctx.requestId, {}) + const workspace = ctx.workspaces[0] + await workspace.with(async (pipeline, communicationApi) => { + await communicationApi.unsubscribeQuery(this.getCommunicationCtx(workspace.wsId), id) + await ctx.sendResponse(ctx.requestId, {}) + }) } - private getCommunicationCtx (): CommunicationSession { + private getCommunicationCtx (workspaceId: WorkspaceIds): CommunicationSession { return { sessionId: this.sessionId, - // TODO: We should decide what to do with communications package and remove this workaround - account: this.account as any + account: { + ...this.account, + // TODO: Fix me, Undetermined role is missing in communication API + role: this.account.workspaces[workspaceId.uuid].role ?? this.account.role, + fullSocialIds: Array.from(this.account.fullSocialIds.values()) // TODO: Fix me, fix types in communication API + } } } } diff --git a/server/server/src/endpoint.ts b/server/server/src/endpoint.ts new file mode 100644 index 00000000000..6d455d4657b --- /dev/null +++ b/server/server/src/endpoint.ts @@ -0,0 +1,350 @@ +import type { + SessionData as CommunicationSession, + EventResult, + RequestEvent, + ServerApi +} from '@hcengineering/communication-sdk-types' +import type { + Collaborator, + FindCollaboratorsParams, + FindLabelsParams, + FindMessagesGroupsParams, + FindMessagesParams, + FindNotificationContextParams, + FindNotificationsParams, + Label, + Message, + MessagesGroup, + Notification, + NotificationContext +} from '@hcengineering/communication-types' +import { + systemAccountUuid, + type AccountUuid, + type Branding, + type Class, + type ClientConnection, + type Doc, + type DocumentQuery, + type EndpointInfo, + type FindOptions, + type FindResult, + type Hierarchy, + type LoadModelResponse, + type MeasureContext, + type ModelDb, + type Ref, + type SearchOptions, + type SearchQuery, + type SearchResult, + type SessionData, + type Timestamp, + type Tx, + type TxResult, + type WorkspaceIds, + type WorkspaceUuid +} from '@hcengineering/core' +import { type EndpointConnectionFactory, type Pipeline } from '@hcengineering/server-core' +import type { Session, SessionManager } from './types' +import type { Workspace } from './workspace' + +interface EndpointWsInfo { + lastHash?: string + lastTx?: string +} + +interface CommunicationConnection extends ClientConnection { + findMessages: (params: FindMessagesParams, queryId?: number) => Promise + findMessagesGroups: (params: FindMessagesGroupsParams) => Promise + findNotificationContexts: (params: FindNotificationContextParams, queryId?: number) => Promise + findNotifications: (params: FindNotificationsParams, queryId?: number) => Promise + findLabels: (params: FindLabelsParams) => Promise + findCollaborators: (params: FindCollaboratorsParams) => Promise + sendEvent: (event: RequestEvent) => Promise + unsubscribeQuery: (id: number) => Promise +} + +export class EndpointClient { + connection: ClientConnection + + accounts: Set = new Set() + + workspaces: Map = new Map() + + constructor ( + readonly ctx: MeasureContext, + readonly region: string, + readonly endpoint: EndpointInfo, + readonly factory: EndpointConnectionFactory, + readonly sessionManager: SessionManager + ) { + this.connection = this.factory( + this.ctx, + this.region === this.endpoint.region ? this.endpoint.internalUrl : this.endpoint.externalUrl, + (txes: Tx[], workspace?: WorkspaceUuid, target?: AccountUuid, exclude?: AccountUuid[]) => { + if (workspace !== undefined) { + this.sessionManager.broadcast(workspace, txes, target, exclude) + } + }, + { + // TODO: Pass model filter, to ignore all model + onConnect: async (event, lastTx, data) => { + // We need to pass a list of accounts and workspaces we manage on every reconnect. + await this.doSubscribe() + }, + onHello (serverVersion) { + return true + } + } + ) + } + + private async doSubscribe (): Promise { + if (this.connection === undefined) { + // No need to subscribe, case for tests + return + } + const info = await this.connection.subscribe({ + accounts: Array.from(this.accounts), + workspaces: Array.from(this.workspaces.keys()) + }) + for (const ws of Array.from(this.workspaces.keys())) { + const wsinfo = this.workspaces.get(ws) + const rinfo = info[ws] + if (wsinfo !== undefined && rinfo !== undefined) { + wsinfo.lastHash = rinfo.lastHash + wsinfo.lastTx = rinfo.lastTx + } + } + } + + async addWorkspace (workspace: WorkspaceUuid): Promise { + const info = await this.connection.subscribe({ + workspaces: [workspace] + }) + this.workspaces.set(workspace, { + lastHash: info[workspace].lastHash, // Will be updated on connect + lastTx: info[workspace].lastTx + }) + } + + async removeWorkspace (workspace: WorkspaceUuid): Promise { + this.workspaces.delete(workspace) + await this.connection.unsubscribe({ + workspaces: [workspace] + }) + } + + async addUser (account: AccountUuid): Promise { + this.accounts.add(account) + await this.connection.subscribe({ + accounts: [account] + }) + } + + async removeUser (account: AccountUuid): Promise { + this.accounts.delete(account) + await this.connection.unsubscribe({ + accounts: [account] + }) + } + + getLastHash (workspace: WorkspaceUuid): string | undefined { + const info = this.workspaces.get(workspace) + if (info === undefined) { + return undefined + } + return info.lastHash + } + + getLastTx (workspace: WorkspaceUuid): string | undefined { + const info = this.workspaces.get(workspace) + if (info === undefined) { + return undefined + } + return info.lastTx + } +} + +export class EndpointWorkspace implements Workspace { + softShutdown: number + + sessions = new Map() + + operations: number = 0 + + maintenance: boolean = false + + lastTx: string | undefined // TODO: Do not cache for proxy case + lastHash: string | undefined // TODO: Do not cache for proxy case + + constructor ( + readonly context: MeasureContext, + + readonly endpoint: EndpointClient, + + readonly token: string, + + readonly tickHash: number, + + softShutdown: number, + + readonly wsId: WorkspaceIds, + readonly branding: Branding | null + ) { + this.softShutdown = softShutdown + } + + async addWorkspace (workspace: WorkspaceUuid): Promise { + await this.endpoint.addWorkspace(workspace) + } + + async addSession (session: Session): Promise { + this.sessions.set(session.sessionId, session) + await this.endpoint.addUser(session.getRawAccount().uuid) + } + + async removeSession (session: Session): Promise { + this.sessions.delete(session.sessionId) + await this.endpoint.addUser(session.getRawAccount().uuid) + } + + open (): void { + // Not required + } + + async close (ctx: MeasureContext): Promise { + // Not required + } + + checkHasUser (): boolean { + for (const val of this.sessions.values()) { + if (val.getUser() !== systemAccountUuid || val.subscribedUsers.size > 0) { + return true + } + } + return false + } + + getLastHash (): string | undefined { + return this.endpoint.getLastHash(this.wsId.uuid) + } + + getLastTx (): string | undefined { + return this.endpoint.getLastTx(this.wsId.uuid) + } + + async with(op: (pipeline: Pipeline, communicationApi: ServerApi) => Promise): Promise { + const endpoint = this.endpoint + const connection = endpoint.connection + if (connection === undefined) { + throw new Error('Endpoint not connected') + } + const wsPipeline: Pipeline = { + context: { + branding: null, + communicationApi: null, + modelDb: {} as any as ModelDb, + contextVars: {}, + hierarchy: {} as any as Hierarchy, + workspace: this.wsId + }, + findAll: async ( + ctx: MeasureContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> => { + return await connection.findAll(_class, query, { + ...options, + user: ctx.contextData.account.uuid, + workspace: this.wsId.uuid + }) + }, + searchFulltext: async ( + ctx: MeasureContext, + query: SearchQuery, + options: SearchOptions + ): Promise => { + return await connection.searchFulltext(query, { + ...options, + user: ctx.contextData.account.uuid, + workspace: this.wsId.uuid + }) + }, + tx: async (ctx: MeasureContext, tx: Tx[]): Promise => { + const result: TxResult[] = [] + // It should be one from endpoint operation. + for (const t of tx) { + result.push(await connection.tx(t, { user: ctx.contextData.account.uuid })) + } + return result[result.length - 1] + }, + close: async (): Promise => { + // Ignore + }, + loadModel: async ( + ctx: MeasureContext, + lastModelTx: Timestamp, + hash?: string + ): Promise => { + return await connection.loadModel(lastModelTx, hash, this.wsId.uuid) + }, + handleBroadcast: async (ctx: MeasureContext): Promise => { + // ignore + } + } + const communicationClient = connection as CommunicationConnection + const serverApi: ServerApi = { + findMessages: async ( + session: CommunicationSession, + params: FindMessagesParams, + queryId?: number + ): Promise => { + return await communicationClient.findMessages(params, queryId) + }, + findMessagesGroups: function ( + session: CommunicationSession, + params: FindMessagesGroupsParams + ): Promise { + throw new Error('Function not implemented.') + }, + findNotificationContexts: function ( + session: CommunicationSession, + params: FindNotificationContextParams, + queryId?: number | string + ): Promise { + throw new Error('Function not implemented.') + }, + findNotifications: function ( + session: CommunicationSession, + params: FindNotificationsParams, + queryId?: number | string + ): Promise { + throw new Error('Function not implemented.') + }, + findLabels: function (session: CommunicationSession, params: FindLabelsParams): Promise { + throw new Error('Function not implemented.') + }, + findCollaborators: function ( + session: CommunicationSession, + params: FindCollaboratorsParams + ): Promise { + throw new Error('Function not implemented.') + }, + event: function (session: CommunicationSession, event: RequestEvent): Promise { + throw new Error('Function not implemented.') + }, + unsubscribeQuery: function (session: CommunicationSession, id: number): Promise { + throw new Error('Function not implemented.') + }, + closeSession: function (sessionId: string): Promise { + throw new Error('Function not implemented.') + }, + close: function (): Promise { + throw new Error('Function not implemented.') + } + } + return await op(wsPipeline, serverApi) + } +} diff --git a/server/server/src/index.ts b/server/server/src/index.ts index 2fe0c8e1473..8c453a7c7c7 100644 --- a/server/server/src/index.ts +++ b/server/server/src/index.ts @@ -20,3 +20,4 @@ export * from './sessionManager' export * from './starter' export * from './stats' export * from './utils' +export * from './types' diff --git a/server/server/src/lookup.ts b/server/server/src/lookup.ts new file mode 100644 index 00000000000..fa9fb03d9ce --- /dev/null +++ b/server/server/src/lookup.ts @@ -0,0 +1,91 @@ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type Doc, type DocumentQuery, type FindOptions, type FindResult, clone, toFindResult } from '@hcengineering/core' +/** + * @public + */ +export function mapLookup ( + query: DocumentQuery, + result: FindResult, + options?: FindOptions +): FindResult { + // Fill lookup map to make more compact representation + + if (options?.lookup !== undefined) { + const newResult: T[] = [] + let counter = 0 + const idClassMap: Record = {} + + function mapDoc (doc: Doc): number { + const key = doc._class + '@' + doc._id + let docRef = idClassMap[key] + if (docRef === undefined) { + docRef = { id: ++counter, doc, count: -1 } + idClassMap[key] = docRef + } + docRef.count++ + return docRef.id + } + + for (const d of result) { + const newDoc: any = { ...d } + if (d.$lookup !== undefined) { + newDoc.$lookup = clone(d.$lookup) + newResult.push(newDoc) + for (const [k, v] of Object.entries(d.$lookup)) { + if (!Array.isArray(v)) { + newDoc.$lookup[k] = v != null ? mapDoc(v) : v + } else { + newDoc.$lookup[k] = v.map((it) => (it != null ? mapDoc(it) : it)) + } + } + } else { + newResult.push(newDoc) + } + } + const lookupMap = Object.fromEntries(Array.from(Object.values(idClassMap)).map((it) => [it.id, it.doc])) + return cleanQuery(toFindResult(newResult, result.total, lookupMap), query, lookupMap) + } + + // We need to get rid of simple query parameters matched in documents + return cleanQuery(result, query) +} + +function cleanQuery ( + result: FindResult, + query: DocumentQuery, + lookupMap?: Record +): FindResult { + const newResult: T[] = [] + for (const doc of result) { + let _doc = doc + let cloned = false + for (const [k, v] of Object.entries(query)) { + if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') { + if ((_doc as any)[k] === v) { + if (!cloned) { + _doc = { ...doc } as any + cloned = true + } + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete (_doc as any)[k] + } + } + } + newResult.push(_doc) + } + return toFindResult(newResult, result.total, lookupMap) +} diff --git a/server/server/src/sessionManager.ts b/server/server/src/sessionManager.ts index 32b9cf4b7b6..90237e8a64c 100644 --- a/server/server/src/sessionManager.ts +++ b/server/server/src/sessionManager.ts @@ -19,8 +19,9 @@ import { type LoginInfoWorkspace } from '@hcengineering/account-client' import { Analytics } from '@hcengineering/analytics' -import { type ServerApi as CommunicationApi } from '@hcengineering/communication-sdk-types' import core, { + type EndpointInfo, + type Account, AccountRole, type AccountUuid, type Branding, @@ -38,9 +39,11 @@ import core, { platformNow, platformNowDiff, SocialIdType, + type SubscribedWorkspaceInfo, systemAccountUuid, type Tx, TxFactory, + type TxHandler, type TxWorkspaceEvent, type Version, versionToString, @@ -49,9 +52,10 @@ import core, { WorkspaceEvent, type WorkspaceIds, type WorkspaceInfoWithStatus, + type WorkspaceMode, type WorkspaceUuid } from '@hcengineering/core' -import { type Status, unknownError } from '@hcengineering/platform' +import platform, { Severity, Status, unknownError } from '@hcengineering/platform' import { type HelloRequest, type HelloResponse, @@ -61,31 +65,29 @@ import { SlidingWindowRateLimitter } from '@hcengineering/rpc' import { - type AddSessionResponse, - type ClientSessionCtx, type CommunicationApiFactory, type ConnectionSocket, - type GetWorkspaceResponse, + type EndpointConnectionFactory, LOGGING_ENABLED, pingConst, - type Pipeline, type PipelineFactory, type PlatformQueue, type PlatformQueueProducer, QueueTopic, type QueueUserMessage, type QueueWorkspaceMessage, - type Session, - type SessionManager, userEvents, type UserStatistics, workspaceEvents, type WorkspaceStatistics } from '@hcengineering/server-core' import { generateToken, type Token } from '@hcengineering/server-token' -import { type PipelinePair, Workspace } from './workspace' +import { type ClientSessionCtx, type Session, type SessionInfoRecord, type SessionManager } from './types' +import { type PipelinePair, type Workspace, WorkspaceImpl, type WorkspacePipelineFactory } from './workspace' + import { ClientSession } from './client' -import { sendResponse } from './utils' +import { EndpointClient, EndpointWorkspace } from './endpoint' +import { getLastHashInfo, sendResponse } from './utils' const ticksPerSecond = 20 const workspaceSoftShutdownTicks = 15 * ticksPerSecond @@ -101,12 +103,24 @@ export interface Timeouts { reconnectTimeout: number // Default 3 seconds } +interface TickHandler { + ticks: number + operation: () => Promise +} + export class TSessionManager implements SessionManager { private readonly statusPromises = new Map>() readonly workspaces = new Map() checkInterval: any - sessions = new Map() + sessions = new Map() + + accountIdToSessions = new Map() + + accounts = new Map() + socialStringsToUsers = new Map() + + tickHandlers = new Map() reconnectIds = new Set() maintenanceTimer: any @@ -118,9 +132,10 @@ export class TSessionManager implements SessionManager { workspaceProducer: PlatformQueueProducer usersProducer: PlatformQueueProducer + broadcastHandlers: TxHandler[] = [] + now: number = Date.now() - ticksContext: MeasureContext constructor ( readonly ctx: MeasureContext, readonly timeouts: Timeouts, @@ -133,20 +148,29 @@ export class TSessionManager implements SessionManager { | undefined, readonly accountsUrl: string, readonly enableCompression: boolean, - readonly doHandleTick: boolean = true, readonly queue: PlatformQueue, readonly pipelineFactory: PipelineFactory, - readonly communicationApiFactory: CommunicationApiFactory + readonly endpointFactory: EndpointConnectionFactory, + readonly communicationApiFactory: CommunicationApiFactory, + readonly region: string, + readonly endpointName: string, + readonly _handleTick: boolean ) { - if (this.doHandleTick) { + if (this._handleTick) { this.checkInterval = setInterval(() => { this.handleTick() }, 1000 / ticksPerSecond) } this.workspaceProducer = this.queue.getProducer(ctx.newChild('queue', {}), QueueTopic.Workspace) this.usersProducer = this.queue.getProducer(ctx.newChild('queue', {}), QueueTopic.Users) + } + + addBroadcastHandler (handler: TxHandler): void { + this.broadcastHandlers.push(handler) + } - this.ticksContext = ctx.newChild('ticks', {}) + removeBroadcastHandler (handler: TxHandler): void { + this.broadcastHandlers = this.broadcastHandlers.filter((it) => it !== handler) } scheduleMaintenance (timeMinutes: number): void { @@ -176,21 +200,22 @@ export class TSessionManager implements SessionManager { if (this.timeMinutes === 0) { return } - const event: TxWorkspaceEvent = this.createMaintenanceWarning() for (const ws of this.workspaces.values()) { + const event: TxWorkspaceEvent = this.createMaintenanceWarning(ws.wsId.uuid) this.doBroadcast(ws, [event]) } } - private createMaintenanceWarning (): TxWorkspaceEvent { + private createMaintenanceWarning (workspace: WorkspaceUuid): TxWorkspaceEvent { return { _id: generateId(), _class: core.class.TxWorkspaceEvent, + _uuid: workspace, event: WorkspaceEvent.MaintenanceNotification, + space: core.space.Workspace, + objectSpace: core.space.DerivedTx, modifiedBy: core.account.System, modifiedOn: Date.now(), - objectSpace: core.space.DerivedTx, - space: core.space.DerivedTx, createdBy: core.account.System, params: { timeMinutes: this.timeMinutes @@ -198,6 +223,21 @@ export class TSessionManager implements SessionManager { } } + private createWorkspaceEvent(workspace: WorkspaceUuid, event: WorkspaceEvent, params?: T): TxWorkspaceEvent { + return { + _uuid: workspace, + _id: generateId(), + _class: core.class.TxWorkspaceEvent, + space: core.space.Workspace, + objectSpace: core.space.DerivedTx, + event, + modifiedBy: core.account.System, + modifiedOn: Date.now(), + createdBy: core.account.System, + params + } + } + ticks = 0 handleTick (): void { @@ -210,60 +250,80 @@ export class TSessionManager implements SessionManager { private handleWorkspaceTick (): void { for (const [wsId, workspace] of this.workspaces.entries()) { - if (this.ticks % (60 * ticksPerSecond) === workspace.tickHash) { + if (this.ticks % ((workspace.maintenance ? 5 : 60) * ticksPerSecond) === workspace.tickHash) { try { - // update account lastVisit every minute per every workspace.∏ - let connected: boolean = false - for (const val of workspace.sessions.values()) { - if (val.session.getUser() !== systemAccountUuid) { - connected = true - break - } - } - void this.getWorkspaceInfo(this.ticksContext, workspace.token, connected).catch(() => { - // Ignore - }) + // TODO: Rework on Queue update listen??? + void this.getWorkspaceInfo(workspace.token, workspace.checkHasUser()) + .then((info) => { + if (info !== undefined) { + const wsVersion: Data = { + major: info.versionMajor, + minor: info.versionMinor, + patch: info.versionPatch + } + + const maintenance = this.getMaintenance(info.mode, versionToString(wsVersion)) + if (workspace.maintenance && !maintenance) { + workspace.maintenance = false + this.ctx.warn('Workspace is back to normal', { + workspace: workspace.wsId.url, + wsId: workspace.wsId.uuid + }) + // Need to inform clients about workspace back to normal + this.broadcastAll(workspace.wsId.uuid, [ + this.createWorkspaceEvent(workspace.wsId.uuid, WorkspaceEvent.WorkpaceActive) + ]) + } + } + }) + .catch(() => { + // Ignore + }) } catch (err: any) { // Ignore } } - for (const [k, v] of Array.from(workspace.tickHandlers.entries())) { - v.ticks-- - if (v.ticks === 0) { - workspace.tickHandlers.delete(k) - try { - v.operation() - } catch (err: any) { - Analytics.handleError(err) - } - } - } - for (const s of workspace.sessions) { if (this.ticks % (5 * 60 * ticksPerSecond) === workspace.tickHash) { - s[1].session.mins5.find = s[1].session.current.find - s[1].session.mins5.tx = s[1].session.current.tx + s[1].mins5.find = s[1].current.find + s[1].mins5.tx = s[1].current.tx - s[1].session.current = { find: 0, tx: 0 } + s[1].current = { find: 0, tx: 0 } } } // Wait some time for new client to appear before closing workspace. - if (workspace.sessions.size === 0 && workspace.closing === undefined && workspace.workspaceInitCompleted) { + if (workspace.sessions.size === 0) { workspace.softShutdown-- if (workspace.softShutdown <= 0) { this.ctx.warn('closing workspace, no users', { workspace: workspace.wsId.url, - wsId, - upgrade: workspace.upgrade + wsId + }) + void this.performWorkspaceCloseCheck(workspace).catch((err) => { + console.error({ message: 'Failed to perform workspace close check', err }) }) - workspace.closing = this.performWorkspaceCloseCheck(workspace) } } else { workspace.softShutdown = workspaceSoftShutdownTicks } } + + for (const [k, v] of Array.from(this.tickHandlers.entries())) { + v.ticks-- + if (v.ticks === 0) { + this.tickHandlers.delete(k) + try { + void v.operation().catch((err) => { + Analytics.handleError(err) + this.ctx.error('error during tick handler', { error: err }) + }) + } catch (err: any) { + Analytics.handleError(err) + } + } + } } private handleSessionTick (now: number): void { @@ -271,7 +331,6 @@ export class TSessionManager implements SessionManager { const isCurrentUserTick = this.ticks % ticksPerSecond === s.tickHash if (isCurrentUserTick) { - const wsId = s.session.workspace.uuid const lastRequestDiff = now - s.session.lastRequest let timeout = 60000 @@ -279,10 +338,10 @@ export class TSessionManager implements SessionManager { timeout = timeout * 10 } if (lastRequestDiff > timeout) { - this.ctx.warn('session hang, closing...', { wsId, user: s.session.getUser() }) + this.ctx.warn('session hang, closing...', { user: s.session.getUser(), sessionId: s.session.sessionId }) // Force close workspace if only one client and it hang. - void this.close(this.ticksContext, s.socket, wsId).catch((err) => { + void this.close(this.ctx, s.session).catch((err) => { this.ctx.error('failed to close', err) }) continue @@ -295,7 +354,7 @@ export class TSessionManager implements SessionManager { // And ping other wize s.session.lastPing = now if (s.socket.checkState()) { - void s.socket.send(this.ticksContext, { result: pingConst }, s.session.binaryMode, s.session.useCompression) + void s.socket.send(this.ctx, { result: pingConst }, s.session.binaryMode, s.session.useCompression) } } for (const r of s.session.requests.values()) { @@ -303,7 +362,7 @@ export class TSessionManager implements SessionManager { if (sec > 0 && sec % 30 === 0) { this.ctx.warn('request hang found', { sec, - wsId, + wsId: r.workspaceId, total: s.session.requests.size, user: s.session.getUser(), ...cutObjectArray(r.params) @@ -314,43 +373,7 @@ export class TSessionManager implements SessionManager { } } - createSession (token: Token, workspace: WorkspaceIds, info: LoginInfoWithWorkspaces): Session { - let primarySocialId: PersonId - let role: AccountRole = info.workspaces[workspace.uuid]?.role ?? AccountRole.User - switch (info.account) { - case systemAccountUuid: - primarySocialId = core.account.System - role = AccountRole.Owner - break - case guestAccount: - primarySocialId = '' as PersonId - role = AccountRole.DocGuest - break - default: - primarySocialId = pickPrimarySocialId(info.socialIds)._id - } - - return new ClientSession( - token, - workspace, - { - uuid: info.account, - socialIds: info.socialIds.map((it) => it._id), - primarySocialId, - fullSocialIds: info.socialIds, - role - }, - info, - token.extra?.mode === 'backup' - ) - } - - @withContext('🧭 get-workspace-info') - async getWorkspaceInfo ( - ctx: MeasureContext, - token: string, - updateLastVisit = true - ): Promise { + async getWorkspaceInfo (token: string, updateLastVisit = true): Promise { try { return await getAccountClient(this.accountsUrl, token).getWorkspaceInfo(updateLastVisit) } catch (err: any) { @@ -361,8 +384,7 @@ export class TSessionManager implements SessionManager { } } - @withContext('🧭 get-login-with-workspace-info') - async getLoginWithWorkspaceInfo (ctx: MeasureContext, token: string): Promise { + async getLoginWithWorkspaceInfo (token: string): Promise { try { const accountClient = getAccountClient(this.accountsUrl, token) return await accountClient.getLoginWithWorkspaceInfo() @@ -376,234 +398,297 @@ export class TSessionManager implements SessionManager { countUserSessions (workspace: Workspace, accountUuid: AccountUuid): number { return Array.from(workspace.sessions.values()) - .filter((it) => it.session.getUser() === accountUuid) + .filter((it) => it.getUser() === accountUuid) .reduce((acc) => acc + 1, 0) } tickCounter = 0 - @withContext('🧭 get-workspace') - async getWorkspace ( + getMaintenance (mode: WorkspaceMode, version: string): boolean { + return ( + isWorkspaceCreating(mode) || + isMigrationMode(mode) || + isRestoringMode(mode) || + (this.modelVersion !== '' && this.modelVersion !== version) + ) + } + + getWorkspace ( ctx: MeasureContext, workspaceUuid: WorkspaceUuid, - workspaceInfo: LoginInfoWorkspace | undefined, - token: Token, - ws: ConnectionSocket - ): Promise<{ workspace?: Workspace, resp?: GetWorkspaceResponse }> { - if (workspaceInfo === undefined) { - return { resp: { error: new Error('Workspace not found or not available'), terminate: true } } - } - + workspaceInfo: LoginInfoWorkspace + ): { workspace?: Workspace, open?: Promise } { if (isArchivingMode(workspaceInfo.mode)) { // No access to disabled workspaces for regular users - return { resp: { error: new Error('Workspace is archived'), terminate: true, specialError: 'archived' } } - } - if (isMigrationMode(workspaceInfo.mode)) { - // No access to disabled workspaces for regular users - return { - resp: { error: new Error('Workspace is in region migration'), terminate: true, specialError: 'migration' } - } - } - if (isRestoringMode(workspaceInfo.mode)) { - // No access to disabled workspaces for regular users - return { - resp: { error: new Error('Workspace is in backup restore'), terminate: true, specialError: 'migration' } - } - } - - if (isWorkspaceCreating(workspaceInfo.mode)) { - // No access to workspace for token. - return { resp: { error: new Error(`Workspace during creation phase...${workspaceUuid}`) } } + return {} } - const wsVersion: Data = { major: workspaceInfo.version.versionMajor, minor: workspaceInfo.version.versionMinor, patch: workspaceInfo.version.versionPatch } - if ( - this.modelVersion !== '' && - this.modelVersion !== versionToString(wsVersion) && - token.extra?.model !== 'upgrade' && - token.extra?.mode !== 'backup' - ) { - ctx.warn('Model version mismatch', { - version: this.modelVersion, - workspaceVersion: versionToString(wsVersion), - workspace: workspaceUuid - }) - // Version mismatch, return upgrading. - return { resp: { upgrade: true, progress: workspaceInfo.mode === 'upgrading' ? workspaceInfo.progress ?? 0 : 0 } } - } + const maintenance = this.getMaintenance(workspaceInfo.mode, versionToString(wsVersion)) let workspace = this.workspaces.get(workspaceUuid) - if (workspace?.closing !== undefined) { - await workspace?.closing - } - - workspace = this.workspaces.get(workspaceUuid) + let open: Promise | undefined const branding = null if (workspace === undefined) { ctx.warn('open workspace', { - account: token.account, - workspace: workspaceUuid, - ...token.extra + url: workspaceInfo.url, + workspace: workspaceUuid + }) + ;({ workspace, open } = this.createWorkspace( + ctx.parent ?? ctx, + ctx, + workspaceUuid, + workspaceInfo.endpoint, + workspaceInfo.url, + workspaceInfo.dataId, + branding + )) + + void this.workspaceProducer.send(workspaceUuid, [workspaceEvents.open()]).catch((err) => { + ctx.warn('failed to send workspace open event', { err }) + }) + } + workspace.maintenance = maintenance + if (maintenance) { + ctx.warn('Workspace in maintenance', { + version: this.modelVersion, + workspaceVersion: versionToString(wsVersion), + workspace: workspaceUuid }) + } + return { workspace, open } + } - workspace = this.createWorkspace(ctx.parent ?? ctx, ctx, token, workspaceInfo.url, workspaceInfo.dataId, branding) - await this.workspaceProducer.send(workspaceUuid, [workspaceEvents.open()]) + async registerAccountByUuid (account: AccountUuid, session: Session): Promise { + const acc = this.accounts.get(account) + if (acc !== undefined) { + this.registerAccount(account, acc, session) + return } - if (token.extra?.model === 'upgrade') { - if (workspace.upgrade) { - ctx.warn('reconnect workspace in upgrade', { - account: token.account, - workspace: workspaceUuid, - wsUrl: workspaceInfo.url - }) - } else { - ctx.warn('reconnect workspace in upgrade switch', { - email: token.account, - workspace: workspaceUuid, - wsUrl: workspaceInfo.url - }) + const loginInfo = await this.getLoginWithWorkspaceInfo(generateToken(account, '' as WorkspaceUuid)) + if (loginInfo !== undefined) { + this.registerAccount(account, createAccountFromInfo(loginInfo), session) + } + } - // We need to wait in case previous upgrade connection is already closing. - await this.switchToUpgradeSession(token, ctx.parent ?? ctx, workspace, ws) + registerAccount (account: AccountUuid, loginInfo: Account, session: Session): void { + this.accounts.set(account, loginInfo) + const existingSessions = this.accountIdToSessions.get(account) ?? [] + if (existingSessions.length === 0) { + for (const sid of loginInfo.fullSocialIds.values()) { + this.socialStringsToUsers.set(sid._id, account) } - } else { - if (workspace.upgrade) { - ctx.warn('connect during upgrade', { - account: token.account, - workspace: workspace.wsId.url, - sessionUsers: Array.from(workspace.sessions.values()).map((it) => it.session.getUser()), - sessionData: Array.from(workspace.sessions.values()).map((it) => it.socket.data()) - }) + } + this.accountIdToSessions.set(account, existingSessions.concat(session)) + } + + unregisterAccount (account: AccountUuid, sessionRef: Session): void { + const filteredSessions = (this.accountIdToSessions.get(account) ?? []).filter( + (it) => it.sessionId !== sessionRef.sessionId + ) + + if (filteredSessions.length === 0) { + // In case no sessions left, remove account from map + this.accountIdToSessions.delete(account) + this.accounts.delete(account) - return { resp: { upgrade: true } } + // Also remove sessionId mapping + for (const sid of sessionRef.getUserSocialIds()) { + this.socialStringsToUsers.delete(sid) } + } else { + this.accountIdToSessions.set(account, filteredSessions) } - return { workspace } } + @withContext('📲 add-session') async addSession ( ctx: MeasureContext, ws: ConnectionSocket, token: Token, rawToken: string, sessionId: string | undefined - ): Promise { - return await ctx.with('📲 add-session', { source: token.extra?.service ?? '🤦‍♂️user' }, async (ctx) => { - let account: LoginInfoWithWorkspaces | undefined + ): Promise { + let account: LoginInfoWithWorkspaces | undefined - try { - account = await this.getLoginWithWorkspaceInfo(ctx, rawToken) - } catch (err: any) { - return { error: err } - } + try { + account = (token.account === systemAccountUuid) + ? { + account: token.account, + name: 'System', + personalWorkspace: core.workspace.Any, + workspaces: {}, + socialIds: [] + } + : await this.getLoginWithWorkspaceInfo(rawToken) + } catch (err: any) { + ctx.error('failed to get login info', { err }) + } - if (account === undefined) { - return { error: new Error('Account not found or not available'), terminate: true } - } + if (account === undefined) { + void ws.send( + ctx, + { + id: -1, + error: new Status(Severity.ERROR, platform.status.AccountNotFound, { + account: token.account + }), + terminate: true + }, + false, + false + ) + throw new Error('Account not found or not available') + } - let wsInfo = account.workspaces[token.workspace] + let targetInfo = account.workspaces[token.workspace] + let targetWorkspace: WorkspaceUuid | undefined + if (targetInfo !== undefined) { + targetWorkspace = token.workspace + } - if (wsInfo === undefined) { - // In case of guest or system account - // We need to get workspace info for system account. - const workspaceInfo = await this.getWorkspaceInfo(ctx, rawToken, false) - if (workspaceInfo === undefined) { - return { error: new Error('Workspace not found or not available'), terminate: true } - } - wsInfo = { - url: workspaceInfo.url, - mode: workspaceInfo.mode, - dataId: workspaceInfo.dataId, - version: { - versionMajor: workspaceInfo.versionMajor, - versionMinor: workspaceInfo.versionMinor, - versionPatch: workspaceInfo.versionPatch - }, - role: AccountRole.Owner, - endpoint: { externalUrl: '', internalUrl: '', region: workspaceInfo.region ?? '' }, - progress: workspaceInfo.processingProgress - } - } - const { workspace, resp } = await this.getWorkspace(ctx.parent ?? ctx, token.workspace, wsInfo, token, ws) - if (resp !== undefined) { - return resp + if (targetInfo === undefined && token.workspace != null && token.workspace !== '') { + // In case of guest or system account + // We need to get workspace info for system account. + const workspaceInfo = await this.getWorkspaceInfo(rawToken, false) + if (workspaceInfo === undefined) { + ctx.warn('Workspace not found or not available', { token }) + } else { + targetInfo = this.toLoginInfoWorkspace(workspaceInfo) + account.workspaces[token.workspace] = targetInfo } + } - if (workspace === undefined || account === undefined) { - // Should not happen - return { error: new Error('Workspace not found or not available'), terminate: true } - } + const oldSession = sessionId !== undefined ? this.sessions.get(sessionId) : undefined + if (oldSession !== undefined) { + // Just close old socket for old session id. + await this.close(ctx, oldSession.session) + } - const oldSession = sessionId !== undefined ? workspace.sessions?.get(sessionId) : undefined - if (oldSession !== undefined) { - // Just close old socket for old session id. - await this.close(ctx, oldSession.socket, workspace.wsId.uuid) - } + // Create sesson for all workspaces, or for selected workspace only + const accountRef: Account = createAccountFromInfo(account, token.workspace) + + const session = new ClientSession( + (ctx.parent ?? ctx).newChild('🧲 session', {}), + token, + ws, + targetInfo !== undefined && targetWorkspace !== undefined + ? new Set([targetWorkspace]) + : new Set(Object.keys(account.workspaces) as WorkspaceUuid[]), + accountRef, + account, + token.extra?.mode === 'backup' + ) - const session = this.createSession(token, workspace.wsId, account) + session.sessionId = sessionId !== undefined && (sessionId ?? '').trim().length > 0 ? sessionId : generateId() - session.sessionId = sessionId !== undefined && (sessionId ?? '').trim().length > 0 ? sessionId : generateId() - session.sessionInstanceId = generateId() - const tickHash = this.tickCounter % ticksPerSecond + const tickHash = this.tickCounter % ticksPerSecond - this.sessions.set(ws.id, { session, socket: ws, tickHash }) - // We need to delete previous session with Id if found. - this.tickCounter++ - workspace.sessions.set(session.sessionId, { session, socket: ws, tickHash }) + this.sessions.set(session.sessionId, { session, socket: ws, tickHash }) - const accountUuid = account.account - if (accountUuid !== systemAccountUuid && accountUuid !== guestAccount) { - await this.usersProducer.send(workspace.wsId.uuid, [ - userEvents.login({ - user: accountUuid, - sessions: this.countUserSessions(workspace, accountUuid), - socialIds: account.socialIds.map((it) => it._id) - }) - ]) - } + this.registerAccount(session.getUser(), accountRef, session) - // Mark workspace as init completed and we had at least one client. - if (!workspace.workspaceInitCompleted) { - workspace.workspaceInitCompleted = true - } + this.tickCounter++ - if (this.timeMinutes > 0) { - void ws - .send(ctx, { result: this.createMaintenanceWarning() }, session.binaryMode, session.useCompression) - .catch((err) => { - ctx.error('failed to send maintenance warning', err) + const accountUuid = account.account + + for (const workspaceRef of session.workspaces) { + const info = account.workspaces[workspaceRef] + if (info === undefined) { + continue + } + const { workspace, open } = this.getWorkspace(ctx, workspaceRef, info) + if (workspace !== undefined) { + if (workspace.maintenance && token.account !== systemAccountUuid) { + // We need to trigger workspace to be updated, to workspace service to perform upgrade. + // TODO: Rework on request in QUEUE + void this.getWorkspaceInfo(workspaceRef, true).catch(() => { + // Ignore }) + } + await workspace.addSession(session) + if (open !== undefined) { + await open + } + + if (accountUuid !== systemAccountUuid && accountUuid !== guestAccount) { + await this.usersProducer.send(workspace.wsId.uuid, [ + userEvents.login({ + user: accountUuid, + sessions: this.countUserSessions(workspace, accountUuid), + socialIds: account.socialIds.map((it) => it._id) + }) + ]) + } } - return { session, context: workspace.context, workspaceId: workspace.wsId.uuid } - }) + } + if (this.timeMinutes > 0) { + void ws + .send( + ctx, + { result: this.createMaintenanceWarning(Object.keys(account.workspaces)[0] as WorkspaceUuid) }, + session.binaryMode, + session.useCompression + ) + .catch((err) => { + ctx.error('failed to send maintenance warning', err) + }) + } + return session } - private async switchToUpgradeSession ( - token: Token, - ctx: MeasureContext, - workspace: Workspace, - ws: ConnectionSocket - ): Promise { - if (LOGGING_ENABLED) { - ctx.info('reloading workspace', { url: workspace.wsId.url, token: JSON.stringify(token) }) + private toLoginInfoWorkspace (workspaceInfo: WorkspaceInfoWithStatus): LoginInfoWorkspace { + return { + url: workspaceInfo.url, + mode: workspaceInfo.mode, + name: workspaceInfo.name, + dataId: workspaceInfo.dataId, + version: { + versionMajor: workspaceInfo.versionMajor, + versionMinor: workspaceInfo.versionMinor, + versionPatch: workspaceInfo.versionPatch + }, + endpoint: workspaceInfo.endpoint, + role: AccountRole.Owner, + progress: workspaceInfo.processingProgress + } + } + + async getLastTxHash ( + workspaceId: WorkspaceUuid + ): Promise<{ lastTx: string | undefined, lastHash: string | undefined }> { + const ws = this.workspaces.get(workspaceId) + if (ws !== undefined) { + return { lastHash: ws.getLastHash(), lastTx: ws.getLastTx() } + } + + const info = await this.getWorkspaceInfo(workspaceId, false) + if (info === undefined) { + return { lastHash: undefined, lastTx: undefined } } + const { workspace } = this.getWorkspace(this.ctx, workspaceId, this.toLoginInfoWorkspace(info)) - // Mark as upgrade, to prevent any new clients to connect during close - workspace.upgrade = true - // If upgrade client is used. - // Drop all existing clients - await this.doCloseAll(workspace, 0, 'upgrade', ws) + if (workspace === undefined) { + return { lastHash: undefined, lastTx: undefined } + } + return await workspace.with(async (pipeline) => { + return { lastTx: pipeline.context.lastTx, lastHash: pipeline.context.lastHash } + }) } - broadcastAll (workspace: WorkspaceUuid, tx: Tx[], target?: string | string[], exclude?: string[]): void { + broadcastAll ( + workspace: WorkspaceUuid, + tx: Tx[], + target?: AccountUuid | AccountUuid[], + exclude?: AccountUuid[] + ): void { const ws = this.workspaces.get(workspace) if (ws === undefined) { return @@ -611,10 +696,7 @@ export class TSessionManager implements SessionManager { this.doBroadcast(ws, tx, target, exclude) } - doBroadcast (ws: Workspace, tx: Tx[], target?: string | string[], exclude?: string[]): void { - if (ws.upgrade) { - return - } + doBroadcast (ws: Workspace, tx: Tx[], target?: AccountUuid | AccountUuid[], exclude?: AccountUuid[]): void { if (target !== undefined && !Array.isArray(target)) { target = [target] } @@ -623,13 +705,13 @@ export class TSessionManager implements SessionManager { if (it === undefined) { return false } - const tt = it.session.getUser() + const tt = it.getUser() return (target === undefined && !(exclude ?? []).includes(tt)) || (target?.includes(tt) ?? false) }) function send (): void { for (const session of sessions) { try { - void sendResponse(ctx, session.session, session.socket, { result: tx }).catch((err) => { + void sendResponse(ctx, session, session.socket, { result: tx }).catch((err) => { ctx.error('failed to send', err) }) } catch (err: any) { @@ -648,21 +730,18 @@ export class TSessionManager implements SessionManager { } broadcastSessions (measure: MeasureContext, workspace: Workspace, sessionIds: string[], result: any): void { - if (workspace.upgrade) { - return - } const ctx = measure.newChild('📬 broadcast sessions', {}) const sessions = [...workspace.sessions.values()].filter((it) => { - if (it === undefined || it.session.sessionId === '') { + if (it === undefined || it.sessionId === '') { return false } - return sessionIds.includes(it.session.sessionId) + return sessionIds.includes(it.sessionId) }) function send (): void { for (const session of sessions) { try { - void sendResponse(ctx, session.session, session.socket, { result }) + void sendResponse(ctx, session, session.socket, { result }) } catch (err: any) { Analytics.handleError(err) ctx.error('error during send', { error: err }) @@ -678,36 +757,38 @@ export class TSessionManager implements SessionManager { } } - broadcast ( - from: Session | null, - workspaceId: WorkspaceUuid, - resp: Tx[], - target: string | undefined, - exclude?: string[] - ): void { + broadcast (workspaceId: WorkspaceUuid, resp: Tx[], target: AccountUuid | undefined, exclude?: AccountUuid[]): void { + if (this.broadcastHandlers.length > 0) { + for (const handler of this.broadcastHandlers) { + handler(resp, workspaceId, target, exclude) + } + } const workspace = this.workspaces.get(workspaceId) if (workspace === undefined) { this.ctx.error('internal: cannot find sessions', { workspaceId, - target, - userId: from?.getUser() ?? '$unknown' + target }) return } - if (workspace?.upgrade ?? false) { - return - } const sessions = [...workspace.sessions.values()] const ctx = this.ctx.newChild('📭 broadcast', {}) const send = (): void => { for (const sessionRef of sessions) { - const tt = sessionRef.session.getUser() - if ((target === undefined && !(exclude ?? []).includes(tt)) || (target?.includes(tt) ?? false)) { - sessionRef.session.broadcast(ctx, sessionRef.socket, resp) + if (sessionRef.subscribedUsers.size > 0) { + if (target == null || sessionRef.subscribedUsers.has(target)) { + // Will be handled by session on endpoint. + sessionRef.broadcast(ctx, sessionRef.socket, resp, target, exclude) + } + } else { + const tt = sessionRef.getUser() + if ((target === undefined && !(exclude ?? []).includes(tt)) || (target?.includes(tt) ?? false)) { + sessionRef.broadcast(ctx, sessionRef.socket, resp) + } } + ctx.end() } - ctx.end() } if (sessions.length > 0) { // We need to send broadcast after our client response so put it after all IO @@ -717,69 +798,94 @@ export class TSessionManager implements SessionManager { } } + // A map of endpoint external url to endpoint client. + endpointClients = new Map() + private createWorkspace ( ctx: MeasureContext, pipelineCtx: MeasureContext, - token: Token, + uuid: WorkspaceUuid, + endpoint: EndpointInfo, workspaceUrl: string, workspaceDataId: WorkspaceDataId | undefined, branding: Branding | null - ): Workspace { - const upgrade = token.extra?.model === 'upgrade' + ): { workspace: Workspace, open?: Promise } { const context = ctx.newChild('🧲 session', {}) const workspaceIds: WorkspaceIds = { - uuid: token.workspace, + uuid, dataId: workspaceDataId, url: workspaceUrl } - const factory = async (): Promise => { - const communicationApi = await this.communicationApiFactory( - pipelineCtx, + let factory: WorkspacePipelineFactory + + let open: Promise | undefined + + if (this.endpointName !== endpoint.internalUrl || this.endpointName !== endpoint.externalUrl) { + const wsToken = generateToken(systemAccountUuid, uuid) + const epClient = + this.endpointClients.get(endpoint.externalUrl) ?? + new EndpointClient(ctx, this.region, endpoint, this.endpointFactory, this) + this.endpointClients.set(endpoint.externalUrl, epClient) + + open = epClient.addWorkspace(uuid) + const workspace = new EndpointWorkspace( + ctx, + epClient, + wsToken, + this.tickCounter % ticksPerSecond, + workspaceSoftShutdownTicks, workspaceIds, - (ctx, sessionIds, result) => { - this.broadcastSessions(ctx, workspace, sessionIds, result) - } + branding ) - const pipeline = await this.pipelineFactory( - pipelineCtx, + this.workspaces.set(uuid, workspace) + return { workspace, open } + } else { + factory = async (): Promise => { + const communicationApi = await this.communicationApiFactory( + pipelineCtx, + workspaceIds, + (ctx, sessionIds, result) => { + this.broadcastSessions(ctx, workspace, sessionIds, result) + } + ) + const pipeline = await this.pipelineFactory( + pipelineCtx, + workspaceIds, + (ctx, tx, targets, exclude) => { + this.broadcastAll(workspaceIds.uuid, tx, targets, exclude) + }, + branding, + communicationApi + ) + return { pipeline, communicationApi } + } + const workspace: Workspace = new WorkspaceImpl( + context, + generateToken(systemAccountUuid, uuid), + factory, + this.tickCounter % ticksPerSecond, + workspaceSoftShutdownTicks, workspaceIds, - (ctx, tx, targets, exclude) => { - this.broadcastAll(workspaceIds.uuid, tx, targets, exclude) - }, - branding, - communicationApi + branding ) - return { pipeline, communicationApi } + this.workspaces.set(uuid, workspace) + workspace.open() // Trigger opening of workspace + return { workspace, open } } - const workspace: Workspace = new Workspace( - context, - generateToken(systemAccountUuid, token.workspace, { service: 'transactor' }), - factory, - this.tickCounter % ticksPerSecond, - workspaceSoftShutdownTicks, - workspaceIds, - branding - ) - workspace.upgrade = upgrade - this.workspaces.set(token.workspace, workspace) - - return workspace } private async trySetStatus ( ctx: MeasureContext, - pipeline: Pipeline, - communicationApi: CommunicationApi, + workspaces: Workspace[], session: Session, - online: boolean, - workspaceId: WorkspaceUuid + online: boolean ): Promise { const current = this.statusPromises.get(session.getUser()) if (current !== undefined) { await current } - const promise = this.setStatus(ctx, pipeline, communicationApi, session, online, workspaceId) + const promise = this.setStatus(ctx, workspaces, session, online) this.statusPromises.set(session.getUser(), promise) await promise this.statusPromises.delete(session.getUser()) @@ -787,45 +893,55 @@ export class TSessionManager implements SessionManager { private async setStatus ( ctx: MeasureContext, - pipeline: Pipeline, - communicationApi: CommunicationApi, + workspaces: Workspace[], session: Session, - online: boolean, - workspaceId: WorkspaceUuid + online: boolean ): Promise { try { const user = session.getUser() - const userRawAccount = session.getRawAccount() - if (user === undefined || userRawAccount.role === AccountRole.ReadOnlyGuest) return + if (user === undefined) return const clientCtx: ClientSessionCtx = { + workspaces, requestId: undefined, - pipeline, - communicationApi, sendResponse: async () => { // No response }, ctx, - socialStringsToUsers: this.getActiveSocialStringsToUsersMap(workspaceId, session), + socialStringsToUsers: this.socialStringsToUsers, sendError: async () => { // Assume no error send }, - sendPong: () => {} + sendPong: () => {}, + getAccount: (uuid) => { + const result = this.accounts.get(uuid) + if (result === undefined) { + throw new Error('Account not found') + } + return result + } } - const status = (await session.findAllRaw(clientCtx, core.class.UserStatus, { user }, { limit: 1 }))[0] - const txFactory = new TxFactory(userRawAccount.primarySocialId, true) - if (status === undefined) { - const tx = txFactory.createTxCreateDoc(core.class.UserStatus, core.space.Space, { - online, - user - }) - await session.tx(clientCtx, tx) - } else if (status.online !== online) { - const tx = txFactory.createTxUpdateDoc(status._class, status.space, status._id, { - online - }) - await session.tx(clientCtx, tx) + const statuses = new Map( + (await session.findAllRaw(clientCtx, core.class.UserStatus, { user }, {})).map((it) => [it._uuid, it]) + ) + for (const ws of workspaces) { + const txFactory = new TxFactory(session.getRawAccount().primarySocialId, ws.wsId.uuid, true) + const status = statuses.get(ws.wsId.uuid) + + if (status === undefined) { + await session.tx( + { ...clientCtx, workspaces: [ws] }, + txFactory.createTxCreateDoc(core.class.UserStatus, core.space.Space, { online, user }) + ) + } else if (status.online !== online) { + await session.tx( + { ...clientCtx, workspaces: [ws] }, + txFactory.createTxUpdateDoc(status._class, core.space.Space, status._id, { + online + }) + ) + } } } catch (err: any) { ctx.error('failed to set status', { err }) @@ -833,152 +949,85 @@ export class TSessionManager implements SessionManager { } } - async close (ctx: MeasureContext, ws: ConnectionSocket, workspaceUuid: WorkspaceUuid): Promise { - const workspace = this.workspaces.get(workspaceUuid) - - const sessionRef = this.sessions.get(ws.id) + async close (ctx: MeasureContext, sessionRef: Session): Promise { if (sessionRef !== undefined) { ctx.info('bye happen', { - workspace: workspace?.wsId.url, - userId: sessionRef.session.getUser(), - user: sessionRef.session.getSocialIds().find((it) => it.type !== SocialIdType.HULY)?.value, - binary: sessionRef.session.binaryMode, - compression: sessionRef.session.useCompression, - totalTime: this.now - sessionRef.session.createTime, - workspaceUsers: workspace?.sessions?.size, + userId: sessionRef.getUser(), + user: sessionRef.getSocialIds().find((it) => it.type !== SocialIdType.HULY)?.value, + binary: sessionRef.binaryMode, + compression: sessionRef.useCompression, + totalTime: Date.now() - sessionRef.createTime, totalUsers: this.sessions.size }) - this.sessions.delete(ws.id) + this.sessions.delete(sessionRef.sessionId) + const account = sessionRef.getUser() - if (workspace !== undefined) { - workspace.sessions.delete(sessionRef.session.sessionId) - - const userUuid = sessionRef.session.getUser() - await this.usersProducer.send(workspaceUuid, [ - userEvents.logout({ - user: userUuid, - sessions: this.countUserSessions(workspace, userUuid), - socialIds: sessionRef.session.getUserSocialIds() - }) - ]) + this.unregisterAccount(account, sessionRef) - if (this.doHandleTick) { - workspace.tickHandlers.set(sessionRef.session.sessionId, { - ticks: this.timeouts.reconnectTimeout * ticksPerSecond, - operation: () => { - this.reconnectIds.delete(sessionRef.session.sessionId) - const user = sessionRef.session.getUser() - if (workspace !== undefined) { - const another = Array.from(workspace.sessions.values()).findIndex((p) => p.session.getUser() === user) - if (another === -1 && !workspace.upgrade) { - void workspace.with(async (pipeline, communicationApi) => { - await communicationApi.closeSession(sessionRef.session.sessionId) - if (user !== guestAccount && user !== systemAccountUuid) { - await this.trySetStatus( - workspace.context, - pipeline, - communicationApi, - sessionRef.session, - false, - workspaceUuid - ).catch(() => {}) - } - }) - } - } - } - }) + if (sessionRef.subscribedUsers.size > 0) { + for (const uuid of Array.from(sessionRef.subscribedUsers)) { + this.unregisterAccount(uuid, sessionRef) } - this.reconnectIds.add(sessionRef.session.sessionId) - } - try { - sessionRef.socket.close() - } catch (err) { - // Ignore if closed } - } - } - async forceClose (wsId: WorkspaceUuid, ignoreSocket?: ConnectionSocket): Promise { - const ws = this.workspaces.get(wsId) - if (ws !== undefined) { - this.ctx.warn('force-close', { name: ws.wsId.url }) - ws.upgrade = true // We need to similare upgrade to refresh all clients. - ws.closing = this.doCloseAll(ws, 99, 'force-close', ignoreSocket) - this.workspaces.delete(wsId) - await ws.closing - ws.closing = undefined - } else { - this.ctx.warn('force-close-unknown', { wsId }) - } - } + const workspaces = this.getSessionWorkspaces(sessionRef) - async doCloseAll ( - workspace: Workspace, - code: number, - reason: 'upgrade' | 'shutdown' | 'force-close', - ignoreSocket?: ConnectionSocket - ): Promise { - if (LOGGING_ENABLED) { - this.ctx.info('closing workspace', { - url: workspace.wsId.url, - uuid: workspace.wsId.uuid, - code, - reason - }) - } + const userUuid = sessionRef.getUser() + for (const workspace of workspaces) { + if (workspace !== undefined) { + await workspace.removeSession(sessionRef) - const sessions = Array.from(workspace.sessions) - workspace.sessions.clear() + this.tickHandlers.set(sessionRef.sessionId, { + ticks: this.timeouts.reconnectTimeout * ticksPerSecond, + operation: async () => { + if (!this.reconnectIds.delete(sessionRef.sessionId)) { + // Already reconnected + return + } + for (const u of [account, ...sessionRef.subscribedUsers]) { + // All other sessions, not for this workspaces. + const allSessions = (this.accountIdToSessions.get(u) ?? []).filter( + (it) => it.sessionId !== sessionRef.sessionId && !it.workspaces.has(workspace.wsId.uuid) + ) + if (allSessions.length === 0) { + // No other sessions for this user + if (account !== guestAccount && u !== systemAccountUuid) { + await this.trySetStatus(ctx, workspaces, sessionRef, false).catch(() => {}) + } + await this.usersProducer.send(workspace.wsId.uuid, [ + userEvents.logout({ + user: u, + sessions: this.countUserSessions(workspace, userUuid), + socialIds: sessionRef.getUserSocialIds() + }) + ]) + } + } - const closeS = (s: Session, webSocket: ConnectionSocket): void => { - s.workspaceClosed = true - if (reason === 'upgrade' || reason === 'force-close') { - // Override message handler, to wait for upgrading response from clients. - this.sendUpgrade(workspace.context, webSocket, s.binaryMode, s.useCompression) + // Need to close all sessions for this user + for (const ws of workspaces) { + // TODO: Make handler async? + await ws.with(async (pipeline, communicationApi) => { + await communicationApi.closeSession(sessionRef.sessionId) + }) + } + } + }) + } } - webSocket.close() - this.reconnectIds.delete(s.sessionId) - } - - if (LOGGING_ENABLED) { - this.ctx.warn('Clients disconnected. Closing Workspace...', { - url: workspace.wsId.url, - uuid: workspace.wsId.uuid - }) + this.reconnectIds.add(sessionRef.sessionId) } - - sessions - .filter((it) => it[1].socket.id !== ignoreSocket?.id) - .forEach((s) => { - closeS(s[1].session, s[1].socket) - }) - - if (reason !== 'upgrade') { - await workspace.close(this.ctx) - if (LOGGING_ENABLED) { - this.ctx.warn('Workspace closed...', { uuid: workspace.wsId.uuid, url: workspace.wsId.url }) - } + try { + sessionRef.socket.close() + } catch (err) { + // Ignore if closed } } - private sendUpgrade (ctx: MeasureContext, webSocket: ConnectionSocket, binary: boolean, compression: boolean): void { - void webSocket.send( - ctx, - { - result: { - _class: core.class.TxModelUpgrade - } - }, - binary, - compression - ) - } - async closeWorkspaces (ctx: MeasureContext): Promise { clearInterval(this.checkInterval) - for (const w of this.workspaces) { - await this.doCloseAll(w[1], 1, 'shutdown') + for (const w of this.workspaces.values()) { + await w.close(this.ctx) } await this.workspaceProducer.close() await this.usersProducer.close() @@ -993,9 +1042,10 @@ export class TSessionManager implements SessionManager { } try { if (workspace.sessions.size === 0) { + this.workspaces.delete(uuid) + await workspace.close(this.ctx) - this.workspaces.delete(uuid) workspace.context.end() if (LOGGING_ENABLED) { this.ctx.warn('Closed workspace', logParams) @@ -1005,7 +1055,7 @@ export class TSessionManager implements SessionManager { } } catch (err: any) { Analytics.handleError(err) - this.workspaces.delete(uuid) + if (LOGGING_ENABLED) { this.ctx.error('failed', { ...logParams, error: err }) } @@ -1023,8 +1073,7 @@ export class TSessionManager implements SessionManager { createOpContext ( ctx: MeasureContext, sendCtx: MeasureContext, - pipeline: Pipeline, - communicationApi: CommunicationApi, + workspaces: Workspace[], requestId: Request['id'], service: Session, ws: ConnectionSocket, @@ -1033,8 +1082,7 @@ export class TSessionManager implements SessionManager { const st = platformNow() return { ctx, - pipeline, - communicationApi, + workspaces, requestId, sendResponse: (reqId, msg) => sendResponse(sendCtx, service, ws, { @@ -1048,7 +1096,7 @@ export class TSessionManager implements SessionManager { sendPong: () => { ws.sendPong() }, - socialStringsToUsers: this.getActiveSocialStringsToUsersMap(service.workspace.uuid), + socialStringsToUsers: this.socialStringsToUsers, sendError: (reqId, msg, error: Status) => sendResponse(sendCtx, service, ws, { id: reqId, @@ -1058,25 +1106,38 @@ export class TSessionManager implements SessionManager { rateLimit, bfst: this.now, queue: service.requests.size - }) + }), + getAccount: (uuid) => { + const result = this.accounts.get(uuid) + if (result === undefined) { + throw new Error('Account not found') + } + return result + } } } // TODO: cache this map and update when sessions created/closed - getActiveSocialStringsToUsersMap (workspace: WorkspaceUuid, ...extra: Session[]): Map { - const ws = this.workspaces.get(workspace) - if (ws === undefined) { - return new Map() - } + getActiveSocialStringsToUsersMap (workspaces: WorkspaceUuid[], ...extra: Session[]): Map { const res = new Map() - for (const s of [...Array.from(ws.sessions.values()).map((it) => it.session), ...extra]) { - const sessionAccount = s.getUser() - if (sessionAccount === systemAccountUuid) { - continue + + for (const workspace of workspaces) { + const ws = this.workspaces.get(workspace) + if (ws === undefined) { + return new Map() } - const userSocialIds = s.getUserSocialIds() - for (const id of userSocialIds) { - res.set(id, sessionAccount) + + for (const s of [...Array.from(ws.sessions.values()).map((it) => it), ...extra]) { + const sessionAccounts = [s.getUser(), ...s.subscribedUsers] + for (const sessionAccount of sessionAccounts) { + if (sessionAccount === systemAccountUuid) { + continue + } + const userSocialIds = s.getUserSocialIds() + for (const id of userSocialIds) { + res.set(id, sessionAccount) + } + } } } return res @@ -1088,18 +1149,23 @@ export class TSessionManager implements SessionManager { () => Date.now() ) - async handleRequest( + async forceCloseWorkspace (ctx: MeasureContext, workspace: WorkspaceUuid): Promise { + const wsRef = this.workspaces.get(workspace) + if (wsRef !== undefined) { + // Just force pipeline close/open + await wsRef.close(ctx) + } + } + + handleRequest ( requestCtx: MeasureContext, - service: S, + session: Session, ws: ConnectionSocket, - request: Request, - workspaceId: WorkspaceUuid + request: Request ): Promise { - const userCtx = requestCtx.newChild('📞 client', { - source: service.token.extra?.service ?? '🤦‍♂️user', - mode: '🧭 handleRequest' - }) - const rateLimit = this.limitter.checkRateLimit(service.getUser() + (service.token.extra?.service ?? '')) + const userCtx = requestCtx.newChild('📞 client', {}) + const rateLimit = + session.getUser() === systemAccountUuid ? undefined : this.limitter.checkRateLimit(session.getUser()) // If remaining is 0, rate limit is exceeded if (rateLimit?.remaining === 0) { void ws.send( @@ -1109,173 +1175,245 @@ export class TSessionManager implements SessionManager { rateLimit, error: unknownError('Rate limit') }, - service.binaryMode, - service.useCompression + session.binaryMode, + session.useCompression ) - return + return Promise.resolve() } // Calculate total number of clients const reqId = generateId() const st = Date.now() - try { - if (request.time != null) { - const delta = Date.now() - request.time - requestCtx.measure('msg-receive-delta', delta) - } - const workspace = this.workspaces.get(workspaceId) - if (workspace === undefined || workspace.closing !== undefined) { - await ws.send( - userCtx, - { - id: request.id, - error: unknownError('Workspace is closing') - }, - service.binaryMode, - service.useCompression - ) - return - } - if (request.id === -1 && request.method === 'hello') { - await this.handleHello(request, service, userCtx, workspace, ws, requestCtx) - return - } - if (request.id === -2 && request.method === 'forceClose') { - // TODO: we chould allow this only for admin or system accounts - let done = false - const wsRef = this.workspaces.get(workspaceId) - if (wsRef?.upgrade ?? false) { - done = true - this.ctx.warn('FORCE CLOSE', { workspace: workspaceId }) - // In case of upgrade, we need to force close workspace not in interval handler - await this.forceClose(workspaceId, ws) + return userCtx + .with('🧭 handleRequest', {}, async (ctx) => { + if (request.time != null) { + const delta = Date.now() - request.time + requestCtx.measure('msg-receive-delta', delta) } - const forceCloseResponse: Response = { - id: request.id, - result: done + if (request.id === -1 && request.method === 'hello') { + await this.handleHello(request, session, ctx, ws, requestCtx) + return + } + if ( + request.id === -2 && + request.method === 'forceClose' && + session.getRawAccount().uuid === systemAccountUuid + ) { + // TODO: we should allow this only for admin or system accounts + let done = false + const wsRef = this.workspaces.get(request.params[0] as WorkspaceUuid) + if (wsRef !== undefined) { + done = true + // Just force pipeline close/open + void wsRef.close(ctx) + } + const forceCloseResponse: Response = { + id: request.id, + result: done + } + await ws.send(ctx, forceCloseResponse, session.binaryMode, session.useCompression) + return } - await ws.send(userCtx, forceCloseResponse, service.binaryMode, service.useCompression) - return - } - service.requests.set(reqId, { - id: reqId, - params: request, - start: st - }) - if (request.id === -1 && request.method === '#upgrade') { - ws.close() - return - } + if ( + (request.method === 'subscribe' || request.method === 'unsubscribe') && + session.getRawAccount().uuid === systemAccountUuid + ) { + const subscription: { accounts?: AccountUuid[], workspaces?: WorkspaceUuid[] } = request.params[0] - const f = (service as any)[request.method] - try { - const params = [...request.params] + const info = await this.handleSubcribe(ctx, session, subscription, request.method === 'subscribe') - if (ws.isBackpressure()) { - await ws.backpressure(userCtx) + const resp: Response = { + id: request.id, + result: request.method === 'subscribe' ? info : true + } + await ws.send(ctx, resp, session.binaryMode, session.useCompression) + return } - await workspace.with(async (pipeline, communicationApi) => { - await userCtx.with('🧨 process', {}, (callTx) => - f.apply(service, [ - this.createOpContext(callTx, userCtx, pipeline, communicationApi, request.id, service, ws, rateLimit), + session.requests.set(reqId, { + id: reqId, + params: request, + start: st + }) + if (request.id === -1 && request.method === '#upgrade') { + ws.close() + return + } + + const f = (session as any)[request.method] + try { + const params = [...request.params] + + if (ws.isBackpressure()) { + await ws.backpressure(ctx) + } + + const workspaces = this.getSessionWorkspaces(session) + + await ctx.with('🧨 process', {}, (callTx) => + f.apply(session, [ + this.createOpContext(callTx, userCtx, workspaces, request.id, session, ws, rateLimit), ...params ]) ) + } catch (err: any) { + Analytics.handleError(err) + if (LOGGING_ENABLED) { + this.ctx.error('error handle request', { error: err, request }) + } + await ws.send( + userCtx, + { + id: request.id, + error: unknownError(err), + result: JSON.parse(JSON.stringify(err?.stack)) + }, + session.binaryMode, + session.useCompression + ) + } + }) + .finally(() => { + userCtx.end() + session.requests.delete(reqId) + }) + } + + async handleSubcribe ( + ctx: MeasureContext, + session: Session, + subscription: { accounts?: AccountUuid[], workspaces?: WorkspaceUuid[] }, + subscribe: boolean + ): Promise { + const result: SubscribedWorkspaceInfo = {} + + if (subscription.accounts !== undefined) { + for (const a of subscription.accounts) { + if (subscribe) { + // Find some session with this account already pressent + await this.registerAccountByUuid(a, session) + } else { + session.subscribedUsers.delete(a) + this.unregisterAccount(a, session) + } + } + } + + if (subscription.workspaces !== undefined) { + for (const ws of subscription.workspaces) { + let workspace = this.workspaces.get(ws) + let open: Promise | undefined + if (workspace === undefined && !subscribe) { + // No workspace and no need unsubscribe + continue + } + if (workspace === undefined) { + // Retrieve info about workspace and make it up. + // Edpoint itself will update last visit. + const rawToken = generateToken(systemAccountUuid, ws, { service: 'server', mode: 'server' }) + const workspaceInfo = await this.getWorkspaceInfo(rawToken, false) + if (workspaceInfo === undefined) { + // No workspace, no need to subscribe + continue + } + ;({ workspace, open } = this.getWorkspace(ctx, ws, this.toLoginInfoWorkspace(workspaceInfo))) + + if (workspace === undefined) { + continue + } + if (open !== undefined) { + await open + } + } + await workspace.with(async (pipeline, communicationApi) => { + result[ws] = { + lastHash: pipeline.context.lastHash, + lastTx: pipeline.context.lastTx + } }) - } catch (err: any) { - Analytics.handleError(err) - if (LOGGING_ENABLED) { - this.ctx.error('error handle request', { error: err, request }) + // We had workspace + if (!subscribe) { + session.workspaces.delete(ws) + await workspace.removeSession(session) + } else { + session.workspaces.add(ws) + await workspace.addSession(session) } - await ws.send( - userCtx, - { - id: request.id, - error: unknownError(err), - result: JSON.parse(JSON.stringify(err?.stack)) - }, - service.binaryMode, - service.useCompression - ) } - } finally { - userCtx.end() - service.requests.delete(reqId) } + return result } - async handleRPC( + private getSessionWorkspaces (session: Session): Workspace[] { + let workspaces = Array.from(session.workspaces) + .map((it) => this.workspaces.get(it)) + .filter((it) => it !== undefined) as Workspace[] + + if (session.getRawAccount().uuid !== systemAccountUuid) { + // Filter all workspaces that are not in maintenance mode + workspaces = workspaces.filter((it) => !it.maintenance) + } + return workspaces + } + + handleRPC ( requestCtx: MeasureContext, - service: S, + workspaceId: WorkspaceUuid, + service: Session, ws: ConnectionSocket, operation: (ctx: ClientSessionCtx, rateLimit: RateLimitInfo | undefined) => Promise ): Promise { - const rateLimitStatus = this.limitter.checkRateLimit(service.getUser() + (service.token.extra?.service ?? '')) + const rateLimitStatus = this.limitter.checkRateLimit(service.getUser()) // If remaining is 0, rate limit is exceeded if (rateLimitStatus?.remaining === 0) { - return await Promise.resolve(rateLimitStatus) + return Promise.resolve(rateLimitStatus) } - const userCtx = requestCtx.newChild('📞 client', { - source: service.token.extra?.service ?? '🤦‍♂️user', - mode: '🧭 handleRPC' - }) + const userCtx = requestCtx.newChild('📞 client', {}) // Calculate total number of clients const reqId = generateId() const st = Date.now() - try { - const workspace = this.workspaces.get(service.workspace.uuid) - if (workspace === undefined || workspace.closing !== undefined) { - throw new Error('Workspace is closing') - } + return userCtx + .with('🧭 handleRPC', {}, async (ctx) => { + const workspaces = this.getSessionWorkspaces(service).filter((it) => it.wsId.uuid === workspaceId) - service.requests.set(reqId, { - id: reqId, - params: {}, - start: st - }) + service.requests.set(reqId, { + id: reqId, + params: {}, + start: st + }) - try { - await workspace.with(async (pipeline, communicationApi) => { - const uctx = this.createOpContext( - userCtx, + try { + // TODO: Select workspaces + const uctx = this.createOpContext(ctx, userCtx, workspaces, reqId, service, ws, rateLimitStatus) + await operation(uctx, rateLimitStatus) + } catch (err: any) { + Analytics.handleError(err) + if (LOGGING_ENABLED) { + this.ctx.error('error handle request', { error: err }) + } + await ws.send( userCtx, - pipeline, - communicationApi, - reqId, - service, - ws, - rateLimitStatus + { + id: reqId, + error: unknownError(err), + result: JSON.parse(JSON.stringify(err?.stack)) + }, + service.binaryMode, + service.useCompression ) - await operation(uctx, rateLimitStatus) - }) - } catch (err: any) { - Analytics.handleError(err) - if (LOGGING_ENABLED) { - this.ctx.error('error handle request', { error: err }) + throw err } - await ws.send( - userCtx, - { - id: reqId, - error: unknownError(err), - result: JSON.parse(JSON.stringify(err?.stack)) - }, - service.binaryMode, - service.useCompression - ) - throw err - } - return undefined - } finally { - userCtx.end() - service.requests.delete(reqId) - } + return undefined + }) + .finally(() => { + userCtx.end() + service.requests.delete(reqId) + }) } entryToUserStats = (session: Session, socket: ConnectionSocket): UserStatistics => { @@ -1291,11 +1429,11 @@ export class TSessionManager implements SessionManager { workspaceToWorkspaceStats = (ws: Workspace): WorkspaceStatistics => { return { - clientsTotal: new Set(Array.from(ws.sessions.values()).map((it) => it.session.getUser())).size, + clientsTotal: new Set(Array.from(ws.sessions.values()).map((it) => it.getUser())).size, sessionsTotal: ws.sessions.size, workspaceName: ws.wsId.url, wsId: ws.wsId.uuid, - sessions: Array.from(ws.sessions.values()).map((it) => this.entryToUserStats(it.session, it.socket)) + sessions: Array.from(ws.sessions.values()).map((it) => this.entryToUserStats(it, it.socket)) } } @@ -1303,61 +1441,50 @@ export class TSessionManager implements SessionManager { return Array.from(this.workspaces.values()).map((it) => this.workspaceToWorkspaceStats(it)) } - private async handleHello( + private async handleHello ( request: Request, - service: S, + session: Session, ctx: MeasureContext, - workspace: Workspace, ws: ConnectionSocket, requestCtx: MeasureContext ): Promise { try { const hello = request as HelloRequest - service.binaryMode = hello.binary ?? false - service.useCompression = this.enableCompression ? hello.compression ?? false : false + session.binaryMode = hello.binary ?? false + session.useCompression = this.enableCompression ? hello.compression ?? false : false if (LOGGING_ENABLED) { ctx.info('hello happen', { - workspace: workspace.wsId.url, - workspaceId: workspace.wsId.uuid, - userId: service.getUser(), - user: service.getSocialIds().find((it) => it.type !== SocialIdType.HULY)?.value, - binary: service.binaryMode, - compression: service.useCompression, - timeToHello: Date.now() - service.createTime, - workspaceUsers: workspace.sessions.size, + userId: session.getUser(), + user: session.getSocialIds().find((it) => it.type !== SocialIdType.HULY)?.value, + binary: session.binaryMode, + compression: session.useCompression, + timeToHello: Date.now() - session.createTime, totalUsers: this.sessions.size }) } - const reconnect = this.reconnectIds.has(service.sessionId) - if (reconnect) { - this.reconnectIds.delete(service.sessionId) + const reconnect = this.reconnectIds.delete(session.sessionId) + + const account = session.getRawAccount() + + const workspaces = this.getSessionWorkspaces(session) + + const helloResponse: HelloResponse = { + id: -1, + result: 'hello', + binary: session.binaryMode, + reconnect, + serverVersion: this.serverVersion, + ...getLastHashInfo(workspaces), + account, + useCompression: session.useCompression } + await ws.send(requestCtx, helloResponse, false, false) - const account = service.getRawAccount() - await workspace.with(async (pipeline, communicationApi) => { - const helloResponse: HelloResponse = { - id: -1, - result: 'hello', - binary: service.binaryMode, - reconnect, - serverVersion: this.serverVersion, - lastTx: pipeline.context.lastTx, - lastHash: pipeline.context.lastHash, - account, - useCompression: service.useCompression - } - await ws.send(requestCtx, helloResponse, false, false) - }) + // TODO: Status passwing should not depend on pipeline. if (account.uuid !== guestAccount && account.uuid !== systemAccountUuid) { - void workspace.with(async (pipeline, communicationApi) => { - // We do not need to wait for set-status, just return session to client - await ctx - .with('set-status', {}, (ctx) => - this.trySetStatus(ctx, pipeline, communicationApi, service, true, service.workspace.uuid) - ) - .catch(() => {}) - }) + // We do not need to wait for set-status, just return session to client + await ctx.with('set-status', {}, (ctx) => this.trySetStatus(ctx, workspaces, session, true)) } } catch (err: any) { ctx.error('error', { err }) @@ -1365,8 +1492,50 @@ export class TSessionManager implements SessionManager { } } +function createAccountFromInfo (info: LoginInfoWithWorkspaces, targetWs?: WorkspaceUuid): Account { + let primarySocialId: PersonId + let role: AccountRole = AccountRole.User + switch (info.account) { + case systemAccountUuid: + primarySocialId = core.account.System + role = AccountRole.Owner + break + case guestAccount: + primarySocialId = '' as PersonId + role = AccountRole.DocGuest + break + default: + primarySocialId = pickPrimarySocialId(info.socialIds)._id + } + + const account: Account = { + uuid: info.account, + socialIds: info.socialIds.map((it) => it._id), + primarySocialId, + fullSocialIds: new Map(info.socialIds.map((it) => [it._id, it])), + socialIdsByValue: new Map(info.socialIds.map((it) => [it.value, it])), + targetWorkspace: targetWs ?? info.personalWorkspace, + personalWorkspace: info.personalWorkspace, + role, + workspaces: {} + } + + for (const [uuid, wsInfo] of Object.entries(info.workspaces)) { + account.workspaces[uuid as WorkspaceUuid] = { + url: wsInfo.url, + name: wsInfo.name, + role: wsInfo.role ?? AccountRole.User, + maintenance: false, + enabled: uuid === targetWs || targetWs === '' + } + } + return account +} + export function createSessionManager ( ctx: MeasureContext, + region: string, // A endpoint region, will be used to perform combined find requests. + endpointName: string | '#endpoint', // If '#endpoint' special name is used, then will be endpoint only transactor, will not hold local workspaces. brandingMap: BrandingMap, timeouts: Timeouts, profiling: @@ -1377,10 +1546,11 @@ export function createSessionManager ( | undefined, accountsUrl: string, enableCompression: boolean, - doHandleTick: boolean = true, queue: PlatformQueue, pipelineFactory: PipelineFactory, - communicationApiFactory: CommunicationApiFactory + endpointFactory: EndpointConnectionFactory, + communicationApiFactory: CommunicationApiFactory, + handleTick: boolean = true ): SessionManager { return new TSessionManager( ctx, @@ -1389,15 +1559,19 @@ export function createSessionManager ( profiling, accountsUrl, enableCompression, - doHandleTick, queue, pipelineFactory, - communicationApiFactory + endpointFactory, + communicationApiFactory, + region, + endpointName, + handleTick ) } export interface SessionManagerOptions extends Partial { pipelineFactory: PipelineFactory + endpointFactory: EndpointConnectionFactory communicationApiFactory: CommunicationApiFactory brandingMap: BrandingMap enableCompression?: boolean @@ -1407,6 +1581,9 @@ export interface SessionManagerOptions extends Partial { stop: () => Promise } queue: PlatformQueue + + region: string + endpointName: string } /** @@ -1415,6 +1592,8 @@ export interface SessionManagerOptions extends Partial { export function startSessionManager (ctx: MeasureContext, opt: SessionManagerOptions): SessionManager { const sessions = createSessionManager( ctx, + opt.region, + opt.endpointName, opt.brandingMap, { pingTimeout: opt.pingTimeout ?? 10000, @@ -1423,9 +1602,9 @@ export function startSessionManager (ctx: MeasureContext, opt: SessionManagerOpt opt.profiling, opt.accountsUrl, opt.enableCompression ?? false, - true, opt.queue, opt.pipelineFactory, + opt.endpointFactory, opt.communicationApiFactory ) return sessions diff --git a/server/server/src/stats.ts b/server/server/src/stats.ts index f3814ca4f35..e0951777e36 100644 --- a/server/server/src/stats.ts +++ b/server/server/src/stats.ts @@ -5,8 +5,8 @@ import { metricsAggregate, type MetricsData } from '@hcengineering/core' -import { type SessionManager } from '@hcengineering/server-core' import os from 'node:os' +import { type SessionManager } from './types' /** * @public diff --git a/server/server/src/types.ts b/server/server/src/types.ts new file mode 100644 index 00000000000..616a82a6b0b --- /dev/null +++ b/server/server/src/types.ts @@ -0,0 +1,245 @@ +// +// Copyright © 2022, 2023 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type RequestEvent as CommunicationEvent, type EventResult } from '@hcengineering/communication-sdk-types' +import { + type FindMessagesGroupsParams, + type FindMessagesParams, + type Message, + type MessagesGroup +} from '@hcengineering/communication-types' +import { + type Account, + type AccountUuid, + type Class, + type Doc, + type DocumentQuery, + type Domain, + type FindOptions, + type FindResult, + type LoadModelResponse, + type MeasureContext, + type PersonId, + type Ref, + type SearchOptions, + type SearchQuery, + type SearchResult, + type SocialId, + type Timestamp, + type Tx, + type TxResult, + type WorkspaceUuid +} from '@hcengineering/core' +import type { RateLimitInfo, ReqId, Request, Response } from '@hcengineering/rpc' +import type { Token } from '@hcengineering/server-token' + +import type { StatisticsElement, WorkspaceStatistics } from '@hcengineering/server-core' +import type { Workspace } from './workspace' + +/** + * @public + */ +export interface SessionRequest { + id: string + params: any + start: number + + workspaceId?: WorkspaceUuid +} + +export interface ClientSessionCtx { + ctx: MeasureContext + workspaces: Workspace[] + socialStringsToUsers: Map + requestId: ReqId | undefined + sendResponse: (id: ReqId | undefined, msg: any) => Promise + sendPong: () => void + sendError: (id: ReqId | undefined, msg: any, error: any) => Promise + + getAccount: (uuid: AccountUuid) => Account +} + +/** + * @public + */ +export interface Session { + workspaces: Set + + socket: ConnectionSocket + createTime: number + + // Session restore information + sessionId: string + + requests: Map + + binaryMode: boolean + useCompression: boolean + total: StatisticsElement + current: StatisticsElement + mins5: StatisticsElement + + lastRequest: number + lastPing: number + + isUpgradeClient: () => boolean + + getMode: () => string + + broadcast: ( + ctx: MeasureContext, + socket: ConnectionSocket, + tx: Tx[], + target?: AccountUuid, + exclude?: AccountUuid[] + ) => void + + // Client methods + ping: (ctx: ClientSessionCtx) => Promise + getUser: () => AccountUuid + + subscribedUsers: Set // A set of accounts a session is listening for + + getUserSocialIds: () => PersonId[] + + getSocialIds: () => SocialId[] + + loadModel: (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string) => Promise + loadModelRaw: (ctx: ClientSessionCtx, lastModelTx: Timestamp, hash?: string) => Promise + getRawAccount: () => Account + findAll: ( + ctx: ClientSessionCtx, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise + findAllRaw: ( + ctx: ClientSessionCtx, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise> + searchFulltext: (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions) => Promise + searchFulltextRaw: (ctx: ClientSessionCtx, query: SearchQuery, options: SearchOptions) => Promise + tx: (ctx: ClientSessionCtx, tx: Tx) => Promise + + txRaw: ( + ctx: ClientSessionCtx, + tx: Tx + ) => Promise<{ + result: TxResult + broadcastPromise: Promise + asyncsPromise: Promise | undefined + }> + + loadChunk: (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain, idx?: number) => Promise + + getDomainHash: (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain) => Promise + closeChunk: (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, idx: number) => Promise + loadDocs: (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]) => Promise + upload: (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain, docs: Doc[]) => Promise + clean: (ctx: ClientSessionCtx, workspaceId: WorkspaceUuid, domain: Domain, docs: Ref[]) => Promise + + eventRaw: (ctx: ClientSessionCtx, event: CommunicationEvent) => Promise + findMessagesRaw: (ctx: ClientSessionCtx, params: FindMessagesParams) => Promise + findMessagesGroupsRaw: (ctx: ClientSessionCtx, params: FindMessagesGroupsParams) => Promise +} + +/** + * @public + */ +export interface ConnectionSocket { + id: string + isClosed: boolean + close: () => void + send: (ctx: MeasureContext, msg: Response, binary: boolean, compression: boolean) => Promise + + sendPong: () => void + data: () => Record + + readRequest: (buffer: Buffer, binary: boolean) => Request + + isBackpressure: () => boolean // In bytes + backpressure: (ctx: MeasureContext) => Promise + checkState: () => boolean +} + +export interface SessionInfoRecord { + session: Session + socket: ConnectionSocket + tickHash: number +} + +/** + * @public + */ +export interface SessionManager { + // workspaces: Map + sessions: Map + + addSession: ( + ctx: MeasureContext, + ws: ConnectionSocket, + token: Token, + rawToken: string, + sessionId: string | undefined + ) => Promise + + broadcastAll: (workspace: WorkspaceUuid, tx: Tx[], targets?: AccountUuid[]) => void + + broadcast: (workspaceId: WorkspaceUuid, resp: Tx[], target: AccountUuid | undefined, exclude?: AccountUuid[]) => void + + close: (ctx: MeasureContext, sessionRef: Session) => Promise + + closeWorkspaces: (ctx: MeasureContext) => Promise + + scheduleMaintenance: (timeMinutes: number) => void + + profiling?: { + start: () => void + stop: () => Promise + } + + handleRequest: ( + requestCtx: MeasureContext, + service: Session, + ws: ConnectionSocket, + request: Request + ) => Promise + + handleRPC: ( + requestCtx: MeasureContext, + workspaceId: WorkspaceUuid, + service: Session, + ws: ConnectionSocket, + operation: (ctx: ClientSessionCtx, rateLimit?: RateLimitInfo | undefined) => Promise + ) => Promise + + createOpContext: ( + ctx: MeasureContext, + sendCtx: MeasureContext, + workspaces: Workspace[], + requestId: Request['id'], + service: Session, + ws: ConnectionSocket, + rateLimit: RateLimitInfo | undefined + ) => ClientSessionCtx + + getStatistics: () => WorkspaceStatistics[] + + forceCloseWorkspace: (ctx: MeasureContext, workspace: WorkspaceUuid) => Promise + + getLastTxHash: (workspaceId: WorkspaceUuid) => Promise<{ lastTx: string | undefined, lastHash: string | undefined }> +} diff --git a/server/server/src/utils.ts b/server/server/src/utils.ts index af482a9e502..5f936eccca9 100644 --- a/server/server/src/utils.ts +++ b/server/server/src/utils.ts @@ -13,64 +13,53 @@ // limitations under the License. // -import { type WorkspaceUuid, type MeasureContext } from '@hcengineering/core' +import { type MeasureContext, type WorkspaceUuid } from '@hcengineering/core' -import type { - AddSessionActive, - AddSessionResponse, - ConnectionSocket, - Session, - SessionManager -} from '@hcengineering/server-core' +import type { ConnectionSocket } from '@hcengineering/server-core' import { type Response } from '@hcengineering/rpc' import type { Token } from '@hcengineering/server-token' +import type { Session, SessionManager } from './types' +import type { Workspace } from './workspace' export interface WebsocketData { connectionSocket?: ConnectionSocket payload: Token token: string - session: Promise | AddSessionResponse | undefined + session: Promise | Session | undefined url: string } -export function doSessionOp ( - data: WebsocketData, - op: (session: AddSessionActive, msg: Buffer) => void, - msg: Buffer -): void { +export function doSessionOp (data: WebsocketData, op: (session: Session, msg: Buffer) => void, msg: Buffer): void { if (data.session instanceof Promise) { // We need to copy since we will out of protected buffer area const msgCopy = Buffer.copyBytesFrom(new Uint8Array(msg)) void data.session .then((_session) => { data.session = _session - if ('session' in _session) { - op(_session, msgCopy) - } + op(data.session, msgCopy) }) .catch((err) => { console.error({ message: 'Failed to process session operation', err }) }) } else { - if (data.session !== undefined && 'session' in data.session) { + if (data.session !== undefined) { op(data.session, msg) } } } export function processRequest ( + ctx: MeasureContext, session: Session, cs: ConnectionSocket, - context: MeasureContext, - workspaceId: WorkspaceUuid, buff: any, sessions: SessionManager ): void { try { const request = cs.readRequest(buff, session.binaryMode) - void sessions.handleRequest(context, session, cs, request, workspaceId).catch((err) => { - context.error('failed to handle request', { err, request }) + void sessions.handleRequest(ctx, session, cs, request).catch((err) => { + ctx.error('failed to handle request', { err }) }) } catch (err: any) { if (((err.message as string) ?? '').includes('Data read, but end of buffer not reached')) { @@ -89,3 +78,19 @@ export function sendResponse ( ): Promise { return socket.send(ctx, resp, session.binaryMode, session.useCompression) } + +export function getLastHashInfo (workspaces: Workspace[]): { + lastTx: Record + lastHash: Record +} { + const lastTx: Record = {} + for (const workspace of workspaces) { + lastTx[workspace.wsId.uuid] = workspace.getLastTx() + } + + const lastHash: Record = {} + for (const workspace of workspaces) { + lastHash[workspace.wsId.uuid] = workspace.getLastHash() + } + return { lastTx, lastHash } +} diff --git a/server/server/src/workspace.ts b/server/server/src/workspace.ts index 575dac09091..95d364c3e3f 100644 --- a/server/server/src/workspace.ts +++ b/server/server/src/workspace.ts @@ -15,13 +15,9 @@ import { Analytics } from '@hcengineering/analytics' import { type ServerApi as CommunicationApi } from '@hcengineering/communication-sdk-types' -import { type Branding, type MeasureContext, type WorkspaceIds } from '@hcengineering/core' -import type { ConnectionSocket, Pipeline, Session } from '@hcengineering/server-core' - -interface TickHandler { - ticks: number - operation: () => void -} +import { systemAccountUuid, type Branding, type MeasureContext, type WorkspaceIds } from '@hcengineering/core' +import type { Pipeline } from '@hcengineering/server-core' +import type { Session } from './types' export interface PipelinePair { pipeline: Pipeline @@ -29,23 +25,58 @@ export interface PipelinePair { } export type WorkspacePipelineFactory = () => Promise +export interface Workspace { + sessions: Map + + operations: number + + maintenance: boolean + + lastTx: string | undefined // TODO: Do not cache for proxy case + lastHash: string | undefined // TODO: Do not cache for proxy case + + context: MeasureContext + token: string // Account workspace update token. + + tickHash: number + + softShutdown: number + + wsId: WorkspaceIds + branding: Branding | null + + open: () => void + + getLastTx: () => string | undefined + + getLastHash: () => string | undefined + + with: (op: (pipeline: Pipeline, communicationApi: CommunicationApi) => Promise) => Promise + + close: (ctx: MeasureContext) => Promise + + addSession: (session: Session) => Promise + removeSession: (session: Session) => Promise + + checkHasUser: () => boolean +} /** * @public */ -export class Workspace { +export class WorkspaceImpl implements Workspace { pipeline?: PipelinePair | Promise - upgrade: boolean = false - closing?: Promise - - workspaceInitCompleted: boolean = false softShutdown: number - sessions = new Map() - tickHandlers = new Map() + sessions = new Map() operations: number = 0 + maintenance: boolean = false + + lastTx: string | undefined // TODO: Do not cache for proxy case + lastHash: string | undefined // TODO: Do not cache for proxy case + constructor ( readonly context: MeasureContext, readonly token: string, // Account workspace update token. @@ -61,6 +92,24 @@ export class Workspace { this.softShutdown = softShutdown } + open (): void { + const pair = this.getPipelinePair() + if (pair instanceof Promise) { + void pair.then((it) => { + this.lastHash = it.pipeline.context.lastHash + this.lastTx = it.pipeline.context.lastTx + }) + } + } + + getLastTx (): string | undefined { + return this.lastTx + } + + getLastHash (): string | undefined { + return this.lastHash + } + private getPipelinePair (): PipelinePair | Promise { if (this.pipeline === undefined) { this.pipeline = this.factory() @@ -76,7 +125,10 @@ export class Workspace { this.pipeline = pair } try { - return await op(pair.pipeline, pair.communicationApi) + const result = await op(pair.pipeline, pair.communicationApi) + this.lastHash = pair.pipeline.context.lastHash + this.lastTx = pair.pipeline.context.lastTx + return result } finally { this.operations-- } @@ -115,8 +167,24 @@ export class Workspace { to.cancelHandle() }) } -} + async addSession (session: Session): Promise { + this.sessions.set(session.sessionId, session) + } + + async removeSession (session: Session): Promise { + this.sessions.delete(session.sessionId) + } + + checkHasUser (): boolean { + for (const val of this.sessions.values()) { + if (val.getUser() !== systemAccountUuid || val.subscribedUsers.size > 0) { + return true + } + } + return false + } +} function timeoutPromise (time: number): { promise: Promise, cancelHandle: () => void } { let timer: any return { diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index b8753f4de95..d62a50cb5d4 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -27,32 +27,31 @@ import core, { type MeasureContext, type MigrationState, ModelDb, + type PersonInfo, platformNow, platformNowDiff, + type Ref, systemAccountUuid, type Tx, type TxOperations, + type WithLookup, type WorkspaceIds, - type WorkspaceUuid, - type Client, - type PersonInfo, - type Ref, - type WithLookup + type WorkspaceUuid } from '@hcengineering/core' import { consoleModelLogger, + type MigrateMode, type MigrateOperation, type ModelLogger, - tryMigrate, - type MigrateMode + tryMigrate } from '@hcengineering/model' import { + type DbAdapter, DomainIndexHelperImpl, type Pipeline, - type StorageAdapter, - type DbAdapter, type PlatformQueueProducer, - type QueueWorkspaceMessage + type QueueWorkspaceMessage, + type StorageAdapter } from '@hcengineering/server-core' import { type InitScript, WorkspaceInitializer } from './initializer' import toolPlugin from './plugin' @@ -128,6 +127,7 @@ export async function initModel ( try { logger.log('creating database...', { workspaceId }) const firstTx: Tx = { + _uuid: workspaceId, _class: core.class.Tx, _id: 'first-tx' as Ref, modifiedBy: core.account.System, @@ -255,7 +255,7 @@ export async function upgradeModel ( wsIds: WorkspaceIds, txes: Tx[], pipeline: Pipeline, - connection: Client, + connection: TxOperations, storageAdapter: StorageAdapter, accountClient: AccountClient, queue: PlatformQueueProducer, diff --git a/server/workspace-service/src/ws-operations.ts b/server/workspace-service/src/ws-operations.ts index ca31a074ca0..25abd905fbb 100644 --- a/server/workspace-service/src/ws-operations.ts +++ b/server/workspace-service/src/ws-operations.ts @@ -7,7 +7,6 @@ import core, { TxOperations, versionToString, type Branding, - type Client, type Data, type MeasureContext, type Tx, @@ -104,7 +103,7 @@ export async function createWorkspace ( initModel(ctx, wsId, txes, txAdapter, storageAdapter, ctxModellogger, async (value) => {}) ) - const client = new TxOperations(wrapPipeline(ctx, pipeline, wsIds), core.account.ConfigUser) + const client = new TxOperations(wrapPipeline(ctx, pipeline, wsIds), core.account.ConfigUser, wsIds.uuid) await updateModel( childLogger, @@ -125,7 +124,7 @@ export async function createWorkspace ( if (creatorUuid != null) { const personInfo = await accountClient.getPersonInfo(creatorUuid) - if (personInfo?.socialIds.length > 0) { + if (personInfo?.socialIds.length > 0 && !(workspaceInfo.personal === true && branding == null)) { await initializeWorkspace( childLogger, branding, @@ -233,7 +232,7 @@ export async function upgradeWorkspace ( url: ws.url ?? '', dataId: ws.dataId } - + const client = new TxOperations(wrapPipeline(ctx, pipeline, wsUrl), core.account.System, ws.uuid) await upgradeWorkspaceWith( ctx, version, @@ -241,7 +240,7 @@ export async function upgradeWorkspace ( migrationOperation, ws, pipeline, - wrapPipeline(ctx, pipeline, wsUrl), + client, storageAdapter, accountClient, queue, @@ -268,7 +267,7 @@ export async function upgradeWorkspaceWith ( migrationOperation: [string, MigrateOperation][], ws: WorkspaceInfoWithStatus, pipeline: Pipeline, - connection: Client, + connection: TxOperations, storageAdapter: StorageAdapter, accountClient: AccountClient, queue: PlatformQueueProducer, diff --git a/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts b/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts index 8f41040d9d0..061ae514da4 100644 --- a/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts +++ b/services/ai-bot/pod-ai-bot/src/workspace/workspaceClient.ts @@ -31,6 +31,7 @@ import contact, { import core, { type Account, AccountRole, + AccountUuid, Blob, Class, Client, @@ -38,6 +39,7 @@ import core, { MeasureContext, PersonId, PersonUuid, + pickPrimarySocialId, RateLimiter, Ref, SocialId, @@ -45,30 +47,28 @@ import core, { Tx, TxCUD, TxOperations, - type WorkspaceUuid, type WorkspaceIds, - AccountUuid, - pickPrimarySocialId + type WorkspaceUuid } from '@hcengineering/core' import { Room } from '@hcengineering/love' import { WorkspaceInfoRecord } from '@hcengineering/server-ai-bot' import fs from 'fs' +import { Tiktoken } from 'js-tiktoken' import { WithId } from 'mongodb' import OpenAI from 'openai' -import { Tiktoken } from 'js-tiktoken' +import { countTokens } from '@hcengineering/openai' +import { getAccountClient } from '@hcengineering/server-client' import { StorageAdapter } from '@hcengineering/server-core' +import { jsonToMarkup, markupToText } from '@hcengineering/text' +import { markdownToMarkup } from '@hcengineering/text-markdown' import config from '../config' +import { DbStorage } from '../storage' import { HistoryRecord } from '../types' +import { getGlobalPerson } from '../utils/account' import { createChatCompletionWithTools, requestSummary } from '../utils/openai' import { connectPlatform } from '../utils/platform' import { LoveController } from './love' -import { DbStorage } from '../storage' -import { jsonToMarkup, markupToText } from '@hcengineering/text' -import { markdownToMarkup } from '@hcengineering/text-markdown' -import { countTokens } from '@hcengineering/openai' -import { getAccountClient } from '@hcengineering/server-client' -import { getGlobalPerson } from '../utils/account' export class WorkspaceClient { client: Client | undefined @@ -109,17 +109,28 @@ export class WorkspaceClient { private async ensureEmployee (client: Client): Promise { const me: Account = { uuid: this.personUuid, + targetWorkspace: this.wsIds.uuid, + workspaces: {}, + personalWorkspace: core.workspace.Personal, // TODO: Do we need a personal workspace here? role: AccountRole.User, primarySocialId: this.primarySocialId._id, socialIds: this.socialIds.map((it) => it._id), - fullSocialIds: this.socialIds + fullSocialIds: new Map(this.socialIds.map((it) => [it._id, it])), + socialIdsByValue: new Map(this.socialIds.map((it) => [it.value, it])) } - await ensureEmployee(this.ctx, me, client, this.socialIds, async () => await getGlobalPerson(this.token)) + await ensureEmployee( + this.ctx, + me, + client, + this.wsIds.uuid, + this.socialIds, + async () => await getGlobalPerson(this.token) + ) } private async initClient (): Promise { this.client = await connectPlatform(this.token, this.transactorUrl) - const opClient = new TxOperations(this.client, this.primarySocialId._id) + const opClient = new TxOperations(this.client, this.primarySocialId._id, this.wsIds.uuid) await this.ensureEmployee(this.client) await this.checkEmployeeInfo(opClient) @@ -134,7 +145,7 @@ export class WorkspaceClient { ) } - this.client.notify = (...txes: Tx[]) => { + this.client.notify = (txes: Tx[]) => { void this.txHandler(opClient, txes as TxCUD[]) } this.ctx.info('Initialized workspace', { workspace: this.wsIds }) diff --git a/services/analytics-collector/pod-analytics-collector/src/supportWsClient.ts b/services/analytics-collector/pod-analytics-collector/src/supportWsClient.ts index cd202d34aba..d9f348729e3 100644 --- a/services/analytics-collector/pod-analytics-collector/src/supportWsClient.ts +++ b/services/analytics-collector/pod-analytics-collector/src/supportWsClient.ts @@ -57,7 +57,7 @@ export class SupportWsClient extends WorkspaceClient { this.generalChannel = await createGeneralOnboardingChannel(this.ctx, client) if (this.client != null) { - this.client.notify = (...txes) => { + this.client.notify = (txes) => { this.handleTx(client, ...txes) } } diff --git a/services/analytics-collector/pod-analytics-collector/src/workspaceClient.ts b/services/analytics-collector/pod-analytics-collector/src/workspaceClient.ts index 19e07f4d0a4..bd7d992c188 100644 --- a/services/analytics-collector/pod-analytics-collector/src/workspaceClient.ts +++ b/services/analytics-collector/pod-analytics-collector/src/workspaceClient.ts @@ -47,7 +47,7 @@ export class WorkspaceClient { }) this.client = await connectPlatform(token) - return new TxOperations(this.client, core.account.System) + return new TxOperations(this.client, core.account.System, this.workspace) } async getPerson (personUuid: PersonUuid): Promise { diff --git a/services/calendar/pod-calendar/src/auth.ts b/services/calendar/pod-calendar/src/auth.ts index 3e99cdf3322..fa572aaaacd 100644 --- a/services/calendar/pod-calendar/src/auth.ts +++ b/services/calendar/pod-calendar/src/auth.ts @@ -62,7 +62,7 @@ export class AuthController { const mutex = await lock(`${state.workspace}:${state.userId}`) try { const client = await getClient(getWorkspaceToken(state.workspace)) - const txOp = new TxOperations(client, core.account.System) + const txOp = new TxOperations(client, core.account.System, state.workspace) const controller = new AuthController(ctx, accountClient, txOp, state) await controller.process(code) } finally { @@ -87,7 +87,7 @@ export class AuthController { const mutex = await lock(`${workspace}:${userId}`) try { const client = await getClient(getWorkspaceToken(workspace)) - const txOp = new TxOperations(client, core.account.System) + const txOp = new TxOperations(client, core.account.System, workspace) const controller = new AuthController(ctx, accountClient, txOp, { userId, workspace diff --git a/services/calendar/pod-calendar/src/calendar.ts b/services/calendar/pod-calendar/src/calendar.ts index a5e659a1719..56c47ec6176 100644 --- a/services/calendar/pod-calendar/src/calendar.ts +++ b/services/calendar/pod-calendar/src/calendar.ts @@ -39,7 +39,7 @@ export class CalendarClient { client: Client, private readonly workspace: WorkspaceClient ) { - this.client = new TxOperations(client, this.user.userId) + this.client = new TxOperations(client, this.user.userId, this.user.workspace) this.rateLimiter = getRateLimitter(this.user.email) const res = getGoogleClient() this.calendar = res.google diff --git a/services/calendar/pod-calendar/src/pushHandler.ts b/services/calendar/pod-calendar/src/pushHandler.ts index 2b28c4bd99e..2e4337edcd5 100644 --- a/services/calendar/pod-calendar/src/pushHandler.ts +++ b/services/calendar/pod-calendar/src/pushHandler.ts @@ -33,7 +33,7 @@ export class PushHandler { {}, async () => { const client = await getClient(getWorkspaceToken(token.workspace)) - const txOp = new TxOperations(client, core.account.System) + const txOp = new TxOperations(client, core.account.System, token.workspace) const res = getGoogleClient() res.auth.setCredentials(token) await IncomingSyncManager.push(this.ctx, this.accountClient, txOp, token, res.google, calendarId) diff --git a/services/calendar/pod-calendar/src/workspaceClient.ts b/services/calendar/pod-calendar/src/workspaceClient.ts index cf611b4bcb0..a2139dbe822 100644 --- a/services/calendar/pod-calendar/src/workspaceClient.ts +++ b/services/calendar/pod-calendar/src/workspaceClient.ts @@ -48,7 +48,7 @@ export class WorkspaceClient { static async run (ctx: MeasureContext, accountClient: AccountClient, workspace: WorkspaceUuid): Promise { const client = await getClient(getWorkspaceToken(workspace)) - const txOp = new TxOperations(client, core.account.System) + const txOp = new TxOperations(client, core.account.System, workspace) const instance = new WorkspaceClient(ctx, accountClient, txOp, workspace) await instance.init() @@ -142,7 +142,7 @@ export class WorkspaceClient { type: 'create' | 'update' | 'delete' ): Promise { const client = await getClient(getWorkspaceToken(workspace)) - const txOp = new TxOperations(client, core.account.System) + const txOp = new TxOperations(client, core.account.System, workspace) const token = await getTokenByEvent(accountClient, txOp, event, workspace) if (token != null) { const instance = new WorkspaceClient(ctx, accountClient, txOp, workspace) diff --git a/services/export/pod-export/src/server.ts b/services/export/pod-export/src/server.ts index f3d3ae6aa44..f914381c345 100644 --- a/services/export/pod-export/src/server.ts +++ b/services/export/pod-export/src/server.ts @@ -189,7 +189,7 @@ export function createServer (storageConfig: StorageConfiguration): { app: Expre const platformClient = await createPlatformClient(token) const account = decodedToken.account - const txOperations = new TxOperations(platformClient, socialId) + const txOperations = new TxOperations(platformClient, socialId, wsIds.uuid) res.status(200).send({ message: 'Export started' }) @@ -242,7 +242,7 @@ export function createServer (storageConfig: StorageConfiguration): { app: Expre } const platformClient = await createPlatformClient(token) - const txOperations = new TxOperations(platformClient, socialId) + const txOperations = new TxOperations(platformClient, socialId, wsIds.uuid) const exportDir = await fs.mkdtemp(join(tmpdir(), 'export-')) try { diff --git a/services/github/pod-github/src/platform.ts b/services/github/pod-github/src/platform.ts index c0168ee770b..8972682e90c 100644 --- a/services/github/pod-github/src/platform.ts +++ b/services/github/pod-github/src/platform.ts @@ -252,13 +252,13 @@ export class PlatformWorker { const oldWorker = this.clients.get(oldWorkspace) as GithubWorker if (oldWorker !== undefined) { - await this.removeInstallationFromWorkspace(oldWorker.client, installationId) + await this.removeInstallationFromWorkspace(oldWorker.client, installationId, oldWorkspace) await oldWorker.reloadRepositories(installationId) } else { let client: Client | undefined try { ;({ client } = await createPlatformClient(oldWorkspace, 30000)) - await this.removeInstallationFromWorkspace(oldWorker, installationId) + await this.removeInstallationFromWorkspace(oldWorker, installationId, oldWorkspace) await client.close() } catch (err: any) { ctx.error('failed to remove old installation from workspace', { workspace: oldWorkspace, installationId }) @@ -307,10 +307,14 @@ export class PlatformWorker { this.triggerCheckWorkspaces() } - private async removeInstallationFromWorkspace (client: Client, installationId: number): Promise { + private async removeInstallationFromWorkspace ( + client: Client, + installationId: number, + workspace: WorkspaceUuid + ): Promise { const wsIntegerations = await client.findAll(github.class.GithubIntegration, { installationId }) - const ops = new TxOperations(client, core.account.System) + const ops = new TxOperations(client, core.account.System, workspace) for (const intValue of wsIntegerations) { await ops.remove(intValue) } @@ -427,7 +431,7 @@ export class PlatformWorker { shouldClose = true ;({ client: platformClient } = await createPlatformClient(payload.workspace, 30000)) } - const client = new TxOperations(platformClient, payload.accountId) + const client = new TxOperations(platformClient, payload.accountId, payload.workspace) let personAuths = await client.findAll(github.class.GithubAuthentication, { attachedTo: payload.accountId @@ -790,7 +794,7 @@ export class PlatformWorker { integeration.enabled = false integeration.synchronized = new Set() - await this.removeInstallationFromWorkspace(worker._client, installId) + await this.removeInstallationFromWorkspace(worker._client, installId, worker.workspace.uuid) await worker._client.remove(integeration.integration) } diff --git a/services/github/pod-github/src/sync/pullrequests.ts b/services/github/pod-github/src/sync/pullrequests.ts index b9a018495fc..585d6dec5d9 100644 --- a/services/github/pod-github/src/sync/pullrequests.ts +++ b/services/github/pod-github/src/sync/pullrequests.ts @@ -1164,6 +1164,7 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS return { ...value, + _uuid: info._uuid, _id: prId, _class: github.class.GithubPullRequest, space: info.space as any, diff --git a/services/github/pod-github/src/sync/repository.ts b/services/github/pod-github/src/sync/repository.ts index 4b338cee1d7..89642baf84f 100644 --- a/services/github/pod-github/src/sync/repository.ts +++ b/services/github/pod-github/src/sync/repository.ts @@ -349,7 +349,7 @@ export class RepositorySyncMapper implements DocSyncManager { ): Promise { // We need to update urls for all sync documents belong to this repository. - const derivedClient = new TxOperations(this.client, core.account.System, true) + const derivedClient = new TxOperations(this.client, core.account.System, this.provider.getWorkspaceId(), true) const processingId = generateId() // Wait previous sync to finish diff --git a/services/github/pod-github/src/worker.ts b/services/github/pod-github/src/worker.ts index 23699c68c6f..7d00784f880 100644 --- a/services/github/pod-github/src/worker.ts +++ b/services/github/pod-github/src/worker.ts @@ -394,7 +394,7 @@ export class GithubWorker implements IntegrationManager { const token = generateToken(systemAccountUuid, this.workspace.uuid, { service: 'github', mode: 'github' }) this.accountClient = getAccountClient(token, 30000) - this._client = new TxOperations(this.client, core.account.System) + this._client = new TxOperations(this.client, core.account.System, this.workspace.uuid) this.liveQuery = new LiveQuery(client) this.repositoryManager = new RepositorySyncMapper(this.ctx.newChild('repository', {}), this._client, this.app) @@ -579,7 +579,7 @@ export class GithubWorker implements IntegrationManager { try { await this.platform.checkRefreshToken(record, true) - const ops = new TxOperations(this.client, account._id) + const ops = new TxOperations(this.client, account._id, this.workspace.uuid) await syncUser(ctx, record, userAuth, ops, account._id) } catch (err: any) { try { @@ -876,7 +876,7 @@ export class GithubWorker implements IntegrationManager { } private registerNotifyHandler (): void { - this.client.notify = (...tx: Tx[]) => { + this.client.notify = (tx: Tx[]) => { void this.liveQuery .tx(...tx) .then(() => { @@ -1007,7 +1007,7 @@ export class GithubWorker implements IntegrationManager { const byRepository = this.groupByRepository(docs) const ints = Array.from(this.integrations.values()) - const derivedClient = new TxOperations(this.client, core.account.System, true) + const derivedClient = new TxOperations(this.client, core.account.System, this.workspace.uuid, true) for (const [repository, docs] of byRepository.entries()) { const integration = ints.find((it) => repositories.find((q) => q._id === repository)) if (integration?.octokit === undefined) { @@ -1073,7 +1073,7 @@ export class GithubWorker implements IntegrationManager { // We need to apply migrations if required. const migrations = await this.client.findAll(core.class.MigrationState, {}) - const derivedClient = new TxOperations(this.client, core.account.System, true) + const derivedClient = new TxOperations(this.client, core.account.System, this.workspace.uuid, true) if (migrations.find((it) => it.plugin === githubId && it.state === key) === undefined) { let modifiedOn = 0 @@ -1378,7 +1378,7 @@ export class GithubWorker implements IntegrationManager { _id: { $in: attachedTo } }) - const derivedClient = new TxOperations(this.client, core.account.System, true) + const derivedClient = new TxOperations(this.client, core.account.System, this.workspace.uuid, true) const docsMap = new Map, Doc>(externalDocs.map((it) => [it._id as Ref, it])) const orderedSyncInfo = [...syncInfo] @@ -1585,7 +1585,7 @@ export class GithubWorker implements IntegrationManager { if (!enabled) { return } - const derivedClient = new TxOperations(this.client, core.account.System, true) + const derivedClient = new TxOperations(this.client, core.account.System, this.workspace.uuid, true) const { projects, repositories } = await this.collectActiveProjects() @@ -1673,7 +1673,7 @@ export class GithubWorker implements IntegrationManager { if (integration === undefined) { return } - const derivedClient = new TxOperations(this.client, core.account.System, true) + const derivedClient = new TxOperations(this.client, core.account.System, this.workspace.uuid, true) for (const { _class, mapper } of this.mappers) { if (_class.includes(requestClass)) { try { @@ -1723,7 +1723,7 @@ export class GithubWorker implements IntegrationManager { return undefined } - await GithubWorker.checkIntegrations(client, installations) + await GithubWorker.checkIntegrations(client, installations, workspace.uuid) const worker = new GithubWorker( ctx, @@ -1746,11 +1746,15 @@ export class GithubWorker implements IntegrationManager { } } - static async checkIntegrations (client: Client, installations: Map): Promise { + static async checkIntegrations ( + client: Client, + installations: Map, + workspace: WorkspaceUuid + ): Promise { const wsIntegerations = await client.findAll(github.class.GithubIntegration, {}) for (const intValue of wsIntegerations) { if (!installations.has(intValue.installationId)) { - const ops = new TxOperations(client, core.account.System) + const ops = new TxOperations(client, core.account.System, workspace) await ops.remove(intValue) } } diff --git a/services/gmail/pod-gmail/src/__tests__/attachments.test.ts b/services/gmail/pod-gmail/src/__tests__/attachments.test.ts index 34983df25d6..9e86305d6c5 100644 --- a/services/gmail/pod-gmail/src/__tests__/attachments.test.ts +++ b/services/gmail/pod-gmail/src/__tests__/attachments.test.ts @@ -65,6 +65,7 @@ describe('AttachmentHandler', () => { } as unknown as TxOperations const getBaseAttachment = (): Attachment => ({ + _uuid: 'test-uuid' as WorkspaceUuid, _id: 'test-attachment' as Ref, name: 'test.txt', type: 'text/plain', diff --git a/services/gmail/pod-gmail/src/gmail.ts b/services/gmail/pod-gmail/src/gmail.ts index a768a1e1e64..a7da486254d 100644 --- a/services/gmail/pod-gmail/src/gmail.ts +++ b/services/gmail/pod-gmail/src/gmail.ts @@ -109,9 +109,11 @@ export class GmailClient { private socialId: SocialId ) { this.email = email + this.integrationToken = serviceToken(wsInfo.workspace) this.tokenStorage = new TokenStorage(this.ctx, wsInfo.workspace, this.integrationToken) - this.client = new TxOperations(client, this.socialId._id) + this.client = new TxOperations(client, this.socialId._id, wsInfo.workspace) + this.account = this.user.userId this.attachmentHandler = new AttachmentHandler(ctx, wsInfo, storageAdapter, this.gmail, this.client) const keyValueClient = getKvsClient(this.integrationToken) diff --git a/services/gmail/pod-gmail/src/message/v1/message.ts b/services/gmail/pod-gmail/src/message/v1/message.ts index ff9be48cf14..de3feda23e5 100644 --- a/services/gmail/pod-gmail/src/message/v1/message.ts +++ b/services/gmail/pod-gmail/src/message/v1/message.ts @@ -70,7 +70,7 @@ export class MessageManagerV1 implements IMessageManager { const channels = this.findChannels(res) if (channels.length === 0) return const attachments = await this.attachmentHandler.getPartFiles(message.data.payload, message.data.id ?? '') - const factory = new TxFactory(this.socialId) + const factory = new TxFactory(this.socialId, this.client.workspaceUuid) for (const channel of channels) { const current = await this.client.findOne(gmail.class.Message, { messageId: res.messageId, diff --git a/services/gmail/pod-gmail/src/workspaceClient.ts b/services/gmail/pod-gmail/src/workspaceClient.ts index 08d9fc546b9..1f14098ff05 100644 --- a/services/gmail/pod-gmail/src/workspaceClient.ts +++ b/services/gmail/pod-gmail/src/workspaceClient.ts @@ -44,7 +44,7 @@ export class WorkspaceClient { private messageSubscribed: boolean = false private channels: Map = new Map() private channelsById: Map, Channel> = new Map, Channel>() - private readonly txHandlers: ((...tx: Tx[]) => Promise)[] = [] + private readonly txHandlers: ((tx: Tx[]) => Promise)[] = [] private client!: Client private readonly clients: Map = new Map() @@ -109,7 +109,7 @@ export class WorkspaceClient { } if (!deleted && socialIds.length > 0) { this.ctx.info('Clean up integrations without clients') - const tx = new TxOperations(this.client, socialIds[0]._id) + const tx = new TxOperations(this.client, socialIds[0]._id, this.workspace) await cleanIntegrations(this.ctx, tx, userId, this.workspace) } @@ -133,8 +133,8 @@ export class WorkspaceClient { const token = generateToken(systemAccountUuid, workspace, { service: 'gmail' }) this.ctx.info('Init client', { workspaceUuid: workspace }) const client = await getClient(token) - client.notify = (...tx: Tx[]) => { - void this.txHandler(...tx) + client.notify = (tx: Tx[]) => { + void this.txHandler(tx) } this.client = client @@ -142,10 +142,10 @@ export class WorkspaceClient { return this.client } - private async txHandler (...tx: Tx[]): Promise { + private async txHandler (tx: Tx[]): Promise { await Promise.all( this.txHandlers.map(async (handler) => { - await handler(...tx) + await handler(tx) }) ) } @@ -154,7 +154,7 @@ export class WorkspaceClient { async subscribeMessages (): Promise { if (this.messageSubscribed) return - this.txHandlers.push(async (...txes: Tx[]) => { + this.txHandlers.push(async (txes: Tx[]) => { for (const tx of txes) { await this.txMessageHandler(tx) } @@ -243,7 +243,7 @@ export class WorkspaceClient { }) ) - this.txHandlers.push(async (...txes: Tx[]) => { + this.txHandlers.push(async (txes: Tx[]) => { for (const tx of txes) { await this.txChannelHandler(tx) } @@ -342,7 +342,7 @@ export class WorkspaceClient { for (const person of removedEmployees) { await this.deactivateUser(person) } - this.txHandlers.push(async (...txes: Tx[]) => { + this.txHandlers.push(async (txes: Tx[]) => { for (const tx of txes) { await this.txEmployeeHandler(tx) } diff --git a/services/love/src/workspaceClient.ts b/services/love/src/workspaceClient.ts index 1eaf93f6b21..d47c8a201a0 100644 --- a/services/love/src/workspaceClient.ts +++ b/services/love/src/workspaceClient.ts @@ -49,7 +49,7 @@ export class WorkspaceClient { private async initClient (workspace: WorkspaceUuid): Promise { const token = generateToken(systemAccountUuid, workspace, { service: 'love' }) const client = await getClient(token) - this.client = new TxOperations(client, core.account.System) + this.client = new TxOperations(client, core.account.System, workspace) return this.client } diff --git a/services/msg2file/src/worker.ts b/services/msg2file/src/worker.ts index 54880524023..f8207f6ddf5 100644 --- a/services/msg2file/src/worker.ts +++ b/services/msg2file/src/worker.ts @@ -198,7 +198,7 @@ async function applyPatchesToGroup ( parsedFile.metadata.toDate, updatedMessages.length ) - await removeGroup(client, group.card, group.blobId) + await removeGroup(client, group.card, group.blobId as any) await removePatches(db, workspace, card, Array.from(patchesByMessage.keys())) await removeFile(storage, ctx, workspace, group.blobId) } catch (error) { @@ -367,7 +367,7 @@ async function pushMessagesToGroup ( .sort((a, b) => a.created.getTime() - b.created.getTime()) const blob = await uploadGroupFile(ctx, storage, workspace, parsedFile.metadata, newMessages) await removeFile(storage, ctx, workspace, group.blobId) - await removeGroup(client, group.card, group.blobId) + await removeGroup(client, group.card, group.blobId as any) await createGroup( client, group.card, diff --git a/services/telegram-bot/pod-telegram-bot/src/workspace.ts b/services/telegram-bot/pod-telegram-bot/src/workspace.ts index d7893f74543..e2d054d10fe 100644 --- a/services/telegram-bot/pod-telegram-bot/src/workspace.ts +++ b/services/telegram-bot/pod-telegram-bot/src/workspace.ts @@ -149,7 +149,7 @@ export class WorkspaceClient { text: string, files: TelegramFileInfo[] ): Promise { - const txFactory = new TxFactory(socialId) + const txFactory = new TxFactory(socialId, this.workspace) const hierarchy = this.hierarchy const isAvailable = await this.isReplyAvailable(account, message) @@ -298,7 +298,7 @@ export class WorkspaceClient { return undefined } - const txFactory = new TxFactory(socialId) + const txFactory = new TxFactory(socialId, this.workspace) const messageId = generateId() const attachments = await this.createAttachments( txFactory, diff --git a/services/telegram/pod-telegram/src/workspace.ts b/services/telegram/pod-telegram/src/workspace.ts index 75cfdd0e41a..7ed8a11eb23 100644 --- a/services/telegram/pod-telegram/src/workspace.ts +++ b/services/telegram/pod-telegram/src/workspace.ts @@ -1,13 +1,14 @@ import attachment, { Attachment } from '@hcengineering/attachment' -import contact, { Channel, Contact as PContact, getFirstName, getLastName } from '@hcengineering/contact' +import contact, { Channel, getFirstName, getLastName, Contact as PContact } from '@hcengineering/contact' import core, { - PersonId, Blob, Client, Doc, Hierarchy, MeasureContext, + PersonId, Ref, + systemAccountUuid, Tx, TxCreateDoc, TxCUD, @@ -16,8 +17,7 @@ import core, { TxProcessor, TxRemoveDoc, TxUpdateDoc, - systemAccountUuid, - WorkspaceDataId + type WorkspaceUuid } from '@hcengineering/core' import type { StorageAdapter } from '@hcengineering/server-core' import { generateToken } from '@hcengineering/server-token' @@ -50,13 +50,13 @@ export class WorkspaceWorker { private readonly ctx: MeasureContext, private readonly client: Client, private readonly storageAdapter: StorageAdapter, - private readonly workspace: WorkspaceDataId, + private readonly workspace: WorkspaceUuid, private readonly userStorage: Collection, private readonly lastMsgStorage: Collection, private readonly channelsStorage: Collection ) { // eslint-disable-next-line - this.client.notify = (...tx) => void this.txHandler(...tx) + this.client.notify = (tx) => void this.txHandler(tx) this.hierarchy = this.client.getHierarchy() } @@ -81,7 +81,7 @@ export class WorkspaceWorker { this.clients.set(rec.phone, { conn }) } - private async txHandler (...txes: Tx[]): Promise { + private async txHandler (txes: Tx[]): Promise { for (const tx of txes) { switch (tx._class) { case core.class.TxCreateDoc: { @@ -140,12 +140,12 @@ export class WorkspaceWorker { static async create ( ctx: MeasureContext, storageAdapter: StorageAdapter, - workspace: WorkspaceDataId, + workspace: WorkspaceUuid, userStorage: Collection, lastMsgStorage: Collection, channelsStorage: Collection ): Promise { - const token = generateToken(systemAccountUuid, workspace as any, { service: 'telegram' }) // TODO: FIXME + const token = generateToken(systemAccountUuid, workspace, { service: 'telegram' }) // TODO: FIXME const client = await createPlatformClient(token) const worker = new WorkspaceWorker( @@ -285,7 +285,7 @@ export class WorkspaceWorker { return } - const txOp = new TxOperations(this.client, core.account.System) + const txOp = new TxOperations(this.client, core.account.System, this.workspace) if (signOut) { console.log('Signout', this.workspace, phone) @@ -358,7 +358,7 @@ export class WorkspaceWorker { } } - const factory = new TxFactory(msg.modifiedBy) + const factory = new TxFactory(msg.modifiedBy, this.workspace) const tx = factory.createTxUpdateDoc(msg._class, msg.space, msg._id, { status: 'sent' }) @@ -543,7 +543,7 @@ export class WorkspaceWorker { } private makePlatformMsg (event: Event, record: UserRecord, channel: Channel): TxCUD { - const factory = new TxFactory(record.userId as any) // TODO: FIXME + const factory = new TxFactory(record.userId as any, this.workspace) // TODO: FIXME const modifiedOn = event.msg.date * 1000 const tx = factory.createTxCollectionCUD( channel._class, @@ -657,7 +657,7 @@ export class WorkspaceWorker { createTx: TxCUD ): Promise { const msg = TxProcessor.createDoc2Doc(createTx as TxCreateDoc) - const factory = new TxFactory(record.userId as any) // TODO: FIXME + const factory = new TxFactory(record.userId as any, this.workspace) // TODO: FIXME const files = await getFiles(event.msg) for (const file of files) { try { diff --git a/ws-tests/api-tests/src/__tests__/mem.test.ts b/ws-tests/api-tests/src/__tests__/mem.test.ts new file mode 100644 index 00000000000..4f888ecd64c --- /dev/null +++ b/ws-tests/api-tests/src/__tests__/mem.test.ts @@ -0,0 +1,42 @@ +import core, { generateId, Hierarchy, MeasureMetricsContext, ModelDb, platformNow } from '@hcengineering/core' +import contact, { AvatarType, type Contact } from '@hcengineering/contact' +import { faker } from '@faker-js/faker' +import buildModel from '@hcengineering/model-all' + +const model = buildModel().getTxes() +describe('mem-objects', () => { + it('check add session', async () => { + const hierarchy = new Hierarchy() + for (const tx of model) { + hierarchy.tx(tx) + } + const memdb = new ModelDb(hierarchy) + memdb.addTxes(new MeasureMetricsContext('test', {}), model, false) + + const before = process.memoryUsage().heapUsed + for (let i = 0; i < 100000; i++) { + const d: Contact = { + _class: contact.class.Contact, + _id: generateId(), + _uuid: core.workspace.Any, + space: core.space.Model, + modifiedBy: core.account.System, + modifiedOn: Date.now(), + name: faker.person.fullName(), + avatarType: AvatarType.GRAVATAR, + avatarProps: { + url: faker.internet.username() + } + } + memdb.addDoc(d) + } + const after = process.memoryUsage().heapUsed + console.log('memdb size', after - before) + + const t0 = platformNow() + const t1 = await memdb.findAll(contact.class.Contact, { + name: faker.person.fullName() + }) + console.log('findAll', platformNow() - t0, t1.length) + }) +}) diff --git a/ws-tests/api-tests/src/__tests__/rest.test.ts b/ws-tests/api-tests/src/__tests__/rest.test.ts index a0698bf884d..da584c82f4e 100644 --- a/ws-tests/api-tests/src/__tests__/rest.test.ts +++ b/ws-tests/api-tests/src/__tests__/rest.test.ts @@ -13,6 +13,7 @@ // limitations under the License. // +import { type AccountClient, getClient as getAccountClient } from '@hcengineering/account-client' import { createRestClient, createRestTxOperations, @@ -21,6 +22,8 @@ import { type RestClient, type WorkspaceToken } from '@hcengineering/api-client' +import chunter from '@hcengineering/chunter' +import contact, { ensureEmployee, type Person, type SocialIdentityRef } from '@hcengineering/contact' import core, { buildSocialIdString, generateId, @@ -28,17 +31,14 @@ import core, { type PersonId, type PersonUuid, pickPrimarySocialId, - SocialIdType, - systemAccountUuid, type Ref, type SocialId, + SocialIdType, type Space, + systemAccountUuid, type TxCreateDoc, type TxOperations } from '@hcengineering/core' -import { type AccountClient, getClient as getAccountClient } from '@hcengineering/account-client' -import chunter from '@hcengineering/chunter' -import contact, { ensureEmployee, type SocialIdentityRef, type Person } from '@hcengineering/contact' import { generateToken } from '@hcengineering/server-token' describe('rest-api-server', () => { @@ -82,12 +82,17 @@ describe('rest-api-server', () => { testCtx, { uuid: apiWorkspace1.info.account, + workspaces: {}, + personalWorkspace: apiWorkspace1.workspaceId, role: apiWorkspace1.info.role, + targetWorkspace: apiWorkspace1.workspaceId, primarySocialId: pickPrimarySocialId(socialIds)._id, socialIds: socialIds.map((si) => si._id), - fullSocialIds: socialIds + fullSocialIds: new Map(socialIds.map((i) => [i._id, i])), + socialIdsByValue: new Map(socialIds.map((i) => [i.value, i])) }, connect(), + apiWorkspace1.workspaceId, socialIds, async () => person ) @@ -96,12 +101,17 @@ describe('rest-api-server', () => { testCtx, { uuid: apiWorkspace2.info.account, + workspaces: {}, role: apiWorkspace2.info.role, + personalWorkspace: apiWorkspace2.workspaceId, + targetWorkspace: apiWorkspace2.workspaceId, primarySocialId: pickPrimarySocialId(socialIds)._id, socialIds: socialIds.map((si) => si._id), - fullSocialIds: socialIds + fullSocialIds: new Map(socialIds.map((i) => [i._id, i])), + socialIdsByValue: new Map(socialIds.map((i) => [i.value, i])) }, connect(apiWorkspace2), + apiWorkspace2.workspaceId, socialIds, async () => person ) @@ -124,7 +134,7 @@ describe('rest-api-server', () => { const account = await conn.getAccount() expect(account.primarySocialId).toEqual(expect.any(String)) - expect(account.role).toBe('USER') + // expect(account.role).toBe('USER') // expect(account.space).toBe(core.space.Model) // expect(account.modifiedBy).toBe(core.account.System) // expect(account.createdBy).toBe(core.account.System) @@ -183,6 +193,7 @@ describe('rest-api-server', () => { const account = await conn.getAccount() const spaceName = generateId() const tx: TxCreateDoc = { + _uuid: apiWorkspace1.workspaceId, _class: core.class.TxCreateDoc, space: core.space.Tx, _id: generateId(),