Few fixes

Signed-off-by: Andrey Sobolev <[email protected]>

Author: haiodo

dev/docker-compose.yaml

@@ -301,6 +301,7 @@ services:
     environment:
       - QUEUE_CONFIG=${QUEUE_CONFIG}
       - REGION=
+      - ENDPOINT=ws://huly.local:3333
       - SERVER_PORT=3333
       - SERVER_SECRET=secret
       - ENABLE_COMPRESSION=true
@@ -337,6 +338,7 @@ services:
       - QUEUE_CONFIG=${QUEUE_CONFIG}
       - SERVER_PORT=3332
       - REGION=cockroach
+      - ENDPOINT=ws://huly.local:3332
       - SERVER_SECRET=secret
       - ENABLE_COMPRESSION=true
       - FULLTEXT_URL=http://huly.local:4702

dev/tool/src/index.ts

@@ -1363,14 +1363,15 @@ export function devTool (
 
   program
     .command('backup-s3-download <bucketName> <dirName> <storeIn>')
+    .option('-s, --skip <skip>', 'A list of ; separated domain names to skip during backup', '')
     .description('Download a full backup from s3 to local dir')
-    .action(async (bucketName: string, dirName: string, storeIn: string, cmd) => {
+    .action(async (bucketName: string, dirName: string, storeIn: string, cmd: { skip: string }) => {
       const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE)
       const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0])
       const backupIds = { uuid: bucketName as WorkspaceUuid, dataId: bucketName as WorkspaceDataId, url: '' }
       try {
         const storage = await createStorageBackupStorage(toolCtx, storageAdapter, backupIds, dirName)
-        await backupDownload(storage, storeIn)
+        await backupDownload(storage, storeIn, new Set(cmd.skip.split(';')))
       } catch (err: any) {
         toolCtx.error('failed to size backup', { err })
       }

plugins/client-resources/src/connection.ts

@@ -512,7 +512,7 @@ class Connection implements ClientConnection {
         }
       }
       this.handlers.forEach((handler) => {
-        handler(...txArr)
+        handler(txArr)
       })
 
       clearTimeout(this.incomingTimer)

pods/server/src/__start.ts

@@ -25,6 +25,8 @@ import serverToken from '@hcengineering/server-token'
 import { join } from 'path'
 import { start } from '.'
 import { profileStart, profileStop } from './inspector'
+import client from '@hcengineering/client'
+import { WebSocket } from 'ws'
 
 configureAnalytics(process.env.SENTRY_DSN, {})
 Analytics.setTag('application', 'transactor')
@@ -87,6 +89,10 @@ setMetadata(serverCalendar.metadata.EndpointURL, process.env.CALENDAR_URL)
 const region = process.env.REGION ?? ''
 const endpointName = process.env.ENDPOINT_NAME ?? `ws://huly.local:${config.serverPort}`
 
+setMetadata(client.metadata.ClientSocketFactory, (url) => {
+  return new WebSocket(url) as any
+})
+
 const { shutdown, sessionManager } = start(metricsContext, config.dbUrl, {
   fulltextUrl: config.fulltextUrl,
   storageConfig,

server/account/src/operations.ts

@@ -27,6 +27,7 @@ import core, {
   type PersonId,
   type PersonUuid,
   SocialIdType,
+  systemAccount,
   systemAccountUuid,
   type WorkspaceMemberInfo,
   type WorkspaceUuid
@@ -1448,12 +1449,22 @@ export async function getLoginWithWorkspaceInfo (
     throw new PlatformError(new Status(Severity.ERROR, platform.status.InternalServerError, {}))
   }
 
-  const account = await db.account.findOne({ uuid: accountUuid })
+  const account = (accountUuid === systemAccountUuid) ? systemAccount : await db.account.findOne({ uuid: accountUuid })
 
   if (account == null) {
     throw new PlatformError(new Status(Severity.ERROR, platform.status.InternalServerError, {}))
   }
 
+  if (accountUuid === systemAccountUuid) {
+    return {
+      account: accountUuid,
+      name: 'System',
+      personalWorkspace: core.workspace.Any,
+      workspaces: {},
+      socialIds: []
+    }
+  }
+
   const personalWorkspace = await getPersonalWorkspace(db, account)
 
   const userWorkspaces = (await db.getAccountWorkspaces(accountUuid)).filter((it) => isActiveMode(it.status.mode))

server/backup/src/backup.ts

@@ -1645,7 +1645,7 @@ export async function backupSize (storage: BackupStorage): Promise<void> {
 /**
  * @public
  */
-export async function backupDownload (storage: BackupStorage, storeIn: string): Promise<void> {
+export async function backupDownload (storage: BackupStorage, storeIn: string, skipDomains: Set<string>): Promise<void> {
   const infoFile = 'backup.json.gz'
   const sizeFile = 'backup.size.gz'
 
@@ -1655,15 +1655,16 @@ export async function backupDownload (storage: BackupStorage, storeIn: string):
   let size = 0
 
   const backupInfo: BackupInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(infoFile))).toString())
-  console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)
+  console.log('Downloading workspace:', backupInfo.workspace ?? '', backupInfo.version, backupInfo.snapshots.length)
 
   let sizeInfo: Record<string, number> = {}
   if (await storage.exists(sizeFile)) {
+    console.log('Parse size file')
     sizeInfo = JSON.parse(gunzipSync(new Uint8Array(await storage.loadFile(sizeFile))).toString())
   }
-  console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)
 
-  const addFileSize = async (file: string | undefined | null, force: boolean = false): Promise<void> => {
+  const downloadFile = async (file: string | undefined | null, force: boolean = false): Promise<void> => {
+    console.log('Download file', file)
     if (file != null) {
       const target = join(storeIn, file)
       const dir = dirname(target)
@@ -1700,17 +1701,21 @@ export async function backupDownload (storage: BackupStorage, storeIn: string):
 
   // Let's calculate data size for backup
   for (const sn of backupInfo.snapshots) {
-    for (const [, d] of Object.entries(sn.domains)) {
-      await addFileSize(d.snapshot)
+    console.log('processing', sn.date)
+    for (const [k, d] of Object.entries(sn.domains)) {
+      if (skipDomains.has(k)) {
+        continue
+      }
+      await downloadFile(d.snapshot)
       for (const snp of d.snapshots ?? []) {
-        await addFileSize(snp)
+        await downloadFile(snp)
       }
       for (const snp of d.storage ?? []) {
-        await addFileSize(snp)
+        await downloadFile(snp)
       }
     }
   }
-  await addFileSize(infoFile, true)
+  await downloadFile(infoFile, true)
 
   console.log('Backup size', size / (1024 * 1024), 'Mb')
 }

server/server/src/sessionManager.ts

@@ -519,7 +519,15 @@ export class TSessionManager implements SessionManager {
     let account: LoginInfoWithWorkspaces | undefined
 
     try {
-      account = await this.getLoginWithWorkspaceInfo(rawToken)
+      account = (token.account === systemAccountUuid)
+        ? {
+            account: token.account,
+            name: 'System',
+            personalWorkspace: core.workspace.Any,
+            workspaces: {},
+            socialIds: []
+          }
+        : await this.getLoginWithWorkspaceInfo(rawToken)
     } catch (err: any) {
       ctx.error('failed to get login info', { err })
     }
@@ -541,6 +549,10 @@ export class TSessionManager implements SessionManager {
     }
 
     let targetInfo = account.workspaces[token.workspace]
+    let targetWorkspace: WorkspaceUuid | undefined
+    if (targetInfo !== undefined) {
+      targetWorkspace = token.workspace
+    }
 
     if (targetInfo === undefined && token.workspace != null && token.workspace !== '') {
       // In case of guest or system account
@@ -567,7 +579,9 @@ export class TSessionManager implements SessionManager {
       (ctx.parent ?? ctx).newChild('🧲 session', {}),
       token,
       ws,
-      new Set(Object.keys(account.workspaces) as WorkspaceUuid[]),
+      targetInfo !== undefined && targetWorkspace !== undefined
+        ? new Set([targetWorkspace])
+        : new Set(Object.keys(account.workspaces) as WorkspaceUuid[]),
       accountRef,
       account,
       token.extra?.mode === 'backup'
@@ -599,10 +613,10 @@ export class TSessionManager implements SessionManager {
             // Ignore
           })
         }
+        await workspace.addSession(session)
         if (open !== undefined) {
           await open
         }
-        await workspace.addSession(session)
 
         if (accountUuid !== systemAccountUuid && accountUuid !== guestAccount) {
           await this.usersProducer.send(workspace.wsId.uuid, [

ws-tests/api-tests/src/__tests__/mem.test.ts

@@ -0,0 +1,42 @@
+import core, { generateId, Hierarchy, MeasureMetricsContext, ModelDb, platformNow } from '@hcengineering/core'
+import contact, { AvatarType, type Contact } from '@hcengineering/contact'
+import { faker } from '@faker-js/faker'
+import buildModel from '@hcengineering/model-all'
+
+const model = buildModel().getTxes()
+describe('mem-objects', () => {
+  it('check add session', async () => {
+    const hierarchy = new Hierarchy()
+    for (const tx of model) {
+      hierarchy.tx(tx)
+    }
+    const memdb = new ModelDb(hierarchy)
+    memdb.addTxes(new MeasureMetricsContext('test', {}), model, false)
+
+    const before = process.memoryUsage().heapUsed
+    for (let i = 0; i < 100000; i++) {
+      const d: Contact = {
+        _class: contact.class.Contact,
+        _id: generateId(),
+        _uuid: core.workspace.Any,
+        space: core.space.Model,
+        modifiedBy: core.account.System,
+        modifiedOn: Date.now(),
+        name: faker.person.fullName(),
+        avatarType: AvatarType.GRAVATAR,
+        avatarProps: {
+          url: faker.internet.username()
+        }
+      }
+      memdb.addDoc(d)
+    }
+    const after = process.memoryUsage().heapUsed
+    console.log('memdb size', after - before)
+
+    const t0 = platformNow()
+    const t1 = await memdb.findAll(contact.class.Contact, {
+      name: faker.person.fullName()
+    })
+    console.log('findAll', platformNow() - t0, t1.length)
+  })
+})