Skip to content

UBERF-9747 #8867

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@
"UPLOAD_URL": "/files",
"AI_BOT_URL": "http://localhost:4010",
"STATS_URL": "http://huly.local:4900",
"QUEUE_CONFIG": "localhost:19092"
"QUEUE_CONFIG": "localhost:19092",
"RATE_LIMIT_MAX": "25000"
},
"runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
"runtimeVersion": "20",
Expand All @@ -120,9 +121,9 @@
"args": ["src/__start.ts"],
"env": {
"FULLTEXT_URL": "http://localhost:4710",
// "DB_URL": "mongodb://localhost:27018",
"DB_URL": "mongodb://localhost:27018",
// "DB_URL": "postgresql://postgres:example@localhost:5432",
"DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
// "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
// "GREEN_URL": "http://huly.local:6767?token=secret",
"SERVER_PORT": "3335",
"METRICS_CONSOLE": "false",
Expand Down Expand Up @@ -183,7 +184,7 @@
"request": "launch",
"args": ["src/__start.ts"],
"env": {
"MONGO_URL": "mongodb://localhost:27017",
// "MONGO_URL": "mongodb://localhost:27017",
// "DB_URL": "mongodb://localhost:27017",
"DB_URL": "postgresql://[email protected]:26257/defaultdb?sslmode=disable",
"SERVER_SECRET": "secret",
Expand Down Expand Up @@ -217,12 +218,13 @@
"args": ["src/__start.ts"],
"env": {
// "MONGO_URL": "mongodb://localhost:27018",
// "DB_URL": "mongodb://localhost:27018",
"DB_URL": "mongodb://localhost:27018",
// "DB_URL": "postgresql://postgres:example@localhost:5432",
"DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
// "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
"SERVER_SECRET": "secret",
"REGION_INFO": "|Mongo;pg|Postgres;cockroach|CockroachDB",
"TRANSACTOR_URL": "ws://transactor:3334;ws://localhost:3334",
"QUEUE_CONFIG": "huly.local:19092",
"REGION_INFO": "|America;europe|Europe",
"TRANSACTOR_URL": "ws://huly.local:3334;ws://huly.local:3334,ws://huly.local:3335;ws://huly.local:3335;europe",
"ACCOUNTS_URL": "http://localhost:3003",
"ACCOUNT_PORT": "3003",
"FRONT_URL": "http://localhost:8083",
Expand Down
1 change: 0 additions & 1 deletion common/config/rush/command-line.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
*/
{
"$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json",

"phases": [
{
"name": "_phase:build",
Expand Down
3 changes: 3 additions & 0 deletions common/scripts/rush_docker.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
export RUSH_ALLOW_UNSUPPORTED_NODEJS=1
rush docker:build -p 20 \
--to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext --to @hcengineering/pod-love --to @hcengineering/pod-mail --to @hcengineering/pod-datalake --to @hcengineering/pod-inbound-mail --to @hcengineering/pod-export --to @hcengineering/pod-msg2file
2 changes: 1 addition & 1 deletion dev/doc-import-tool/src/import.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ export default async function importExtractedFile (

try {
console.log(`Connected to ${transactorUrl}`)
const txops = new TxOperations(connection, core.account.System)
const txops = new TxOperations(connection, core.account.System, workspaceId)

try {
const docId = await createDocument(txops, extractedFile, config)
Expand Down
2 changes: 2 additions & 0 deletions dev/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,7 @@ services:
environment:
- QUEUE_CONFIG=${QUEUE_CONFIG}
- REGION=
- ENDPOINT=ws://huly.local:3333
- SERVER_PORT=3333
- SERVER_SECRET=secret
- ENABLE_COMPRESSION=true
Expand Down Expand Up @@ -338,6 +339,7 @@ services:
- QUEUE_CONFIG=${QUEUE_CONFIG}
- SERVER_PORT=3332
- REGION=cockroach
- ENDPOINT=ws://huly.local:3332
- SERVER_SECRET=secret
- ENABLE_COMPRESSION=true
- FULLTEXT_URL=http://huly.local:4702
Expand Down
2 changes: 1 addition & 1 deletion dev/import-tool/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ export function importTool (): void {

console.log('Connecting to Transactor URL: ', selectedWs.endpoint)
const connection = await createClient(selectedWs.endpoint, selectedWs.token)
const client = new TxOperations(connection, socialId)
const client = new TxOperations(connection, socialId, selectedWs.workspace)
const fileUploader = new FrontFileUploader(
getFrontUrl(),
selectedWs.workspace,
Expand Down
4 changes: 3 additions & 1 deletion dev/prod/src/platform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ import { uploaderId } from '@hcengineering/uploader'
import { mediaId } from '@hcengineering/media'
import recorder, { recorderId } from '@hcengineering/recorder'
import { viewId } from '@hcengineering/view'
import workbench, { workbenchId } from '@hcengineering/workbench'
import workbench, { workbenchId, workbenchAppsId } from '@hcengineering/workbench'
import { mailId } from '@hcengineering/mail'
import { chatId } from '@hcengineering/chat'
import github, { githubId } from '@hcengineering/github'
Expand Down Expand Up @@ -147,6 +147,7 @@ import { preferenceId } from '@hcengineering/preference'
import { uiId } from '@hcengineering/ui/src/plugin'
import { configureAnalytics } from './analytics'


export interface Config {
ACCOUNTS_URL: string
UPLOAD_URL: string
Expand Down Expand Up @@ -487,6 +488,7 @@ export async function configurePlatform() {
uiPlugin.metadata.Routes,
new Map([
[workbenchId, workbench.component.WorkbenchApp],
[workbenchAppsId, workbench.component.WorkbenchApps],
[loginId, login.component.LoginApp],
[onboardId, onboard.component.OnboardApp],
[githubId, github.component.ConnectApp],
Expand Down
6 changes: 3 additions & 3 deletions dev/tool/src/benchmark.ts
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,7 @@ export function benchmarkWorker (): void {

if (msg.options.mode === 'find-all') {
const benchmarkPersonId = (core.account.System + '_benchmark') as PersonId
const opt = new TxOperations(connection, benchmarkPersonId)
const opt = new TxOperations(connection, benchmarkPersonId, msg.workspaceId)
parentPort?.postMessage({
type: 'operate',
workId: msg.workId
Expand Down Expand Up @@ -544,7 +544,7 @@ export async function stressBenchmark (transactor: string, mode: StressBenchmark
export async function testFindAll (endpoint: string, workspace: WorkspaceUuid, account: PersonUuid): Promise<void> {
const connection = await connect(endpoint, workspace, account)
try {
const client = new TxOperations(connection, core.account.System)
const client = new TxOperations(connection, core.account.System, workspace)
const start = platformNow()
const res = await client.findAll(
recruit.class.Applicant,
Expand All @@ -569,7 +569,7 @@ export async function generateWorkspaceData (
email: string
): Promise<void> {
const connection = await connect(endpoint, workspace)
const client = new TxOperations(connection, core.account.System)
const client = new TxOperations(connection, core.account.System, workspace)
try {
const emailSocialString = buildSocialIdString({ type: SocialIdType.EMAIL, value: email })
const person = await getPersonBySocialKey(client, emailSocialString)
Expand Down
42 changes: 25 additions & 17 deletions dev/tool/src/clean.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,10 @@ export async function cleanWorkspace (
opt: { recruit: boolean, tracker: boolean, removedTx: boolean }
): Promise<void> {
const connection = (await connect(transactorUrl, workspaceId, undefined, {
mode: 'backup',
model: 'upgrade'
mode: 'backup'
})) as unknown as CoreClient & BackupClient
try {
const ops = new TxOperations(connection, core.account.System)
const ops = new TxOperations(connection, core.account.System, workspaceId)

const hierarchy = ops.getHierarchy()

Expand Down Expand Up @@ -164,6 +163,7 @@ export async function cleanWorkspace (
client.close()
}
} catch (err: any) {
// TODO: Add force-close
console.trace(err)
} finally {
await connection.close()
Expand Down Expand Up @@ -214,6 +214,7 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceUuid, tran
objectId: { $in: removedDocs.map((it) => it.objectId) }
})
await connection.clean(
workspaceId,
DOMAIN_TX,
toRemove.map((it) => it._id)
)
Expand All @@ -232,8 +233,7 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceUuid, tran

export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl: string): Promise<void> {
const connection = (await connect(transactorUrl, workspaceId, undefined, {
mode: 'backup',
model: 'upgrade'
mode: 'backup'
})) as unknown as CoreClient & BackupClient
try {
let count = 0
Expand Down Expand Up @@ -289,7 +289,7 @@ export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl:
}
}

await connection.clean(DOMAIN_TX, toRemove)
await connection.clean(workspaceId, DOMAIN_TX, toRemove)

count += toRemove.length
console.log('processed', count)
Expand All @@ -298,6 +298,8 @@ export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl:
} catch (err: any) {
console.trace(err)
} finally {
// TODO: Add force-close
await connection.sendForceClose(workspaceId)
await connection.close()
}
}
Expand All @@ -307,7 +309,7 @@ export async function cleanArchivedSpaces (workspaceId: WorkspaceUuid, transacto
})) as unknown as CoreClient & BackupClient
try {
const count = 0
const ops = new TxOperations(connection, core.account.System)
const ops = new TxOperations(connection, core.account.System, workspaceId)
while (true) {
const spaces = await connection.findAll(core.class.Space, { archived: true }, { limit: 1000 })
if (spaces.length === 0) {
Expand Down Expand Up @@ -370,7 +372,7 @@ export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceUuid, tran
// We have found duplicate one, let's rename it.
const doc = TxProcessor.createDoc2Doc<ChatMessage>(c as unknown as TxCreateDoc<ChatMessage>)
if (doc.message !== '' && doc.message.trim() !== '<p></p>') {
await connection.clean(DOMAIN_TX, [c._id])
await connection.clean(workspaceId, DOMAIN_TX, [c._id])
if (oldValue.get(cid) === doc.message.trim()) {
console.log('delete tx', cid, doc.message)
} else {
Expand All @@ -379,9 +381,9 @@ export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceUuid, tran
// Remove previous transaction.
c.objectId = generateId()
doc._id = c.objectId as Ref<ChatMessage>
await connection.upload(DOMAIN_TX, [c])
await connection.upload(workspaceId, DOMAIN_TX, [c])
// Also we need to create snapsot
await connection.upload(DOMAIN_ACTIVITY, [doc])
await connection.upload(workspaceId, DOMAIN_ACTIVITY, [doc])
}
}
}
Expand Down Expand Up @@ -475,7 +477,7 @@ export async function fixSkills (
// fix skills with + and -
if (step === '3') {
console.log('STEP 3')
const ops = new TxOperations(connection, core.account.System)
const ops = new TxOperations(connection, core.account.System, workspaceId)
const regex = /\S+(?:[-+]\S+)+/g
const tagsToClean = (await connection.findAll(tags.class.TagElement, {
category: {
Expand Down Expand Up @@ -532,7 +534,7 @@ export async function fixSkills (
}
})) as TagElement[]
goodTags = goodTags.sort((a, b) => b.title.length - a.title.length).filter((t) => t.title.length > 2)
const ops = new TxOperations(connection, core.account.System)
const ops = new TxOperations(connection, core.account.System, workspaceId)
const tagsToClean = (await connection.findAll(tags.class.TagElement, {
category: {
$in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref<TagCategory>[]
Expand Down Expand Up @@ -669,8 +671,7 @@ export async function restoreRecruitingTaskTypes (
transactorUrl: string
): Promise<void> {
const connection = (await connect(transactorUrl, workspaceId, undefined, {
mode: 'backup',
model: 'upgrade'
mode: 'backup'
})) as unknown as CoreClient & BackupClient
const client = getMongoClient(mongoUrl)
try {
Expand Down Expand Up @@ -772,6 +773,7 @@ export async function restoreRecruitingTaskTypes (
statusCategories.sort(compareCategories)

const createTxNew: TxCreateDoc<TaskType> = {
_uuid: workspaceId,
_id: generateId(),
_class: core.class.TxCreateDoc,
space: core.space.Tx,
Expand Down Expand Up @@ -822,7 +824,9 @@ export async function restoreRecruitingTaskTypes (
} catch (err: any) {
console.trace(err)
} finally {
// TODO: Add force-close
client.close()
await connection.sendForceClose(workspaceId)
await connection.close()
}
}
Expand All @@ -833,8 +837,7 @@ export async function restoreHrTaskTypesFromUpdates (
transactorUrl: string
): Promise<void> {
const connection = (await connect(transactorUrl, workspaceId, undefined, {
mode: 'backup',
model: 'upgrade'
mode: 'backup'
})) as unknown as CoreClient & BackupClient
const client = getMongoClient(mongoUrl)
try {
Expand Down Expand Up @@ -927,6 +930,7 @@ export async function restoreHrTaskTypesFromUpdates (
const ofClassClass = hierarchy.getClass(recruit.class.Applicant)

await db.collection<TxCreateDoc<Doc>>(DOMAIN_TX).insertOne({
_uuid: workspaceId,
_id: generateId(),
_class: core.class.TxCreateDoc,
space: core.space.Tx,
Expand All @@ -946,6 +950,7 @@ export async function restoreHrTaskTypesFromUpdates (
})

createTaskTypeTx = {
_uuid: workspaceId,
_id: generateId(),
_class: core.class.TxCreateDoc,
space: core.space.Tx,
Expand Down Expand Up @@ -980,6 +985,7 @@ export async function restoreHrTaskTypesFromUpdates (
const ofClassClass = hierarchy.getClass(recruit.class.Vacancy)

await db.collection<TxCreateDoc<Doc>>(DOMAIN_TX).insertOne({
_uuid: workspaceId,
_id: generateId(),
_class: core.class.TxCreateDoc,
space: core.space.Tx,
Expand All @@ -999,6 +1005,7 @@ export async function restoreHrTaskTypesFromUpdates (
})

const createProjectTypeTx: TxCreateDoc<ProjectType> = {
_uuid: workspaceId,
_id: generateId(),
_class: core.class.TxCreateDoc,
space: core.space.Tx,
Expand Down Expand Up @@ -1031,6 +1038,7 @@ export async function restoreHrTaskTypesFromUpdates (
console.trace(err)
} finally {
client.close()
await connection.sendForceClose(workspaceId)
await connection.close()
}
}
Expand Down Expand Up @@ -1127,7 +1135,7 @@ export async function removeDuplicateIds (
// await updateId(ctx, wsClient, db, storageAdapter, wsDataId, doc)
// }
// }
// await wsClient.sendForceClose()
// await wsClient.sendForceClose(workspaceId)
// await wsClient.close()
// await db.collection<MigrationState>(DOMAIN_MIGRATION).insertOne({
// _id: generateId(),
Expand Down
2 changes: 1 addition & 1 deletion dev/tool/src/configuration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export async function changeConfiguration (
}
const enable = (cmd.enable ?? '').trim().split(',')
console.log('enable', enable)
const ops = new TxFactory(core.account.ConfigUser)
const ops = new TxFactory(core.account.ConfigUser, workspaceId)
if (enable.length > 0) {
const p = config.filter((it) => enable.includes(it.pluginId) || enable.includes('*'))
for (const pp of p) {
Expand Down
4 changes: 2 additions & 2 deletions dev/tool/src/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ async function moveWorkspace (
const token = generateToken(systemAccountUuid, wsId, { service: 'tool' })
const endpoint = await getTransactorEndpoint(token, 'external')
const connection = (await connect(endpoint, wsId, undefined, {
model: 'upgrade'
mode: 'backup'
})) as unknown as Client & BackupClient
for (const collection of collections) {
const domain = translateDomain(collection.collectionName)
Expand Down Expand Up @@ -158,7 +158,7 @@ async function moveWorkspace (
}
// TODO: FIXME
// await updateWorkspace(accountDb, ws, { region })
await connection.sendForceClose()
await connection.sendForceClose(wsId)
await connection.close()
} catch (err) {
console.log('Error when move workspace', ws.name ?? ws.url, err)
Expand Down
6 changes: 3 additions & 3 deletions dev/tool/src/github.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,15 +94,15 @@ export async function performGithubAccountMigrations (db: Db, region: string | n
let idx: number | undefined

while (true) {
const info = await client.loadChunk(DOMAIN_MODEL_TX, idx)
const info = await client.loadChunk(ws.uuid, DOMAIN_MODEL_TX, idx)
idx = info.idx
const ids = Array.from(info.docs.map((it) => it.id as Ref<Doc>))
const docs = (await client.loadDocs(DOMAIN_MODEL_TX, ids)).filter((it) =>
const docs = (await client.loadDocs(ws.uuid, DOMAIN_MODEL_TX, ids)).filter((it) =>
TxProcessor.isExtendsCUD(it._class)
) as TxCUD<Doc>[]
accountsTxes.push(...docs)
if (info.finished && idx !== undefined) {
await client.closeChunk(info.idx)
await client.closeChunk(ws.uuid, info.idx)
break
}
}
Expand Down
Loading
Loading