UBERF-9747

Signed-off-by: Andrey Sobolev <[email protected]>

Author: haiodo

.vscode/launch.json

@@ -103,7 +103,8 @@
         "UPLOAD_URL": "/files",
         "AI_BOT_URL": "http://localhost:4010",
         "STATS_URL": "http://huly.local:4900",
-        "QUEUE_CONFIG": "localhost:19092"
+        "QUEUE_CONFIG": "localhost:19092",
+        "RATE_LIMIT_MAX": "25000"
       },
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
       "runtimeVersion": "20",
@@ -120,9 +121,9 @@
       "args": ["src/__start.ts"],
       "env": {
         "FULLTEXT_URL": "http://localhost:4710",
-        // "DB_URL": "mongodb://localhost:27018",
+        "DB_URL": "mongodb://localhost:27018",
         // "DB_URL": "postgresql://postgres:example@localhost:5432",
-        "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
+        // "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
         // "GREEN_URL": "http://huly.local:6767?token=secret",
         "SERVER_PORT": "3335",
         "METRICS_CONSOLE": "false",
@@ -183,7 +184,7 @@
       "request": "launch",
       "args": ["src/__start.ts"],
       "env": {
-        "MONGO_URL": "mongodb://localhost:27017",
+        // "MONGO_URL": "mongodb://localhost:27017",
         // "DB_URL": "mongodb://localhost:27017",
         "DB_URL": "postgresql://[email protected]:26257/defaultdb?sslmode=disable",
         "SERVER_SECRET": "secret",
@@ -217,12 +218,13 @@
       "args": ["src/__start.ts"],
       "env": {
         // "MONGO_URL": "mongodb://localhost:27018",
-        // "DB_URL": "mongodb://localhost:27018",
+        "DB_URL": "mongodb://localhost:27018",
         // "DB_URL": "postgresql://postgres:example@localhost:5432",
-        "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
+        // "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
         "SERVER_SECRET": "secret",
-        "REGION_INFO": "|Mongo;pg|Postgres;cockroach|CockroachDB",
-        "TRANSACTOR_URL": "ws://transactor:3334;ws://localhost:3334",
+        "QUEUE_CONFIG": "huly.local:19092",
+        "REGION_INFO": "|America;europe|Europe",
+        "TRANSACTOR_URL": "ws://huly.local:3334;ws://huly.local:3334,ws://huly.local:3335;ws://huly.local:3335;europe",
         "ACCOUNTS_URL": "http://localhost:3003",
         "ACCOUNT_PORT": "3003",
         "FRONT_URL": "http://localhost:8083",

common/config/rush/command-line.json

@@ -4,7 +4,6 @@
  */
 {
   "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json",
-
   "phases": [
     {
       "name": "_phase:build",

common/scripts/rush_docker.sh

@@ -0,0 +1,3 @@
+export RUSH_ALLOW_UNSUPPORTED_NODEJS=1
+rush docker:build -p 20 \
+  --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext --to @hcengineering/pod-love --to @hcengineering/pod-mail --to @hcengineering/pod-datalake --to @hcengineering/pod-inbound-mail --to @hcengineering/pod-export --to @hcengineering/pod-msg2file

dev/doc-import-tool/src/import.ts

@@ -47,7 +47,7 @@ export default async function importExtractedFile (
 
   try {
     console.log(`Connected to ${transactorUrl}`)
-    const txops = new TxOperations(connection, core.account.System)
+    const txops = new TxOperations(connection, core.account.System, workspaceId)
 
     try {
       const docId = await createDocument(txops, extractedFile, config)

dev/docker-compose.yaml

@@ -302,6 +302,7 @@ services:
     environment:
       - QUEUE_CONFIG=${QUEUE_CONFIG}
       - REGION=
+      - ENDPOINT=ws://huly.local:3333
       - SERVER_PORT=3333
       - SERVER_SECRET=secret
       - ENABLE_COMPRESSION=true
@@ -338,6 +339,7 @@ services:
       - QUEUE_CONFIG=${QUEUE_CONFIG}
       - SERVER_PORT=3332
       - REGION=cockroach
+      - ENDPOINT=ws://huly.local:3332
       - SERVER_SECRET=secret
       - ENABLE_COMPRESSION=true
       - FULLTEXT_URL=http://huly.local:4702

dev/import-tool/src/index.ts

@@ -95,7 +95,7 @@ export function importTool (): void {
 
     console.log('Connecting to Transactor URL: ', selectedWs.endpoint)
     const connection = await createClient(selectedWs.endpoint, selectedWs.token)
-    const client = new TxOperations(connection, socialId)
+    const client = new TxOperations(connection, socialId, selectedWs.workspace)
     const fileUploader = new FrontFileUploader(
       getFrontUrl(),
       selectedWs.workspace,

dev/prod/src/platform.ts

@@ -67,7 +67,7 @@ import { uploaderId } from '@hcengineering/uploader'
 import { mediaId } from '@hcengineering/media'
 import recorder, { recorderId } from '@hcengineering/recorder'
 import { viewId } from '@hcengineering/view'
-import workbench, { workbenchId } from '@hcengineering/workbench'
+import workbench, { workbenchId, workbenchAppsId } from '@hcengineering/workbench'
 import { mailId } from '@hcengineering/mail'
 import { chatId } from '@hcengineering/chat'
 import github, { githubId } from '@hcengineering/github'
@@ -147,6 +147,7 @@ import { preferenceId } from '@hcengineering/preference'
 import { uiId } from '@hcengineering/ui/src/plugin'
 import { configureAnalytics } from './analytics'
 
+
 export interface Config {
   ACCOUNTS_URL: string
   UPLOAD_URL: string
@@ -487,6 +488,7 @@ export async function configurePlatform() {
     uiPlugin.metadata.Routes,
     new Map([
       [workbenchId, workbench.component.WorkbenchApp],
+      [workbenchAppsId, workbench.component.WorkbenchApps],
       [loginId, login.component.LoginApp],
       [onboardId, onboard.component.OnboardApp],
       [githubId, github.component.ConnectApp],

dev/tool/src/benchmark.ts

@@ -403,7 +403,7 @@ export function benchmarkWorker (): void {
 
       if (msg.options.mode === 'find-all') {
         const benchmarkPersonId = (core.account.System + '_benchmark') as PersonId
-        const opt = new TxOperations(connection, benchmarkPersonId)
+        const opt = new TxOperations(connection, benchmarkPersonId, msg.workspaceId)
         parentPort?.postMessage({
           type: 'operate',
           workId: msg.workId
@@ -544,7 +544,7 @@ export async function stressBenchmark (transactor: string, mode: StressBenchmark
 export async function testFindAll (endpoint: string, workspace: WorkspaceUuid, account: PersonUuid): Promise<void> {
   const connection = await connect(endpoint, workspace, account)
   try {
-    const client = new TxOperations(connection, core.account.System)
+    const client = new TxOperations(connection, core.account.System, workspace)
     const start = platformNow()
     const res = await client.findAll(
       recruit.class.Applicant,
@@ -569,7 +569,7 @@ export async function generateWorkspaceData (
   email: string
 ): Promise<void> {
   const connection = await connect(endpoint, workspace)
-  const client = new TxOperations(connection, core.account.System)
+  const client = new TxOperations(connection, core.account.System, workspace)
   try {
     const emailSocialString = buildSocialIdString({ type: SocialIdType.EMAIL, value: email })
     const person = await getPersonBySocialKey(client, emailSocialString)

dev/tool/src/clean.ts

@@ -86,11 +86,10 @@ export async function cleanWorkspace (
   opt: { recruit: boolean, tracker: boolean, removedTx: boolean }
 ): Promise<void> {
   const connection = (await connect(transactorUrl, workspaceId, undefined, {
-    mode: 'backup',
-    model: 'upgrade'
+    mode: 'backup'
   })) as unknown as CoreClient & BackupClient
   try {
-    const ops = new TxOperations(connection, core.account.System)
+    const ops = new TxOperations(connection, core.account.System, workspaceId)
 
     const hierarchy = ops.getHierarchy()
 
@@ -164,6 +163,7 @@ export async function cleanWorkspace (
       client.close()
     }
   } catch (err: any) {
+    // TODO: Add force-close
     console.trace(err)
   } finally {
     await connection.close()
@@ -214,6 +214,7 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceUuid, tran
         objectId: { $in: removedDocs.map((it) => it.objectId) }
       })
       await connection.clean(
+        workspaceId,
         DOMAIN_TX,
         toRemove.map((it) => it._id)
       )
@@ -232,8 +233,7 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceUuid, tran
 
 export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl: string): Promise<void> {
   const connection = (await connect(transactorUrl, workspaceId, undefined, {
-    mode: 'backup',
-    model: 'upgrade'
+    mode: 'backup'
   })) as unknown as CoreClient & BackupClient
   try {
     let count = 0
@@ -289,7 +289,7 @@ export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl:
       }
     }
 
-    await connection.clean(DOMAIN_TX, toRemove)
+    await connection.clean(workspaceId, DOMAIN_TX, toRemove)
 
     count += toRemove.length
     console.log('processed', count)
@@ -298,6 +298,8 @@ export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl:
   } catch (err: any) {
     console.trace(err)
   } finally {
+    // TODO: Add force-close
+    await connection.sendForceClose(workspaceId)
     await connection.close()
   }
 }
@@ -307,7 +309,7 @@ export async function cleanArchivedSpaces (workspaceId: WorkspaceUuid, transacto
   })) as unknown as CoreClient & BackupClient
   try {
     const count = 0
-    const ops = new TxOperations(connection, core.account.System)
+    const ops = new TxOperations(connection, core.account.System, workspaceId)
     while (true) {
       const spaces = await connection.findAll(core.class.Space, { archived: true }, { limit: 1000 })
       if (spaces.length === 0) {
@@ -370,7 +372,7 @@ export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceUuid, tran
         // We have found duplicate one, let's rename it.
         const doc = TxProcessor.createDoc2Doc<ChatMessage>(c as unknown as TxCreateDoc<ChatMessage>)
         if (doc.message !== '' && doc.message.trim() !== '<p></p>') {
-          await connection.clean(DOMAIN_TX, [c._id])
+          await connection.clean(workspaceId, DOMAIN_TX, [c._id])
           if (oldValue.get(cid) === doc.message.trim()) {
             console.log('delete tx', cid, doc.message)
           } else {
@@ -379,9 +381,9 @@ export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceUuid, tran
             // Remove previous transaction.
             c.objectId = generateId()
             doc._id = c.objectId as Ref<ChatMessage>
-            await connection.upload(DOMAIN_TX, [c])
+            await connection.upload(workspaceId, DOMAIN_TX, [c])
             // Also we need to create snapsot
-            await connection.upload(DOMAIN_ACTIVITY, [doc])
+            await connection.upload(workspaceId, DOMAIN_ACTIVITY, [doc])
           }
         }
       }
@@ -475,7 +477,7 @@ export async function fixSkills (
     // fix skills with + and -
     if (step === '3') {
       console.log('STEP 3')
-      const ops = new TxOperations(connection, core.account.System)
+      const ops = new TxOperations(connection, core.account.System, workspaceId)
       const regex = /\S+(?:[-+]\S+)+/g
       const tagsToClean = (await connection.findAll(tags.class.TagElement, {
         category: {
@@ -532,7 +534,7 @@ export async function fixSkills (
         }
       })) as TagElement[]
       goodTags = goodTags.sort((a, b) => b.title.length - a.title.length).filter((t) => t.title.length > 2)
-      const ops = new TxOperations(connection, core.account.System)
+      const ops = new TxOperations(connection, core.account.System, workspaceId)
       const tagsToClean = (await connection.findAll(tags.class.TagElement, {
         category: {
           $in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref<TagCategory>[]
@@ -669,8 +671,7 @@ export async function restoreRecruitingTaskTypes (
   transactorUrl: string
 ): Promise<void> {
   const connection = (await connect(transactorUrl, workspaceId, undefined, {
-    mode: 'backup',
-    model: 'upgrade'
+    mode: 'backup'
   })) as unknown as CoreClient & BackupClient
   const client = getMongoClient(mongoUrl)
   try {
@@ -772,6 +773,7 @@ export async function restoreRecruitingTaskTypes (
         statusCategories.sort(compareCategories)
 
         const createTxNew: TxCreateDoc<TaskType> = {
+          _uuid: workspaceId,
           _id: generateId(),
           _class: core.class.TxCreateDoc,
           space: core.space.Tx,
@@ -822,7 +824,9 @@ export async function restoreRecruitingTaskTypes (
   } catch (err: any) {
     console.trace(err)
   } finally {
+    // TODO: Add force-close
     client.close()
+    await connection.sendForceClose(workspaceId)
     await connection.close()
   }
 }
@@ -833,8 +837,7 @@ export async function restoreHrTaskTypesFromUpdates (
   transactorUrl: string
 ): Promise<void> {
   const connection = (await connect(transactorUrl, workspaceId, undefined, {
-    mode: 'backup',
-    model: 'upgrade'
+    mode: 'backup'
   })) as unknown as CoreClient & BackupClient
   const client = getMongoClient(mongoUrl)
   try {
@@ -927,6 +930,7 @@ export async function restoreHrTaskTypesFromUpdates (
         const ofClassClass = hierarchy.getClass(recruit.class.Applicant)
 
         await db.collection<TxCreateDoc<Doc>>(DOMAIN_TX).insertOne({
+          _uuid: workspaceId,
           _id: generateId(),
           _class: core.class.TxCreateDoc,
           space: core.space.Tx,
@@ -946,6 +950,7 @@ export async function restoreHrTaskTypesFromUpdates (
         })
 
         createTaskTypeTx = {
+          _uuid: workspaceId,
           _id: generateId(),
           _class: core.class.TxCreateDoc,
           space: core.space.Tx,
@@ -980,6 +985,7 @@ export async function restoreHrTaskTypesFromUpdates (
       const ofClassClass = hierarchy.getClass(recruit.class.Vacancy)
 
       await db.collection<TxCreateDoc<Doc>>(DOMAIN_TX).insertOne({
+        _uuid: workspaceId,
         _id: generateId(),
         _class: core.class.TxCreateDoc,
         space: core.space.Tx,
@@ -999,6 +1005,7 @@ export async function restoreHrTaskTypesFromUpdates (
       })
 
       const createProjectTypeTx: TxCreateDoc<ProjectType> = {
+        _uuid: workspaceId,
         _id: generateId(),
         _class: core.class.TxCreateDoc,
         space: core.space.Tx,
@@ -1031,6 +1038,7 @@ export async function restoreHrTaskTypesFromUpdates (
     console.trace(err)
   } finally {
     client.close()
+    await connection.sendForceClose(workspaceId)
     await connection.close()
   }
 }
@@ -1127,7 +1135,7 @@ export async function removeDuplicateIds (
   //         await updateId(ctx, wsClient, db, storageAdapter, wsDataId, doc)
   //       }
   //     }
-  //     await wsClient.sendForceClose()
+  //     await wsClient.sendForceClose(workspaceId)
   //     await wsClient.close()
   //     await db.collection<MigrationState>(DOMAIN_MIGRATION).insertOne({
   //       _id: generateId(),

dev/tool/src/configuration.ts

@@ -41,7 +41,7 @@ export async function changeConfiguration (
     }
     const enable = (cmd.enable ?? '').trim().split(',')
     console.log('enable', enable)
-    const ops = new TxFactory(core.account.ConfigUser)
+    const ops = new TxFactory(core.account.ConfigUser, workspaceId)
     if (enable.length > 0) {
       const p = config.filter((it) => enable.includes(it.pluginId) || enable.includes('*'))
       for (const pp of p) {