Update unique fields on standard field - include soft deleted records

packages/twenty-server/src/database/commands/upgrade-version-command/1-7/1-7-deduplicate-unique-fields.command.ts

@@ -0,0 +1,360 @@
+import { Logger } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { IsNull, Not, Repository } from 'typeorm';
+
+import {
+  ActiveOrSuspendedWorkspacesMigrationCommandRunner,
+  type RunOnWorkspaceArgs,
+} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
+import { IndexMetadataService } from 'src/engine/metadata-modules/index-metadata/index-metadata.service';
+import { generateDeterministicIndexName } from 'src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
+import {
+  WorkspaceMigrationIndexAction,
+  WorkspaceMigrationIndexActionType,
+  WorkspaceMigrationTableAction,
+  WorkspaceMigrationTableActionType,
+} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
+import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
+import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
+import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service';
+
+@Command({
+  name: 'upgrade:1-7:deduplicate-unique-fields',
+  description:
+    'Deduplicate unique fields for workspaceMembers, companies and people because we changed the unique constraint',
+})
+export class DeduplicateUniqueFieldsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  protected readonly logger = new Logger(DeduplicateUniqueFieldsCommand.name);
+  constructor(
+    @InjectRepository(Workspace)
+    protected readonly workspaceRepository: Repository<Workspace>,
+    protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+    protected readonly indexMetadataService: IndexMetadataService,
+    protected readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
+    protected readonly workspaceMigrationService: WorkspaceMigrationService,
+    @InjectRepository(ObjectMetadataEntity)
+    protected readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
+    @InjectRepository(IndexMetadataEntity)
+    protected readonly indexMetadataRepository: Repository<IndexMetadataEntity>,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager);
+  }
+
+  override async runOnWorkspace({
+    workspaceId,
+    dataSource,
+    options,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    this.logger.log(
+      `Deduplicating indexed fields for workspace ${workspaceId}`,
+    );
+
+    await this.deduplicateUniqueUserEmailFieldForWorkspaceMembers({
+      dataSource,
+      dryRun: options.dryRun ?? false,
+    });
+
+    await this.deduplicateUniqueDomainNameFieldForCompanies({
+      dataSource,
+      dryRun: options.dryRun ?? false,
+    });
+
+    await this.deduplicateUniqueEmailFieldForPeople({
+      dataSource,
+      dryRun: options.dryRun ?? false,
+    });
+
+    if (!options.dryRun) {
+      await this.updateExistingIndexedFields({
+        workspaceId,
+        objectMetadataNameSingular: 'workspaceMember',
+        columnName: 'userEmail',
+      });
+      await this.updateExistingIndexedFields({
+        workspaceId,
+        objectMetadataNameSingular: 'company',
+        columnName: 'domainNamePrimaryLinkUrl',
+      });
+      await this.updateExistingIndexedFields({
+        workspaceId,
+        objectMetadataNameSingular: 'person',
+        columnName: 'emailsPrimaryEmail',
+      });
+    }
+  }
+
+  private computeExistingUniqueIndexName({
+    objectMetadata,
+    fieldMetadataToIndex,
+  }: {
+    objectMetadata: ObjectMetadataEntity;
+    fieldMetadataToIndex: Partial<FieldMetadataEntity>[];
+  }) {
+    const tableName = computeObjectTargetTable(objectMetadata);
+    const columnNames: string[] = fieldMetadataToIndex.map(
+      (fieldMetadata) => fieldMetadata.name as string,
+    );
+
+    return `IDX_UNIQUE_${generateDeterministicIndexName([tableName, ...columnNames])}`;
+  }
+
+  private async computeExistingIndexDeletionMigration({
+    objectMetadata,
+    fieldMetadataToIndex,
+  }: {
+    objectMetadata: ObjectMetadataEntity;
+    fieldMetadataToIndex: Partial<FieldMetadataEntity>[];
+  }) {
+    const tableName = computeObjectTargetTable(objectMetadata);
+
+    const indexName = this.computeExistingUniqueIndexName({
+      objectMetadata,
+      fieldMetadataToIndex,
+    });
+
+    return {
+      name: tableName,
+      action: WorkspaceMigrationTableActionType.ALTER_INDEXES,
+      indexes: [
+        {
+          action: WorkspaceMigrationIndexActionType.DROP,
+          name: indexName,
+          columns: [],
+          isUnique: true,
+        } satisfies WorkspaceMigrationIndexAction,
+      ],
+    } satisfies WorkspaceMigrationTableAction;
+  }
+
+  private async updateExistingIndexedFields({
+    workspaceId,
+    objectMetadataNameSingular,
+    columnName,
+  }: {
+    workspaceId: string;
+    objectMetadataNameSingular: string;
+    columnName: string;
+  }) {
+    this.logger.log(
+      `Updating existing indexed fields for workspace members for workspace ${workspaceId}`,
+    );
+
+    const workspaceMemberObjectMetadata =
+      await this.objectMetadataRepository.findOneByOrFail({
+        nameSingular: objectMetadataNameSingular,
+      });
+
+    await this.indexMetadataRepository.delete({
+      workspaceId,
+      name: this.computeExistingUniqueIndexName({
+        objectMetadata: workspaceMemberObjectMetadata,
+        fieldMetadataToIndex: [{ name: columnName }],
+      }),
+    });
+
+    const indexDeletionMigration =
+      await this.computeExistingIndexDeletionMigration({
+        objectMetadata: workspaceMemberObjectMetadata,
+        fieldMetadataToIndex: [{ name: columnName }],
+      });
+
+    await this.workspaceMigrationService.createCustomMigration(
+      generateMigrationName(`delete-${objectMetadataNameSingular}-index`),
+      workspaceId,
+      [indexDeletionMigration],
+    );
+
+    await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
+      workspaceId,
+    );
+  }
+
+  private async deduplicateUniqueUserEmailFieldForWorkspaceMembers({
+    dataSource,
+    dryRun,
+  }: {
+    dataSource: WorkspaceDataSource;
+    dryRun: boolean;
+  }) {
+    const workspaceMemberRepository = dataSource.getRepository(
+      'workspaceMember',
+      true,
+    );
+
+    const duplicates = await workspaceMemberRepository
+      .createQueryBuilder('workspaceMember')
+      .select('workspaceMember.userEmail', 'userEmail')
+      .addSelect('COUNT(*)', 'count')
+      .andWhere('workspaceMember.userEmail IS NOT NULL')
+      .andWhere("workspaceMember.userEmail != ''")
+      .withDeleted()
+      .groupBy('workspaceMember.userEmail')
+      .having('COUNT(*) > 1')
+      .getRawMany();
+
+    for (const duplicate of duplicates) {
+      const { userEmail } = duplicate;
+
+      const softDeletedWorkspaceMembers = await workspaceMemberRepository.find({
+        where: {
+          userEmail,
+          deletedAt: Not(IsNull()),
+        },
+        withDeleted: true,
+      });
+
+      for (const [
+        i,
+        softDeletedWorkspaceMember,
+      ] of softDeletedWorkspaceMembers.entries()) {
+        const newUserEmail = this.computeNewFieldValues(
+          softDeletedWorkspaceMember.userEmail,
+          i,
+        );
+
+        if (!dryRun) {
+          await workspaceMemberRepository
+            .createQueryBuilder('workspaceMember')
+            .update()
+            .set({
+              userEmail: newUserEmail,
+            })
+            .where('id = :id', { id: softDeletedWorkspaceMember.id })
+            .execute();
+        }
+        this.logger.log(
+          `Updated workspaceMember ${softDeletedWorkspaceMembers[i].id} userEmail from ${userEmail} to ${newUserEmail}`,
+        );
+      }
+    }
+  }
+
+  private async deduplicateUniqueDomainNameFieldForCompanies({
+    dataSource,
+    dryRun,
+  }: {
+    dataSource: WorkspaceDataSource;
+    dryRun: boolean;
+  }) {
+    const companyRepository = dataSource.getRepository('company', true);
+
+    const duplicates = await companyRepository
+      .createQueryBuilder('company')
+      .select('company.domainNamePrimaryLinkUrl', 'domainNamePrimaryLinkUrl')
+      .addSelect('COUNT(*)', 'count')
+      .andWhere('company.domainNamePrimaryLinkUrl IS NOT NULL')
+      .andWhere("company.domainNamePrimaryLinkUrl != ''")
+      .withDeleted()
+      .groupBy('company.domainNamePrimaryLinkUrl')
+      .having('COUNT(*) > 1')
+      .getRawMany();
+
+    for (const duplicate of duplicates) {
+      const { domainNamePrimaryLinkUrl } = duplicate;
+
+      const softDeletedCompanies = await companyRepository.find({
+        where: {
+          domainName: {
+            primaryLinkUrl: domainNamePrimaryLinkUrl,
+          },
+          deletedAt: Not(IsNull()),
+        },
+        withDeleted: true,
+      });
+
+      for (const [i, softDeletedCompany] of softDeletedCompanies.entries()) {
+        const newDomainNamePrimaryLinkUrl = this.computeNewFieldValues(
+          softDeletedCompany.domainName.primaryLinkUrl,
+          i,
+        );
+
+        if (!dryRun) {
+          await companyRepository
+            .createQueryBuilder('company')
+            .update()
+            .set({
+              domainName: {
+                primaryLinkUrl: newDomainNamePrimaryLinkUrl,
+              },
+            })
+            .where('id = :id', { id: softDeletedCompany.id })
+            .execute();
+        }
+        this.logger.log(
+          `Updated company ${softDeletedCompany.id} domainNamePrimaryLinkUrl from ${domainNamePrimaryLinkUrl} to ${newDomainNamePrimaryLinkUrl}`,
+        );
+      }
+    }
+  }
+
+  private async deduplicateUniqueEmailFieldForPeople({
+    dataSource,
+    dryRun,
+  }: {
+    dataSource: WorkspaceDataSource;
+    dryRun: boolean;
+  }) {
+    const personRepository = dataSource.getRepository('person', true);
+
+    const duplicates = await personRepository
+      .createQueryBuilder('person')
+      .select('person.emailsPrimaryEmail', 'emailsPrimaryEmail')
+      .addSelect('COUNT(*)', 'count')
+      .andWhere('person.emailsPrimaryEmail IS NOT NULL')
+      .andWhere("person.emailsPrimaryEmail != ''")
+      .withDeleted()
+      .groupBy('person.emailsPrimaryEmail')
+      .having('COUNT(*) > 1')
+      .getRawMany();
+
+    for (const duplicate of duplicates) {
+      const { emailsPrimaryEmail } = duplicate;
+
+      const softDeletedPersons = await personRepository.find({
+        where: {
+          emails: {
+            primaryEmail: emailsPrimaryEmail,
+          },
+          deletedAt: Not(IsNull()),
+        },
+        withDeleted: true,
+      });
+
+      for (const [i, softDeletedPerson] of softDeletedPersons.entries()) {
+        const newEmailsPrimaryEmail = this.computeNewFieldValues(
+          softDeletedPerson.emails.primaryEmail,
+          i,
+        );
+
+        if (!dryRun) {
+          await personRepository
+            .createQueryBuilder('person')
+            .update()
+            .set({
+              emails: {
+                primaryEmail: newEmailsPrimaryEmail,
+              },
+            })
+            .where('id = :id', { id: softDeletedPerson.id })
+            .execute();
+        }
+        this.logger.log(
+          `Updated person ${softDeletedPerson.id} emailsPrimaryEmail from ${emailsPrimaryEmail} to ${newEmailsPrimaryEmail}`,
+        );
+      }
+    }
+  }
+
+  private computeNewFieldValues(fieldValue: string, i: number) {
+    return `${fieldValue}-old-${i}`;
+  }
+}

packages/twenty-server/src/database/commands/upgrade-version-command/1-7/1-7-upgrade-version-command.module.ts

@@ -0,0 +1,38 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+
+import { DeduplicateUniqueFieldsCommand } from 'src/database/commands/upgrade-version-command/1-7/1-7-deduplicate-unique-fields.command';
+import { ViewFieldEntity } from 'src/engine/core-modules/view/entities/view-field.entity';
+import { ViewEntity } from 'src/engine/core-modules/view/entities/view.entity';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
+import { IndexMetadataModule } from 'src/engine/metadata-modules/index-metadata/index-metadata.module';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
+import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
+import { WorkspaceSchemaManagerModule } from 'src/engine/twenty-orm/workspace-schema-manager/workspace-schema-manager.module';
+import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
+import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
+
+@Module({
+  imports: [
+    TypeOrmModule.forFeature([
+      Workspace,
+      FieldMetadataEntity,
+      ObjectMetadataEntity,
+      ViewEntity,
+      ViewFieldEntity,
+      IndexMetadataEntity,
+    ]),
+    WorkspaceDataSourceModule,
+    WorkspaceMigrationRunnerModule,
+    WorkspaceSchemaManagerModule,
+    IndexMetadataModule,
+    WorkspaceMigrationModule,
+    WorkspaceMetadataVersionModule,
+  ],
+  providers: [DeduplicateUniqueFieldsCommand],
+  exports: [DeduplicateUniqueFieldsCommand],
+})
+export class V1_7_UpgradeVersionCommandModule {}