Size increase for namespace column (#2639)

* Increase size of namespace column * MariaDB migration * MySQL migration * MSSQL migration * Postgres migration * Planetscale migration * fixed indexNamespace * Paginate --------- Co-authored-by: Deepak Prabhakara <[email protected]>
boxyhq · May 21, 2024 · 25b29d7 · 25b29d7
1 parent a30efd8
commit 25b29d7
Show file tree

Hide file tree

Showing 10 changed files with 97 additions and 16 deletions.
diff --git a/npm/migration/mariadb/1714417013715-md_namespace.ts b/npm/migration/mariadb/1714417013715-md_namespace.ts
@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MdNamespace1714417013715 implements MigrationInterface {
+    name = 'MdNamespace1714417013715'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` MODIFY \`namespace\` varchar(256) NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` MODIFY \`namespace\` varchar(64) NULL`);
+    }
+
+}
diff --git a/npm/migration/mssql/1714421718208-mss_namespace.ts b/npm/migration/mssql/1714421718208-mss_namespace.ts
@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MssNamespace1714421718208 implements MigrationInterface {
+    name = 'MssNamespace1714421718208'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "jackson_store" ALTER COLUMN "namespace" varchar(256)`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "jackson_store" ALTER COLUMN "namespace" varchar(64)`);
+    }
+
+}
diff --git a/npm/migration/mysql/1714419315556-ms_namespace.ts b/npm/migration/mysql/1714419315556-ms_namespace.ts
@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MsNamespace1714419315556 implements MigrationInterface {
+    name = 'MsNamespace1714419315556'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` MODIFY \`namespace\` varchar(256) NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` MODIFY \`namespace\` varchar(64) NULL`);
+    }
+
+}
diff --git a/npm/migration/planetscale/1714457285484-ms_namespace.ts b/npm/migration/planetscale/1714457285484-ms_namespace.ts
@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MsNamespace1714457285484 implements MigrationInterface {
+    name = 'MsNamespace1714457285484'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` MODIFY \`namespace\` varchar(256) NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` MODIFY \`namespace\` varchar(64) NULL`);
+    }
+
+}
diff --git a/npm/migration/postgres/1714452929542-pg_namespace.ts b/npm/migration/postgres/1714452929542-pg_namespace.ts
@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class PgNamespace1714452929542 implements MigrationInterface {
+    name = 'PgNamespace1714452929542'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "jackson_store" ALTER COLUMN "namespace" TYPE VARCHAR(256)`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "jackson_store" ALTER COLUMN "namespace" TYPE VARCHAR(64)`);
+    }
+
+}
diff --git a/npm/src/db/planetscale/entity/JacksonStore.ts b/npm/src/db/planetscale/entity/JacksonStore.ts
@@ -46,7 +46,7 @@ export class JacksonStore {
   @Index('_jackson_store_namespace')
   @Column({
     type: 'varchar',
-    length: 64,
+    length: 256,
     nullable: true,
   })
   namespace?: string;

diff --git a/npm/src/db/sql/entity/JacksonStore.ts b/npm/src/db/sql/entity/JacksonStore.ts
@@ -44,7 +44,7 @@ export class JacksonStore {
   @Index('_jackson_store_namespace')
   @Column({
     type: 'varchar',
-    length: 64,
+    length: 256,
     nullable: true,
   })
   namespace?: string;

diff --git a/npm/src/db/sql/mariadb/entity/JacksonStore.ts b/npm/src/db/sql/mariadb/entity/JacksonStore.ts
@@ -46,7 +46,7 @@ export class JacksonStore {
   @Index('_jackson_store_namespace')
   @Column({
     type: 'varchar',
-    length: 64,
+    length: 256,
     nullable: true,
   })
   namespace?: string;

diff --git a/npm/src/db/sql/mssql/entity/JacksonStore.ts b/npm/src/db/sql/mssql/entity/JacksonStore.ts
@@ -44,7 +44,7 @@ export class JacksonStore {
   @Index('_jackson_store_namespace')
   @Column({
     type: 'varchar',
-    length: 64,
+    length: 256,
     nullable: true,
   })
   namespace?: string;

diff --git a/npm/src/db/sql/sql.ts b/npm/src/db/sql/sql.ts
@@ -131,19 +131,30 @@ class Sql implements DatabaseDriver {
   }
 
   async indexNamespace() {
-    const res = await this.storeRepository.find({
-      where: {
-        namespace: IsNull(),
-      },
-      select: ['key'],
-    });
-    const searchTerm = ':';
+    try {
+      const take = 1000;
+      while (true) {
+        const res = await this.storeRepository.find({
+          where: {
+            namespace: IsNull(),
+          },
+          select: ['key'],
+          take,
+        });
+        const searchTerm = ':';
 
-    for (const r of res) {
-      const key = r.key;
-      const tokens2 = key.split(searchTerm).slice(0, 2);
-      const value = tokens2.join(searchTerm);
-      await this.storeRepository.update({ key }, { namespace: value });
+        if (res.length === 0) {
+          break;
+        }
+
+        for (const r of res) {
+          const key = r.key;
+          const lastIndex = r.key.lastIndexOf(searchTerm);
+          await this.storeRepository.update({ key }, { namespace: r.key.substring(0, lastIndex) });
+        }
+      }
+    } catch (err) {
+      console.error('Error running indexNamespace:', err);
     }
   }