Merge pull request #8590 from nocodb/nc-fix/mysql-index-long

Nc fix/Comment schema migration

packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts

@@ -33,6 +33,7 @@ import * as nc_043_user_refresh_token from '~/meta/migrations/v2/nc_043_user_ref
 import * as nc_044_view_column_index from '~/meta/migrations/v2/nc_044_view_column_index';
 import * as nc_045_extensions from '~/meta/migrations/v2/nc_045_extensions';
 import * as nc_046_comment_mentions from '~/meta/migrations/v2/nc_046_comment_mentions';
+import * as nc_047_comment_migration from '~/meta/migrations/v2/nc_047_comment_migration';
 
 // Create a custom migration source class
 export default class XcMigrationSourcev2 {
@@ -77,6 +78,7 @@ export default class XcMigrationSourcev2 {
       'nc_044_view_column_index',
       'nc_045_extensions',
       'nc_046_comment_mentions',
+      'nc_047_comment_migration',
     ]);
   }
 
@@ -156,6 +158,8 @@ export default class XcMigrationSourcev2 {
         return nc_045_extensions;
       case 'nc_046_comment_mentions':
         return nc_046_comment_mentions;
+      case 'nc_047_comment_migration':
+        return nc_047_comment_migration;
     }
   }
 }

packages/nocodb/src/meta/migrations/v2/nc_046_comment_mentions.ts

@@ -2,21 +2,19 @@ import { Logger } from '@nestjs/common';
 import type { Knex } from 'knex';
 import { MetaTable } from '~/utils/globals';
 
-const logger = new Logger('nc_046_comment_mentions');
-
 const up = async (knex: Knex) => {
   await knex.schema.createTable(MetaTable.COMMENTS, (table) => {
     table.string('id', 20).primary();
 
     table.string('row_id', 255);
 
-    table.string('comment', 3000);
+    table.text('comment');
 
-    table.string('created_by', 255);
+    table.string('created_by', 20);
 
     table.string('created_by_email', 255);
 
-    table.string('resolved_by', 255);
+    table.string('resolved_by', 20);
 
     table.string('resolved_by_email', 255);
 
@@ -42,7 +40,7 @@ const up = async (knex: Knex) => {
 
       table.string('row_id', 255);
 
-      table.string('user_id', 255);
+      table.string('user_id', 20);
 
       table.string('fk_model_id', 20);
 
@@ -70,7 +68,7 @@ const up = async (knex: Knex) => {
 
     table.string('source_id', 20);
 
-    table.string('fk_modal_id', 20);
+    table.string('fk_model_id', 20);
 
     table.string('base_id', 128);
 
@@ -80,55 +78,6 @@ const up = async (knex: Knex) => {
 
     table.timestamps(true, true);
   });
-
-  logger.log('nc_046_comment_mentions: Tables Created');
-
-  knex
-    .select(
-      `${MetaTable.AUDIT}.id`,
-      `${MetaTable.AUDIT}.row_id`,
-      `${MetaTable.AUDIT}.description`,
-      `${MetaTable.AUDIT}.user as user_email`,
-      `${MetaTable.AUDIT}.source_id`,
-      `${MetaTable.AUDIT}.base_id`,
-      `${MetaTable.AUDIT}.fk_model_id`,
-      `${MetaTable.AUDIT}.created_at`,
-      `${MetaTable.AUDIT}.updated_at`,
-      `${MetaTable.USERS}.id as user_id`,
-    )
-    .from(MetaTable.AUDIT)
-    .where(`${MetaTable.AUDIT}.op_type`, 'COMMENT')
-    .leftJoin(
-      MetaTable.USERS,
-      `${MetaTable.AUDIT}.user`,
-      `${MetaTable.USERS}.email`,
-    )
-    .then(async (rows) => {
-      if (!rows.length) return;
-
-      logger.log('nc_046_comment_mentions: Data from Audit Table Selected');
-
-      const formattedRows = rows.map((row) => ({
-        id: row.id,
-        row_id: row.row_id,
-        comment: row.description
-          .substring(row.description.indexOf(':') + 1)
-          .trim(),
-        created_by: row.user_id,
-        created_by_email: row.user_email,
-        source_id: row.source_id,
-        base_id: row.base_id,
-        fk_model_id: row.fk_model_id,
-        created_at: row.created_at,
-        updated_at: row.updated_at,
-      }));
-
-      logger.log('nc_046_comment_mentions: Data from Audit Table Formatted');
-
-      await knex(MetaTable.COMMENTS).insert(formattedRows);
-
-      logger.log('nc_046_comment_mentions: Data from Audit Table Migrated');
-    });
 };
 
 const down = async (knex: Knex) => {

packages/nocodb/src/meta/migrations/v2/nc_047_comment_migration.ts

@@ -0,0 +1,84 @@
+import { Logger } from '@nestjs/common';
+import type { Knex } from 'knex';
+import { MetaTable } from '~/utils/globals';
+
+const logger = new Logger('nc_046_comment_mentions');
+
+const READ_BATCH_SIZE = 1000;
+const INSERT_BATCH_SIZE = 200;
+
+const up = async (knex: Knex) => {
+  logger.log('Migration Started');
+
+  let fetchNextBatch = true;
+
+  for (let offset = 0; fetchNextBatch; offset += READ_BATCH_SIZE) {
+    const rows = await knex
+      .select(
+        `${MetaTable.AUDIT}.id`,
+        `${MetaTable.AUDIT}.row_id`,
+        `${MetaTable.AUDIT}.description`,
+        `${MetaTable.AUDIT}.user as user_email`,
+        `${MetaTable.AUDIT}.source_id`,
+        `${MetaTable.AUDIT}.base_id`,
+        `${MetaTable.AUDIT}.fk_model_id`,
+        `${MetaTable.AUDIT}.created_at`,
+        `${MetaTable.AUDIT}.updated_at`,
+        `${MetaTable.USERS}.id as user_id`,
+      )
+      .from(MetaTable.AUDIT)
+      .where(`${MetaTable.AUDIT}.op_type`, 'COMMENT')
+      .leftJoin(
+        MetaTable.USERS,
+        `${MetaTable.AUDIT}.user`,
+        `${MetaTable.USERS}.email`,
+      )
+      .offset(offset)
+      // increase limit by 1 to check if there are more rows
+      .limit(READ_BATCH_SIZE + 1);
+
+    logger.log(
+       `Data from Audit Table fetched, batch: ${offset} - ${
+        offset + READ_BATCH_SIZE
+      }`,
+    );
+
+    const formattedRows = rows
+      // exclude the last row since it was used to check if there are more rows
+      .slice(0, READ_BATCH_SIZE)
+      .map((row) => ({
+        id: row.id,
+        row_id: row.row_id,
+        comment: (row.description ?? '')
+          .substring((row.description ?? '').indexOf(':') + 1)
+          .trim(),
+        created_by: row.user_id,
+        created_by_email: row.user_email,
+        source_id: row.source_id,
+        base_id: row.base_id,
+        fk_model_id: row.fk_model_id,
+        created_at: row.created_at,
+        updated_at: row.updated_at,
+      }));
+
+    logger.log('Data from Audit Table formatted');
+
+    await knex.batchInsert(
+      MetaTable.COMMENTS,
+      formattedRows,
+      INSERT_BATCH_SIZE,
+    );
+
+    // check if there are more rows to fetch
+    fetchNextBatch = rows.length > READ_BATCH_SIZE;
+  }
+  logger.log(
+    'Data migrated from Audit Table to Comments Table',
+  );
+};
+
+const down = async (knex: Knex) => {
+  await knex.schema.dropTable(MetaTable.COMMENTS);
+};
+
+export { up, down };