diff --git a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts index 22c21739db..df2cf7aa9a 100644 --- a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts +++ b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts @@ -33,6 +33,7 @@ import * as nc_043_user_refresh_token from '~/meta/migrations/v2/nc_043_user_ref import * as nc_044_view_column_index from '~/meta/migrations/v2/nc_044_view_column_index'; import * as nc_045_extensions from '~/meta/migrations/v2/nc_045_extensions'; import * as nc_046_comment_mentions from '~/meta/migrations/v2/nc_046_comment_mentions'; +import * as nc_047_comment_migration from '~/meta/migrations/v2/nc_047_comment_migration'; // Create a custom migration source class export default class XcMigrationSourcev2 { @@ -77,6 +78,7 @@ export default class XcMigrationSourcev2 { 'nc_044_view_column_index', 'nc_045_extensions', 'nc_046_comment_mentions', + 'nc_047_comment_migration', ]); } @@ -156,6 +158,8 @@ export default class XcMigrationSourcev2 { return nc_045_extensions; case 'nc_046_comment_mentions': return nc_046_comment_mentions; + case 'nc_047_comment_migration': + return nc_047_comment_migration; } } } diff --git a/packages/nocodb/src/meta/migrations/v2/nc_046_comment_mentions.ts b/packages/nocodb/src/meta/migrations/v2/nc_046_comment_mentions.ts index 653f3be43a..9276a283d0 100644 --- a/packages/nocodb/src/meta/migrations/v2/nc_046_comment_mentions.ts +++ b/packages/nocodb/src/meta/migrations/v2/nc_046_comment_mentions.ts @@ -2,21 +2,19 @@ import { Logger } from '@nestjs/common'; import type { Knex } from 'knex'; import { MetaTable } from '~/utils/globals'; -const logger = new Logger('nc_046_comment_mentions'); - const up = async (knex: Knex) => { await knex.schema.createTable(MetaTable.COMMENTS, (table) => { table.string('id', 20).primary(); table.string('row_id', 255); - table.string('comment', 3000); + table.text('comment'); - table.string('created_by', 255); + table.string('created_by', 20); table.string('created_by_email', 255); - table.string('resolved_by', 255); + table.string('resolved_by', 20); table.string('resolved_by_email', 255); @@ -42,7 +40,7 @@ const up = async (knex: Knex) => { table.string('row_id', 255); - table.string('user_id', 255); + table.string('user_id', 20); table.string('fk_model_id', 20); @@ -70,7 +68,7 @@ const up = async (knex: Knex) => { table.string('source_id', 20); - table.string('fk_modal_id', 20); + table.string('fk_model_id', 20); table.string('base_id', 128); @@ -80,55 +78,6 @@ const up = async (knex: Knex) => { table.timestamps(true, true); }); - - logger.log('nc_046_comment_mentions: Tables Created'); - - knex - .select( - `${MetaTable.AUDIT}.id`, - `${MetaTable.AUDIT}.row_id`, - `${MetaTable.AUDIT}.description`, - `${MetaTable.AUDIT}.user as user_email`, - `${MetaTable.AUDIT}.source_id`, - `${MetaTable.AUDIT}.base_id`, - `${MetaTable.AUDIT}.fk_model_id`, - `${MetaTable.AUDIT}.created_at`, - `${MetaTable.AUDIT}.updated_at`, - `${MetaTable.USERS}.id as user_id`, - ) - .from(MetaTable.AUDIT) - .where(`${MetaTable.AUDIT}.op_type`, 'COMMENT') - .leftJoin( - MetaTable.USERS, - `${MetaTable.AUDIT}.user`, - `${MetaTable.USERS}.email`, - ) - .then(async (rows) => { - if (!rows.length) return; - - logger.log('nc_046_comment_mentions: Data from Audit Table Selected'); - - const formattedRows = rows.map((row) => ({ - id: row.id, - row_id: row.row_id, - comment: row.description - .substring(row.description.indexOf(':') + 1) - .trim(), - created_by: row.user_id, - created_by_email: row.user_email, - source_id: row.source_id, - base_id: row.base_id, - fk_model_id: row.fk_model_id, - created_at: row.created_at, - updated_at: row.updated_at, - })); - - logger.log('nc_046_comment_mentions: Data from Audit Table Formatted'); - - await knex(MetaTable.COMMENTS).insert(formattedRows); - - logger.log('nc_046_comment_mentions: Data from Audit Table Migrated'); - }); }; const down = async (knex: Knex) => { diff --git a/packages/nocodb/src/meta/migrations/v2/nc_047_comment_migration.ts b/packages/nocodb/src/meta/migrations/v2/nc_047_comment_migration.ts new file mode 100644 index 0000000000..6c02d8fcad --- /dev/null +++ b/packages/nocodb/src/meta/migrations/v2/nc_047_comment_migration.ts @@ -0,0 +1,84 @@ +import { Logger } from '@nestjs/common'; +import type { Knex } from 'knex'; +import { MetaTable } from '~/utils/globals'; + +const logger = new Logger('nc_046_comment_mentions'); + +const READ_BATCH_SIZE = 1000; +const INSERT_BATCH_SIZE = 200; + +const up = async (knex: Knex) => { + logger.log('Migration Started'); + + let fetchNextBatch = true; + + for (let offset = 0; fetchNextBatch; offset += READ_BATCH_SIZE) { + const rows = await knex + .select( + `${MetaTable.AUDIT}.id`, + `${MetaTable.AUDIT}.row_id`, + `${MetaTable.AUDIT}.description`, + `${MetaTable.AUDIT}.user as user_email`, + `${MetaTable.AUDIT}.source_id`, + `${MetaTable.AUDIT}.base_id`, + `${MetaTable.AUDIT}.fk_model_id`, + `${MetaTable.AUDIT}.created_at`, + `${MetaTable.AUDIT}.updated_at`, + `${MetaTable.USERS}.id as user_id`, + ) + .from(MetaTable.AUDIT) + .where(`${MetaTable.AUDIT}.op_type`, 'COMMENT') + .leftJoin( + MetaTable.USERS, + `${MetaTable.AUDIT}.user`, + `${MetaTable.USERS}.email`, + ) + .offset(offset) + // increase limit by 1 to check if there are more rows + .limit(READ_BATCH_SIZE + 1); + + logger.log( + `Data from Audit Table fetched, batch: ${offset} - ${ + offset + READ_BATCH_SIZE + }`, + ); + + const formattedRows = rows + // exclude the last row since it was used to check if there are more rows + .slice(0, READ_BATCH_SIZE) + .map((row) => ({ + id: row.id, + row_id: row.row_id, + comment: (row.description ?? '') + .substring((row.description ?? '').indexOf(':') + 1) + .trim(), + created_by: row.user_id, + created_by_email: row.user_email, + source_id: row.source_id, + base_id: row.base_id, + fk_model_id: row.fk_model_id, + created_at: row.created_at, + updated_at: row.updated_at, + })); + + logger.log('Data from Audit Table formatted'); + + await knex.batchInsert( + MetaTable.COMMENTS, + formattedRows, + INSERT_BATCH_SIZE, + ); + + // check if there are more rows to fetch + fetchNextBatch = rows.length > READ_BATCH_SIZE; + } + logger.log( + 'Data migrated from Audit Table to Comments Table', + ); +}; + +const down = async (knex: Knex) => { + await knex.schema.dropTable(MetaTable.COMMENTS); +}; + +export { up, down };