diff --git a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts index b6f946ced4..8606cda5c0 100644 --- a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts +++ b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts @@ -40,6 +40,7 @@ import * as nc_050_tenant_isolation from '~/meta/migrations/v2/nc_050_tenant_iso import * as nc_051_source_readonly_columns from '~/meta/migrations/v2/nc_051_source_readonly_columns'; import * as nc_052_field_aggregation from '~/meta/migrations/v2/nc_052_field_aggregation'; import * as nc_053_jobs from '~/meta/migrations/v2/nc_053_jobs'; +import * as nc_054_id_length from '~/meta/migrations/v2/nc_054_id_length'; // Create a custom migration source class export default class XcMigrationSourcev2 { @@ -91,6 +92,7 @@ export default class XcMigrationSourcev2 { 'nc_051_source_readonly_columns', 'nc_052_field_aggregation', 'nc_053_jobs', + 'nc_054_id_length', ]); } @@ -184,6 +186,8 @@ export default class XcMigrationSourcev2 { return nc_052_field_aggregation; case 'nc_053_jobs': return nc_053_jobs; + case 'nc_054_id_length': + return nc_054_id_length; } } } diff --git a/packages/nocodb/src/meta/migrations/v2/nc_050_tenant_isolation.ts b/packages/nocodb/src/meta/migrations/v2/nc_050_tenant_isolation.ts index 270bfe174e..7473429145 100644 --- a/packages/nocodb/src/meta/migrations/v2/nc_050_tenant_isolation.ts +++ b/packages/nocodb/src/meta/migrations/v2/nc_050_tenant_isolation.ts @@ -1,5 +1,6 @@ import type { Knex } from 'knex'; import { MetaTable } from '~/utils/globals'; +import { replaceLongBaseIds } from '~/meta/migrations/v2/nc_054_id_length'; /* Add base_id to: @@ -181,6 +182,9 @@ const listIndexesOnColumn = async ( const up = async (knex: Knex) => { log('Migration started'); + // Replace long base_ids before adding new columns to avoid value too long error + await replaceLongBaseIds(knex); + log('Adding missing base_id columns'); const addBaseId = [ diff --git a/packages/nocodb/src/meta/migrations/v2/nc_054_id_length.ts b/packages/nocodb/src/meta/migrations/v2/nc_054_id_length.ts new file mode 100644 index 0000000000..3ac502cc84 --- /dev/null +++ b/packages/nocodb/src/meta/migrations/v2/nc_054_id_length.ts @@ -0,0 +1,200 @@ +import { customAlphabet } from 'nanoid'; +import type { Knex } from 'knex'; +import { MetaTable } from '~/utils/globals'; + +const log = (message: string) => { + console.log(`nc_054_id_length: ${message}`); +}; + +let hrTime = process.hrtime(); + +const logExecutionTime = (message: string) => { + const [seconds, nanoseconds] = process.hrtime(hrTime); + const elapsedSeconds = seconds + nanoseconds / 1e9; + log(`${message} in ${elapsedSeconds}s`); +}; + +const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14); + +const generateUniqueBaseId = async (knex: Knex) => { + const baseId = `p${nanoidv2()}`; + + const base = await knex(MetaTable.PROJECT).where('id', baseId).first(); + + if (base) { + return generateUniqueBaseId(knex); + } + + return baseId; +}; + +const listBasesWithLongIds = async (knex: Knex) => { + const sourceType = knex.client.driverName; + + switch (sourceType) { + case 'pg': { + const bases = await knex.raw( + `SELECT id FROM ?? WHERE LENGTH(id) > 20`, + MetaTable.PROJECT, + ); + + return bases.rows.map((row: any) => row.id); + } + case 'mysql': + case 'mysql2': { + const bases = await knex.raw( + `SELECT id FROM ?? WHERE CHAR_LENGTH(id) > 20`, + MetaTable.PROJECT, + ); + + return bases[0].map((row: any) => row.id); + } + case 'sqlite3': { + const bases = await knex.raw( + `SELECT id FROM ?? WHERE LENGTH(id) > 20`, + MetaTable.PROJECT, + ); + + return bases.map((row: any) => row.id); + } + case 'mssql': { + const bases = await knex.raw( + `SELECT id FROM ?? WHERE LEN(id) > 20`, + MetaTable.PROJECT, + ); + + return bases.map((row: any) => row.id); + } + + default: + throw new Error(`Unsupported database: ${sourceType}`); + } +}; + +export const replaceLongBaseIds = async (knex: Knex) => { + const basesWithLongIds = await listBasesWithLongIds(knex); + + for (const baseId of basesWithLongIds) { + const newBaseId = await generateUniqueBaseId(knex); + + if (!baseId || !newBaseId) { + throw new Error(`Failed to replace ${baseId} with ${newBaseId}`); + } + + const tablesToChangeBaseId = [ + MetaTable.API_TOKENS, + MetaTable.AUDIT, + MetaTable.PROJECT_USERS, + MetaTable.CALENDAR_VIEW_COLUMNS, + MetaTable.CALENDAR_VIEW, + MetaTable.COLUMNS, + MetaTable.COMMENTS_REACTIONS, + MetaTable.COMMENTS, + MetaTable.MODEL_ROLE_VISIBILITY, + MetaTable.EXTENSIONS, + MetaTable.FILTER_EXP, + MetaTable.FORM_VIEW_COLUMNS, + MetaTable.FORM_VIEW, + MetaTable.GALLERY_VIEW_COLUMNS, + MetaTable.GALLERY_VIEW, + MetaTable.GRID_VIEW_COLUMNS, + MetaTable.GRID_VIEW, + MetaTable.HOOK_LOGS, + MetaTable.HOOKS, + MetaTable.KANBAN_VIEW_COLUMNS, + MetaTable.KANBAN_VIEW, + MetaTable.MAP_VIEW_COLUMNS, + MetaTable.MAP_VIEW, + MetaTable.MODELS, + MetaTable.SORT, + MetaTable.BASES, + MetaTable.SYNC_LOGS, + MetaTable.SYNC_SOURCE, + MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE, + MetaTable.VIEWS, + ]; + + for (const table of tablesToChangeBaseId) { + await knex(table).where('base_id', baseId).update({ base_id: newBaseId }); + } + + await knex(MetaTable.PROJECT).where('id', baseId).update({ id: newBaseId }); + + log(`Replaced ${baseId} with ${newBaseId} (because it was too long)`); + } +}; + +const tablesToAlterBaseId = [ + MetaTable.API_TOKENS, + MetaTable.AUDIT, + MetaTable.PROJECT_USERS, + MetaTable.COLUMNS, + MetaTable.COMMENTS_REACTIONS, + MetaTable.COMMENTS, + MetaTable.MODEL_ROLE_VISIBILITY, + MetaTable.FILTER_EXP, + MetaTable.FORM_VIEW_COLUMNS, + MetaTable.FORM_VIEW, + MetaTable.GALLERY_VIEW_COLUMNS, + MetaTable.GALLERY_VIEW, + MetaTable.GRID_VIEW_COLUMNS, + MetaTable.GRID_VIEW, + MetaTable.HOOK_LOGS, + MetaTable.HOOKS, + MetaTable.KANBAN_VIEW_COLUMNS, + MetaTable.KANBAN_VIEW, + MetaTable.MAP_VIEW, + MetaTable.MAP_VIEW_COLUMNS, + MetaTable.MODELS, + MetaTable.SORT, + MetaTable.BASES, + MetaTable.SYNC_LOGS, + MetaTable.SYNC_SOURCE, + MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE, + MetaTable.VIEWS, +]; + +const tablesToAlterSourceId = [ + MetaTable.CALENDAR_VIEW, + MetaTable.CALENDAR_VIEW_COLUMNS, +]; + +const up = async (knex: Knex) => { + hrTime = process.hrtime(); + + await replaceLongBaseIds(knex); + + logExecutionTime('Replaced long base IDs'); + + for (const table of tablesToAlterBaseId) { + hrTime = process.hrtime(); + await knex.schema.alterTable(table, (tableQb) => { + tableQb.string('base_id', 20).alter(); + }); + logExecutionTime(`Altered ${table}.base_id to 20 characters`); + } + + for (const table of tablesToAlterSourceId) { + hrTime = process.hrtime(); + await knex.schema.alterTable(table, (tableQb) => { + tableQb.string('source_id', 20).alter(); + }); + logExecutionTime(`Altered ${table}.source_id to 20 characters`); + } +}; + +const down = async (knex: Knex) => { + for (const table of tablesToAlterBaseId) { + await knex.schema.alterTable(table, (tableQb) => { + tableQb.string('base_id', 128).alter(); + }); + } + + for (const table of tablesToAlterSourceId) { + await knex.schema.alterTable(table, (tableQb) => { + tableQb.string('source_id', 128).alter(); + }); + } +}; + +export { up, down };