From 9b5f82b7e6f6a7ee1d03c74de3d3790ccd6a65a1 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 5 Nov 2024 18:08:05 +0000 Subject: [PATCH 1/4] refactor: move link recover and duplicate remove upgrader to migration job --- packages/nocodb/src/db/BaseModelSqlv2.ts | 10 +- packages/nocodb/src/interface/Jobs.ts | 2 + .../nocodb/src/modules/jobs/jobs.module.ts | 4 + .../migration-jobs/init-migration-jobs.ts | 14 + .../nc_job_003_recover_links.ts | 328 ++++++++++++++++++ .../nc_job_004_cleanup_duplicate_column.ts | 30 ++ .../nocodb/src/services/columns.service.ts | 1 - .../nocodb/src/version-upgrader/NcUpgrader.ts | 5 - .../upgraders/0227000_dupColMergeUpgrader.ts | 6 - .../upgraders/0227002_ncBrokenLinkRecovery.ts | 297 ---------------- 10 files changed, 385 insertions(+), 312 deletions(-) create mode 100644 packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts create mode 100644 packages/nocodb/src/modules/jobs/migration-jobs/nc_job_004_cleanup_duplicate_column.ts delete mode 100644 packages/nocodb/src/version-upgrader/upgraders/0227000_dupColMergeUpgrader.ts delete mode 100644 packages/nocodb/src/version-upgrader/upgraders/0227002_ncBrokenLinkRecovery.ts diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index ce12efc89d..55fa1c5223 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -10230,9 +10230,13 @@ export function getCompositePkValue(primaryKeys: Column[], row) { if (typeof row !== 'object') return row; if (primaryKeys.length > 1) { - return primaryKeys.map((c) => - (row[c.title] ?? row[c.column_name])?.toString?.().replaceAll('_', '\\_'), - ).join('___'); + return primaryKeys + .map((c) => + (row[c.title] ?? row[c.column_name]) + ?.toString?.() + .replaceAll('_', '\\_'), + ) + .join('___'); } return ( diff --git a/packages/nocodb/src/interface/Jobs.ts b/packages/nocodb/src/interface/Jobs.ts index d4dd99d8b6..1a67c10770 100644 --- a/packages/nocodb/src/interface/Jobs.ts +++ b/packages/nocodb/src/interface/Jobs.ts @@ -5,6 +5,8 @@ export const JOBS_QUEUE = 'jobs'; export enum MigrationJobTypes { Attachment = 'attachment', Thumbnail = 'thumbnail', + RecoverLinks = 'recover-links', + CleanupDuplicateColumns = 'cleanup-duplicate-columns', } export enum JobTypes { diff --git a/packages/nocodb/src/modules/jobs/jobs.module.ts b/packages/nocodb/src/modules/jobs/jobs.module.ts index 8f9079956e..46bb4d8c23 100644 --- a/packages/nocodb/src/modules/jobs/jobs.module.ts +++ b/packages/nocodb/src/modules/jobs/jobs.module.ts @@ -41,6 +41,8 @@ import { JobsEventService } from '~/modules/jobs/jobs-event.service'; import { JobsService as FallbackJobsService } from '~/modules/jobs/fallback/jobs.service'; import { QueueService as FallbackQueueService } from '~/modules/jobs/fallback/fallback-queue.service'; import { JOBS_QUEUE } from '~/interface/Jobs'; +import { RecoverLinksMigration } from '~/modules/jobs/migration-jobs/nc_job_003_recover_links'; +import { CleanupDuplicateColumnMigration } from '~/modules/jobs/migration-jobs/nc_job_004_cleanup_duplicate_column'; export const JobsModuleMetadata = { imports: [ @@ -101,6 +103,8 @@ export const JobsModuleMetadata = { InitMigrationJobs, AttachmentMigration, ThumbnailMigration, + RecoverLinksMigration, + CleanupDuplicateColumnMigration, ], exports: ['JobsService'], }; diff --git a/packages/nocodb/src/modules/jobs/migration-jobs/init-migration-jobs.ts b/packages/nocodb/src/modules/jobs/migration-jobs/init-migration-jobs.ts index d0f8ab4d49..a9241f8fdd 100644 --- a/packages/nocodb/src/modules/jobs/migration-jobs/init-migration-jobs.ts +++ b/packages/nocodb/src/modules/jobs/migration-jobs/init-migration-jobs.ts @@ -11,6 +11,8 @@ import { setMigrationJobsStallInterval, updateMigrationJobsState, } from '~/helpers/migrationJobs'; +import { RecoverLinksMigration } from '~/modules/jobs/migration-jobs/nc_job_003_recover_links'; +import { CleanupDuplicateColumnMigration } from '~/modules/jobs/migration-jobs/nc_job_004_cleanup_duplicate_column'; @Injectable() export class InitMigrationJobs { @@ -25,6 +27,16 @@ export class InitMigrationJobs { job: MigrationJobTypes.Thumbnail, service: this.thumbnailMigration, }, + { + version: '3', + job: MigrationJobTypes.RecoverLinks, + service: this.recoverLinksMigration, + }, + { + version: '4', + job: MigrationJobTypes.CleanupDuplicateColumns, + service: this.cleanupDuplicateColumnMigration, + }, ]; private readonly debugLog = debug('nc:migration-jobs:init'); @@ -34,6 +46,8 @@ export class InitMigrationJobs { private readonly jobsService: IJobsService, private readonly attachmentMigration: AttachmentMigration, private readonly thumbnailMigration: ThumbnailMigration, + private readonly recoverLinksMigration: RecoverLinksMigration, + private readonly cleanupDuplicateColumnMigration: CleanupDuplicateColumnMigration, ) {} log = (...msgs: string[]) => { diff --git a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts new file mode 100644 index 0000000000..5d19f04cc2 --- /dev/null +++ b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts @@ -0,0 +1,328 @@ +import debug from 'debug'; +import { Injectable } from '@nestjs/common'; +import { RelationTypes, UITypes } from 'nocodb-sdk'; +import type { MetaService } from '~/meta/meta.service'; +import Noco from '~/Noco'; +import { MetaTable } from '~/utils/globals'; +import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor'; +import { isEE } from '~/utils'; +import { Column } from '~/models'; + +/** + * This migration look for any broken link and try to recover it + * using the existing data + */ +@Injectable() +export class RecoverLinksMigration { + private readonly debugLog = debug('nc:migration-jobs:recover-links'); + + constructor( + private readonly thumbnailGeneratorProcessor: ThumbnailGeneratorProcessor, + ) {} + + log = (...msgs: string[]) => { + console.log('[nc_job_003_thumbnail]: ', ...msgs); + }; + + async job() { + // start transaction + const ncMeta = await Noco.ncMeta.startTransaction(); + + try { + // Get all broken link columns which doesn't have colOptions + const columns = await ncMeta + .knex(MetaTable.COLUMNS) + .select(`${MetaTable.COLUMNS}.*`) + .leftJoin( + MetaTable.COL_RELATIONS, + `${MetaTable.COLUMNS}.id`, + `${MetaTable.COL_RELATIONS}.fk_column_id`, + ) + .where(`${MetaTable.COLUMNS}.uidt`, UITypes.LinkToAnotherRecord) + .whereNull(`${MetaTable.COL_RELATIONS}.id`); + + // Recover broken link + for (const column of columns) { + this.log(`Recovering column '${column.title}' (ID: '${column.id}')`); + + let relatedTableId; + + // check any lookup or rollup column is using this column + const lookupColumns = await ncMeta + .knex(MetaTable.COL_LOOKUP) + .select(`${MetaTable.COL_LOOKUP}.*`) + .where(`${MetaTable.COL_LOOKUP}.fk_relation_column_id`, column.id); + + for (const lookupColumn of lookupColumns) { + const lookupCol = await ncMeta + .knex(MetaTable.COLUMNS) + .select(`${MetaTable.COLUMNS}.fk_model_id`) + .where(`${MetaTable.COLUMNS}.id`, lookupColumn.fk_lookup_column_id) + .first(); + if (lookupCol) { + relatedTableId = lookupCol.fk_model_id; + break; + } + } + + if (!relatedTableId) { + const rollupColumns = await ncMeta + .knex(MetaTable.COL_ROLLUP) + .select(`${MetaTable.COL_ROLLUP}.*`) + .where(`${MetaTable.COL_ROLLUP}.fk_relation_column_id`, column.id); + + for (const rollupColumn of rollupColumns) { + const rollupCol = await ncMeta + .knex(MetaTable.COLUMNS) + .select(`${MetaTable.COLUMNS}.fk_model_id`) + .where( + `${MetaTable.COLUMNS}.id`, + rollupColumn.fk_rollup_column_id, + ) + .first(); + if (rollupCol) { + relatedTableId = rollupCol.fk_model_id; + break; + } + } + } + + // if related table is not found then iterate over all links which is related to current table + const linksQb = ncMeta + .knex(MetaTable.COL_RELATIONS) + .select(`${MetaTable.COL_RELATIONS}.*`) + .select(`${MetaTable.COLUMNS}.fk_model_id`) + .where( + `${MetaTable.COL_RELATIONS}.fk_related_model_id`, + column.fk_model_id, + ) + .join( + MetaTable.COLUMNS, + `${MetaTable.COL_RELATIONS}.fk_column_id`, + `${MetaTable.COLUMNS}.id`, + ); + if (relatedTableId) { + linksQb.where(`${MetaTable.COLUMNS}.fk_model_id`, relatedTableId); + } + + const links = await linksQb; + let foundAndMapped = false; + + // iterate over all links which is related to current table and if found relation which doesn't have link in the related table then use it to populate colOptions + for (const link of links) { + const relatedTableId = link.fk_model_id; + let columnInCurrTable = null; + if (link.type === RelationTypes.HAS_MANY) { + // check for bt column in current table + columnInCurrTable = await ncMeta + .knex(MetaTable.COL_RELATIONS) + .join( + MetaTable.COLUMNS, + `${MetaTable.COL_RELATIONS}.fk_column_id`, + `${MetaTable.COLUMNS}.id`, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_related_model_id`, + relatedTableId, + ) + .where( + `${MetaTable.COL_RELATIONS}.type`, + RelationTypes.BELONGS_TO, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_child_column_id`, + link.fk_child_column_id, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_parent_column_id`, + link.fk_parent_column_id, + ) + .first(); + } else if (link.type === RelationTypes.ONE_TO_ONE) { + // check for one to one column in current table and confirm type in meta + columnInCurrTable = await ncMeta + .knex(MetaTable.COL_RELATIONS) + .join( + MetaTable.COLUMNS, + `${MetaTable.COL_RELATIONS}.fk_column_id`, + `${MetaTable.COLUMNS}.id`, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_related_model_id`, + relatedTableId, + ) + .where( + `${MetaTable.COL_RELATIONS}.type`, + RelationTypes.ONE_TO_ONE, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_child_column_id`, + link.fk_child_column_id, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_parent_column_id`, + link.fk_parent_column_id, + ) + .first(); + } else if (link.type === RelationTypes.BELONGS_TO) { + // check for hm column in current table + columnInCurrTable = await ncMeta + .knex(MetaTable.COL_RELATIONS) + .join( + MetaTable.COLUMNS, + `${MetaTable.COL_RELATIONS}.fk_column_id`, + `${MetaTable.COLUMNS}.id`, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_related_model_id`, + relatedTableId, + ) + .where(`${MetaTable.COL_RELATIONS}.type`, RelationTypes.HAS_MANY) + .where( + `${MetaTable.COL_RELATIONS}.fk_child_column_id`, + link.fk_child_column_id, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_parent_column_id`, + link.fk_parent_column_id, + ) + .first(); + } else if (link.type === RelationTypes.MANY_TO_MANY) { + // check for mtm column in current table + columnInCurrTable = await ncMeta + .knex(MetaTable.COL_RELATIONS) + .join( + MetaTable.COLUMNS, + `${MetaTable.COL_RELATIONS}.fk_column_id`, + `${MetaTable.COLUMNS}.id`, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_related_model_id`, + relatedTableId, + ) + .where( + `${MetaTable.COL_RELATIONS}.type`, + RelationTypes.BELONGS_TO, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_child_column_id`, + link.fk_parent_column_id, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_parent_column_id`, + link.fk_parent_column_id, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_mm_model_id`, + link.fk_mm_model_id, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_mm_child_column_id`, + link.fk_mm_parent_column_id, + ) + .where( + `${MetaTable.COL_RELATIONS}.fk_mm_parent_column_id`, + link.fk_mm_child_column_id, + ) + .first(); + } + + if (!columnInCurrTable) { + // generate meta and insert into colOptions + + const commonProps: Record = { + id: await (ncMeta as MetaService).genNanoid( + MetaTable.COL_RELATIONS, + ), + fk_column_id: column.id, + fk_related_model_id: relatedTableId, + created_at: link.created_at, + updated_at: link.updated_at, + virtual: link.virtual, + base_id: link.base_id, + }; + + if (isEE) { + commonProps.fk_workspace_id = link.fk_workspace_id; + } + + // based on type insert data into colOptions + switch (link.type) { + case RelationTypes.HAS_MANY: + // insert data into colOptions + await ncMeta.knex(MetaTable.COL_RELATIONS).insert({ + ...commonProps, + type: RelationTypes.BELONGS_TO, + fk_child_column_id: link.fk_child_column_id, + fk_parent_column_id: link.fk_parent_column_id, + }); + break; + case RelationTypes.ONE_TO_ONE: + { + // insert data into colOptions + await ncMeta.knex(MetaTable.COL_RELATIONS).insert({ + ...commonProps, + type: RelationTypes.ONE_TO_ONE, + fk_child_column_id: link.fk_child_column_id, + fk_parent_column_id: link.fk_parent_column_id, + }); + } + break; + case RelationTypes.BELONGS_TO: + // insert data into colOptions + + await ncMeta.knex(MetaTable.COL_RELATIONS).insert({ + ...commonProps, + type: RelationTypes.HAS_MANY, + fk_child_column_id: link.fk_child_column_id, + fk_parent_column_id: link.fk_parent_column_id, + }); + break; + case RelationTypes.MANY_TO_MANY: + // insert data into colOptions + + await ncMeta.knex(MetaTable.COL_RELATIONS).insert({ + ...commonProps, + type: RelationTypes.MANY_TO_MANY, + fk_child_column_id: link.fk_parent_column_id, + fk_parent_column_id: link.fk_child_column_id, + + fk_mm_model_id: link.fk_mm_model_id, + fk_mm_child_column_id: link.fk_mm_parent_column_id, + fk_mm_parent_column_id: link.fk_mm_child_column_id, + }); + break; + } + + foundAndMapped = true; + break; + } + } + + if (!foundAndMapped) { + this.log( + `No related column found for link column '${column.title}' (ID: '${column.id}'). Deleting it.`, + ); + + // delete the link column since it's not useful anymore and not recoverable + await Column.delete( + { + workspace_id: column.workspace_id, + base_id: column.base_id, + }, + column.id, + ncMeta, + ); + } else { + this.log(`Recovered column '${column.title}' (ID: '${column.id}')`); + } + } + await (await ncMeta).commit(); + } catch (e) { + await (await ncMeta).rollback(e); + this.log('Error recovering links', e); + return false; + } + return true; + } +} diff --git a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_004_cleanup_duplicate_column.ts b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_004_cleanup_duplicate_column.ts new file mode 100644 index 0000000000..1c7afb06cd --- /dev/null +++ b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_004_cleanup_duplicate_column.ts @@ -0,0 +1,30 @@ +import path from 'path'; +import debug from 'debug'; +import { Injectable } from '@nestjs/common'; +import PQueue from 'p-queue'; +import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2'; +import Noco from '~/Noco'; +import mimetypes from '~/utils/mimeTypes'; +import { RootScopes } from '~/utils/globals'; +import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor'; +import { getPathFromUrl } from '~/helpers/attachmentHelpers'; + +@Injectable() +export class CleanupDuplicateColumnMigration { + private readonly debugLog = debug( + 'nc:migration-jobs:cleanup-duplicate-column', + ); + + constructor( + private readonly thumbnailGeneratorProcessor: ThumbnailGeneratorProcessor, + ) {} + + log = (...msgs: string[]) => { + console.log('[nc_job_004_cleanup_duplicate_column]: ', ...msgs); + }; + + async job() { + // cloud only migration so keep as empty + return true; + } +} diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index c280beb54e..b4de119103 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -47,7 +47,6 @@ import { createHmAndBtColumn, createOOColumn, generateFkName, - randomID, sanitizeColumnName, validateLookupPayload, validatePayload, diff --git a/packages/nocodb/src/version-upgrader/NcUpgrader.ts b/packages/nocodb/src/version-upgrader/NcUpgrader.ts index 10dab59b0f..09b2e72276 100644 --- a/packages/nocodb/src/version-upgrader/NcUpgrader.ts +++ b/packages/nocodb/src/version-upgrader/NcUpgrader.ts @@ -12,8 +12,6 @@ import ncXcdbLTARUpgrader from './upgraders/0108002_ncXcdbLTARUpgrader'; import ncXcdbLTARIndexUpgrader from './upgraders/0111002_ncXcdbLTARIndexUpgrader'; import ncXcdbCreatedAndUpdatedSystemFieldsUpgrader from './upgraders/0111005_ncXcdbCreatedAndUpdatedSystemFieldsUpgrader'; import ncDatasourceDecrypt from './upgraders/0225002_ncDatasourceDecrypt'; -// import ncBrokenLinkRecovery from './upgraders/0227002_ncBrokenLinkRecovery'; -// import dupColMergeUpgrader from './upgraders/0227003_dupColMergeUpgrader'; import type { MetaService } from '~/meta/meta.service'; import type { NcConfig } from '~/interface/config'; import { T } from '~/utils'; @@ -152,9 +150,6 @@ export default class NcUpgrader { { name: '0111002', handler: ncXcdbLTARIndexUpgrader }, { name: '0111005', handler: ncXcdbCreatedAndUpdatedSystemFieldsUpgrader }, { name: '0225002', handler: ncDatasourceDecrypt }, - // disable for now - // { name: '0257002', handler: ncBrokenLinkRecovery }, - // { name: '0227000', handler: dupColMergeUpgrader }, ]; } } diff --git a/packages/nocodb/src/version-upgrader/upgraders/0227000_dupColMergeUpgrader.ts b/packages/nocodb/src/version-upgrader/upgraders/0227000_dupColMergeUpgrader.ts deleted file mode 100644 index 1df1b046e0..0000000000 --- a/packages/nocodb/src/version-upgrader/upgraders/0227000_dupColMergeUpgrader.ts +++ /dev/null @@ -1,6 +0,0 @@ -// this migration is only applicable in ee -import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader'; - -export default async (_: NcUpgraderCtx) => { - // do nothing -}; diff --git a/packages/nocodb/src/version-upgrader/upgraders/0227002_ncBrokenLinkRecovery.ts b/packages/nocodb/src/version-upgrader/upgraders/0227002_ncBrokenLinkRecovery.ts deleted file mode 100644 index f58541b6c0..0000000000 --- a/packages/nocodb/src/version-upgrader/upgraders/0227002_ncBrokenLinkRecovery.ts +++ /dev/null @@ -1,297 +0,0 @@ -import { RelationTypes, UITypes } from 'nocodb-sdk'; -import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader'; -import type { MetaService } from '~/meta/meta.service'; -import { MetaTable } from '~/utils/globals'; -import { Column } from '~/models'; -import { isEE } from '~/utils'; - -/** - * This upgrader look for any broken link and try to recover it - * using the existing data - */ - -const logger = { - log: (message: string) => { - console.log(`[0227002_ncBrokenLinkRecovery ${Date.now()}] ` + message); - }, - error: (message: string) => { - console.error(`[0227002_ncBrokenLinkRecovery ${Date.now()}] ` + message); - }, -}; - -export default async function ({ ncMeta }: NcUpgraderCtx) { - // Get all broken link columns which doesn't have colOptions - const columns = await ncMeta - .knex(MetaTable.COLUMNS) - .select(`${MetaTable.COLUMNS}.*`) - .leftJoin( - MetaTable.COL_RELATIONS, - `${MetaTable.COLUMNS}.id`, - `${MetaTable.COL_RELATIONS}.fk_column_id`, - ) - .where(`${MetaTable.COLUMNS}.uidt`, UITypes.LinkToAnotherRecord) - .whereNull(`${MetaTable.COL_RELATIONS}.id`); - - // Recover broken link - for (const column of columns) { - logger.log(`Recovering column '${column.title}' (ID: '${column.id}')`); - - let relatedTableId; - - // check any lookup or rollup column is using this column - const lookupColumns = await ncMeta - .knex(MetaTable.COL_LOOKUP) - .select(`${MetaTable.COL_LOOKUP}.*`) - .where(`${MetaTable.COL_LOOKUP}.fk_relation_column_id`, column.id); - - for (const lookupColumn of lookupColumns) { - const lookupCol = await ncMeta - .knex(MetaTable.COLUMNS) - .select(`${MetaTable.COLUMNS}.fk_model_id`) - .where(`${MetaTable.COLUMNS}.id`, lookupColumn.fk_lookup_column_id) - .first(); - if (lookupCol) { - relatedTableId = lookupCol.fk_model_id; - break; - } - } - - if (!relatedTableId) { - const rollupColumns = await ncMeta - .knex(MetaTable.COL_ROLLUP) - .select(`${MetaTable.COL_ROLLUP}.*`) - .where(`${MetaTable.COL_ROLLUP}.fk_relation_column_id`, column.id); - - for (const rollupColumn of rollupColumns) { - const rollupCol = await ncMeta - .knex(MetaTable.COLUMNS) - .select(`${MetaTable.COLUMNS}.fk_model_id`) - .where(`${MetaTable.COLUMNS}.id`, rollupColumn.fk_rollup_column_id) - .first(); - if (rollupCol) { - relatedTableId = rollupCol.fk_model_id; - break; - } - } - } - - // if related table is not found then iterate over all links which is related to current table - const linksQb = ncMeta - .knex(MetaTable.COL_RELATIONS) - .select(`${MetaTable.COL_RELATIONS}.*`) - .select(`${MetaTable.COLUMNS}.fk_model_id`) - .where( - `${MetaTable.COL_RELATIONS}.fk_related_model_id`, - column.fk_model_id, - ) - .join( - MetaTable.COLUMNS, - `${MetaTable.COL_RELATIONS}.fk_column_id`, - `${MetaTable.COLUMNS}.id`, - ); - if (relatedTableId) { - linksQb.where(`${MetaTable.COLUMNS}.fk_model_id`, relatedTableId); - } - - const links = await linksQb; - let foundAndMapped = false; - - // iterate over all links which is related to current table and if found relation which doesn't have link in the related table then use it to populate colOptions - for (const link of links) { - const relatedTableId = link.fk_model_id; - let columnInCurrTable = null; - if (link.type === RelationTypes.HAS_MANY) { - // check for bt column in current table - columnInCurrTable = await ncMeta - .knex(MetaTable.COL_RELATIONS) - .join( - MetaTable.COLUMNS, - `${MetaTable.COL_RELATIONS}.fk_column_id`, - `${MetaTable.COLUMNS}.id`, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_related_model_id`, - relatedTableId, - ) - .where(`${MetaTable.COL_RELATIONS}.type`, RelationTypes.BELONGS_TO) - .where( - `${MetaTable.COL_RELATIONS}.fk_child_column_id`, - link.fk_child_column_id, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_parent_column_id`, - link.fk_parent_column_id, - ) - .first(); - } else if (link.type === RelationTypes.ONE_TO_ONE) { - // check for one to one column in current table and confirm type in meta - columnInCurrTable = await ncMeta - .knex(MetaTable.COL_RELATIONS) - .join( - MetaTable.COLUMNS, - `${MetaTable.COL_RELATIONS}.fk_column_id`, - `${MetaTable.COLUMNS}.id`, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_related_model_id`, - relatedTableId, - ) - .where(`${MetaTable.COL_RELATIONS}.type`, RelationTypes.ONE_TO_ONE) - .where( - `${MetaTable.COL_RELATIONS}.fk_child_column_id`, - link.fk_child_column_id, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_parent_column_id`, - link.fk_parent_column_id, - ) - .first(); - } else if (link.type === RelationTypes.BELONGS_TO) { - // check for hm column in current table - columnInCurrTable = await ncMeta - .knex(MetaTable.COL_RELATIONS) - .join( - MetaTable.COLUMNS, - `${MetaTable.COL_RELATIONS}.fk_column_id`, - `${MetaTable.COLUMNS}.id`, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_related_model_id`, - relatedTableId, - ) - .where(`${MetaTable.COL_RELATIONS}.type`, RelationTypes.HAS_MANY) - .where( - `${MetaTable.COL_RELATIONS}.fk_child_column_id`, - link.fk_child_column_id, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_parent_column_id`, - link.fk_parent_column_id, - ) - .first(); - } else if (link.type === RelationTypes.MANY_TO_MANY) { - // check for mtm column in current table - columnInCurrTable = await ncMeta - .knex(MetaTable.COL_RELATIONS) - .join( - MetaTable.COLUMNS, - `${MetaTable.COL_RELATIONS}.fk_column_id`, - `${MetaTable.COLUMNS}.id`, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_related_model_id`, - relatedTableId, - ) - .where(`${MetaTable.COL_RELATIONS}.type`, RelationTypes.BELONGS_TO) - .where( - `${MetaTable.COL_RELATIONS}.fk_child_column_id`, - link.fk_parent_column_id, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_parent_column_id`, - link.fk_parent_column_id, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_mm_model_id`, - link.fk_mm_model_id, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_mm_child_column_id`, - link.fk_mm_parent_column_id, - ) - .where( - `${MetaTable.COL_RELATIONS}.fk_mm_parent_column_id`, - link.fk_mm_child_column_id, - ) - .first(); - } - - if (!columnInCurrTable) { - // generate meta and insert into colOptions - - const commonProps: Record = { - id: await (ncMeta as MetaService).genNanoid(MetaTable.COL_RELATIONS), - fk_column_id: column.id, - fk_related_model_id: relatedTableId, - created_at: link.created_at, - updated_at: link.updated_at, - virtual: link.virtual, - base_id: link.base_id, - }; - - if (isEE) { - commonProps.fk_workspace_id = link.fk_workspace_id; - } - - // based on type insert data into colOptions - switch (link.type) { - case RelationTypes.HAS_MANY: - // insert data into colOptions - await ncMeta.knex(MetaTable.COL_RELATIONS).insert({ - ...commonProps, - type: RelationTypes.BELONGS_TO, - fk_child_column_id: link.fk_child_column_id, - fk_parent_column_id: link.fk_parent_column_id, - }); - break; - case RelationTypes.ONE_TO_ONE: - { - // insert data into colOptions - await ncMeta.knex(MetaTable.COL_RELATIONS).insert({ - ...commonProps, - type: RelationTypes.ONE_TO_ONE, - fk_child_column_id: link.fk_child_column_id, - fk_parent_column_id: link.fk_parent_column_id, - }); - } - break; - case RelationTypes.BELONGS_TO: - // insert data into colOptions - - await ncMeta.knex(MetaTable.COL_RELATIONS).insert({ - ...commonProps, - type: RelationTypes.HAS_MANY, - fk_child_column_id: link.fk_child_column_id, - fk_parent_column_id: link.fk_parent_column_id, - }); - break; - case RelationTypes.MANY_TO_MANY: - // insert data into colOptions - - await ncMeta.knex(MetaTable.COL_RELATIONS).insert({ - ...commonProps, - type: RelationTypes.MANY_TO_MANY, - fk_child_column_id: link.fk_parent_column_id, - fk_parent_column_id: link.fk_child_column_id, - - fk_mm_model_id: link.fk_mm_model_id, - fk_mm_child_column_id: link.fk_mm_parent_column_id, - fk_mm_parent_column_id: link.fk_mm_child_column_id, - }); - break; - } - - foundAndMapped = true; - break; - } - } - - if (!foundAndMapped) { - logger.error( - `No related column found for link column '${column.title}' (ID: '${column.id}'). Deleting it.`, - ); - - // delete the link column since it's not useful anymore and not recoverable - await Column.delete( - { - workspace_id: column.workspace_id, - base_id: column.base_id, - }, - column.id, - ncMeta, - ); - } else { - logger.log(`Recovered column '${column.title}' (ID: '${column.id}')`); - } - } -} From 326aecad15f3fdb887a77a921b50e10ba82b0e6d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 5 Nov 2024 18:08:05 +0000 Subject: [PATCH 2/4] refactor: cleanup and add logs --- .../jobs/migration-jobs/nc_job_003_recover_links.ts | 8 +++----- .../nc_job_004_cleanup_duplicate_column.ts | 12 +----------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts index 5d19f04cc2..812f82d985 100644 --- a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts +++ b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts @@ -4,7 +4,6 @@ import { RelationTypes, UITypes } from 'nocodb-sdk'; import type { MetaService } from '~/meta/meta.service'; import Noco from '~/Noco'; import { MetaTable } from '~/utils/globals'; -import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor'; import { isEE } from '~/utils'; import { Column } from '~/models'; @@ -16,12 +15,10 @@ import { Column } from '~/models'; export class RecoverLinksMigration { private readonly debugLog = debug('nc:migration-jobs:recover-links'); - constructor( - private readonly thumbnailGeneratorProcessor: ThumbnailGeneratorProcessor, - ) {} + constructor() {} log = (...msgs: string[]) => { - console.log('[nc_job_003_thumbnail]: ', ...msgs); + console.log('[nc_job_003_recover_links]: ', ...msgs); }; async job() { @@ -317,6 +314,7 @@ export class RecoverLinksMigration { this.log(`Recovered column '${column.title}' (ID: '${column.id}')`); } } + this.log('Recovery completed'); await (await ncMeta).commit(); } catch (e) { await (await ncMeta).rollback(e); diff --git a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_004_cleanup_duplicate_column.ts b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_004_cleanup_duplicate_column.ts index 1c7afb06cd..c28357dd5b 100644 --- a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_004_cleanup_duplicate_column.ts +++ b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_004_cleanup_duplicate_column.ts @@ -1,13 +1,5 @@ -import path from 'path'; import debug from 'debug'; import { Injectable } from '@nestjs/common'; -import PQueue from 'p-queue'; -import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2'; -import Noco from '~/Noco'; -import mimetypes from '~/utils/mimeTypes'; -import { RootScopes } from '~/utils/globals'; -import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor'; -import { getPathFromUrl } from '~/helpers/attachmentHelpers'; @Injectable() export class CleanupDuplicateColumnMigration { @@ -15,9 +7,7 @@ export class CleanupDuplicateColumnMigration { 'nc:migration-jobs:cleanup-duplicate-column', ); - constructor( - private readonly thumbnailGeneratorProcessor: ThumbnailGeneratorProcessor, - ) {} + constructor() {} log = (...msgs: string[]) => { console.log('[nc_job_004_cleanup_duplicate_column]: ', ...msgs); From 1f6a314835a79945d3868f4a15f59096035b23b0 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 5 Nov 2024 23:57:44 +0530 Subject: [PATCH 3/4] Update packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../modules/jobs/migration-jobs/nc_job_003_recover_links.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts index 812f82d985..deadf9a472 100644 --- a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts +++ b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts @@ -315,9 +315,9 @@ export class RecoverLinksMigration { } } this.log('Recovery completed'); - await (await ncMeta).commit(); + await ncMeta.commit(); } catch (e) { - await (await ncMeta).rollback(e); + await ncMeta.rollback(e); this.log('Error recovering links', e); return false; } From 44c1b37a4fc5e8584251a977068ff6c495f235ea Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 5 Nov 2024 23:58:12 +0530 Subject: [PATCH 4/4] Update packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts index deadf9a472..68de2ef506 100644 --- a/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts +++ b/packages/nocodb/src/modules/jobs/migration-jobs/nc_job_003_recover_links.ts @@ -15,7 +15,6 @@ import { Column } from '~/models'; export class RecoverLinksMigration { private readonly debugLog = debug('nc:migration-jobs:recover-links'); - constructor() {} log = (...msgs: string[]) => { console.log('[nc_job_003_recover_links]: ', ...msgs);