From 2ef19094576827c9ba923f94662bd8192a1e544a Mon Sep 17 00:00:00 2001 From: mertmit Date: Thu, 4 May 2023 15:46:06 +0300 Subject: [PATCH] feat: duplicate table back-end Signed-off-by: mertmit --- .../export-import/duplicate.controller.ts | 45 +- .../jobs/export-import/duplicate.processor.ts | 561 +++++++++++++----- .../jobs/export-import/export.service.ts | 25 +- .../jobs/export-import/import.service.ts | 414 ++++++++++++- .../modules/jobs/fallback-queue.service.ts | 4 + 5 files changed, 875 insertions(+), 174 deletions(-) diff --git a/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts index 5dbbd94eed..a2df328a48 100644 --- a/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts +++ b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts @@ -15,7 +15,7 @@ import { ExtractProjectIdMiddleware, } from '../../../middlewares/extract-project-id/extract-project-id.middleware'; import { ProjectsService } from '../../../services/projects.service'; -import { Base, Project } from '../../../models'; +import { Base, Model, Project } from '../../../models'; import { generateUniqueName } from '../../../helpers/exportImportHelpers'; import { QueueService } from '../fallback-queue.service'; import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs'; @@ -80,4 +80,47 @@ export class DuplicateController { return { id: job.id, name: job.name }; } + + @Post('/api/v1/db/meta/duplicate/:projectId/model/:modelId') + @HttpCode(200) + @Acl('duplicateModel') + async duplicateModel( + @Request() req, + @Param('projectId') projectId: string, + @Param('modelId') modelId?: string, + ) { + const project = await Project.get(projectId); + + if (!project) { + throw new Error(`Project not found for id '${projectId}'`); + } + + const model = await Model.get(modelId); + + if (!model) { + throw new Error(`Model not found!`); + } + + const base = await Base.get(model.base_id); + + const models = await base.getModels(); + + const uniqueTitle = generateUniqueName( + `${model.title} copy`, + models.map((p) => p.title), + ); + + const job = await this.activeQueue.add(JobTypes.DuplicateModel, { + projectId: project.id, + baseId: base.id, + modelId: model.id, + req: { + user: req.user, + clientIp: req.clientIp, + }, + title: uniqueTitle, + }); + + return { id: job.id, name: job.name }; + } } diff --git a/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts index 2e7fcba3ca..55d812ce8c 100644 --- a/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts +++ b/packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts @@ -2,6 +2,7 @@ import { Readable } from 'stream'; import { Process, Processor } from '@nestjs/bull'; import { Job } from 'bull'; import papaparse from 'papaparse'; +import { UITypes } from 'nocodb-sdk'; import { Base, Column, Model, Project } from '../../../models'; import { ProjectsService } from '../../../services/projects.service'; import { findWithIdentifier } from '../../../helpers/exportImportHelpers'; @@ -13,6 +14,28 @@ import type { LinkToAnotherRecordColumn } from '../../../models'; const DEBUG = false; +const debugLog = function (...args: any[]) { + if (DEBUG) { + console.log(...args); + } +}; + +const initTime = function () { + return { + hrTime: process.hrtime(), + }; +}; + +const elapsedTime = function ( + time: { hrTime: [number, number] }, + label?: string, +) { + const elapsedS = process.hrtime(time.hrTime)[0].toFixed(3); + const elapsedMs = process.hrtime(time.hrTime)[1] / 1000000; + if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`); + time.hrTime = process.hrtime(); +}; + @Processor(JOBS_QUEUE) export class DuplicateProcessor { constructor( @@ -24,6 +47,8 @@ export class DuplicateProcessor { @Process(JobTypes.DuplicateBase) async duplicateBase(job: Job) { + const hrTime = initTime(); + const { projectId, baseId, dupProjectId, req } = job.data; const project = await Project.get(projectId); @@ -35,21 +60,6 @@ export class DuplicateProcessor { throw new Error(`Project or base not found!`); } - let start = process.hrtime(); - - const debugLog = function (...args: any[]) { - if (DEBUG) { - console.log(...args); - } - }; - - const elapsedTime = function (label?: string) { - const elapsedS = process.hrtime(start)[0].toFixed(3); - const elapsedMs = process.hrtime(start)[1] / 1000000; - if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`); - start = process.hrtime(); - }; - const user = (req as any).user; const models = (await base.getModels()).filter( @@ -61,7 +71,7 @@ export class DuplicateProcessor { modelIds: models.map((m) => m.id), }); - elapsedTime('serializeModels'); + elapsedTime(hrTime, 'serializeModels'); if (!exportedModels) { throw new Error(`Export failed for base '${base.id}'`); @@ -69,48 +79,384 @@ export class DuplicateProcessor { await dupProject.getBases(); - const dupBaseId = dupProject.bases[0].id; + const dupBase = dupProject.bases[0]; - elapsedTime('projectCreate'); + elapsedTime(hrTime, 'projectCreate'); const idMap = await this.importService.importModels({ user, projectId: dupProject.id, - baseId: dupBaseId, + baseId: dupBase.id, data: exportedModels, req: req, }); - elapsedTime('importModels'); + elapsedTime(hrTime, 'importModels'); if (!idMap) { throw new Error(`Import failed for base '${base.id}'`); } - const handledLinks = []; - const lChunks: Record = {}; // fk_mm_model_id: { rowId, childId }[] - - const insertChunks = async () => { - for (const [k, v] of Object.entries(lChunks)) { - try { - if (v.length === 0) continue; - await this.bulkDataService.bulkDataInsert({ - projectName: dupProject.id, - tableName: k, - body: v, - cookie: null, - chunkSize: 1000, - foreign_key_checks: false, - raw: true, - }); - lChunks[k] = []; - } catch (e) { - console.log(e); - } + await this.importModelsData({ + idMap, + sourceProject: project, + sourceModels: models, + destProject: dupProject, + destBase: dupBase, + hrTime, + }); + + await this.projectsService.projectUpdate({ + projectId: dupProject.id, + project: { + status: null, + }, + }); + } catch (e) { + if (dupProject?.id) { + await this.projectsService.projectSoftDelete({ + projectId: dupProject.id, + }); + } + throw e; + } + } + + @Process(JobTypes.DuplicateModel) + async duplicateModel(job: Job) { + const hrTime = initTime(); + + const { projectId, baseId, modelId, title, req } = job.data; + + const project = await Project.get(projectId); + const base = await Base.get(baseId); + + const user = (req as any).user; + + const models = (await base.getModels()).filter( + (m) => !m.mm && m.type === 'table', + ); + + const sourceModel = models.find((m) => m.id === modelId); + + await sourceModel.getColumns(); + + const relatedModelIds = sourceModel.columns + .filter((col) => col.uidt === UITypes.LinkToAnotherRecord) + .map((col) => col.colOptions.fk_related_model_id) + .filter((id) => id); + + const relatedModels = models.filter((m) => relatedModelIds.includes(m.id)); + + const exportedModel = ( + await this.exportService.serializeModels({ + modelIds: [modelId], + }) + )[0]; + + elapsedTime(hrTime, 'serializeModel'); + + if (!exportedModel) { + throw new Error(`Export failed for base '${base.id}'`); + } + + exportedModel.model.title = title; + exportedModel.model.table_name = title.toLowerCase().replace(/ /g, '_'); + + const idMap = await this.importService.importModels({ + projectId, + baseId, + data: [exportedModel], + user, + req, + externalModels: relatedModels, + }); + + elapsedTime(hrTime, 'reimportModelSchema'); + + if (!idMap) { + throw new Error(`Import failed for model '${modelId}'`); + } + + const fields: Record = {}; + + for (const md of relatedModels) { + const bts = md.columns + .filter( + (c) => + c.uidt === UITypes.LinkToAnotherRecord && + c.colOptions.type === 'bt' && + c.colOptions.fk_related_model_id === modelId, + ) + .map((c) => c.id); + + if (bts.length > 0) { + fields[md.id] = [md.primaryKey.id]; + fields[md.id].push(...bts); + } + } + + await this.importModelsData({ + idMap, + sourceProject: project, + sourceModels: [sourceModel], + destProject: project, + destBase: base, + hrTime, + modelFieldIds: fields, + externalModels: relatedModels, + }); + + elapsedTime(hrTime, 'reimportModelData'); + + // console.log('exportedModel', exportedModel); + } + + async importModelsData(param: { + idMap: Map; + sourceProject: Project; + sourceModels: Model[]; + destProject: Project; + destBase: Base; + hrTime: { hrTime: [number, number] }; + modelFieldIds?: Record; + externalModels?: Model[]; + }) { + const { + idMap, + sourceProject, + sourceModels, + destProject, + destBase, + hrTime, + modelFieldIds, + externalModels, + } = param; + + const handledLinks = []; + const lChunks: Record = {}; // fk_mm_model_id: { rowId, childId }[] + + const insertChunks = async () => { + for (const [k, v] of Object.entries(lChunks)) { + try { + if (v.length === 0) continue; + await this.bulkDataService.bulkDataInsert({ + projectName: destProject.id, + tableName: k, + body: v, + cookie: null, + chunkSize: 1000, + foreign_key_checks: false, + raw: true, + }); + lChunks[k] = []; + } catch (e) { + console.log(e); } - }; + } + }; + + for (const sourceModel of sourceModels) { + const dataStream = new Readable({ + read() {}, + }); + + const linkStream = new Readable({ + read() {}, + }); + + this.exportService.streamModelData({ + dataStream, + linkStream, + projectId: sourceProject.id, + modelId: sourceModel.id, + handledMmList: handledLinks, + }); + + const headers: string[] = []; + let chunk = []; + + const model = await Model.get(findWithIdentifier(idMap, sourceModel.id)); + + await new Promise((resolve) => { + papaparse.parse(dataStream, { + newline: '\r\n', + step: async (results, parser) => { + if (!headers.length) { + parser.pause(); + for (const header of results.data) { + const id = idMap.get(header); + if (id) { + const col = await Column.get({ + base_id: destBase.id, + colId: id, + }); + if (col.colOptions?.type === 'bt') { + const childCol = await Column.get({ + base_id: destBase.id, + colId: col.colOptions.fk_child_column_id, + }); + headers.push(childCol.column_name); + } else { + headers.push(col.column_name); + } + } else { + debugLog('header not found', header); + } + } + parser.resume(); + } else { + if (results.errors.length === 0) { + const row = {}; + for (let i = 0; i < headers.length; i++) { + if (results.data[i] !== '') { + row[headers[i]] = results.data[i]; + } + } + chunk.push(row); + if (chunk.length > 1000) { + parser.pause(); + try { + await this.bulkDataService.bulkDataInsert({ + projectName: destProject.id, + tableName: model.id, + body: chunk, + cookie: null, + chunkSize: chunk.length + 1, + foreign_key_checks: false, + raw: true, + }); + } catch (e) { + console.log(e); + } + chunk = []; + parser.resume(); + } + } + } + }, + complete: async () => { + if (chunk.length > 0) { + try { + await this.bulkDataService.bulkDataInsert({ + projectName: destProject.id, + tableName: model.id, + body: chunk, + cookie: null, + chunkSize: chunk.length + 1, + foreign_key_checks: false, + raw: true, + }); + } catch (e) { + console.log(e); + } + chunk = []; + } + resolve(null); + }, + }); + }); + + let headersFound = false; + + let childIndex = -1; + let parentIndex = -1; + let columnIndex = -1; + + const mmColumns: Record = {}; + const mmParentChild: any = {}; + + await new Promise((resolve) => { + papaparse.parse(linkStream, { + newline: '\r\n', + step: async (results, parser) => { + if (!headersFound) { + for (const [i, header] of Object.entries(results.data)) { + if (header === 'child') { + childIndex = parseInt(i); + } else if (header === 'parent') { + parentIndex = parseInt(i); + } else if (header === 'column') { + columnIndex = parseInt(i); + } + } + headersFound = true; + } else { + if (results.errors.length === 0) { + const child = results.data[childIndex]; + const parent = results.data[parentIndex]; + const columnId = results.data[columnIndex]; + if (child && parent && columnId) { + if (mmColumns[columnId]) { + // push to chunk + const mmModelId = + mmColumns[columnId].colOptions.fk_mm_model_id; + const mm = mmParentChild[mmModelId]; + lChunks[mmModelId].push({ + [mm.parent]: parent, + [mm.child]: child, + }); + } else { + // get column for the first time + parser.pause(); + + await insertChunks(); + + const col = await Column.get({ + base_id: destBase.id, + colId: findWithIdentifier(idMap, columnId), + }); + + const colOptions = + await col.getColOptions(); + + const vChildCol = await colOptions.getMMChildColumn(); + const vParentCol = await colOptions.getMMParentColumn(); + + mmParentChild[col.colOptions.fk_mm_model_id] = { + parent: vParentCol.column_name, + child: vChildCol.column_name, + }; + + mmColumns[columnId] = col; + + handledLinks.push(col.colOptions.fk_mm_model_id); + + const mmModelId = col.colOptions.fk_mm_model_id; + + // create chunk + lChunks[mmModelId] = []; + + // push to chunk + const mm = mmParentChild[mmModelId]; + lChunks[mmModelId].push({ + [mm.parent]: parent, + [mm.child]: child, + }); + + parser.resume(); + } + } + } + } + }, + complete: async () => { + await insertChunks(); + resolve(null); + }, + }); + }); + + elapsedTime(hrTime, model.title); + } + + // update external models (has bt to this model) + if (externalModels) { + for (const sourceModel of externalModels) { + const fields = modelFieldIds?.[sourceModel.id]; + + if (!fields) continue; - for (const sourceModel of models) { const dataStream = new Readable({ read() {}, }); @@ -122,17 +468,16 @@ export class DuplicateProcessor { this.exportService.streamModelData({ dataStream, linkStream, - projectId: project.id, + projectId: sourceProject.id, modelId: sourceModel.id, handledMmList: handledLinks, + _fieldIds: fields, }); const headers: string[] = []; let chunk = []; - const model = await Model.get( - findWithIdentifier(idMap, sourceModel.id), - ); + const model = await Model.get(sourceModel.id); await new Promise((resolve) => { papaparse.parse(dataStream, { @@ -144,12 +489,12 @@ export class DuplicateProcessor { const id = idMap.get(header); if (id) { const col = await Column.get({ - base_id: dupBaseId, + base_id: destBase.id, colId: id, }); if (col.colOptions?.type === 'bt') { const childCol = await Column.get({ - base_id: dupBaseId, + base_id: destBase.id, colId: col.colOptions.fk_child_column_id, }); headers.push(childCol.column_name); @@ -173,13 +518,11 @@ export class DuplicateProcessor { if (chunk.length > 1000) { parser.pause(); try { - await this.bulkDataService.bulkDataInsert({ - projectName: dupProject.id, + await this.bulkDataService.bulkDataUpdate({ + projectName: destProject.id, tableName: model.id, body: chunk, cookie: null, - chunkSize: chunk.length + 1, - foreign_key_checks: false, raw: true, }); } catch (e) { @@ -193,14 +536,13 @@ export class DuplicateProcessor { }, complete: async () => { if (chunk.length > 0) { + console.log('chunk', chunk); try { - await this.bulkDataService.bulkDataInsert({ - projectName: dupProject.id, + await this.bulkDataService.bulkDataUpdate({ + projectName: destProject.id, tableName: model.id, body: chunk, cookie: null, - chunkSize: chunk.length + 1, - foreign_key_checks: false, raw: true, }); } catch (e) { @@ -213,113 +555,8 @@ export class DuplicateProcessor { }); }); - let headersFound = false; - - let childIndex = -1; - let parentIndex = -1; - let columnIndex = -1; - - const mmColumns: Record = {}; - const mmParentChild: any = {}; - - await new Promise((resolve) => { - papaparse.parse(linkStream, { - newline: '\r\n', - step: async (results, parser) => { - if (!headersFound) { - for (const [i, header] of Object.entries(results.data)) { - if (header === 'child') { - childIndex = parseInt(i); - } else if (header === 'parent') { - parentIndex = parseInt(i); - } else if (header === 'column') { - columnIndex = parseInt(i); - } - } - headersFound = true; - } else { - if (results.errors.length === 0) { - const child = results.data[childIndex]; - const parent = results.data[parentIndex]; - const columnId = results.data[columnIndex]; - if (child && parent && columnId) { - if (mmColumns[columnId]) { - // push to chunk - const mmModelId = - mmColumns[columnId].colOptions.fk_mm_model_id; - const mm = mmParentChild[mmModelId]; - lChunks[mmModelId].push({ - [mm.parent]: parent, - [mm.child]: child, - }); - } else { - // get column for the first time - parser.pause(); - - await insertChunks(); - - const col = await Column.get({ - base_id: dupBaseId, - colId: findWithIdentifier(idMap, columnId), - }); - - const colOptions = - await col.getColOptions(); - - const vChildCol = await colOptions.getMMChildColumn(); - const vParentCol = await colOptions.getMMParentColumn(); - - mmParentChild[col.colOptions.fk_mm_model_id] = { - parent: vParentCol.column_name, - child: vChildCol.column_name, - }; - - mmColumns[columnId] = col; - - handledLinks.push(col.colOptions.fk_mm_model_id); - - const mmModelId = col.colOptions.fk_mm_model_id; - - // create chunk - lChunks[mmModelId] = []; - - // push to chunk - const mm = mmParentChild[mmModelId]; - lChunks[mmModelId].push({ - [mm.parent]: parent, - [mm.child]: child, - }); - - parser.resume(); - } - } - } - } - }, - complete: async () => { - await insertChunks(); - resolve(null); - }, - }); - }); - - elapsedTime(model.title); + elapsedTime(hrTime, `external bt ${model.title}`); } - - elapsedTime('links'); - await this.projectsService.projectUpdate({ - projectId: dupProject.id, - project: { - status: null, - }, - }); - } catch (e) { - if (dupProject?.id) { - await this.projectsService.projectSoftDelete({ - projectId: dupProject.id, - }); - } - throw e; } } } diff --git a/packages/nocodb-nest/src/modules/jobs/export-import/export.service.ts b/packages/nocodb-nest/src/modules/jobs/export-import/export.service.ts index 79c9e2de47..6af00e0458 100644 --- a/packages/nocodb-nest/src/modules/jobs/export-import/export.service.ts +++ b/packages/nocodb-nest/src/modules/jobs/export-import/export.service.ts @@ -241,6 +241,7 @@ export class ExportService { modelId: string; viewId?: string; handledMmList?: string[]; + _fieldIds?: string[]; }) { const { dataStream, linkStream, handledMmList } = param; @@ -254,13 +255,6 @@ export class ExportService { await model.getColumns(); - const pkMap = new Map(); - - const fields = model.columns - .filter((c) => c.uidt !== UITypes.LinkToAnotherRecord) - .map((c) => c.title) - .join(','); - const btMap = new Map(); for (const column of model.columns.filter( @@ -273,6 +267,13 @@ export class ExportService { (c) => c.id === column.colOptions?.fk_child_column_id, ); if (fkCol) { + // replace bt column with fk column if it is in _fieldIds + if (param._fieldIds && param._fieldIds.includes(column.id)) { + param._fieldIds.push(fkCol.id); + const btIndex = param._fieldIds.indexOf(column.id); + param._fieldIds.splice(btIndex, 1); + } + btMap.set( fkCol.id, `${column.project_id}::${column.base_id}::${column.fk_model_id}::${column.id}`, @@ -280,6 +281,16 @@ export class ExportService { } } + const fields = param._fieldIds + ? model.columns + .filter((c) => param._fieldIds?.includes(c.id)) + .map((c) => c.title) + .join(',') + : model.columns + .filter((c) => c.uidt !== UITypes.LinkToAnotherRecord) + .map((c) => c.title) + .join(','); + const mmColumns = model.columns.filter( (col) => col.uidt === UITypes.LinkToAnotherRecord && diff --git a/packages/nocodb-nest/src/modules/jobs/export-import/import.service.ts b/packages/nocodb-nest/src/modules/jobs/export-import/import.service.ts index fa3e441445..0052a8b22c 100644 --- a/packages/nocodb-nest/src/modules/jobs/export-import/import.service.ts +++ b/packages/nocodb-nest/src/modules/jobs/export-import/import.service.ts @@ -3,6 +3,8 @@ import { Injectable } from '@nestjs/common'; import papaparse from 'papaparse'; import { findWithIdentifier, + generateUniqueName, + getEntityIdentifier, getParentIdentifier, reverseGet, withoutId, @@ -51,9 +53,11 @@ export class ImportService { | { models: { model: any; views: any[] }[] } | { model: any; views: any[] }[]; req: any; + externalModels?: Model[]; }) { // structured id to db id const idMap = new Map(); + const externalIdMap = new Map(); const project = await Project.get(param.projectId); @@ -72,6 +76,22 @@ export class ImportService { param.data = Array.isArray(param.data) ? param.data : param.data.models; + // allow existing models to be linked + if (param.externalModels) { + for (const model of param.externalModels) { + externalIdMap.set( + `${model.project_id}::${model.base_id}::${model.id}`, + model.id, + ); + + await model.getColumns(); + + for (const col of model.columns) { + externalIdMap.set(`${idMap.get(model.id)}::${col.id}`, col.id); + } + } + } + // create tables with static columns for (const data of param.data) { const modelData = data.model; @@ -124,10 +144,7 @@ export class ImportService { for (const col of linkedColumnSet) { if (col.colOptions) { const colOptions = col.colOptions; - if ( - col.uidt === UITypes.LinkToAnotherRecord && - idMap.has(colOptions.fk_related_model_id) - ) { + if (idMap.has(colOptions.fk_related_model_id)) { if (colOptions.type === 'mm') { if (!linkMap.has(colOptions.fk_mm_model_id)) { // delete col.column_name as it is not required and will cause ajv error (null for LTAR) @@ -296,6 +313,395 @@ export class ImportService { } } } + } else if (externalIdMap.has(colOptions.fk_related_model_id)) { + if (colOptions.type === 'mm') { + if (!linkMap.has(colOptions.fk_mm_model_id)) { + // delete col.column_name as it is not required and will cause ajv error (null for LTAR) + delete col.column_name; + + const freshModelData = await this.columnsService.columnAdd({ + tableId: table.id, + column: withoutId({ + ...col, + ...{ + parentId: + idMap.get( + getParentIdentifier(colOptions.fk_child_column_id), + ) || + externalIdMap.get( + getParentIdentifier(colOptions.fk_child_column_id), + ), + childId: + idMap.get( + getParentIdentifier(colOptions.fk_parent_column_id), + ) || + externalIdMap.get( + getParentIdentifier(colOptions.fk_parent_column_id), + ), + type: colOptions.type, + virtual: colOptions.virtual, + ur: colOptions.ur, + dr: colOptions.dr, + }, + }), + req: param.req, + }); + + for (const nColumn of freshModelData.columns) { + if (nColumn.title === col.title) { + idMap.set(col.id, nColumn.id); + linkMap.set( + colOptions.fk_mm_model_id, + nColumn.colOptions.fk_mm_model_id, + ); + break; + } + } + + const childModel = + getParentIdentifier(colOptions.fk_parent_column_id) === + modelData.id + ? freshModelData + : await Model.get( + idMap.get( + getParentIdentifier(colOptions.fk_parent_column_id), + ) || + externalIdMap.get( + getParentIdentifier(colOptions.fk_parent_column_id), + ), + ); + + if ( + getParentIdentifier(colOptions.fk_parent_column_id) !== + modelData.id + ) + await childModel.getColumns(); + + const childColumn = ( + param.data.find( + (a) => + a.model.id === + getParentIdentifier(colOptions.fk_parent_column_id), + )?.model || + param.externalModels.find( + (a) => + a.id === + getEntityIdentifier( + getParentIdentifier(colOptions.fk_parent_column_id), + ), + ) + ).columns.find( + (a) => + (a.colOptions?.fk_mm_model_id === + colOptions.fk_mm_model_id && + a.id !== col.id) || + (a.colOptions?.fk_mm_model_id === + getEntityIdentifier(colOptions.fk_mm_model_id) && + a.id !== getEntityIdentifier(col.id)), + ); + + for (const nColumn of childModel.columns) { + if ( + nColumn?.colOptions?.fk_mm_model_id === + linkMap.get(colOptions.fk_mm_model_id) && + nColumn.id !== idMap.get(col.id) && + nColumn.id !== externalIdMap.get(col.id) + ) { + if (childColumn.id.includes('::')) { + idMap.set(childColumn.id, nColumn.id); + } else { + idMap.set( + `${childColumn.project_id}::${childColumn.base_id}::${childColumn.fk_model_id}::${childColumn.id}`, + nColumn.id, + ); + } + + childColumn.title = `${childColumn.title} copy`; + + childColumn.title = generateUniqueName( + childColumn.title, + childModel.columns.map((a) => a.title), + ); + + if (nColumn.title !== childColumn.title) { + await this.columnsService.columnUpdate({ + columnId: nColumn.id, + column: { + ...nColumn, + column_name: childColumn.title, + title: childColumn.title, + }, + }); + } + break; + } + } + } + } else if (colOptions.type === 'hm') { + if ( + !linkMap.has( + `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`, + ) + ) { + // delete col.column_name as it is not required and will cause ajv error (null for LTAR) + delete col.column_name; + + const freshModelData = await this.columnsService.columnAdd({ + tableId: table.id, + column: withoutId({ + ...col, + ...{ + parentId: + idMap.get( + getParentIdentifier(colOptions.fk_parent_column_id), + ) || + externalIdMap.get( + getParentIdentifier(colOptions.fk_parent_column_id), + ), + childId: + idMap.get( + getParentIdentifier(colOptions.fk_child_column_id), + ) || + externalIdMap.get( + getParentIdentifier(colOptions.fk_child_column_id), + ), + type: colOptions.type, + virtual: colOptions.virtual, + ur: colOptions.ur, + dr: colOptions.dr, + }, + }), + req: param.req, + }); + + linkMap.set( + `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`, + `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`, + ); + + for (const nColumn of freshModelData.columns) { + if (nColumn.title === col.title) { + idMap.set(col.id, nColumn.id); + idMap.set( + colOptions.fk_parent_column_id, + nColumn.colOptions.fk_parent_column_id, + ); + idMap.set( + colOptions.fk_child_column_id, + nColumn.colOptions.fk_child_column_id, + ); + break; + } + } + + const childModel = + colOptions.fk_related_model_id === modelData.id + ? freshModelData + : await Model.get( + idMap.get(colOptions.fk_related_model_id) || + externalIdMap.get(colOptions.fk_related_model_id), + ); + + if (colOptions.fk_related_model_id !== modelData.id) + await childModel.getColumns(); + + const childColumn = ( + param.data.find( + (a) => a.model.id === colOptions.fk_related_model_id, + )?.model || + param.externalModels.find( + (a) => + a.id === + getEntityIdentifier(colOptions.fk_related_model_id), + ) + ).columns.find( + (a) => + (a.colOptions?.fk_parent_column_id === + colOptions.fk_parent_column_id && + a.colOptions?.fk_child_column_id === + colOptions.fk_child_column_id && + a.id !== col.id) || + (a.colOptions?.fk_parent_column_id === + getEntityIdentifier(colOptions.fk_parent_column_id) && + a.colOptions?.fk_child_column_id === + getEntityIdentifier(colOptions.fk_child_column_id) && + a.id !== getEntityIdentifier(col.id)), + ); + + for (const nColumn of childModel.columns) { + if ( + nColumn.id !== idMap.get(col.id) && + nColumn.id !== externalIdMap.get(col.id) && + (nColumn.colOptions?.fk_parent_column_id === + idMap.get(colOptions.fk_parent_column_id) || + externalIdMap.get(colOptions.fk_parent_column_id)) && + (nColumn.colOptions?.fk_child_column_id === + idMap.get(colOptions.fk_child_column_id) || + externalIdMap.get(colOptions.fk_child_column_id)) + ) { + if (childColumn.id.includes('::')) { + idMap.set(childColumn.id, nColumn.id); + } else { + idMap.set( + `${childColumn.project_id}::${childColumn.base_id}::${childColumn.fk_model_id}::${childColumn.id}`, + nColumn.id, + ); + } + + childColumn.title = `${childColumn.title} copy`; + + childColumn.title = generateUniqueName( + childColumn.title, + childModel.columns.map((a) => a.title), + ); + + if (nColumn.title !== childColumn.title) { + await this.columnsService.columnUpdate({ + columnId: nColumn.id, + column: { + ...nColumn, + column_name: childColumn.title, + title: childColumn.title, + }, + }); + } + break; + } + } + } + } else if (colOptions.type === 'bt') { + if ( + !linkMap.has( + `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`, + ) + ) { + // delete col.column_name as it is not required and will cause ajv error (null for LTAR) + delete col.column_name; + + const freshModelData = await this.columnsService.columnAdd({ + tableId: table.id, + column: withoutId({ + ...col, + ...{ + parentId: + idMap.get( + getParentIdentifier(colOptions.fk_parent_column_id), + ) || + externalIdMap.get( + getParentIdentifier(colOptions.fk_parent_column_id), + ), + childId: + idMap.get( + getParentIdentifier(colOptions.fk_child_column_id), + ) || + externalIdMap.get( + getParentIdentifier(colOptions.fk_child_column_id), + ), + type: colOptions.type, + virtual: colOptions.virtual, + ur: colOptions.ur, + dr: colOptions.dr, + }, + }), + req: param.req, + }); + + linkMap.set( + `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`, + `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`, + ); + + for (const nColumn of freshModelData.columns) { + if (nColumn.title === col.title) { + idMap.set(col.id, nColumn.id); + idMap.set( + colOptions.fk_parent_column_id, + nColumn.colOptions.fk_parent_column_id, + ); + idMap.set( + colOptions.fk_child_column_id, + nColumn.colOptions.fk_child_column_id, + ); + break; + } + } + + const childModel = + colOptions.fk_related_model_id === modelData.id + ? freshModelData + : await Model.get( + idMap.get(colOptions.fk_related_model_id) || + externalIdMap.get(colOptions.fk_related_model_id), + ); + + if (colOptions.fk_related_model_id !== modelData.id) + await childModel.getColumns(); + + const childColumn = ( + param.data.find( + (a) => a.model.id === colOptions.fk_related_model_id, + )?.model || + param.externalModels.find( + (a) => + a.id === + getEntityIdentifier(colOptions.fk_related_model_id), + ) + ).columns.find( + (a) => + (a.colOptions?.fk_parent_column_id === + colOptions.fk_parent_column_id && + a.colOptions?.fk_child_column_id === + colOptions.fk_child_column_id && + a.id !== col.id) || + (a.colOptions?.fk_parent_column_id === + getEntityIdentifier(colOptions.fk_parent_column_id) && + a.colOptions?.fk_child_column_id === + getEntityIdentifier(colOptions.fk_child_column_id) && + a.id !== getEntityIdentifier(col.id)), + ); + + for (const nColumn of childModel.columns) { + if ( + nColumn.id !== idMap.get(col.id) && + nColumn.id !== externalIdMap.get(col.id) && + (nColumn.colOptions?.fk_parent_column_id === + idMap.get(colOptions.fk_parent_column_id) || + externalIdMap.get(colOptions.fk_parent_column_id)) && + (nColumn.colOptions?.fk_child_column_id === + idMap.get(colOptions.fk_child_column_id) || + externalIdMap.get(colOptions.fk_child_column_id)) + ) { + if (childColumn.id.includes('::')) { + idMap.set(childColumn.id, nColumn.id); + } else { + idMap.set( + `${childColumn.project_id}::${childColumn.base_id}::${childColumn.fk_model_id}::${childColumn.id}`, + nColumn.id, + ); + } + + childColumn.title = `${childColumn.title} copy`; + + childColumn.title = generateUniqueName( + childColumn.title, + childModel.columns.map((a) => a.title), + ); + + if (nColumn.title !== childColumn.title) { + await this.columnsService.columnUpdate({ + columnId: nColumn.id, + column: { + ...nColumn, + column_name: childColumn.title, + title: childColumn.title, + }, + }); + } + break; + } + } + } + } } } } diff --git a/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts b/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts index 5a5659480c..a7f51a4b53 100644 --- a/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts +++ b/packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts @@ -59,6 +59,10 @@ export class QueueService { this: this.duplicateProcessor, fn: this.duplicateProcessor.duplicateBase, }, + [JobTypes.DuplicateModel]: { + this: this.duplicateProcessor, + fn: this.duplicateProcessor.duplicateModel, + }, [JobTypes.AtImport]: { this: this.atImportProcessor, fn: this.atImportProcessor.job,