feat: duplicate table back-end

Signed-off-by: mertmit <mertmit99@gmail.com>

packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts

@@ -15,7 +15,7 @@ import {
   ExtractProjectIdMiddleware,
 } from '../../../middlewares/extract-project-id/extract-project-id.middleware';
 import { ProjectsService } from '../../../services/projects.service';
-import { Base, Project } from '../../../models';
+import { Base, Model, Project } from '../../../models';
 import { generateUniqueName } from '../../../helpers/exportImportHelpers';
 import { QueueService } from '../fallback-queue.service';
 import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs';
@@ -80,4 +80,47 @@ export class DuplicateController {
 
     return { id: job.id, name: job.name };
   }
+
+  @Post('/api/v1/db/meta/duplicate/:projectId/model/:modelId')
+  @HttpCode(200)
+  @Acl('duplicateModel')
+  async duplicateModel(
+    @Request() req,
+    @Param('projectId') projectId: string,
+    @Param('modelId') modelId?: string,
+  ) {
+    const project = await Project.get(projectId);
+
+    if (!project) {
+      throw new Error(`Project not found for id '${projectId}'`);
+    }
+
+    const model = await Model.get(modelId);
+
+    if (!model) {
+      throw new Error(`Model not found!`);
+    }
+
+    const base = await Base.get(model.base_id);
+
+    const models = await base.getModels();
+
+    const uniqueTitle = generateUniqueName(
+      `${model.title} copy`,
+      models.map((p) => p.title),
+    );
+
+    const job = await this.activeQueue.add(JobTypes.DuplicateModel, {
+      projectId: project.id,
+      baseId: base.id,
+      modelId: model.id,
+      req: {
+        user: req.user,
+        clientIp: req.clientIp,
+      },
+      title: uniqueTitle,
+    });
+
+    return { id: job.id, name: job.name };
+  }
 }

packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts

@@ -2,6 +2,7 @@ import { Readable } from 'stream';
 import { Process, Processor } from '@nestjs/bull';
 import { Job } from 'bull';
 import papaparse from 'papaparse';
+import { UITypes } from 'nocodb-sdk';
 import { Base, Column, Model, Project } from '../../../models';
 import { ProjectsService } from '../../../services/projects.service';
 import { findWithIdentifier } from '../../../helpers/exportImportHelpers';
@@ -13,6 +14,28 @@ import type { LinkToAnotherRecordColumn } from '../../../models';
 
 const DEBUG = false;
 
+const debugLog = function (...args: any[]) {
+  if (DEBUG) {
+    console.log(...args);
+  }
+};
+
+const initTime = function () {
+  return {
+    hrTime: process.hrtime(),
+  };
+};
+
+const elapsedTime = function (
+  time: { hrTime: [number, number] },
+  label?: string,
+) {
+  const elapsedS = process.hrtime(time.hrTime)[0].toFixed(3);
+  const elapsedMs = process.hrtime(time.hrTime)[1] / 1000000;
+  if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
+  time.hrTime = process.hrtime();
+};
+
 @Processor(JOBS_QUEUE)
 export class DuplicateProcessor {
   constructor(
@@ -24,6 +47,8 @@ export class DuplicateProcessor {
 
   @Process(JobTypes.DuplicateBase)
   async duplicateBase(job: Job) {
+    const hrTime = initTime();
+
     const { projectId, baseId, dupProjectId, req } = job.data;
 
     const project = await Project.get(projectId);
@@ -35,21 +60,6 @@ export class DuplicateProcessor {
         throw new Error(`Project or base not found!`);
       }
 
-      let start = process.hrtime();
-
-      const debugLog = function (...args: any[]) {
-        if (DEBUG) {
-          console.log(...args);
-        }
-      };
-
-      const elapsedTime = function (label?: string) {
-        const elapsedS = process.hrtime(start)[0].toFixed(3);
-        const elapsedMs = process.hrtime(start)[1] / 1000000;
-        if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
-        start = process.hrtime();
-      };
-
       const user = (req as any).user;
 
       const models = (await base.getModels()).filter(
@@ -61,7 +71,7 @@ export class DuplicateProcessor {
         modelIds: models.map((m) => m.id),
       });
 
-      elapsedTime('serializeModels');
+      elapsedTime(hrTime, 'serializeModels');
 
       if (!exportedModels) {
         throw new Error(`Export failed for base '${base.id}'`);
@@ -69,48 +79,384 @@ export class DuplicateProcessor {
 
       await dupProject.getBases();
 
-      const dupBaseId = dupProject.bases[0].id;
+      const dupBase = dupProject.bases[0];
 
-      elapsedTime('projectCreate');
+      elapsedTime(hrTime, 'projectCreate');
 
       const idMap = await this.importService.importModels({
         user,
         projectId: dupProject.id,
-        baseId: dupBaseId,
+        baseId: dupBase.id,
         data: exportedModels,
         req: req,
       });
 
-      elapsedTime('importModels');
+      elapsedTime(hrTime, 'importModels');
 
       if (!idMap) {
         throw new Error(`Import failed for base '${base.id}'`);
       }
 
-      const handledLinks = [];
-      const lChunks: Record<string, any[]> = {}; // fk_mm_model_id: { rowId, childId }[]
-
-      const insertChunks = async () => {
-        for (const [k, v] of Object.entries(lChunks)) {
-          try {
-            if (v.length === 0) continue;
-            await this.bulkDataService.bulkDataInsert({
-              projectName: dupProject.id,
-              tableName: k,
-              body: v,
-              cookie: null,
-              chunkSize: 1000,
-              foreign_key_checks: false,
-              raw: true,
-            });
-            lChunks[k] = [];
-          } catch (e) {
-            console.log(e);
-          }
+      await this.importModelsData({
+        idMap,
+        sourceProject: project,
+        sourceModels: models,
+        destProject: dupProject,
+        destBase: dupBase,
+        hrTime,
+      });
+
+      await this.projectsService.projectUpdate({
+        projectId: dupProject.id,
+        project: {
+          status: null,
+        },
+      });
+    } catch (e) {
+      if (dupProject?.id) {
+        await this.projectsService.projectSoftDelete({
+          projectId: dupProject.id,
+        });
+      }
+      throw e;
+    }
+  }
+
+  @Process(JobTypes.DuplicateModel)
+  async duplicateModel(job: Job) {
+    const hrTime = initTime();
+
+    const { projectId, baseId, modelId, title, req } = job.data;
+
+    const project = await Project.get(projectId);
+    const base = await Base.get(baseId);
+
+    const user = (req as any).user;
+
+    const models = (await base.getModels()).filter(
+      (m) => !m.mm && m.type === 'table',
+    );
+
+    const sourceModel = models.find((m) => m.id === modelId);
+
+    await sourceModel.getColumns();
+
+    const relatedModelIds = sourceModel.columns
+      .filter((col) => col.uidt === UITypes.LinkToAnotherRecord)
+      .map((col) => col.colOptions.fk_related_model_id)
+      .filter((id) => id);
+
+    const relatedModels = models.filter((m) => relatedModelIds.includes(m.id));
+
+    const exportedModel = (
+      await this.exportService.serializeModels({
+        modelIds: [modelId],
+      })
+    )[0];
+
+    elapsedTime(hrTime, 'serializeModel');
+
+    if (!exportedModel) {
+      throw new Error(`Export failed for base '${base.id}'`);
+    }
+
+    exportedModel.model.title = title;
+    exportedModel.model.table_name = title.toLowerCase().replace(/ /g, '_');
+
+    const idMap = await this.importService.importModels({
+      projectId,
+      baseId,
+      data: [exportedModel],
+      user,
+      req,
+      externalModels: relatedModels,
+    });
+
+    elapsedTime(hrTime, 'reimportModelSchema');
+
+    if (!idMap) {
+      throw new Error(`Import failed for model '${modelId}'`);
+    }
+
+    const fields: Record<string, string[]> = {};
+
+    for (const md of relatedModels) {
+      const bts = md.columns
+        .filter(
+          (c) =>
+            c.uidt === UITypes.LinkToAnotherRecord &&
+            c.colOptions.type === 'bt' &&
+            c.colOptions.fk_related_model_id === modelId,
+        )
+        .map((c) => c.id);
+
+      if (bts.length > 0) {
+        fields[md.id] = [md.primaryKey.id];
+        fields[md.id].push(...bts);
+      }
+    }
+
+    await this.importModelsData({
+      idMap,
+      sourceProject: project,
+      sourceModels: [sourceModel],
+      destProject: project,
+      destBase: base,
+      hrTime,
+      modelFieldIds: fields,
+      externalModels: relatedModels,
+    });
+
+    elapsedTime(hrTime, 'reimportModelData');
+
+    // console.log('exportedModel', exportedModel);
+  }
+
+  async importModelsData(param: {
+    idMap: Map<string, string>;
+    sourceProject: Project;
+    sourceModels: Model[];
+    destProject: Project;
+    destBase: Base;
+    hrTime: { hrTime: [number, number] };
+    modelFieldIds?: Record<string, string[]>;
+    externalModels?: Model[];
+  }) {
+    const {
+      idMap,
+      sourceProject,
+      sourceModels,
+      destProject,
+      destBase,
+      hrTime,
+      modelFieldIds,
+      externalModels,
+    } = param;
+
+    const handledLinks = [];
+    const lChunks: Record<string, any[]> = {}; // fk_mm_model_id: { rowId, childId }[]
+
+    const insertChunks = async () => {
+      for (const [k, v] of Object.entries(lChunks)) {
+        try {
+          if (v.length === 0) continue;
+          await this.bulkDataService.bulkDataInsert({
+            projectName: destProject.id,
+            tableName: k,
+            body: v,
+            cookie: null,
+            chunkSize: 1000,
+            foreign_key_checks: false,
+            raw: true,
+          });
+          lChunks[k] = [];
+        } catch (e) {
+          console.log(e);
         }
-      };
+      }
+    };
+
+    for (const sourceModel of sourceModels) {
+      const dataStream = new Readable({
+        read() {},
+      });
+
+      const linkStream = new Readable({
+        read() {},
+      });
+
+      this.exportService.streamModelData({
+        dataStream,
+        linkStream,
+        projectId: sourceProject.id,
+        modelId: sourceModel.id,
+        handledMmList: handledLinks,
+      });
+
+      const headers: string[] = [];
+      let chunk = [];
+
+      const model = await Model.get(findWithIdentifier(idMap, sourceModel.id));
+
+      await new Promise((resolve) => {
+        papaparse.parse(dataStream, {
+          newline: '\r\n',
+          step: async (results, parser) => {
+            if (!headers.length) {
+              parser.pause();
+              for (const header of results.data) {
+                const id = idMap.get(header);
+                if (id) {
+                  const col = await Column.get({
+                    base_id: destBase.id,
+                    colId: id,
+                  });
+                  if (col.colOptions?.type === 'bt') {
+                    const childCol = await Column.get({
+                      base_id: destBase.id,
+                      colId: col.colOptions.fk_child_column_id,
+                    });
+                    headers.push(childCol.column_name);
+                  } else {
+                    headers.push(col.column_name);
+                  }
+                } else {
+                  debugLog('header not found', header);
+                }
+              }
+              parser.resume();
+            } else {
+              if (results.errors.length === 0) {
+                const row = {};
+                for (let i = 0; i < headers.length; i++) {
+                  if (results.data[i] !== '') {
+                    row[headers[i]] = results.data[i];
+                  }
+                }
+                chunk.push(row);
+                if (chunk.length > 1000) {
+                  parser.pause();
+                  try {
+                    await this.bulkDataService.bulkDataInsert({
+                      projectName: destProject.id,
+                      tableName: model.id,
+                      body: chunk,
+                      cookie: null,
+                      chunkSize: chunk.length + 1,
+                      foreign_key_checks: false,
+                      raw: true,
+                    });
+                  } catch (e) {
+                    console.log(e);
+                  }
+                  chunk = [];
+                  parser.resume();
+                }
+              }
+            }
+          },
+          complete: async () => {
+            if (chunk.length > 0) {
+              try {
+                await this.bulkDataService.bulkDataInsert({
+                  projectName: destProject.id,
+                  tableName: model.id,
+                  body: chunk,
+                  cookie: null,
+                  chunkSize: chunk.length + 1,
+                  foreign_key_checks: false,
+                  raw: true,
+                });
+              } catch (e) {
+                console.log(e);
+              }
+              chunk = [];
+            }
+            resolve(null);
+          },
+        });
+      });
+
+      let headersFound = false;
+
+      let childIndex = -1;
+      let parentIndex = -1;
+      let columnIndex = -1;
+
+      const mmColumns: Record<string, Column> = {};
+      const mmParentChild: any = {};
+
+      await new Promise((resolve) => {
+        papaparse.parse(linkStream, {
+          newline: '\r\n',
+          step: async (results, parser) => {
+            if (!headersFound) {
+              for (const [i, header] of Object.entries(results.data)) {
+                if (header === 'child') {
+                  childIndex = parseInt(i);
+                } else if (header === 'parent') {
+                  parentIndex = parseInt(i);
+                } else if (header === 'column') {
+                  columnIndex = parseInt(i);
+                }
+              }
+              headersFound = true;
+            } else {
+              if (results.errors.length === 0) {
+                const child = results.data[childIndex];
+                const parent = results.data[parentIndex];
+                const columnId = results.data[columnIndex];
+                if (child && parent && columnId) {
+                  if (mmColumns[columnId]) {
+                    // push to chunk
+                    const mmModelId =
+                      mmColumns[columnId].colOptions.fk_mm_model_id;
+                    const mm = mmParentChild[mmModelId];
+                    lChunks[mmModelId].push({
+                      [mm.parent]: parent,
+                      [mm.child]: child,
+                    });
+                  } else {
+                    // get column for the first time
+                    parser.pause();
+
+                    await insertChunks();
+
+                    const col = await Column.get({
+                      base_id: destBase.id,
+                      colId: findWithIdentifier(idMap, columnId),
+                    });
+
+                    const colOptions =
+                      await col.getColOptions<LinkToAnotherRecordColumn>();
+
+                    const vChildCol = await colOptions.getMMChildColumn();
+                    const vParentCol = await colOptions.getMMParentColumn();
+
+                    mmParentChild[col.colOptions.fk_mm_model_id] = {
+                      parent: vParentCol.column_name,
+                      child: vChildCol.column_name,
+                    };
+
+                    mmColumns[columnId] = col;
+
+                    handledLinks.push(col.colOptions.fk_mm_model_id);
+
+                    const mmModelId = col.colOptions.fk_mm_model_id;
+
+                    // create chunk
+                    lChunks[mmModelId] = [];
+
+                    // push to chunk
+                    const mm = mmParentChild[mmModelId];
+                    lChunks[mmModelId].push({
+                      [mm.parent]: parent,
+                      [mm.child]: child,
+                    });
+
+                    parser.resume();
+                  }
+                }
+              }
+            }
+          },
+          complete: async () => {
+            await insertChunks();
+            resolve(null);
+          },
+        });
+      });
+
+      elapsedTime(hrTime, model.title);
+    }
+
+    // update external models (has bt to this model)
+    if (externalModels) {
+      for (const sourceModel of externalModels) {
+        const fields = modelFieldIds?.[sourceModel.id];
+
+        if (!fields) continue;
 
-      for (const sourceModel of models) {
         const dataStream = new Readable({
           read() {},
         });
@@ -122,17 +468,16 @@ export class DuplicateProcessor {
         this.exportService.streamModelData({
           dataStream,
           linkStream,
-          projectId: project.id,
+          projectId: sourceProject.id,
           modelId: sourceModel.id,
           handledMmList: handledLinks,
+          _fieldIds: fields,
         });
 
         const headers: string[] = [];
         let chunk = [];
 
-        const model = await Model.get(
-          findWithIdentifier(idMap, sourceModel.id),
-        );
+        const model = await Model.get(sourceModel.id);
 
         await new Promise((resolve) => {
           papaparse.parse(dataStream, {
@@ -144,12 +489,12 @@ export class DuplicateProcessor {
                   const id = idMap.get(header);
                   if (id) {
                     const col = await Column.get({
-                      base_id: dupBaseId,
+                      base_id: destBase.id,
                       colId: id,
                     });
                     if (col.colOptions?.type === 'bt') {
                       const childCol = await Column.get({
-                        base_id: dupBaseId,
+                        base_id: destBase.id,
                         colId: col.colOptions.fk_child_column_id,
                       });
                       headers.push(childCol.column_name);
@@ -173,13 +518,11 @@ export class DuplicateProcessor {
                   if (chunk.length > 1000) {
                     parser.pause();
                     try {
-                      await this.bulkDataService.bulkDataInsert({
-                        projectName: dupProject.id,
+                      await this.bulkDataService.bulkDataUpdate({
+                        projectName: destProject.id,
                         tableName: model.id,
                         body: chunk,
                         cookie: null,
-                        chunkSize: chunk.length + 1,
-                        foreign_key_checks: false,
                         raw: true,
                       });
                     } catch (e) {
@@ -193,14 +536,13 @@ export class DuplicateProcessor {
             },
             complete: async () => {
               if (chunk.length > 0) {
+                console.log('chunk', chunk);
                 try {
-                  await this.bulkDataService.bulkDataInsert({
-                    projectName: dupProject.id,
+                  await this.bulkDataService.bulkDataUpdate({
+                    projectName: destProject.id,
                     tableName: model.id,
                     body: chunk,
                     cookie: null,
-                    chunkSize: chunk.length + 1,
-                    foreign_key_checks: false,
                     raw: true,
                   });
                 } catch (e) {
@@ -213,113 +555,8 @@ export class DuplicateProcessor {
           });
         });
 
-        let headersFound = false;
-
-        let childIndex = -1;
-        let parentIndex = -1;
-        let columnIndex = -1;
-
-        const mmColumns: Record<string, Column> = {};
-        const mmParentChild: any = {};
-
-        await new Promise((resolve) => {
-          papaparse.parse(linkStream, {
-            newline: '\r\n',
-            step: async (results, parser) => {
-              if (!headersFound) {
-                for (const [i, header] of Object.entries(results.data)) {
-                  if (header === 'child') {
-                    childIndex = parseInt(i);
-                  } else if (header === 'parent') {
-                    parentIndex = parseInt(i);
-                  } else if (header === 'column') {
-                    columnIndex = parseInt(i);
-                  }
-                }
-                headersFound = true;
-              } else {
-                if (results.errors.length === 0) {
-                  const child = results.data[childIndex];
-                  const parent = results.data[parentIndex];
-                  const columnId = results.data[columnIndex];
-                  if (child && parent && columnId) {
-                    if (mmColumns[columnId]) {
-                      // push to chunk
-                      const mmModelId =
-                        mmColumns[columnId].colOptions.fk_mm_model_id;
-                      const mm = mmParentChild[mmModelId];
-                      lChunks[mmModelId].push({
-                        [mm.parent]: parent,
-                        [mm.child]: child,
-                      });
-                    } else {
-                      // get column for the first time
-                      parser.pause();
-
-                      await insertChunks();
-
-                      const col = await Column.get({
-                        base_id: dupBaseId,
-                        colId: findWithIdentifier(idMap, columnId),
-                      });
-
-                      const colOptions =
-                        await col.getColOptions<LinkToAnotherRecordColumn>();
-
-                      const vChildCol = await colOptions.getMMChildColumn();
-                      const vParentCol = await colOptions.getMMParentColumn();
-
-                      mmParentChild[col.colOptions.fk_mm_model_id] = {
-                        parent: vParentCol.column_name,
-                        child: vChildCol.column_name,
-                      };
-
-                      mmColumns[columnId] = col;
-
-                      handledLinks.push(col.colOptions.fk_mm_model_id);
-
-                      const mmModelId = col.colOptions.fk_mm_model_id;
-
-                      // create chunk
-                      lChunks[mmModelId] = [];
-
-                      // push to chunk
-                      const mm = mmParentChild[mmModelId];
-                      lChunks[mmModelId].push({
-                        [mm.parent]: parent,
-                        [mm.child]: child,
-                      });
-
-                      parser.resume();
-                    }
-                  }
-                }
-              }
-            },
-            complete: async () => {
-              await insertChunks();
-              resolve(null);
-            },
-          });
-        });
-
-        elapsedTime(model.title);
+        elapsedTime(hrTime, `external bt ${model.title}`);
       }
-
-      elapsedTime('links');
-      await this.projectsService.projectUpdate({
-        projectId: dupProject.id,
-        project: {
-          status: null,
-        },
-      });
-    } catch (e) {
-      if (dupProject?.id) {
-        await this.projectsService.projectSoftDelete({
-          projectId: dupProject.id,
-        });
-      }
-      throw e;
     }
   }
 }

packages/nocodb-nest/src/modules/jobs/export-import/export.service.ts

@@ -241,6 +241,7 @@ export class ExportService {
     modelId: string;
     viewId?: string;
     handledMmList?: string[];
+    _fieldIds?: string[];
   }) {
     const { dataStream, linkStream, handledMmList } = param;
 
@@ -254,13 +255,6 @@ export class ExportService {
 
     await model.getColumns();
 
-    const pkMap = new Map<string, string>();
-
-    const fields = model.columns
-      .filter((c) => c.uidt !== UITypes.LinkToAnotherRecord)
-      .map((c) => c.title)
-      .join(',');
-
     const btMap = new Map<string, string>();
 
     for (const column of model.columns.filter(
@@ -273,6 +267,13 @@ export class ExportService {
         (c) => c.id === column.colOptions?.fk_child_column_id,
       );
       if (fkCol) {
+        // replace bt column with fk column if it is in _fieldIds
+        if (param._fieldIds && param._fieldIds.includes(column.id)) {
+          param._fieldIds.push(fkCol.id);
+          const btIndex = param._fieldIds.indexOf(column.id);
+          param._fieldIds.splice(btIndex, 1);
+        }
+
         btMap.set(
           fkCol.id,
           `${column.project_id}::${column.base_id}::${column.fk_model_id}::${column.id}`,
@@ -280,6 +281,16 @@ export class ExportService {
       }
     }
 
+    const fields = param._fieldIds
+      ? model.columns
+          .filter((c) => param._fieldIds?.includes(c.id))
+          .map((c) => c.title)
+          .join(',')
+      : model.columns
+          .filter((c) => c.uidt !== UITypes.LinkToAnotherRecord)
+          .map((c) => c.title)
+          .join(',');
+
     const mmColumns = model.columns.filter(
       (col) =>
         col.uidt === UITypes.LinkToAnotherRecord &&

packages/nocodb-nest/src/modules/jobs/export-import/import.service.ts

@@ -3,6 +3,8 @@ import { Injectable } from '@nestjs/common';
 import papaparse from 'papaparse';
 import {
   findWithIdentifier,
+  generateUniqueName,
+  getEntityIdentifier,
   getParentIdentifier,
   reverseGet,
   withoutId,
@@ -51,9 +53,11 @@ export class ImportService {
       | { models: { model: any; views: any[] }[] }
       | { model: any; views: any[] }[];
     req: any;
+    externalModels?: Model[];
   }) {
     // structured id to db id
     const idMap = new Map<string, string>();
+    const externalIdMap = new Map<string, string>();
 
     const project = await Project.get(param.projectId);
 
@@ -72,6 +76,22 @@ export class ImportService {
 
     param.data = Array.isArray(param.data) ? param.data : param.data.models;
 
+    // allow existing models to be linked
+    if (param.externalModels) {
+      for (const model of param.externalModels) {
+        externalIdMap.set(
+          `${model.project_id}::${model.base_id}::${model.id}`,
+          model.id,
+        );
+
+        await model.getColumns();
+
+        for (const col of model.columns) {
+          externalIdMap.set(`${idMap.get(model.id)}::${col.id}`, col.id);
+        }
+      }
+    }
+
     // create tables with static columns
     for (const data of param.data) {
       const modelData = data.model;
@@ -124,10 +144,7 @@ export class ImportService {
       for (const col of linkedColumnSet) {
         if (col.colOptions) {
           const colOptions = col.colOptions;
-          if (
-            col.uidt === UITypes.LinkToAnotherRecord &&
-            idMap.has(colOptions.fk_related_model_id)
-          ) {
+          if (idMap.has(colOptions.fk_related_model_id)) {
             if (colOptions.type === 'mm') {
               if (!linkMap.has(colOptions.fk_mm_model_id)) {
                 // delete col.column_name as it is not required and will cause ajv error (null for LTAR)
@@ -296,6 +313,395 @@ export class ImportService {
                 }
               }
             }
+          } else if (externalIdMap.has(colOptions.fk_related_model_id)) {
+            if (colOptions.type === 'mm') {
+              if (!linkMap.has(colOptions.fk_mm_model_id)) {
+                // delete col.column_name as it is not required and will cause ajv error (null for LTAR)
+                delete col.column_name;
+
+                const freshModelData = await this.columnsService.columnAdd({
+                  tableId: table.id,
+                  column: withoutId({
+                    ...col,
+                    ...{
+                      parentId:
+                        idMap.get(
+                          getParentIdentifier(colOptions.fk_child_column_id),
+                        ) ||
+                        externalIdMap.get(
+                          getParentIdentifier(colOptions.fk_child_column_id),
+                        ),
+                      childId:
+                        idMap.get(
+                          getParentIdentifier(colOptions.fk_parent_column_id),
+                        ) ||
+                        externalIdMap.get(
+                          getParentIdentifier(colOptions.fk_parent_column_id),
+                        ),
+                      type: colOptions.type,
+                      virtual: colOptions.virtual,
+                      ur: colOptions.ur,
+                      dr: colOptions.dr,
+                    },
+                  }),
+                  req: param.req,
+                });
+
+                for (const nColumn of freshModelData.columns) {
+                  if (nColumn.title === col.title) {
+                    idMap.set(col.id, nColumn.id);
+                    linkMap.set(
+                      colOptions.fk_mm_model_id,
+                      nColumn.colOptions.fk_mm_model_id,
+                    );
+                    break;
+                  }
+                }
+
+                const childModel =
+                  getParentIdentifier(colOptions.fk_parent_column_id) ===
+                  modelData.id
+                    ? freshModelData
+                    : await Model.get(
+                        idMap.get(
+                          getParentIdentifier(colOptions.fk_parent_column_id),
+                        ) ||
+                          externalIdMap.get(
+                            getParentIdentifier(colOptions.fk_parent_column_id),
+                          ),
+                      );
+
+                if (
+                  getParentIdentifier(colOptions.fk_parent_column_id) !==
+                  modelData.id
+                )
+                  await childModel.getColumns();
+
+                const childColumn = (
+                  param.data.find(
+                    (a) =>
+                      a.model.id ===
+                      getParentIdentifier(colOptions.fk_parent_column_id),
+                  )?.model ||
+                  param.externalModels.find(
+                    (a) =>
+                      a.id ===
+                      getEntityIdentifier(
+                        getParentIdentifier(colOptions.fk_parent_column_id),
+                      ),
+                  )
+                ).columns.find(
+                  (a) =>
+                    (a.colOptions?.fk_mm_model_id ===
+                      colOptions.fk_mm_model_id &&
+                      a.id !== col.id) ||
+                    (a.colOptions?.fk_mm_model_id ===
+                      getEntityIdentifier(colOptions.fk_mm_model_id) &&
+                      a.id !== getEntityIdentifier(col.id)),
+                );
+
+                for (const nColumn of childModel.columns) {
+                  if (
+                    nColumn?.colOptions?.fk_mm_model_id ===
+                      linkMap.get(colOptions.fk_mm_model_id) &&
+                    nColumn.id !== idMap.get(col.id) &&
+                    nColumn.id !== externalIdMap.get(col.id)
+                  ) {
+                    if (childColumn.id.includes('::')) {
+                      idMap.set(childColumn.id, nColumn.id);
+                    } else {
+                      idMap.set(
+                        `${childColumn.project_id}::${childColumn.base_id}::${childColumn.fk_model_id}::${childColumn.id}`,
+                        nColumn.id,
+                      );
+                    }
+
+                    childColumn.title = `${childColumn.title} copy`;
+
+                    childColumn.title = generateUniqueName(
+                      childColumn.title,
+                      childModel.columns.map((a) => a.title),
+                    );
+
+                    if (nColumn.title !== childColumn.title) {
+                      await this.columnsService.columnUpdate({
+                        columnId: nColumn.id,
+                        column: {
+                          ...nColumn,
+                          column_name: childColumn.title,
+                          title: childColumn.title,
+                        },
+                      });
+                    }
+                    break;
+                  }
+                }
+              }
+            } else if (colOptions.type === 'hm') {
+              if (
+                !linkMap.has(
+                  `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`,
+                )
+              ) {
+                // delete col.column_name as it is not required and will cause ajv error (null for LTAR)
+                delete col.column_name;
+
+                const freshModelData = await this.columnsService.columnAdd({
+                  tableId: table.id,
+                  column: withoutId({
+                    ...col,
+                    ...{
+                      parentId:
+                        idMap.get(
+                          getParentIdentifier(colOptions.fk_parent_column_id),
+                        ) ||
+                        externalIdMap.get(
+                          getParentIdentifier(colOptions.fk_parent_column_id),
+                        ),
+                      childId:
+                        idMap.get(
+                          getParentIdentifier(colOptions.fk_child_column_id),
+                        ) ||
+                        externalIdMap.get(
+                          getParentIdentifier(colOptions.fk_child_column_id),
+                        ),
+                      type: colOptions.type,
+                      virtual: colOptions.virtual,
+                      ur: colOptions.ur,
+                      dr: colOptions.dr,
+                    },
+                  }),
+                  req: param.req,
+                });
+
+                linkMap.set(
+                  `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`,
+                  `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`,
+                );
+
+                for (const nColumn of freshModelData.columns) {
+                  if (nColumn.title === col.title) {
+                    idMap.set(col.id, nColumn.id);
+                    idMap.set(
+                      colOptions.fk_parent_column_id,
+                      nColumn.colOptions.fk_parent_column_id,
+                    );
+                    idMap.set(
+                      colOptions.fk_child_column_id,
+                      nColumn.colOptions.fk_child_column_id,
+                    );
+                    break;
+                  }
+                }
+
+                const childModel =
+                  colOptions.fk_related_model_id === modelData.id
+                    ? freshModelData
+                    : await Model.get(
+                        idMap.get(colOptions.fk_related_model_id) ||
+                          externalIdMap.get(colOptions.fk_related_model_id),
+                      );
+
+                if (colOptions.fk_related_model_id !== modelData.id)
+                  await childModel.getColumns();
+
+                const childColumn = (
+                  param.data.find(
+                    (a) => a.model.id === colOptions.fk_related_model_id,
+                  )?.model ||
+                  param.externalModels.find(
+                    (a) =>
+                      a.id ===
+                      getEntityIdentifier(colOptions.fk_related_model_id),
+                  )
+                ).columns.find(
+                  (a) =>
+                    (a.colOptions?.fk_parent_column_id ===
+                      colOptions.fk_parent_column_id &&
+                      a.colOptions?.fk_child_column_id ===
+                        colOptions.fk_child_column_id &&
+                      a.id !== col.id) ||
+                    (a.colOptions?.fk_parent_column_id ===
+                      getEntityIdentifier(colOptions.fk_parent_column_id) &&
+                      a.colOptions?.fk_child_column_id ===
+                        getEntityIdentifier(colOptions.fk_child_column_id) &&
+                      a.id !== getEntityIdentifier(col.id)),
+                );
+
+                for (const nColumn of childModel.columns) {
+                  if (
+                    nColumn.id !== idMap.get(col.id) &&
+                    nColumn.id !== externalIdMap.get(col.id) &&
+                    (nColumn.colOptions?.fk_parent_column_id ===
+                      idMap.get(colOptions.fk_parent_column_id) ||
+                      externalIdMap.get(colOptions.fk_parent_column_id)) &&
+                    (nColumn.colOptions?.fk_child_column_id ===
+                      idMap.get(colOptions.fk_child_column_id) ||
+                      externalIdMap.get(colOptions.fk_child_column_id))
+                  ) {
+                    if (childColumn.id.includes('::')) {
+                      idMap.set(childColumn.id, nColumn.id);
+                    } else {
+                      idMap.set(
+                        `${childColumn.project_id}::${childColumn.base_id}::${childColumn.fk_model_id}::${childColumn.id}`,
+                        nColumn.id,
+                      );
+                    }
+
+                    childColumn.title = `${childColumn.title} copy`;
+
+                    childColumn.title = generateUniqueName(
+                      childColumn.title,
+                      childModel.columns.map((a) => a.title),
+                    );
+
+                    if (nColumn.title !== childColumn.title) {
+                      await this.columnsService.columnUpdate({
+                        columnId: nColumn.id,
+                        column: {
+                          ...nColumn,
+                          column_name: childColumn.title,
+                          title: childColumn.title,
+                        },
+                      });
+                    }
+                    break;
+                  }
+                }
+              }
+            } else if (colOptions.type === 'bt') {
+              if (
+                !linkMap.has(
+                  `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`,
+                )
+              ) {
+                // delete col.column_name as it is not required and will cause ajv error (null for LTAR)
+                delete col.column_name;
+
+                const freshModelData = await this.columnsService.columnAdd({
+                  tableId: table.id,
+                  column: withoutId({
+                    ...col,
+                    ...{
+                      parentId:
+                        idMap.get(
+                          getParentIdentifier(colOptions.fk_parent_column_id),
+                        ) ||
+                        externalIdMap.get(
+                          getParentIdentifier(colOptions.fk_parent_column_id),
+                        ),
+                      childId:
+                        idMap.get(
+                          getParentIdentifier(colOptions.fk_child_column_id),
+                        ) ||
+                        externalIdMap.get(
+                          getParentIdentifier(colOptions.fk_child_column_id),
+                        ),
+                      type: colOptions.type,
+                      virtual: colOptions.virtual,
+                      ur: colOptions.ur,
+                      dr: colOptions.dr,
+                    },
+                  }),
+                  req: param.req,
+                });
+
+                linkMap.set(
+                  `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`,
+                  `${colOptions.fk_parent_column_id}::${colOptions.fk_child_column_id}`,
+                );
+
+                for (const nColumn of freshModelData.columns) {
+                  if (nColumn.title === col.title) {
+                    idMap.set(col.id, nColumn.id);
+                    idMap.set(
+                      colOptions.fk_parent_column_id,
+                      nColumn.colOptions.fk_parent_column_id,
+                    );
+                    idMap.set(
+                      colOptions.fk_child_column_id,
+                      nColumn.colOptions.fk_child_column_id,
+                    );
+                    break;
+                  }
+                }
+
+                const childModel =
+                  colOptions.fk_related_model_id === modelData.id
+                    ? freshModelData
+                    : await Model.get(
+                        idMap.get(colOptions.fk_related_model_id) ||
+                          externalIdMap.get(colOptions.fk_related_model_id),
+                      );
+
+                if (colOptions.fk_related_model_id !== modelData.id)
+                  await childModel.getColumns();
+
+                const childColumn = (
+                  param.data.find(
+                    (a) => a.model.id === colOptions.fk_related_model_id,
+                  )?.model ||
+                  param.externalModels.find(
+                    (a) =>
+                      a.id ===
+                      getEntityIdentifier(colOptions.fk_related_model_id),
+                  )
+                ).columns.find(
+                  (a) =>
+                    (a.colOptions?.fk_parent_column_id ===
+                      colOptions.fk_parent_column_id &&
+                      a.colOptions?.fk_child_column_id ===
+                        colOptions.fk_child_column_id &&
+                      a.id !== col.id) ||
+                    (a.colOptions?.fk_parent_column_id ===
+                      getEntityIdentifier(colOptions.fk_parent_column_id) &&
+                      a.colOptions?.fk_child_column_id ===
+                        getEntityIdentifier(colOptions.fk_child_column_id) &&
+                      a.id !== getEntityIdentifier(col.id)),
+                );
+
+                for (const nColumn of childModel.columns) {
+                  if (
+                    nColumn.id !== idMap.get(col.id) &&
+                    nColumn.id !== externalIdMap.get(col.id) &&
+                    (nColumn.colOptions?.fk_parent_column_id ===
+                      idMap.get(colOptions.fk_parent_column_id) ||
+                      externalIdMap.get(colOptions.fk_parent_column_id)) &&
+                    (nColumn.colOptions?.fk_child_column_id ===
+                      idMap.get(colOptions.fk_child_column_id) ||
+                      externalIdMap.get(colOptions.fk_child_column_id))
+                  ) {
+                    if (childColumn.id.includes('::')) {
+                      idMap.set(childColumn.id, nColumn.id);
+                    } else {
+                      idMap.set(
+                        `${childColumn.project_id}::${childColumn.base_id}::${childColumn.fk_model_id}::${childColumn.id}`,
+                        nColumn.id,
+                      );
+                    }
+
+                    childColumn.title = `${childColumn.title} copy`;
+
+                    childColumn.title = generateUniqueName(
+                      childColumn.title,
+                      childModel.columns.map((a) => a.title),
+                    );
+
+                    if (nColumn.title !== childColumn.title) {
+                      await this.columnsService.columnUpdate({
+                        columnId: nColumn.id,
+                        column: {
+                          ...nColumn,
+                          column_name: childColumn.title,
+                          title: childColumn.title,
+                        },
+                      });
+                    }
+                    break;
+                  }
+                }
+              }
+            }
           }
         }
       }

packages/nocodb-nest/src/modules/jobs/fallback-queue.service.ts

@@ -59,6 +59,10 @@ export class QueueService {
       this: this.duplicateProcessor,
       fn: this.duplicateProcessor.duplicateBase,
     },
+    [JobTypes.DuplicateModel]: {
+      this: this.duplicateProcessor,
+      fn: this.duplicateProcessor.duplicateModel,
+    },
     [JobTypes.AtImport]: {
       this: this.atImportProcessor,
       fn: this.atImportProcessor.job,