feat: duplicate column

Signed-off-by: mertmit <mertmit99@gmail.com>

packages/nc-gui/components/dlg/ColumnDuplicate.vue

@@ -0,0 +1,153 @@
+<script setup lang="ts">
+import type { ColumnType } from 'nocodb-sdk'
+import { message } from 'ant-design-vue'
+import { useVModel } from '#imports'
+import type { TabType } from '#imports'
+
+const props = defineProps<{
+  modelValue: boolean
+  column: ColumnType
+}>()
+
+const emit = defineEmits(['update:modelValue'])
+
+const { api } = useApi()
+
+const dialogShow = useVModel(props, 'modelValue', emit)
+
+const { addTab } = useTabs()
+
+const { $e, $poller } = useNuxtApp()
+
+const basesStore = useBases()
+
+const { createProject: _createProject } = basesStore
+
+const { openTable } = useTablesStore()
+
+const baseStore = useBase()
+
+const { loadTables } = baseStore
+
+const { tables } = storeToRefs(baseStore)
+
+const { t } = useI18n()
+
+const { activeTable: _activeTable } = storeToRefs(useTablesStore())
+
+const { refreshCommandPalette } = useCommandPalette()
+
+const reloadDataHook = inject(ReloadViewDataHookInj)
+
+const { eventBus } = useSmartsheetStoreOrThrow()
+
+const { getMeta } = useMetas()
+
+const meta = inject(MetaInj, ref())
+
+const options = ref({
+  includeData: true,
+})
+
+const optionsToExclude = computed(() => {
+  const { includeData } = options.value
+  return {
+    excludeData: !includeData,
+  }
+})
+
+const isLoading = ref(false)
+
+const reloadTable = async () => {
+  await getMeta(meta!.value!.id!, true)
+  eventBus.emit(SmartsheetStoreEvents.FIELD_RELOAD)
+  reloadDataHook?.trigger()
+}
+
+const _duplicate = async () => {
+  try {
+    isLoading.value = true
+    const jobData = await api.dbTable.duplicateColumn(props.column.base_id!, props.column.id!, {
+      options: optionsToExclude.value,
+    })
+
+    $poller.subscribe(
+      { id: jobData.id },
+      async (data: {
+        id: string
+        status?: string
+        data?: {
+          error?: {
+            message: string
+          }
+          message?: string
+          result?: any
+        }
+      }) => {
+        if (data.status !== 'close') {
+          if (data.status === JobStatus.COMPLETED) {
+            reloadTable()
+            isLoading.value = false
+            dialogShow.value = false
+          } else if (data.status === JobStatus.FAILED) {
+            message.error(`There was an error duplicating the column.`)
+            reloadTable()
+            isLoading.value = false
+            dialogShow.value = false
+          }
+        }
+      },
+    )
+
+    $e('a:column:duplicate')
+  } catch (e: any) {
+    message.error(await extractSdkResponseErrorMsg(e))
+    isLoading.value = false
+    dialogShow.value = false
+  }
+}
+
+onKeyStroke('Enter', () => {
+  // should only trigger this when our modal is open
+  if (dialogShow.value) {
+    _duplicate()
+  }
+})
+
+const isEaster = ref(false)
+</script>
+
+<template>
+  <GeneralModal
+    v-model:visible="dialogShow"
+    :class="{ active: dialogShow }"
+    :closable="!isLoading"
+    :mask-closable="!isLoading"
+    :keyboard="!isLoading"
+    centered
+    wrap-class-name="nc-modal-column-duplicate"
+    :footer="null"
+    class="!w-[30rem]"
+    @keydown.esc="dialogShow = false"
+  >
+    <div>
+      <div class="prose-xl font-bold self-center" @dblclick="isEaster = !isEaster">
+        {{ $t('general.duplicate') }} {{ $t('objects.column') }}
+      </div>
+
+      <div class="mt-4">{{ $t('msg.warning.duplicateProject') }}</div>
+
+      <div class="prose-md self-center text-gray-500 mt-4">{{ $t('title.advancedSettings') }}</div>
+
+      <a-divider class="!m-0 !p-0 !my-2" />
+
+      <div class="text-xs p-2">
+        <a-checkbox v-model:checked="options.includeData" :disabled="isLoading">{{ $t('labels.includeData') }}</a-checkbox>
+      </div>
+    </div>
+    <div class="flex flex-row gap-x-2 mt-2.5 pt-2.5 justify-end">
+      <NcButton v-if="!isLoading" key="back" type="secondary" @click="dialogShow = false">{{ $t('general.cancel') }}</NcButton>
+      <NcButton key="submit" type="primary" :loading="isLoading" @click="_duplicate">{{ $t('general.confirm') }} </NcButton>
+    </div>
+  </GeneralModal>
+</template>

packages/nc-gui/components/smartsheet/header/Menu.vue

@@ -1,5 +1,5 @@
 <script lang="ts" setup>
-import type { ColumnReqType } from 'nocodb-sdk'
+import type { ColumnType } from 'nocodb-sdk'
 import { RelationTypes, UITypes, isLinksOrLTAR } from 'nocodb-sdk'
 import {
   ActiveViewInj,
@@ -8,8 +8,6 @@ import {
   MetaInj,
   ReloadViewDataHookInj,
   SmartsheetStoreEvents,
-  extractSdkResponseErrorMsg,
-  getUniqueColumnName,
   iconMap,
   inject,
   message,
@@ -112,80 +110,13 @@ const sortByColumn = async (direction: 'asc' | 'desc') => {
   })
 }
 
-const duplicateColumn = async () => {
-  let columnCreatePayload = {}
-
-  // generate duplicate column name
-  const duplicateColumnName = getUniqueColumnName(`${column!.value.title}_copy`, meta!.value!.columns!)
-
-  // construct column create payload
-  switch (column?.value.uidt) {
-    case UITypes.LinkToAnotherRecord:
-    case UITypes.Links:
-    case UITypes.Lookup:
-    case UITypes.Rollup:
-    case UITypes.Formula:
-      return message.info('Not available at the moment')
-    case UITypes.SingleSelect:
-    case UITypes.MultiSelect:
-      columnCreatePayload = {
-        ...column!.value!,
-        title: duplicateColumnName,
-        column_name: duplicateColumnName,
-        id: undefined,
-        order: undefined,
-        colOptions: {
-          options:
-            column.value.colOptions?.options?.map((option: Record<string, any>) => ({
-              ...option,
-              id: undefined,
-            })) ?? [],
-        },
-      }
-      break
-    default:
-      columnCreatePayload = {
-        ...column!.value!,
-        ...(column!.value.colOptions ?? {}),
-        title: duplicateColumnName,
-        column_name: duplicateColumnName,
-        id: undefined,
-        colOptions: undefined,
-        order: undefined,
-      }
-      break
-  }
+const isDuplicateDlgOpen = ref(false)
+const selectedColumnToDuplicate = ref<ColumnType>()
 
-  try {
-    const gridViewColumnList = (await $api.dbViewColumn.list(view.value?.id as string)).list
-
-    const currentColumnIndex = gridViewColumnList.findIndex((f) => f.fk_column_id === column!.value.id)
-    let newColumnOrder
-    if (currentColumnIndex === gridViewColumnList.length - 1) {
-      newColumnOrder = gridViewColumnList[currentColumnIndex].order! + 1
-    } else {
-      newColumnOrder = (gridViewColumnList[currentColumnIndex].order! + gridViewColumnList[currentColumnIndex + 1].order!) / 2
-    }
-
-    await $api.dbTableColumn.create(meta!.value!.id!, {
-      ...columnCreatePayload,
-      pv: false,
-      view_id: view.value!.id as string,
-      column_order: {
-        order: newColumnOrder,
-        view_id: view.value!.id as string,
-      },
-    } as ColumnReqType)
-    await getMeta(meta!.value!.id!, true)
-
-    eventBus.emit(SmartsheetStoreEvents.FIELD_RELOAD)
-    reloadDataHook?.trigger()
-
-    // message.success(t('msg.success.columnDuplicated'))
-  } catch (e) {
-    message.error(await extractSdkResponseErrorMsg(e))
-  }
-  // closing dropdown
+const openDuplicateDlg = () => {
+  if (!column?.value) return
+  selectedColumnToDuplicate.value = column.value
+  isDuplicateDlgOpen.value = true
   isOpen.value = false
 }
 
@@ -336,8 +267,8 @@ const onInsertAfter = () => {
         <a-divider class="!my-0" />
 
         <a-menu-item
-          v-if="column.uidt !== UITypes.LinkToAnotherRecord && column.uidt !== UITypes.Lookup && !column.pk"
-          @click="duplicateColumn"
+          v-if="!column?.pk && column?.uidt !== UITypes.Lookup && column?.uidt !== UITypes.Rollup"
+          @click="openDuplicateDlg"
         >
           <div v-e="['a:field:duplicate']" class="nc-column-duplicate nc-header-menu-item my-0.5">
             <component :is="iconMap.duplicate" class="text-gray-700 mx-0.75" />
@@ -372,6 +303,7 @@ const onInsertAfter = () => {
     </template>
   </a-dropdown>
   <SmartsheetHeaderDeleteColumnModal v-model:visible="showDeleteColumnModal" />
+  <DlgColumnDuplicate v-if="selectedColumnToDuplicate" v-model="isDuplicateDlgOpen" :column="selectedColumnToDuplicate" />
 </template>
 
 <style scoped>

packages/nocodb-sdk/src/lib/Api.ts

@@ -5746,6 +5746,52 @@ export class Api<
         ...params,
       }),
 
+    /**
+ * @description Duplicate a column
+ * 
+ * @tags DB Table
+ * @name DuplicateColumn
+ * @summary Duplicate Column
+ * @request POST:/api/v1/db/meta/duplicate/{baseId}/column/{columnId}
+ * @response `200` `{
+  name?: string,
+  id?: string,
+
+}` OK
+ * @response `400` `{
+  \** @example BadRequest [Error]: <ERROR MESSAGE> *\
+  msg: string,
+
+}`
+ */
+    duplicateColumn: (
+      baseId: IdType,
+      columnId: IdType,
+      data: {
+        options?: {
+          excludeData?: boolean;
+        };
+      },
+      params: RequestParams = {}
+    ) =>
+      this.request<
+        {
+          name?: string;
+          id?: string;
+        },
+        {
+          /** @example BadRequest [Error]: <ERROR MESSAGE> */
+          msg: string;
+        }
+      >({
+        path: `/api/v1/db/meta/duplicate/${baseId}/column/${columnId}`,
+        method: 'POST',
+        body: data,
+        type: ContentType.Json,
+        format: 'json',
+        ...params,
+      }),
+
     /**
  * @description Update the order of the given Table
  * 

packages/nocodb/src/interface/Jobs.ts

@@ -3,6 +3,7 @@ export const JOBS_QUEUE = 'jobs';
 export enum JobTypes {
   DuplicateBase = 'duplicate-base',
   DuplicateModel = 'duplicate-model',
+  DuplicateColumn = 'duplicate-column',
   AtImport = 'at-import',
   MetaSync = 'meta-sync',
   SourceCreate = 'source-create',

packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts

@@ -68,6 +68,10 @@ export class QueueService {
       this: this.duplicateProcessor,
       fn: this.duplicateProcessor.duplicateModel,
     },
+    [JobTypes.DuplicateColumn]: {
+      this: this.duplicateProcessor,
+      fn: this.duplicateProcessor.duplicateColumn,
+    },
     [JobTypes.AtImport]: {
       this: this.atImportProcessor,
       fn: this.atImportProcessor.job,

packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts

@@ -13,7 +13,7 @@ import { ProjectStatus } from 'nocodb-sdk';
 import { GlobalGuard } from '~/guards/global/global.guard';
 import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware';
 import { BasesService } from '~/services/bases.service';
-import { Base, Model, Source } from '~/models';
+import { Base, Column, Model, Source } from '~/models';
 import { generateUniqueName } from '~/helpers/exportImportHelpers';
 import { JobTypes } from '~/interface/Jobs';
 import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard';
@@ -217,4 +217,58 @@ export class DuplicateController {
 
     return { id: job.id };
   }
+
+  @Post([
+    '/api/v1/db/meta/duplicate/:baseId/column/:columnId',
+    '/api/v2/meta/duplicate/:baseId/column/:columnId',
+  ])
+  @HttpCode(200)
+  @Acl('duplicateColumn')
+  async duplicateColumn(
+    @Req() req: Request,
+    @Param('baseId') baseId: string,
+    @Param('columnId') columnId?: string,
+    @Body()
+    body?: {
+      options?: {
+        excludeData?: boolean;
+      };
+    },
+  ) {
+    const base = await Base.get(baseId);
+
+    if (!base) {
+      throw new Error(`Base not found for id '${baseId}'`);
+    }
+
+    const column = await Column.get({
+      source_id: base.id,
+      colId: columnId,
+    });
+
+    if (!column) {
+      throw new Error(`Column not found!`);
+    }
+
+    const model = await Model.get(column.fk_model_id);
+
+    if (!model) {
+      throw new Error(`Model not found!`);
+    }
+
+    const job = await this.jobsService.add(JobTypes.DuplicateColumn, {
+      baseId: base.id,
+      sourceId: column.source_id,
+      modelId: model.id,
+      columnId: column.id,
+      options: body.options || {},
+      req: {
+        user: req.user,
+        clientIp: req.clientIp,
+        headers: req.headers,
+      },
+    });
+
+    return { id: job.id };
+  }
 }

packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@@ -6,7 +6,10 @@ import debug from 'debug';
 import { isLinksOrLTAR } from 'nocodb-sdk';
 import { Base, Column, Model, Source } from '~/models';
 import { BasesService } from '~/services/bases.service';
-import { findWithIdentifier } from '~/helpers/exportImportHelpers';
+import {
+  findWithIdentifier,
+  generateUniqueName,
+} from '~/helpers/exportImportHelpers';
 import { BulkDataAliasService } from '~/services/bulk-data-alias.service';
 import { JOBS_QUEUE, JobTypes } from '~/interface/Jobs';
 import { elapsedTime, initTime } from '~/modules/jobs/helpers';
@@ -227,6 +230,146 @@ export class DuplicateProcessor {
     return await Model.get(findWithIdentifier(idMap, sourceModel.id));
   }
 
+  @Process(JobTypes.DuplicateColumn)
+  async duplicateColumn(job: Job) {
+    this.debugLog(`job started for ${job.id} (${JobTypes.DuplicateColumn})`);
+
+    const hrTime = initTime();
+
+    const { baseId, sourceId, columnId, req, options } = job.data;
+
+    const excludeData = options?.excludeData || false;
+
+    const base = await Base.get(baseId);
+
+    const sourceColumn = await Column.get({
+      source_id: sourceId,
+      colId: columnId,
+    });
+
+    const user = (req as any).user;
+
+    const source = await Source.get(sourceColumn.source_id);
+
+    const models = (await source.getModels()).filter(
+      (m) => !m.mm && m.type === 'table',
+    );
+
+    const sourceModel = models.find((m) => m.id === sourceColumn.fk_model_id);
+
+    const columns = await sourceModel.getColumns();
+
+    const title = generateUniqueName(
+      `${sourceColumn.title} copy`,
+      columns.map((p) => p.title),
+    );
+
+    const relatedModelIds = [sourceColumn]
+      .filter((col) => isLinksOrLTAR(col))
+      .map((col) => col.colOptions.fk_related_model_id)
+      .filter((id) => id);
+
+    const relatedModels = models.filter((m) => relatedModelIds.includes(m.id));
+
+    const exportedModel = (
+      await this.exportService.serializeModels({
+        modelIds: [sourceModel.id],
+        excludeData,
+        excludeHooks: true,
+        excludeViews: true,
+      })
+    )[0];
+
+    elapsedTime(
+      hrTime,
+      `serialize model schema for ${sourceModel.id}`,
+      'duplicateColumn',
+    );
+
+    if (!exportedModel) {
+      throw new Error(`Export failed for model '${sourceModel.id}'`);
+    }
+
+    const serializedPrimaryKey = exportedModel.model.columns.find((c) =>
+      c.id.includes(sourceModel.primaryKey.id),
+    );
+
+    exportedModel.model.columns = exportedModel.model.columns.filter((c) =>
+      c.id.includes(columnId),
+    );
+
+    if (exportedModel.model.columns.length !== 1) {
+      throw new Error(`There was an error duplicating column!`);
+    }
+
+    const replacedColumn = exportedModel.model.columns.find((c) =>
+      c.id.includes(columnId),
+    );
+
+    replacedColumn.title = title;
+    replacedColumn.column_name = title.toLowerCase().replace(/ /g, '_');
+
+    const idMap = await this.importService.importModels({
+      baseId,
+      sourceId: source.id,
+      data: [exportedModel],
+      user,
+      req,
+      externalModels: relatedModels,
+      existingModel: sourceModel,
+    });
+
+    elapsedTime(hrTime, 'import model schema', 'duplicateColumn');
+
+    if (!idMap) {
+      throw new Error(`Import failed for model '${sourceModel.id}'`);
+    }
+
+    if (!excludeData) {
+      const fields: Record<string, string[]> = {};
+
+      idMap.set(serializedPrimaryKey.id, sourceModel.primaryKey.id);
+      fields[sourceModel.id] = [sourceModel.primaryKey.id];
+      fields[sourceModel.id].push(columnId);
+
+      for (const md of relatedModels) {
+        const bts = md.columns
+          .filter(
+            (c) =>
+              isLinksOrLTAR(c) &&
+              c.colOptions.type === 'bt' &&
+              c.colOptions.fk_related_model_id === sourceModel.id,
+          )
+          .map((c) => c.id);
+
+        if (bts.length > 0) {
+          fields[md.id] = [md.primaryKey.id];
+          fields[md.id].push(...bts);
+        }
+      }
+
+      await this.importModelsData({
+        idMap,
+        sourceProject: base,
+        sourceModels: [],
+        destProject: base,
+        destBase: source,
+        hrTime,
+        modelFieldIds: fields,
+        externalModels: [sourceModel, ...relatedModels],
+      });
+
+      elapsedTime(hrTime, 'import model data', 'duplicateColumn');
+    }
+
+    this.debugLog(`job completed for ${job.id} (${JobTypes.DuplicateModel})`);
+
+    return await Column.get({
+      source_id: base.id,
+      colId: findWithIdentifier(idMap, sourceColumn.id),
+    });
+  }
+
   async importModelsData(param: {
     idMap: Map<string, string>;
     sourceProject: Base;
@@ -395,6 +538,9 @@ export class DuplicateProcessor {
                   if (chunk.length > 1000) {
                     parser.pause();
                     try {
+                      // remove empty rows (only pk is present)
+                      chunk = chunk.filter((r) => Object.keys(r).length > 1);
+                      if (chunk.length > 0) {
                         await this.bulkDataService.bulkDataUpdate({
                           baseName: destProject.id,
                           tableName: model.id,
@@ -402,6 +548,7 @@ export class DuplicateProcessor {
                           cookie: null,
                           raw: true,
                         });
+                      }
                     } catch (e) {
                       this.debugLog(e);
                     }
@@ -414,6 +561,9 @@ export class DuplicateProcessor {
             complete: async () => {
               if (chunk.length > 0) {
                 try {
+                  // remove empty rows (only pk is present)
+                  chunk = chunk.filter((r) => Object.keys(r).length > 1);
+                  if (chunk.length > 0) {
                     await this.bulkDataService.bulkDataUpdate({
                       baseName: destProject.id,
                       tableName: model.id,
@@ -421,6 +571,7 @@ export class DuplicateProcessor {
                       cookie: null,
                       raw: true,
                     });
+                  }
                 } catch (e) {
                   this.debugLog(e);
                 }
@@ -433,6 +584,14 @@ export class DuplicateProcessor {
 
         if (error) throw error;
 
+        handledLinks = await this.importService.importLinkFromCsvStream({
+          idMap,
+          linkStream,
+          destProject,
+          destBase,
+          handledLinks,
+        });
+
         elapsedTime(
           hrTime,
           `map existing links to ${model.title}`,

packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@@ -67,6 +67,7 @@ export class ImportService {
       | { model: any; views: any[]; hooks?: any[] }[];
     req: NcRequest;
     externalModels?: Model[];
+    existingModel?: Model;
   }) {
     const hrTime = initTime();
 
@@ -131,7 +132,9 @@ export class ImportService {
       );
 
       // create table with static columns
-      const table = await this.tablesService.tableCreate({
+      const table =
+        param.existingModel ||
+        (await this.tablesService.tableCreate({
           baseId: base.id,
           sourceId: source.id,
           user: param.user,
@@ -139,10 +142,31 @@ export class ImportService {
             ...modelData,
             columns: reducedColumnSet.map((a) => withoutId(a)),
           }),
-      });
+        }));
 
       idMap.set(modelData.id, table.id);
 
+      if (param.existingModel) {
+        if (reducedColumnSet.length) {
+          for (const col of reducedColumnSet) {
+            const freshModelData = await this.columnsService.columnAdd({
+              tableId: getIdOrExternalId(getParentIdentifier(col.id)),
+              column: withoutId({
+                ...col,
+              }) as any,
+              req: param.req,
+              user: param.user,
+            });
+
+            for (const nColumn of freshModelData.columns) {
+              if (nColumn.title === col.title) {
+                idMap.set(col.id, nColumn.id);
+                break;
+              }
+            }
+          }
+        }
+      } else {
         // map column id's with new created column id's
         for (const col of table.columns) {
           const colRef = modelData.columns.find(
@@ -174,6 +198,7 @@ export class ImportService {
             }
           }
         }
+      }
 
       tableReferences.set(modelData.id, table);
     }
@@ -912,6 +937,8 @@ export class ImportService {
 
     // create views
     for (const data of param.data) {
+      if (param.existingModel) break;
+
       const modelData = data.model;
       const viewsData = data.views;
 
@@ -1045,6 +1072,7 @@ export class ImportService {
 
     // create hooks
     for (const data of param.data) {
+      if (param.existingModel) break;
       if (!data?.hooks) break;
       const modelData = data.model;
       const hookData = data.hooks;

packages/nocodb/src/schema/swagger.json

@@ -4589,6 +4589,93 @@
         ]
       }
     },
+    "/api/v1/db/meta/duplicate/{baseId}/column/{columnId}": {
+      "post": {
+        "summary": "Duplicate Column",
+        "operationId": "duplicate-column",
+        "responses": {
+          "200": {
+            "description": "OK",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "name": {
+                      "type": "string"
+                    },
+                    "id": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "400": {
+            "$ref": "#/components/responses/BadRequest"
+          }
+        },
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "options": {
+                    "type": "object",
+                    "properties": {
+                      "excludeData": {
+                        "type": "boolean",
+                        "required": false
+                      }
+                    }
+                  }
+                }
+              },
+              "examples": {
+                "Example 1": {
+                  "value": {
+                    "excludeData": true
+                  }
+                }
+              }
+            }
+          }
+        },
+        "tags": [
+          "DB Table"
+        ],
+        "description": "Duplicate a column",
+        "parameters": [
+          {
+            "$ref": "#/components/parameters/xc-auth"
+          },
+          {
+            "schema": {
+              "$ref": "#/components/schemas/Id",
+              "example": "p_124hhlkbeasewh",
+              "type": "string"
+            },
+            "name": "baseId",
+            "in": "path",
+            "required": true,
+            "description": "Unique Base ID"
+          },
+          {
+            "schema": {
+              "$ref": "#/components/schemas/Id",
+              "example": "md_124hhlkbeasewh",
+              "type": "string"
+            },
+            "name": "columnId",
+            "in": "path",
+            "required": true,
+            "description": "Unique Column ID"
+          }
+        ]
+      }
+    },
     "/api/v1/db/meta/projects/{baseId}/{sourceId}/tables": {
       "parameters": [
         {

packages/nocodb/src/utils/acl.ts

@@ -116,6 +116,7 @@ const permissionScopes = {
     'bulkDataDeleteAll',
     'relationDataRemove',
     'relationDataAdd',
+    'duplicateColumn',
 
     // Base API Tokens
     'baseApiTokenList',