Browse Source

feat: duplicate column

Signed-off-by: mertmit <mertmit99@gmail.com>
pull/6953/head
mertmit 1 year ago
parent
commit
5704f0b9b8
  1. 153
      packages/nc-gui/components/dlg/ColumnDuplicate.vue
  2. 88
      packages/nc-gui/components/smartsheet/header/Menu.vue
  3. 46
      packages/nocodb-sdk/src/lib/Api.ts
  4. 1
      packages/nocodb/src/interface/Jobs.ts
  5. 4
      packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts
  6. 56
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts
  7. 189
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts
  8. 100
      packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts
  9. 87
      packages/nocodb/src/schema/swagger.json
  10. 1
      packages/nocodb/src/utils/acl.ts

153
packages/nc-gui/components/dlg/ColumnDuplicate.vue

@ -0,0 +1,153 @@
<script setup lang="ts">
import type { ColumnType } from 'nocodb-sdk'
import { message } from 'ant-design-vue'
import { useVModel } from '#imports'
import type { TabType } from '#imports'
const props = defineProps<{
modelValue: boolean
column: ColumnType
}>()
const emit = defineEmits(['update:modelValue'])
const { api } = useApi()
const dialogShow = useVModel(props, 'modelValue', emit)
const { addTab } = useTabs()
const { $e, $poller } = useNuxtApp()
const basesStore = useBases()
const { createProject: _createProject } = basesStore
const { openTable } = useTablesStore()
const baseStore = useBase()
const { loadTables } = baseStore
const { tables } = storeToRefs(baseStore)
const { t } = useI18n()
const { activeTable: _activeTable } = storeToRefs(useTablesStore())
const { refreshCommandPalette } = useCommandPalette()
const reloadDataHook = inject(ReloadViewDataHookInj)
const { eventBus } = useSmartsheetStoreOrThrow()
const { getMeta } = useMetas()
const meta = inject(MetaInj, ref())
const options = ref({
includeData: true,
})
const optionsToExclude = computed(() => {
const { includeData } = options.value
return {
excludeData: !includeData,
}
})
const isLoading = ref(false)
const reloadTable = async () => {
await getMeta(meta!.value!.id!, true)
eventBus.emit(SmartsheetStoreEvents.FIELD_RELOAD)
reloadDataHook?.trigger()
}
const _duplicate = async () => {
try {
isLoading.value = true
const jobData = await api.dbTable.duplicateColumn(props.column.base_id!, props.column.id!, {
options: optionsToExclude.value,
})
$poller.subscribe(
{ id: jobData.id },
async (data: {
id: string
status?: string
data?: {
error?: {
message: string
}
message?: string
result?: any
}
}) => {
if (data.status !== 'close') {
if (data.status === JobStatus.COMPLETED) {
reloadTable()
isLoading.value = false
dialogShow.value = false
} else if (data.status === JobStatus.FAILED) {
message.error(`There was an error duplicating the column.`)
reloadTable()
isLoading.value = false
dialogShow.value = false
}
}
},
)
$e('a:column:duplicate')
} catch (e: any) {
message.error(await extractSdkResponseErrorMsg(e))
isLoading.value = false
dialogShow.value = false
}
}
onKeyStroke('Enter', () => {
// should only trigger this when our modal is open
if (dialogShow.value) {
_duplicate()
}
})
const isEaster = ref(false)
</script>
<template>
<GeneralModal
v-model:visible="dialogShow"
:class="{ active: dialogShow }"
:closable="!isLoading"
:mask-closable="!isLoading"
:keyboard="!isLoading"
centered
wrap-class-name="nc-modal-column-duplicate"
:footer="null"
class="!w-[30rem]"
@keydown.esc="dialogShow = false"
>
<div>
<div class="prose-xl font-bold self-center" @dblclick="isEaster = !isEaster">
{{ $t('general.duplicate') }} {{ $t('objects.column') }}
</div>
<div class="mt-4">{{ $t('msg.warning.duplicateProject') }}</div>
<div class="prose-md self-center text-gray-500 mt-4">{{ $t('title.advancedSettings') }}</div>
<a-divider class="!m-0 !p-0 !my-2" />
<div class="text-xs p-2">
<a-checkbox v-model:checked="options.includeData" :disabled="isLoading">{{ $t('labels.includeData') }}</a-checkbox>
</div>
</div>
<div class="flex flex-row gap-x-2 mt-2.5 pt-2.5 justify-end">
<NcButton v-if="!isLoading" key="back" type="secondary" @click="dialogShow = false">{{ $t('general.cancel') }}</NcButton>
<NcButton key="submit" type="primary" :loading="isLoading" @click="_duplicate">{{ $t('general.confirm') }} </NcButton>
</div>
</GeneralModal>
</template>

88
packages/nc-gui/components/smartsheet/header/Menu.vue

@ -1,5 +1,5 @@
<script lang="ts" setup>
import type { ColumnReqType } from 'nocodb-sdk'
import type { ColumnType } from 'nocodb-sdk'
import { RelationTypes, UITypes, isLinksOrLTAR } from 'nocodb-sdk'
import {
ActiveViewInj,
@ -8,8 +8,6 @@ import {
MetaInj,
ReloadViewDataHookInj,
SmartsheetStoreEvents,
extractSdkResponseErrorMsg,
getUniqueColumnName,
iconMap,
inject,
message,
@ -112,80 +110,13 @@ const sortByColumn = async (direction: 'asc' | 'desc') => {
})
}
const duplicateColumn = async () => {
let columnCreatePayload = {}
// generate duplicate column name
const duplicateColumnName = getUniqueColumnName(`${column!.value.title}_copy`, meta!.value!.columns!)
// construct column create payload
switch (column?.value.uidt) {
case UITypes.LinkToAnotherRecord:
case UITypes.Links:
case UITypes.Lookup:
case UITypes.Rollup:
case UITypes.Formula:
return message.info('Not available at the moment')
case UITypes.SingleSelect:
case UITypes.MultiSelect:
columnCreatePayload = {
...column!.value!,
title: duplicateColumnName,
column_name: duplicateColumnName,
id: undefined,
order: undefined,
colOptions: {
options:
column.value.colOptions?.options?.map((option: Record<string, any>) => ({
...option,
id: undefined,
})) ?? [],
},
}
break
default:
columnCreatePayload = {
...column!.value!,
...(column!.value.colOptions ?? {}),
title: duplicateColumnName,
column_name: duplicateColumnName,
id: undefined,
colOptions: undefined,
order: undefined,
}
break
}
try {
const gridViewColumnList = (await $api.dbViewColumn.list(view.value?.id as string)).list
const currentColumnIndex = gridViewColumnList.findIndex((f) => f.fk_column_id === column!.value.id)
let newColumnOrder
if (currentColumnIndex === gridViewColumnList.length - 1) {
newColumnOrder = gridViewColumnList[currentColumnIndex].order! + 1
} else {
newColumnOrder = (gridViewColumnList[currentColumnIndex].order! + gridViewColumnList[currentColumnIndex + 1].order!) / 2
}
await $api.dbTableColumn.create(meta!.value!.id!, {
...columnCreatePayload,
pv: false,
view_id: view.value!.id as string,
column_order: {
order: newColumnOrder,
view_id: view.value!.id as string,
},
} as ColumnReqType)
await getMeta(meta!.value!.id!, true)
const isDuplicateDlgOpen = ref(false)
const selectedColumnToDuplicate = ref<ColumnType>()
eventBus.emit(SmartsheetStoreEvents.FIELD_RELOAD)
reloadDataHook?.trigger()
// message.success(t('msg.success.columnDuplicated'))
} catch (e) {
message.error(await extractSdkResponseErrorMsg(e))
}
// closing dropdown
const openDuplicateDlg = () => {
if (!column?.value) return
selectedColumnToDuplicate.value = column.value
isDuplicateDlgOpen.value = true
isOpen.value = false
}
@ -336,8 +267,8 @@ const onInsertAfter = () => {
<a-divider class="!my-0" />
<a-menu-item
v-if="column.uidt !== UITypes.LinkToAnotherRecord && column.uidt !== UITypes.Lookup && !column.pk"
@click="duplicateColumn"
v-if="!column?.pk && column?.uidt !== UITypes.Lookup && column?.uidt !== UITypes.Rollup"
@click="openDuplicateDlg"
>
<div v-e="['a:field:duplicate']" class="nc-column-duplicate nc-header-menu-item my-0.5">
<component :is="iconMap.duplicate" class="text-gray-700 mx-0.75" />
@ -372,6 +303,7 @@ const onInsertAfter = () => {
</template>
</a-dropdown>
<SmartsheetHeaderDeleteColumnModal v-model:visible="showDeleteColumnModal" />
<DlgColumnDuplicate v-if="selectedColumnToDuplicate" v-model="isDuplicateDlgOpen" :column="selectedColumnToDuplicate" />
</template>
<style scoped>

46
packages/nocodb-sdk/src/lib/Api.ts

@ -5746,6 +5746,52 @@ export class Api<
...params,
}),
/**
* @description Duplicate a column
*
* @tags DB Table
* @name DuplicateColumn
* @summary Duplicate Column
* @request POST:/api/v1/db/meta/duplicate/{baseId}/column/{columnId}
* @response `200` `{
name?: string,
id?: string,
}` OK
* @response `400` `{
\** @example BadRequest [Error]: <ERROR MESSAGE> *\
msg: string,
}`
*/
duplicateColumn: (
baseId: IdType,
columnId: IdType,
data: {
options?: {
excludeData?: boolean;
};
},
params: RequestParams = {}
) =>
this.request<
{
name?: string;
id?: string;
},
{
/** @example BadRequest [Error]: <ERROR MESSAGE> */
msg: string;
}
>({
path: `/api/v1/db/meta/duplicate/${baseId}/column/${columnId}`,
method: 'POST',
body: data,
type: ContentType.Json,
format: 'json',
...params,
}),
/**
* @description Update the order of the given Table
*

1
packages/nocodb/src/interface/Jobs.ts

@ -3,6 +3,7 @@ export const JOBS_QUEUE = 'jobs';
export enum JobTypes {
DuplicateBase = 'duplicate-base',
DuplicateModel = 'duplicate-model',
DuplicateColumn = 'duplicate-column',
AtImport = 'at-import',
MetaSync = 'meta-sync',
SourceCreate = 'source-create',

4
packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts

@ -68,6 +68,10 @@ export class QueueService {
this: this.duplicateProcessor,
fn: this.duplicateProcessor.duplicateModel,
},
[JobTypes.DuplicateColumn]: {
this: this.duplicateProcessor,
fn: this.duplicateProcessor.duplicateColumn,
},
[JobTypes.AtImport]: {
this: this.atImportProcessor,
fn: this.atImportProcessor.job,

56
packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts

@ -13,7 +13,7 @@ import { ProjectStatus } from 'nocodb-sdk';
import { GlobalGuard } from '~/guards/global/global.guard';
import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware';
import { BasesService } from '~/services/bases.service';
import { Base, Model, Source } from '~/models';
import { Base, Column, Model, Source } from '~/models';
import { generateUniqueName } from '~/helpers/exportImportHelpers';
import { JobTypes } from '~/interface/Jobs';
import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard';
@ -217,4 +217,58 @@ export class DuplicateController {
return { id: job.id };
}
@Post([
'/api/v1/db/meta/duplicate/:baseId/column/:columnId',
'/api/v2/meta/duplicate/:baseId/column/:columnId',
])
@HttpCode(200)
@Acl('duplicateColumn')
async duplicateColumn(
@Req() req: Request,
@Param('baseId') baseId: string,
@Param('columnId') columnId?: string,
@Body()
body?: {
options?: {
excludeData?: boolean;
};
},
) {
const base = await Base.get(baseId);
if (!base) {
throw new Error(`Base not found for id '${baseId}'`);
}
const column = await Column.get({
source_id: base.id,
colId: columnId,
});
if (!column) {
throw new Error(`Column not found!`);
}
const model = await Model.get(column.fk_model_id);
if (!model) {
throw new Error(`Model not found!`);
}
const job = await this.jobsService.add(JobTypes.DuplicateColumn, {
baseId: base.id,
sourceId: column.source_id,
modelId: model.id,
columnId: column.id,
options: body.options || {},
req: {
user: req.user,
clientIp: req.clientIp,
headers: req.headers,
},
});
return { id: job.id };
}
}

189
packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@ -6,7 +6,10 @@ import debug from 'debug';
import { isLinksOrLTAR } from 'nocodb-sdk';
import { Base, Column, Model, Source } from '~/models';
import { BasesService } from '~/services/bases.service';
import { findWithIdentifier } from '~/helpers/exportImportHelpers';
import {
findWithIdentifier,
generateUniqueName,
} from '~/helpers/exportImportHelpers';
import { BulkDataAliasService } from '~/services/bulk-data-alias.service';
import { JOBS_QUEUE, JobTypes } from '~/interface/Jobs';
import { elapsedTime, initTime } from '~/modules/jobs/helpers';
@ -227,6 +230,146 @@ export class DuplicateProcessor {
return await Model.get(findWithIdentifier(idMap, sourceModel.id));
}
@Process(JobTypes.DuplicateColumn)
async duplicateColumn(job: Job) {
this.debugLog(`job started for ${job.id} (${JobTypes.DuplicateColumn})`);
const hrTime = initTime();
const { baseId, sourceId, columnId, req, options } = job.data;
const excludeData = options?.excludeData || false;
const base = await Base.get(baseId);
const sourceColumn = await Column.get({
source_id: sourceId,
colId: columnId,
});
const user = (req as any).user;
const source = await Source.get(sourceColumn.source_id);
const models = (await source.getModels()).filter(
(m) => !m.mm && m.type === 'table',
);
const sourceModel = models.find((m) => m.id === sourceColumn.fk_model_id);
const columns = await sourceModel.getColumns();
const title = generateUniqueName(
`${sourceColumn.title} copy`,
columns.map((p) => p.title),
);
const relatedModelIds = [sourceColumn]
.filter((col) => isLinksOrLTAR(col))
.map((col) => col.colOptions.fk_related_model_id)
.filter((id) => id);
const relatedModels = models.filter((m) => relatedModelIds.includes(m.id));
const exportedModel = (
await this.exportService.serializeModels({
modelIds: [sourceModel.id],
excludeData,
excludeHooks: true,
excludeViews: true,
})
)[0];
elapsedTime(
hrTime,
`serialize model schema for ${sourceModel.id}`,
'duplicateColumn',
);
if (!exportedModel) {
throw new Error(`Export failed for model '${sourceModel.id}'`);
}
const serializedPrimaryKey = exportedModel.model.columns.find((c) =>
c.id.includes(sourceModel.primaryKey.id),
);
exportedModel.model.columns = exportedModel.model.columns.filter((c) =>
c.id.includes(columnId),
);
if (exportedModel.model.columns.length !== 1) {
throw new Error(`There was an error duplicating column!`);
}
const replacedColumn = exportedModel.model.columns.find((c) =>
c.id.includes(columnId),
);
replacedColumn.title = title;
replacedColumn.column_name = title.toLowerCase().replace(/ /g, '_');
const idMap = await this.importService.importModels({
baseId,
sourceId: source.id,
data: [exportedModel],
user,
req,
externalModels: relatedModels,
existingModel: sourceModel,
});
elapsedTime(hrTime, 'import model schema', 'duplicateColumn');
if (!idMap) {
throw new Error(`Import failed for model '${sourceModel.id}'`);
}
if (!excludeData) {
const fields: Record<string, string[]> = {};
idMap.set(serializedPrimaryKey.id, sourceModel.primaryKey.id);
fields[sourceModel.id] = [sourceModel.primaryKey.id];
fields[sourceModel.id].push(columnId);
for (const md of relatedModels) {
const bts = md.columns
.filter(
(c) =>
isLinksOrLTAR(c) &&
c.colOptions.type === 'bt' &&
c.colOptions.fk_related_model_id === sourceModel.id,
)
.map((c) => c.id);
if (bts.length > 0) {
fields[md.id] = [md.primaryKey.id];
fields[md.id].push(...bts);
}
}
await this.importModelsData({
idMap,
sourceProject: base,
sourceModels: [],
destProject: base,
destBase: source,
hrTime,
modelFieldIds: fields,
externalModels: [sourceModel, ...relatedModels],
});
elapsedTime(hrTime, 'import model data', 'duplicateColumn');
}
this.debugLog(`job completed for ${job.id} (${JobTypes.DuplicateModel})`);
return await Column.get({
source_id: base.id,
colId: findWithIdentifier(idMap, sourceColumn.id),
});
}
async importModelsData(param: {
idMap: Map<string, string>;
sourceProject: Base;
@ -395,13 +538,17 @@ export class DuplicateProcessor {
if (chunk.length > 1000) {
parser.pause();
try {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
// remove empty rows (only pk is present)
chunk = chunk.filter((r) => Object.keys(r).length > 1);
if (chunk.length > 0) {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
}
} catch (e) {
this.debugLog(e);
}
@ -414,13 +561,17 @@ export class DuplicateProcessor {
complete: async () => {
if (chunk.length > 0) {
try {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
// remove empty rows (only pk is present)
chunk = chunk.filter((r) => Object.keys(r).length > 1);
if (chunk.length > 0) {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
}
} catch (e) {
this.debugLog(e);
}
@ -433,6 +584,14 @@ export class DuplicateProcessor {
if (error) throw error;
handledLinks = await this.importService.importLinkFromCsvStream({
idMap,
linkStream,
destProject,
destBase,
handledLinks,
});
elapsedTime(
hrTime,
`map existing links to ${model.title}`,

100
packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@ -67,6 +67,7 @@ export class ImportService {
| { model: any; views: any[]; hooks?: any[] }[];
req: NcRequest;
externalModels?: Model[];
existingModel?: Model;
}) {
const hrTime = initTime();
@ -131,45 +132,69 @@ export class ImportService {
);
// create table with static columns
const table = await this.tablesService.tableCreate({
baseId: base.id,
sourceId: source.id,
user: param.user,
table: withoutId({
...modelData,
columns: reducedColumnSet.map((a) => withoutId(a)),
}),
});
const table =
param.existingModel ||
(await this.tablesService.tableCreate({
baseId: base.id,
sourceId: source.id,
user: param.user,
table: withoutId({
...modelData,
columns: reducedColumnSet.map((a) => withoutId(a)),
}),
}));
idMap.set(modelData.id, table.id);
// map column id's with new created column id's
for (const col of table.columns) {
const colRef = modelData.columns.find(
(a) =>
a.column_name &&
sanitizeColumnName(a.column_name) === col.column_name,
);
idMap.set(colRef.id, col.id);
// setval for auto increment column in pg
if (source.type === 'pg') {
if (modelData.pgSerialLastVal) {
if (col.ai) {
const baseModel = await Model.getBaseModelSQL({
id: table.id,
viewId: null,
dbDriver: await NcConnectionMgrv2.get(source),
});
const sqlClient = await NcConnectionMgrv2.getSqlClient(source);
await sqlClient.raw(
`SELECT setval(pg_get_serial_sequence('??', ?), ?);`,
[
baseModel.getTnPath(table.table_name),
col.column_name,
modelData.pgSerialLastVal,
],
);
if (param.existingModel) {
if (reducedColumnSet.length) {
for (const col of reducedColumnSet) {
const freshModelData = await this.columnsService.columnAdd({
tableId: getIdOrExternalId(getParentIdentifier(col.id)),
column: withoutId({
...col,
}) as any,
req: param.req,
user: param.user,
});
for (const nColumn of freshModelData.columns) {
if (nColumn.title === col.title) {
idMap.set(col.id, nColumn.id);
break;
}
}
}
}
} else {
// map column id's with new created column id's
for (const col of table.columns) {
const colRef = modelData.columns.find(
(a) =>
a.column_name &&
sanitizeColumnName(a.column_name) === col.column_name,
);
idMap.set(colRef.id, col.id);
// setval for auto increment column in pg
if (source.type === 'pg') {
if (modelData.pgSerialLastVal) {
if (col.ai) {
const baseModel = await Model.getBaseModelSQL({
id: table.id,
viewId: null,
dbDriver: await NcConnectionMgrv2.get(source),
});
const sqlClient = await NcConnectionMgrv2.getSqlClient(source);
await sqlClient.raw(
`SELECT setval(pg_get_serial_sequence('??', ?), ?);`,
[
baseModel.getTnPath(table.table_name),
col.column_name,
modelData.pgSerialLastVal,
],
);
}
}
}
}
@ -912,6 +937,8 @@ export class ImportService {
// create views
for (const data of param.data) {
if (param.existingModel) break;
const modelData = data.model;
const viewsData = data.views;
@ -1045,6 +1072,7 @@ export class ImportService {
// create hooks
for (const data of param.data) {
if (param.existingModel) break;
if (!data?.hooks) break;
const modelData = data.model;
const hookData = data.hooks;

87
packages/nocodb/src/schema/swagger.json

@ -4589,6 +4589,93 @@
]
}
},
"/api/v1/db/meta/duplicate/{baseId}/column/{columnId}": {
"post": {
"summary": "Duplicate Column",
"operationId": "duplicate-column",
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"id": {
"type": "string"
}
}
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest"
}
},
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"excludeData": {
"type": "boolean",
"required": false
}
}
}
}
},
"examples": {
"Example 1": {
"value": {
"excludeData": true
}
}
}
}
}
},
"tags": [
"DB Table"
],
"description": "Duplicate a column",
"parameters": [
{
"$ref": "#/components/parameters/xc-auth"
},
{
"schema": {
"$ref": "#/components/schemas/Id",
"example": "p_124hhlkbeasewh",
"type": "string"
},
"name": "baseId",
"in": "path",
"required": true,
"description": "Unique Base ID"
},
{
"schema": {
"$ref": "#/components/schemas/Id",
"example": "md_124hhlkbeasewh",
"type": "string"
},
"name": "columnId",
"in": "path",
"required": true,
"description": "Unique Column ID"
}
]
}
},
"/api/v1/db/meta/projects/{baseId}/{sourceId}/tables": {
"parameters": [
{

1
packages/nocodb/src/utils/acl.ts

@ -116,6 +116,7 @@ const permissionScopes = {
'bulkDataDeleteAll',
'relationDataRemove',
'relationDataAdd',
'duplicateColumn',
// Base API Tokens
'baseApiTokenList',

Loading…
Cancel
Save