Browse Source

Merge pull request #6953 from nocodb/feat/dup-col

feat: duplicate column with data
pull/6972/head
mertmit 1 year ago committed by GitHub
parent
commit
931cee2e66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 140
      packages/nc-gui/components/dlg/ColumnDuplicate.vue
  2. 6
      packages/nc-gui/components/smartsheet/column/DecimalOptions.vue
  3. 102
      packages/nc-gui/components/smartsheet/header/Menu.vue
  4. 47
      packages/nocodb-sdk/src/lib/Api.ts
  5. 64
      packages/nocodb/src/db/BaseModelSqlv2.ts
  6. 5
      packages/nocodb/src/helpers/columnHelpers.ts
  7. 1
      packages/nocodb/src/interface/Jobs.ts
  8. 4
      packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts
  9. 58
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts
  10. 209
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts
  11. 12
      packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts
  12. 105
      packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts
  13. 90
      packages/nocodb/src/schema/swagger.json
  14. 29
      packages/nocodb/src/services/columns.service.ts
  15. 13
      packages/nocodb/src/services/datas.service.ts
  16. 1
      packages/nocodb/src/utils/acl.ts
  17. 2
      packages/nocodb/tests/unit/factory/row.ts
  18. 6
      tests/playwright/pages/Dashboard/Grid/Column/index.ts
  19. 2
      tests/playwright/tests/db/columns/columnMenuOperations.spec.ts

140
packages/nc-gui/components/dlg/ColumnDuplicate.vue

@ -0,0 +1,140 @@
<script setup lang="ts">
import type { ColumnType } from 'nocodb-sdk'
import { message } from 'ant-design-vue'
import { useVModel } from '#imports'
const props = defineProps<{
modelValue: boolean
column: ColumnType
extra: any
}>()
const emit = defineEmits(['update:modelValue'])
const { api } = useApi()
const dialogShow = useVModel(props, 'modelValue', emit)
const { $e, $poller } = useNuxtApp()
const basesStore = useBases()
const { createProject: _createProject } = basesStore
const { activeTable: _activeTable } = storeToRefs(useTablesStore())
const reloadDataHook = inject(ReloadViewDataHookInj)
const { eventBus } = useSmartsheetStoreOrThrow()
const { getMeta } = useMetas()
const meta = inject(MetaInj, ref())
const options = ref({
includeData: true,
})
const optionsToExclude = computed(() => {
const { includeData } = options.value
return {
excludeData: !includeData,
}
})
const isLoading = ref(false)
const reloadTable = async () => {
await getMeta(meta!.value!.id!, true)
eventBus.emit(SmartsheetStoreEvents.FIELD_RELOAD)
reloadDataHook?.trigger()
}
const _duplicate = async () => {
try {
isLoading.value = true
const jobData = await api.dbTable.duplicateColumn(props.column.base_id!, props.column.id!, {
options: optionsToExclude.value,
extra: props.extra,
})
$poller.subscribe(
{ id: jobData.id },
async (data: {
id: string
status?: string
data?: {
error?: {
message: string
}
message?: string
result?: any
}
}) => {
if (data.status !== 'close') {
if (data.status === JobStatus.COMPLETED) {
reloadTable()
isLoading.value = false
dialogShow.value = false
} else if (data.status === JobStatus.FAILED) {
message.error(`There was an error duplicating the column.`)
reloadTable()
isLoading.value = false
dialogShow.value = false
}
}
},
)
$e('a:column:duplicate')
} catch (e: any) {
message.error(await extractSdkResponseErrorMsg(e))
isLoading.value = false
dialogShow.value = false
}
}
onKeyStroke('Enter', () => {
// should only trigger this when our modal is open
if (dialogShow.value) {
_duplicate()
}
})
const isEaster = ref(false)
</script>
<template>
<GeneralModal
v-model:visible="dialogShow"
:class="{ active: dialogShow }"
:closable="!isLoading"
:mask-closable="!isLoading"
:keyboard="!isLoading"
centered
wrap-class-name="nc-modal-column-duplicate"
:footer="null"
class="!w-[30rem]"
@keydown.esc="dialogShow = false"
>
<div>
<div class="prose-xl font-bold self-center" @dblclick="isEaster = !isEaster">
{{ $t('general.duplicate') }} {{ $t('objects.column') }}
</div>
<div class="mt-4">{{ $t('msg.warning.duplicateProject') }}</div>
<div class="prose-md self-center text-gray-500 mt-4">{{ $t('title.advancedSettings') }}</div>
<a-divider class="!m-0 !p-0 !my-2" />
<div class="text-xs p-2">
<a-checkbox v-model:checked="options.includeData" :disabled="isLoading">{{ $t('labels.includeData') }}</a-checkbox>
</div>
</div>
<div class="flex flex-row gap-x-2 mt-2.5 pt-2.5 justify-end">
<NcButton v-if="!isLoading" key="back" type="secondary" @click="dialogShow = false">{{ $t('general.cancel') }}</NcButton>
<NcButton key="submit" type="primary" :loading="isLoading" @click="_duplicate">{{ $t('general.confirm') }} </NcButton>
</div>
</GeneralModal>
</template>

6
packages/nc-gui/components/smartsheet/column/DecimalOptions.vue

@ -34,7 +34,11 @@ onMounted(() => {
<template>
<a-form-item :label="$t('placeholder.precision')">
<a-select v-model:value="vModel.meta.precision" dropdown-class-name="nc-dropdown-decimal-format">
<a-select
v-if="vModel.meta?.precision"
v-model:value="vModel.meta.precision"
dropdown-class-name="nc-dropdown-decimal-format"
>
<a-select-option v-for="(format, i) of precisionFormats" :key="i" :value="format">
<div class="flex flex-row items-center">
<div class="text-xs">

102
packages/nc-gui/components/smartsheet/header/Menu.vue

@ -1,6 +1,6 @@
<script lang="ts" setup>
import type { ColumnReqType } from 'nocodb-sdk'
import { RelationTypes, UITypes, isLinksOrLTAR } from 'nocodb-sdk'
import type { ColumnReqType, ColumnType } from 'nocodb-sdk'
import { RelationTypes, UITypes, isLinksOrLTAR, isVirtualCol } from 'nocodb-sdk'
import {
ActiveViewInj,
ColumnInj,
@ -8,8 +8,6 @@ import {
MetaInj,
ReloadViewDataHookInj,
SmartsheetStoreEvents,
extractSdkResponseErrorMsg,
getUniqueColumnName,
iconMap,
inject,
message,
@ -112,48 +110,24 @@ const sortByColumn = async (direction: 'asc' | 'desc') => {
})
}
const duplicateColumn = async () => {
const isDuplicateDlgOpen = ref(false)
const selectedColumnToDuplicate = ref<ColumnType>()
const selectedColumnExtra = ref<any>()
const duplicateVirtualColumn = async () => {
let columnCreatePayload = {}
// generate duplicate column name
const duplicateColumnName = getUniqueColumnName(`${column!.value.title}_copy`, meta!.value!.columns!)
// construct column create payload
switch (column?.value.uidt) {
case UITypes.LinkToAnotherRecord:
case UITypes.Links:
case UITypes.Lookup:
case UITypes.Rollup:
case UITypes.Formula:
return message.info('Not available at the moment')
case UITypes.SingleSelect:
case UITypes.MultiSelect:
columnCreatePayload = {
...column!.value!,
title: duplicateColumnName,
column_name: duplicateColumnName,
id: undefined,
order: undefined,
colOptions: {
options:
column.value.colOptions?.options?.map((option: Record<string, any>) => ({
...option,
id: undefined,
})) ?? [],
},
}
break
default:
columnCreatePayload = {
...column!.value!,
...(column!.value.colOptions ?? {}),
title: duplicateColumnName,
column_name: duplicateColumnName,
id: undefined,
colOptions: undefined,
order: undefined,
}
break
// generate duplicate column title
const duplicateColumnTitle = getUniqueColumnName(`${column!.value.title} copy`, meta!.value!.columns!)
columnCreatePayload = {
...column!.value!,
...(column!.value.colOptions ?? {}),
title: duplicateColumnTitle,
column_name: duplicateColumnTitle.replace(/\s/g, '_'),
id: undefined,
colOptions: undefined,
order: undefined,
}
try {
@ -189,6 +163,35 @@ const duplicateColumn = async () => {
isOpen.value = false
}
const openDuplicateDlg = async () => {
if (!column?.value) return
if (column.value.uidt && [UITypes.Formula, UITypes.Lookup, UITypes.Rollup].includes(column.value.uidt as UITypes)) {
duplicateVirtualColumn()
} else {
const gridViewColumnList = (await $api.dbViewColumn.list(view.value?.id as string)).list
const currentColumnIndex = gridViewColumnList.findIndex((f) => f.fk_column_id === column!.value.id)
let newColumnOrder
if (currentColumnIndex === gridViewColumnList.length - 1) {
newColumnOrder = gridViewColumnList[currentColumnIndex].order! + 1
} else {
newColumnOrder = (gridViewColumnList[currentColumnIndex].order! + gridViewColumnList[currentColumnIndex + 1].order!) / 2
}
selectedColumnExtra.value = {
pv: false,
view_id: view.value!.id as string,
column_order: {
order: newColumnOrder,
view_id: view.value!.id as string,
},
}
selectedColumnToDuplicate.value = column.value
isDuplicateDlgOpen.value = true
isOpen.value = false
}
}
// add column before or after current column
const addColumn = async (before = false) => {
const gridViewColumnList = (await $api.dbViewColumn.list(view.value?.id as string)).list
@ -335,10 +338,7 @@ const onInsertAfter = () => {
<a-divider class="!my-0" />
<a-menu-item
v-if="column.uidt !== UITypes.LinkToAnotherRecord && column.uidt !== UITypes.Lookup && !column.pk"
@click="duplicateColumn"
>
<a-menu-item v-if="!column?.pk" @click="openDuplicateDlg">
<div v-e="['a:field:duplicate']" class="nc-column-duplicate nc-header-menu-item my-0.5">
<component :is="iconMap.duplicate" class="text-gray-700 mx-0.75" />
<!-- Duplicate -->
@ -372,6 +372,12 @@ const onInsertAfter = () => {
</template>
</a-dropdown>
<SmartsheetHeaderDeleteColumnModal v-model:visible="showDeleteColumnModal" />
<DlgColumnDuplicate
v-if="selectedColumnToDuplicate"
v-model="isDuplicateDlgOpen"
:column="selectedColumnToDuplicate"
:extra="selectedColumnExtra"
/>
</template>
<style scoped>

47
packages/nocodb-sdk/src/lib/Api.ts

@ -5746,6 +5746,53 @@ export class Api<
...params,
}),
/**
* @description Duplicate a column
*
* @tags DB Table
* @name DuplicateColumn
* @summary Duplicate Column
* @request POST:/api/v1/db/meta/duplicate/{baseId}/column/{columnId}
* @response `200` `{
name?: string,
id?: string,
}` OK
* @response `400` `{
\** @example BadRequest [Error]: <ERROR MESSAGE> *\
msg: string,
}`
*/
duplicateColumn: (
baseId: IdType,
columnId: IdType,
data: {
options?: {
excludeData?: boolean;
};
extra?: object;
},
params: RequestParams = {}
) =>
this.request<
{
name?: string;
id?: string;
},
{
/** @example BadRequest [Error]: <ERROR MESSAGE> */
msg: string;
}
>({
path: `/api/v1/db/meta/duplicate/${baseId}/column/${columnId}`,
method: 'POST',
body: data,
type: ContentType.Json,
format: 'json',
...params,
}),
/**
* @description Update the order of the given Table
*

64
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -265,10 +265,20 @@ class BaseModelSqlv2 {
sort?: string | string[];
fieldsSet?: Set<string>;
} = {},
ignoreViewFilterAndSort = false,
validateFormula = false,
throwErrorIfInvalidParams = false,
options: {
ignoreViewFilterAndSort?: boolean;
ignorePagination?: boolean;
validateFormula?: boolean;
throwErrorIfInvalidParams?: boolean;
} = {},
): Promise<any> {
const {
ignoreViewFilterAndSort = false,
ignorePagination = false,
validateFormula = false,
throwErrorIfInvalidParams = false,
} = options;
const { where, fields, ...rest } = this._getListArgs(args as any);
const qb = this.dbDriver(this.tnPath);
@ -359,7 +369,7 @@ class BaseModelSqlv2 {
qb.orderBy('created_at');
}
if (!ignoreViewFilterAndSort) applyPaginate(qb, rest);
if (!ignorePagination) applyPaginate(qb, rest);
const proto = await this.getProto();
let data;
@ -370,7 +380,11 @@ class BaseModelSqlv2 {
if (validateFormula || !haveFormulaColumn(await this.model.getColumns()))
throw e;
console.log(e);
return this.list(args, ignoreViewFilterAndSort, true);
return this.list(args, {
ignoreViewFilterAndSort,
ignorePagination,
validateFormula: true,
});
}
return data?.map((d) => {
d.__proto__ = proto;
@ -1891,7 +1905,10 @@ class BaseModelSqlv2 {
}),
],
},
true,
{
ignoreViewFilterAndSort: true,
ignorePagination: true,
},
);
const groupedList = groupBy(data, pCol.title);
@ -2131,7 +2148,7 @@ class BaseModelSqlv2 {
aliasToColumnBuilder,
);
qb.select({
[column.column_name]: selectQb.builder,
[column.title]: selectQb.builder,
});
} catch {
continue;
@ -2139,7 +2156,7 @@ class BaseModelSqlv2 {
break;
default: {
qb.select({
[column.column_name]: barcodeValueColumn.column_name,
[column.title]: barcodeValueColumn.column_name,
});
break;
}
@ -2604,7 +2621,20 @@ class BaseModelSqlv2 {
}
const ai = this.model.columns.find((c) => c.ai);
if (
let ag: Column;
if (!ai) ag = this.model.columns.find((c) => c.meta?.ag);
// handle if autogenerated primary key is used
if (ag) {
if (!response) await this.execAndParse(query);
response = await this.readByPk(
data[ag.title],
false,
{},
{ ignoreView: true, getHiddenColumn: true },
);
} else if (
!response ||
(typeof response?.[0] !== 'object' && response?.[0] !== null)
) {
@ -2663,12 +2693,14 @@ class BaseModelSqlv2 {
await Promise.all(postInsertOps.map((f) => f(rowId)));
response = await this.readByPk(
rowId,
false,
{},
{ ignoreView: true, getHiddenColumn: true },
);
if (rowId !== null && rowId !== undefined) {
response = await this.readByPk(
rowId,
false,
{},
{ ignoreView: true, getHiddenColumn: true },
);
}
await this.afterInsert(response, this.dbDriver, cookie);
@ -2974,7 +3006,7 @@ class BaseModelSqlv2 {
}
} else {
response =
this.isPg || this.isMssql
!raw && (this.isPg || this.isMssql)
? await trx
.batchInsert(this.tnPath, insertDatas, chunkSize)
.returning({

5
packages/nocodb/src/helpers/columnHelpers.ts

@ -35,6 +35,7 @@ export async function createHmAndBtColumn(
isSystemCol = false,
columnMeta = null,
isLinks = false,
colExtra?: any,
) {
// save bt column
{
@ -59,6 +60,7 @@ export async function createHmAndBtColumn(
system: isSystemCol || parent.id === child.id,
fk_col_name: fkColName,
fk_index_name: fkColName,
...(type === 'bt' ? colExtra : {}),
});
}
// save hm column
@ -85,6 +87,7 @@ export async function createHmAndBtColumn(
fk_col_name: fkColName,
fk_index_name: fkColName,
meta,
...(type === 'hm' ? colExtra : {}),
});
}
}
@ -264,7 +267,7 @@ export async function populateRollupForLTAR({
}
export const sanitizeColumnName = (name: string) => {
const columnName = name.replace(/\W+/g, '_').trim();
const columnName = name.replace(/\W/g, '_');
// if column name only contains _ then return as 'field'
if (/^_+$/.test(columnName)) return 'field';

1
packages/nocodb/src/interface/Jobs.ts

@ -3,6 +3,7 @@ export const JOBS_QUEUE = 'jobs';
export enum JobTypes {
DuplicateBase = 'duplicate-base',
DuplicateModel = 'duplicate-model',
DuplicateColumn = 'duplicate-column',
AtImport = 'at-import',
MetaSync = 'meta-sync',
SourceCreate = 'source-create',

4
packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts

@ -68,6 +68,10 @@ export class QueueService {
this: this.duplicateProcessor,
fn: this.duplicateProcessor.duplicateModel,
},
[JobTypes.DuplicateColumn]: {
this: this.duplicateProcessor,
fn: this.duplicateProcessor.duplicateColumn,
},
[JobTypes.AtImport]: {
this: this.atImportProcessor,
fn: this.atImportProcessor.job,

58
packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts

@ -13,7 +13,7 @@ import { ProjectStatus } from 'nocodb-sdk';
import { GlobalGuard } from '~/guards/global/global.guard';
import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware';
import { BasesService } from '~/services/bases.service';
import { Base, Model, Source } from '~/models';
import { Base, Column, Model, Source } from '~/models';
import { generateUniqueName } from '~/helpers/exportImportHelpers';
import { JobTypes } from '~/interface/Jobs';
import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard';
@ -217,4 +217,60 @@ export class DuplicateController {
return { id: job.id };
}
@Post([
'/api/v1/db/meta/duplicate/:baseId/column/:columnId',
'/api/v2/meta/duplicate/:baseId/column/:columnId',
])
@HttpCode(200)
@Acl('duplicateColumn')
async duplicateColumn(
@Req() req: Request,
@Param('baseId') baseId: string,
@Param('columnId') columnId?: string,
@Body()
body?: {
options?: {
excludeData?: boolean;
};
extra?: any;
},
) {
const base = await Base.get(baseId);
if (!base) {
throw new Error(`Base not found for id '${baseId}'`);
}
const column = await Column.get({
source_id: base.id,
colId: columnId,
});
if (!column) {
throw new Error(`Column not found!`);
}
const model = await Model.get(column.fk_model_id);
if (!model) {
throw new Error(`Model not found!`);
}
const job = await this.jobsService.add(JobTypes.DuplicateColumn, {
baseId: base.id,
sourceId: column.source_id,
modelId: model.id,
columnId: column.id,
options: body.options || {},
extra: body.extra || {},
req: {
user: req.user,
clientIp: req.clientIp,
headers: req.headers,
},
});
return { id: job.id };
}
}

209
packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@ -6,8 +6,12 @@ import debug from 'debug';
import { isLinksOrLTAR } from 'nocodb-sdk';
import { Base, Column, Model, Source } from '~/models';
import { BasesService } from '~/services/bases.service';
import { findWithIdentifier } from '~/helpers/exportImportHelpers';
import {
findWithIdentifier,
generateUniqueName,
} from '~/helpers/exportImportHelpers';
import { BulkDataAliasService } from '~/services/bulk-data-alias.service';
import { ColumnsService } from '~/services/columns.service';
import { JOBS_QUEUE, JobTypes } from '~/interface/Jobs';
import { elapsedTime, initTime } from '~/modules/jobs/helpers';
import { ExportService } from '~/modules/jobs/jobs/export-import/export.service';
@ -22,6 +26,7 @@ export class DuplicateProcessor {
private readonly importService: ImportService,
private readonly projectsService: BasesService,
private readonly bulkDataService: BulkDataAliasService,
private readonly columnsService: ColumnsService,
) {}
@Process(JobTypes.DuplicateBase)
@ -227,6 +232,164 @@ export class DuplicateProcessor {
return await Model.get(findWithIdentifier(idMap, sourceModel.id));
}
@Process(JobTypes.DuplicateColumn)
async duplicateColumn(job: Job) {
this.debugLog(`job started for ${job.id} (${JobTypes.DuplicateColumn})`);
const hrTime = initTime();
const { baseId, sourceId, columnId, extra, req, options } = job.data;
const excludeData = options?.excludeData || false;
const base = await Base.get(baseId);
const sourceColumn = await Column.get({
source_id: sourceId,
colId: columnId,
});
const user = (req as any).user;
const source = await Source.get(sourceColumn.source_id);
const models = (await source.getModels()).filter(
(m) => !m.mm && m.type === 'table',
);
const sourceModel = models.find((m) => m.id === sourceColumn.fk_model_id);
const columns = await sourceModel.getColumns();
const title = generateUniqueName(
`${sourceColumn.title} copy`,
columns.map((p) => p.title),
);
const relatedModelIds = [sourceColumn]
.filter((col) => isLinksOrLTAR(col))
.map((col) => col.colOptions.fk_related_model_id)
.filter((id) => id);
const relatedModels = models.filter((m) => relatedModelIds.includes(m.id));
const exportedModel = (
await this.exportService.serializeModels({
modelIds: [sourceModel.id],
excludeData,
excludeHooks: true,
excludeViews: true,
})
)[0];
elapsedTime(
hrTime,
`serialize model schema for ${sourceModel.id}`,
'duplicateColumn',
);
if (!exportedModel) {
throw new Error(`Export failed for model '${sourceModel.id}'`);
}
exportedModel.model.columns = exportedModel.model.columns.filter((c) =>
c.id.includes(columnId),
);
if (exportedModel.model.columns.length !== 1) {
throw new Error(`There was an error duplicating column!`);
}
const replacedColumn = exportedModel.model.columns.find((c) =>
c.id.includes(columnId),
);
// save old default value
const oldCdf = replacedColumn.cdf;
replacedColumn.title = title;
replacedColumn.column_name = title.toLowerCase().replace(/ /g, '_');
// remove default value to avoid filling existing empty rows
replacedColumn.cdf = null;
Object.assign(replacedColumn, extra);
const idMap = await this.importService.importModels({
baseId,
sourceId: source.id,
data: [exportedModel],
user,
req,
externalModels: relatedModels,
existingModel: sourceModel,
});
elapsedTime(hrTime, 'import model schema', 'duplicateColumn');
if (!idMap) {
throw new Error(`Import failed for model '${sourceModel.id}'`);
}
if (!excludeData) {
const fields: Record<string, string[]> = {};
fields[sourceModel.id] = [sourceModel.primaryKey.id];
fields[sourceModel.id].push(columnId);
for (const md of relatedModels) {
const bts = md.columns
.filter(
(c) =>
isLinksOrLTAR(c) &&
c.colOptions.type === 'bt' &&
c.colOptions.fk_related_model_id === sourceModel.id,
)
.map((c) => c.id);
if (bts.length > 0) {
fields[md.id] = [md.primaryKey.id];
fields[md.id].push(...bts);
}
}
await this.importModelsData({
idMap,
sourceProject: base,
sourceModels: [],
destProject: base,
destBase: source,
hrTime,
modelFieldIds: fields,
externalModels: [sourceModel, ...relatedModels],
});
elapsedTime(hrTime, 'import model data', 'duplicateColumn');
}
const destColumn = await Column.get({
source_id: base.id,
colId: findWithIdentifier(idMap, sourceColumn.id),
});
// update cdf
await this.columnsService.columnUpdate({
columnId: findWithIdentifier(idMap, sourceColumn.id),
column: {
...destColumn,
cdf: oldCdf,
},
user: req.user,
});
this.debugLog(`job completed for ${job.id} (${JobTypes.DuplicateModel})`);
return await Column.get({
source_id: base.id,
colId: findWithIdentifier(idMap, sourceColumn.id),
});
}
async importModelsData(param: {
idMap: Map<string, string>;
sourceProject: Base;
@ -395,13 +558,17 @@ export class DuplicateProcessor {
if (chunk.length > 1000) {
parser.pause();
try {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
// remove empty rows (only pk is present)
chunk = chunk.filter((r) => Object.keys(r).length > 1);
if (chunk.length > 0) {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
}
} catch (e) {
this.debugLog(e);
}
@ -414,13 +581,17 @@ export class DuplicateProcessor {
complete: async () => {
if (chunk.length > 0) {
try {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
// remove empty rows (only pk is present)
chunk = chunk.filter((r) => Object.keys(r).length > 1);
if (chunk.length > 0) {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
}
} catch (e) {
this.debugLog(e);
}
@ -433,6 +604,14 @@ export class DuplicateProcessor {
if (error) throw error;
handledLinks = await this.importService.importLinkFromCsvStream({
idMap,
linkStream,
destProject,
destBase,
handledLinks,
});
elapsedTime(
hrTime,
`map existing links to ${model.title}`,

12
packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@ -408,9 +408,13 @@ export class ExportService {
.map((c) => c.title)
.join(',');
const mmColumns = model.columns.filter(
(col) => isLinksOrLTAR(col) && col.colOptions?.type === 'mm',
);
const mmColumns = param._fieldIds
? model.columns
.filter((c) => param._fieldIds?.includes(c.id))
.filter((col) => isLinksOrLTAR(col) && col.colOptions?.type === 'mm')
: model.columns.filter(
(col) => isLinksOrLTAR(col) && col.colOptions?.type === 'mm',
);
const hasLink = mmColumns.length > 0;
@ -577,6 +581,7 @@ export class ExportService {
view,
query: { limit, offset, fields },
baseModel,
ignoreViewFilterAndSort: true,
})
.then((result) => {
try {
@ -625,6 +630,7 @@ export class ExportService {
view,
query: { limit, offset, fields },
baseModel,
ignoreViewFilterAndSort: true,
})
.then((result) => {
try {

105
packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@ -67,6 +67,7 @@ export class ImportService {
| { model: any; views: any[]; hooks?: any[] }[];
req: NcRequest;
externalModels?: Model[];
existingModel?: Model;
}) {
const hrTime = initTime();
@ -93,6 +94,10 @@ export class ImportService {
param.data = Array.isArray(param.data) ? param.data : param.data.models;
// allow existing model to be linked
if (param.existingModel)
param.externalModels = [param.existingModel, ...param.externalModels];
// allow existing models to be linked
if (param.externalModels) {
for (const model of param.externalModels) {
@ -131,45 +136,69 @@ export class ImportService {
);
// create table with static columns
const table = await this.tablesService.tableCreate({
baseId: base.id,
sourceId: source.id,
user: param.user,
table: withoutId({
...modelData,
columns: reducedColumnSet.map((a) => withoutId(a)),
}),
});
const table =
param.existingModel ||
(await this.tablesService.tableCreate({
baseId: base.id,
sourceId: source.id,
user: param.user,
table: withoutId({
...modelData,
columns: reducedColumnSet.map((a) => withoutId(a)),
}),
}));
idMap.set(modelData.id, table.id);
// map column id's with new created column id's
for (const col of table.columns) {
const colRef = modelData.columns.find(
(a) =>
a.column_name &&
sanitizeColumnName(a.column_name) === col.column_name,
);
idMap.set(colRef.id, col.id);
// setval for auto increment column in pg
if (source.type === 'pg') {
if (modelData.pgSerialLastVal) {
if (col.ai) {
const baseModel = await Model.getBaseModelSQL({
id: table.id,
viewId: null,
dbDriver: await NcConnectionMgrv2.get(source),
});
const sqlClient = await NcConnectionMgrv2.getSqlClient(source);
await sqlClient.raw(
`SELECT setval(pg_get_serial_sequence('??', ?), ?);`,
[
baseModel.getTnPath(table.table_name),
col.column_name,
modelData.pgSerialLastVal,
],
);
if (param.existingModel) {
if (reducedColumnSet.length) {
for (const col of reducedColumnSet) {
const freshModelData = await this.columnsService.columnAdd({
tableId: getIdOrExternalId(getParentIdentifier(col.id)),
column: withoutId({
...col,
}) as any,
req: param.req,
user: param.user,
});
for (const nColumn of freshModelData.columns) {
if (nColumn.title === col.title) {
idMap.set(col.id, nColumn.id);
break;
}
}
}
}
} else {
// map column id's with new created column id's
for (const col of table.columns) {
const colRef = modelData.columns.find(
(a) =>
a.column_name &&
sanitizeColumnName(a.column_name) === col.column_name,
);
idMap.set(colRef.id, col.id);
// setval for auto increment column in pg
if (source.type === 'pg') {
if (modelData.pgSerialLastVal) {
if (col.ai) {
const baseModel = await Model.getBaseModelSQL({
id: table.id,
viewId: null,
dbDriver: await NcConnectionMgrv2.get(source),
});
const sqlClient = await NcConnectionMgrv2.getSqlClient(source);
await sqlClient.raw(
`SELECT setval(pg_get_serial_sequence('??', ?), ?);`,
[
baseModel.getTnPath(table.table_name),
col.column_name,
modelData.pgSerialLastVal,
],
);
}
}
}
}
@ -885,6 +914,7 @@ export class ImportService {
}
}
} else if (col.uidt === UITypes.Barcode) {
flatCol.validate = null;
const freshModelData = await this.columnsService.columnAdd({
tableId: getIdOrExternalId(getParentIdentifier(col.id)),
column: withoutId({
@ -912,6 +942,8 @@ export class ImportService {
// create views
for (const data of param.data) {
if (param.existingModel) break;
const modelData = data.model;
const viewsData = data.views;
@ -1045,6 +1077,7 @@ export class ImportService {
// create hooks
for (const data of param.data) {
if (param.existingModel) break;
if (!data?.hooks) break;
const modelData = data.model;
const hookData = data.hooks;

90
packages/nocodb/src/schema/swagger.json

@ -4589,6 +4589,96 @@
]
}
},
"/api/v1/db/meta/duplicate/{baseId}/column/{columnId}": {
"post": {
"summary": "Duplicate Column",
"operationId": "duplicate-column",
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"id": {
"type": "string"
}
}
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest"
}
},
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"excludeData": {
"type": "boolean",
"required": false
}
}
},
"extra": {
"type": "object"
}
}
},
"examples": {
"Example 1": {
"value": {
"excludeData": true
}
}
}
}
}
},
"tags": [
"DB Table"
],
"description": "Duplicate a column",
"parameters": [
{
"$ref": "#/components/parameters/xc-auth"
},
{
"schema": {
"$ref": "#/components/schemas/Id",
"example": "p_124hhlkbeasewh",
"type": "string"
},
"name": "baseId",
"in": "path",
"required": true,
"description": "Unique Base ID"
},
{
"schema": {
"$ref": "#/components/schemas/Id",
"example": "md_124hhlkbeasewh",
"type": "string"
},
"name": "columnId",
"in": "path",
"required": true,
"description": "Unique Column ID"
}
]
}
},
"/api/v1/db/meta/projects/{baseId}/{sourceId}/tables": {
"parameters": [
{

29
packages/nocodb/src/services/columns.service.ts

@ -1127,6 +1127,12 @@ export class ColumnsService {
}
let colBody: any = param.column;
const colExtra = {
view_id: colBody.view_id,
column_order: colBody.column_order,
};
switch (colBody.uidt) {
case UITypes.Rollup:
{
@ -1151,7 +1157,13 @@ export class ColumnsService {
case UITypes.Links:
case UITypes.LinkToAnotherRecord:
await this.createLTARColumn({ ...param, source, base, reuse });
await this.createLTARColumn({
...param,
source,
base,
reuse,
colExtra,
});
this.appHooksService.emit(AppEvents.RELATION_DELETE, {
column: {
@ -1818,6 +1830,7 @@ export class ColumnsService {
source: Source;
base: Base;
reuse?: ReusableParams;
colExtra?: any;
}) {
validateParams(['parentId', 'childId', 'type'], param.column);
@ -1837,7 +1850,9 @@ export class ColumnsService {
id: param.source.base_id,
}),
);
const isLinks = param.column.uidt === UITypes.Links;
const isLinks =
param.column.uidt === UITypes.Links ||
(param.column as LinkToAnotherColumnReqType).type === 'bt';
// if xcdb base then treat as virtual relation to avoid creating foreign key
if (param.source.isMeta()) {
@ -1941,6 +1956,7 @@ export class ColumnsService {
null,
param.column['meta'],
isLinks,
param.colExtra,
);
} else if ((param.column as LinkToAnotherColumnReqType).type === 'mm') {
const aTn = `${param.base?.prefix ?? ''}_nc_m2m_${randomID()}`;
@ -2046,6 +2062,9 @@ export class ColumnsService {
foreignKeyName1,
(param.column as LinkToAnotherColumnReqType).virtual,
true,
null,
false,
param.colExtra,
);
await createHmAndBtColumn(
assocModel,
@ -2056,6 +2075,9 @@ export class ColumnsService {
foreignKeyName2,
(param.column as LinkToAnotherColumnReqType).virtual,
true,
null,
false,
param.colExtra,
);
await Column.insert({
@ -2108,6 +2130,9 @@ export class ColumnsService {
plural: param.column['meta']?.plural || pluralize(child.title),
singular: param.column['meta']?.singular || singularize(child.title),
},
// column_order and view_id if provided
...param.colExtra,
});
// todo: create index for virtual relations as well

13
packages/nocodb/src/services/datas.service.ts

@ -134,8 +134,9 @@ export class DatasService {
query: any;
baseModel?: BaseModelSqlv2;
throwErrorIfInvalidParams?: boolean;
ignoreViewFilterAndSort?: boolean;
}) {
const { model, view, query = {} } = param;
const { model, view, query = {}, ignoreViewFilterAndSort = false } = param;
const source = await Source.get(model.source_id);
@ -169,12 +170,10 @@ export class DatasService {
try {
data = await nocoExecute(
ast,
await baseModel.list(
listArgs,
false,
false,
param.throwErrorIfInvalidParams,
),
await baseModel.list(listArgs, {
ignoreViewFilterAndSort,
throwErrorIfInvalidParams: param.throwErrorIfInvalidParams,
}),
{},
listArgs,
);

1
packages/nocodb/src/utils/acl.ts

@ -116,6 +116,7 @@ const permissionScopes = {
'bulkDataDeleteAll',
'relationDataRemove',
'relationDataAdd',
'duplicateColumn',
// Base API Tokens
'baseApiTokenList',

2
packages/nocodb/tests/unit/factory/row.ts

@ -232,7 +232,7 @@ const listRow = async ({
const ignorePagination = !options;
return await baseModel.list(options, ignorePagination);
return await baseModel.list(options, { ignorePagination });
};
const getOneRow = async (

6
tests/playwright/pages/Dashboard/Grid/Column/index.ts

@ -319,12 +319,14 @@ export class ColumnPageObject extends BasePage {
await this.rootPage.locator('.nc-more-options').click();
}
async duplicateColumn({ title, expectedTitle = `${title}_copy` }: { title: string; expectedTitle?: string }) {
async duplicateColumn({ title, expectedTitle = `${title} copy` }: { title: string; expectedTitle?: string }) {
await this.grid.get().locator(`th[data-title="${title}"] .nc-ui-dt-dropdown`).click();
await this.rootPage.locator('li[role="menuitem"]:has-text("Duplicate"):visible').click();
await this.rootPage.locator('.nc-modal-column-duplicate .nc-button:has-text("Confirm"):visible').click();
// await this.verifyToast({ message: 'Column duplicated successfully' });
await this.grid.get().locator(`th[data-title="${expectedTitle}"]`).waitFor({ state: 'visible' });
await this.grid.get().locator(`th[data-title="${expectedTitle}"]`).waitFor({ state: 'visible', timeout: 10000 });
}
async hideColumn({ title, isDisplayValue = false }: { title: string; isDisplayValue?: boolean }) {

2
tests/playwright/tests/db/columns/columnMenuOperations.spec.ts

@ -70,7 +70,7 @@ test.describe('Column menu operations', () => {
});
await dashboard.grid.column.duplicateColumn({
title,
expectedTitle: `${title}_copy_1`,
expectedTitle: `${title} copy_1`,
});
}
await dashboard.closeTab({ title: 'Film' });

Loading…
Cancel
Save