Browse Source

Merge branch 'develop' into fix/runtime-directive-warnings

pull/6954/head
աɨռɢӄաօռɢ 1 year ago
parent
commit
3b6801ae31
  1. 8
      packages/nc-gui/components/cell/MultiSelect.vue
  2. 4
      packages/nc-gui/components/cell/SingleSelect.vue
  3. 140
      packages/nc-gui/components/dlg/ColumnDuplicate.vue
  4. 6
      packages/nc-gui/components/smartsheet/column/DecimalOptions.vue
  5. 102
      packages/nc-gui/components/smartsheet/header/Menu.vue
  6. 47
      packages/nocodb-sdk/src/lib/Api.ts
  7. 77
      packages/nocodb/src/db/BaseModelSqlv2.ts
  8. 2
      packages/nocodb/src/db/sql-client/lib/pg/PgClient.ts
  9. 5
      packages/nocodb/src/helpers/columnHelpers.ts
  10. 1
      packages/nocodb/src/interface/Jobs.ts
  11. 3
      packages/nocodb/src/models/Model.ts
  12. 4
      packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts
  13. 58
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts
  14. 209
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts
  15. 25
      packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts
  16. 103
      packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts
  17. 90
      packages/nocodb/src/schema/swagger.json
  18. 49
      packages/nocodb/src/services/columns.service.ts
  19. 15
      packages/nocodb/src/services/datas.service.ts
  20. 1
      packages/nocodb/src/utils/acl.ts
  21. 2
      packages/nocodb/tests/unit/factory/row.ts
  22. 6
      packages/nocodb/tests/unit/rest/tests/groupby.test.ts
  23. 6
      tests/playwright/pages/Dashboard/Grid/Column/index.ts
  24. 2
      tests/playwright/tests/db/columns/columnMenuOperations.spec.ts

8
packages/nc-gui/components/cell/MultiSelect.vue

@ -144,14 +144,14 @@ const selectedTitles = computed(() =>
}
return 0
})
: modelValue.split(',').map((el) => el.trim())
: modelValue.map((el) => el.trim())
: modelValue.split(',')
: modelValue
: [],
)
onMounted(() => {
selectedIds.value = selectedTitles.value.flatMap((el) => {
const item = options.value.find((op) => op.title === el)
const item = options.value.find((op) => op.title === el || op.title === el?.trim())
const itemIdOrTitle = item?.id || item?.title
if (itemIdOrTitle) {
return [itemIdOrTitle]
@ -165,7 +165,7 @@ watch(
() => modelValue,
() => {
selectedIds.value = selectedTitles.value.flatMap((el) => {
const item = options.value.find((op) => op.title === el)
const item = options.value.find((op) => op.title === el || op.title === el?.trim())
if (item && (item.id || item.title)) {
return [(item.id || item.title)!]
}

4
packages/nc-gui/components/cell/SingleSelect.vue

@ -104,7 +104,7 @@ const hasEditRoles = computed(() => isUIAllowed('dataEdit'))
const editAllowed = computed(() => (hasEditRoles.value || isForm.value) && active.value)
const vModel = computed({
get: () => tempSelectedOptState.value ?? modelValue?.trim(),
get: () => tempSelectedOptState.value ?? modelValue,
set: (val) => {
if (val && isNewOptionCreateEnabled.value && (options.value ?? []).every((op) => op.title !== val)) {
tempSelectedOptState.value = val
@ -259,7 +259,7 @@ const handleClose = (e: MouseEvent) => {
useEventListener(document, 'click', handleClose, true)
const selectedOpt = computed(() => {
return options.value.find((o) => o.value === vModel.value)
return options.value.find((o) => o.value === vModel.value || o.value === vModel.value?.trim())
})
</script>

140
packages/nc-gui/components/dlg/ColumnDuplicate.vue

@ -0,0 +1,140 @@
<script setup lang="ts">
import type { ColumnType } from 'nocodb-sdk'
import { message } from 'ant-design-vue'
import { useVModel } from '#imports'
const props = defineProps<{
modelValue: boolean
column: ColumnType
extra: any
}>()
const emit = defineEmits(['update:modelValue'])
const { api } = useApi()
const dialogShow = useVModel(props, 'modelValue', emit)
const { $e, $poller } = useNuxtApp()
const basesStore = useBases()
const { createProject: _createProject } = basesStore
const { activeTable: _activeTable } = storeToRefs(useTablesStore())
const reloadDataHook = inject(ReloadViewDataHookInj)
const { eventBus } = useSmartsheetStoreOrThrow()
const { getMeta } = useMetas()
const meta = inject(MetaInj, ref())
const options = ref({
includeData: true,
})
const optionsToExclude = computed(() => {
const { includeData } = options.value
return {
excludeData: !includeData,
}
})
const isLoading = ref(false)
const reloadTable = async () => {
await getMeta(meta!.value!.id!, true)
eventBus.emit(SmartsheetStoreEvents.FIELD_RELOAD)
reloadDataHook?.trigger()
}
const _duplicate = async () => {
try {
isLoading.value = true
const jobData = await api.dbTable.duplicateColumn(props.column.base_id!, props.column.id!, {
options: optionsToExclude.value,
extra: props.extra,
})
$poller.subscribe(
{ id: jobData.id },
async (data: {
id: string
status?: string
data?: {
error?: {
message: string
}
message?: string
result?: any
}
}) => {
if (data.status !== 'close') {
if (data.status === JobStatus.COMPLETED) {
reloadTable()
isLoading.value = false
dialogShow.value = false
} else if (data.status === JobStatus.FAILED) {
message.error(`There was an error duplicating the column.`)
reloadTable()
isLoading.value = false
dialogShow.value = false
}
}
},
)
$e('a:column:duplicate')
} catch (e: any) {
message.error(await extractSdkResponseErrorMsg(e))
isLoading.value = false
dialogShow.value = false
}
}
onKeyStroke('Enter', () => {
// should only trigger this when our modal is open
if (dialogShow.value) {
_duplicate()
}
})
const isEaster = ref(false)
</script>
<template>
<GeneralModal
v-model:visible="dialogShow"
:class="{ active: dialogShow }"
:closable="!isLoading"
:mask-closable="!isLoading"
:keyboard="!isLoading"
centered
wrap-class-name="nc-modal-column-duplicate"
:footer="null"
class="!w-[30rem]"
@keydown.esc="dialogShow = false"
>
<div>
<div class="prose-xl font-bold self-center" @dblclick="isEaster = !isEaster">
{{ $t('general.duplicate') }} {{ $t('objects.column') }}
</div>
<div class="mt-4">{{ $t('msg.warning.duplicateProject') }}</div>
<div class="prose-md self-center text-gray-500 mt-4">{{ $t('title.advancedSettings') }}</div>
<a-divider class="!m-0 !p-0 !my-2" />
<div class="text-xs p-2">
<a-checkbox v-model:checked="options.includeData" :disabled="isLoading">{{ $t('labels.includeData') }}</a-checkbox>
</div>
</div>
<div class="flex flex-row gap-x-2 mt-2.5 pt-2.5 justify-end">
<NcButton v-if="!isLoading" key="back" type="secondary" @click="dialogShow = false">{{ $t('general.cancel') }}</NcButton>
<NcButton key="submit" type="primary" :loading="isLoading" @click="_duplicate">{{ $t('general.confirm') }} </NcButton>
</div>
</GeneralModal>
</template>

6
packages/nc-gui/components/smartsheet/column/DecimalOptions.vue

@ -34,7 +34,11 @@ onMounted(() => {
<template>
<a-form-item :label="$t('placeholder.precision')">
<a-select v-model:value="vModel.meta.precision" dropdown-class-name="nc-dropdown-decimal-format">
<a-select
v-if="vModel.meta?.precision"
v-model:value="vModel.meta.precision"
dropdown-class-name="nc-dropdown-decimal-format"
>
<a-select-option v-for="(format, i) of precisionFormats" :key="i" :value="format">
<div class="flex flex-row items-center">
<div class="text-xs">

102
packages/nc-gui/components/smartsheet/header/Menu.vue

@ -1,6 +1,6 @@
<script lang="ts" setup>
import type { ColumnReqType } from 'nocodb-sdk'
import { RelationTypes, UITypes, isLinksOrLTAR } from 'nocodb-sdk'
import type { ColumnReqType, ColumnType } from 'nocodb-sdk'
import { RelationTypes, UITypes, isLinksOrLTAR, isVirtualCol } from 'nocodb-sdk'
import {
ActiveViewInj,
ColumnInj,
@ -8,8 +8,6 @@ import {
MetaInj,
ReloadViewDataHookInj,
SmartsheetStoreEvents,
extractSdkResponseErrorMsg,
getUniqueColumnName,
iconMap,
inject,
message,
@ -112,48 +110,24 @@ const sortByColumn = async (direction: 'asc' | 'desc') => {
})
}
const duplicateColumn = async () => {
const isDuplicateDlgOpen = ref(false)
const selectedColumnToDuplicate = ref<ColumnType>()
const selectedColumnExtra = ref<any>()
const duplicateVirtualColumn = async () => {
let columnCreatePayload = {}
// generate duplicate column name
const duplicateColumnName = getUniqueColumnName(`${column!.value.title}_copy`, meta!.value!.columns!)
// construct column create payload
switch (column?.value.uidt) {
case UITypes.LinkToAnotherRecord:
case UITypes.Links:
case UITypes.Lookup:
case UITypes.Rollup:
case UITypes.Formula:
return message.info('Not available at the moment')
case UITypes.SingleSelect:
case UITypes.MultiSelect:
columnCreatePayload = {
...column!.value!,
title: duplicateColumnName,
column_name: duplicateColumnName,
id: undefined,
order: undefined,
colOptions: {
options:
column.value.colOptions?.options?.map((option: Record<string, any>) => ({
...option,
id: undefined,
})) ?? [],
},
}
break
default:
columnCreatePayload = {
...column!.value!,
...(column!.value.colOptions ?? {}),
title: duplicateColumnName,
column_name: duplicateColumnName,
id: undefined,
colOptions: undefined,
order: undefined,
}
break
// generate duplicate column title
const duplicateColumnTitle = getUniqueColumnName(`${column!.value.title} copy`, meta!.value!.columns!)
columnCreatePayload = {
...column!.value!,
...(column!.value.colOptions ?? {}),
title: duplicateColumnTitle,
column_name: duplicateColumnTitle.replace(/\s/g, '_'),
id: undefined,
colOptions: undefined,
order: undefined,
}
try {
@ -189,6 +163,35 @@ const duplicateColumn = async () => {
isOpen.value = false
}
const openDuplicateDlg = async () => {
if (!column?.value) return
if (column.value.uidt && [UITypes.Formula, UITypes.Lookup, UITypes.Rollup].includes(column.value.uidt as UITypes)) {
duplicateVirtualColumn()
} else {
const gridViewColumnList = (await $api.dbViewColumn.list(view.value?.id as string)).list
const currentColumnIndex = gridViewColumnList.findIndex((f) => f.fk_column_id === column!.value.id)
let newColumnOrder
if (currentColumnIndex === gridViewColumnList.length - 1) {
newColumnOrder = gridViewColumnList[currentColumnIndex].order! + 1
} else {
newColumnOrder = (gridViewColumnList[currentColumnIndex].order! + gridViewColumnList[currentColumnIndex + 1].order!) / 2
}
selectedColumnExtra.value = {
pv: false,
view_id: view.value!.id as string,
column_order: {
order: newColumnOrder,
view_id: view.value!.id as string,
},
}
selectedColumnToDuplicate.value = column.value
isDuplicateDlgOpen.value = true
isOpen.value = false
}
}
// add column before or after current column
const addColumn = async (before = false) => {
const gridViewColumnList = (await $api.dbViewColumn.list(view.value?.id as string)).list
@ -335,10 +338,7 @@ const onInsertAfter = () => {
<a-divider class="!my-0" />
<a-menu-item
v-if="column.uidt !== UITypes.LinkToAnotherRecord && column.uidt !== UITypes.Lookup && !column.pk"
@click="duplicateColumn"
>
<a-menu-item v-if="!column?.pk" @click="openDuplicateDlg">
<div v-e="['a:field:duplicate']" class="nc-column-duplicate nc-header-menu-item my-0.5">
<component :is="iconMap.duplicate" class="text-gray-700 mx-0.75" />
<!-- Duplicate -->
@ -372,6 +372,12 @@ const onInsertAfter = () => {
</template>
</a-dropdown>
<SmartsheetHeaderDeleteColumnModal v-model:visible="showDeleteColumnModal" />
<DlgColumnDuplicate
v-if="selectedColumnToDuplicate"
v-model="isDuplicateDlgOpen"
:column="selectedColumnToDuplicate"
:extra="selectedColumnExtra"
/>
</template>
<style scoped>

47
packages/nocodb-sdk/src/lib/Api.ts

@ -5746,6 +5746,53 @@ export class Api<
...params,
}),
/**
* @description Duplicate a column
*
* @tags DB Table
* @name DuplicateColumn
* @summary Duplicate Column
* @request POST:/api/v1/db/meta/duplicate/{baseId}/column/{columnId}
* @response `200` `{
name?: string,
id?: string,
}` OK
* @response `400` `{
\** @example BadRequest [Error]: <ERROR MESSAGE> *\
msg: string,
}`
*/
duplicateColumn: (
baseId: IdType,
columnId: IdType,
data: {
options?: {
excludeData?: boolean;
};
extra?: object;
},
params: RequestParams = {}
) =>
this.request<
{
name?: string;
id?: string;
},
{
/** @example BadRequest [Error]: <ERROR MESSAGE> */
msg: string;
}
>({
path: `/api/v1/db/meta/duplicate/${baseId}/column/${columnId}`,
method: 'POST',
body: data,
type: ContentType.Json,
format: 'json',
...params,
}),
/**
* @description Update the order of the given Table
*

77
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -265,10 +265,20 @@ class BaseModelSqlv2 {
sort?: string | string[];
fieldsSet?: Set<string>;
} = {},
ignoreViewFilterAndSort = false,
validateFormula = false,
throwErrorIfInvalidParams = false,
options: {
ignoreViewFilterAndSort?: boolean;
ignorePagination?: boolean;
validateFormula?: boolean;
throwErrorIfInvalidParams?: boolean;
} = {},
): Promise<any> {
const {
ignoreViewFilterAndSort = false,
ignorePagination = false,
validateFormula = false,
throwErrorIfInvalidParams = false,
} = options;
const { where, fields, ...rest } = this._getListArgs(args as any);
const qb = this.dbDriver(this.tnPath);
@ -359,7 +369,7 @@ class BaseModelSqlv2 {
qb.orderBy('created_at');
}
if (!ignoreViewFilterAndSort) applyPaginate(qb, rest);
if (!ignorePagination) applyPaginate(qb, rest);
const proto = await this.getProto();
let data;
@ -370,7 +380,11 @@ class BaseModelSqlv2 {
if (validateFormula || !haveFormulaColumn(await this.model.getColumns()))
throw e;
console.log(e);
return this.list(args, ignoreViewFilterAndSort, true);
return this.list(args, {
ignoreViewFilterAndSort,
ignorePagination,
validateFormula: true,
});
}
return data?.map((d) => {
d.__proto__ = proto;
@ -655,14 +669,17 @@ class BaseModelSqlv2 {
qb,
);
if (!sorts)
sorts = args.sortArr?.length
? args.sortArr
: await Sort.list({ viewId: this.viewId });
if (!sorts) {
if (args.sortArr?.length) {
sorts = args.sortArr;
} else if (this.viewId) {
sorts = await Sort.list({ viewId: this.viewId });
}
}
// if sort is provided filter out the group by columns sort and apply
// since we are grouping by the column and applying sort on any other column is not required
for (const sort of sorts) {
for (const sort of sorts || []) {
if (!groupByColumns[sort.fk_column_id]) {
continue;
}
@ -1888,7 +1905,10 @@ class BaseModelSqlv2 {
}),
],
},
true,
{
ignoreViewFilterAndSort: true,
ignorePagination: true,
},
);
const groupedList = groupBy(data, pCol.title);
@ -2128,7 +2148,7 @@ class BaseModelSqlv2 {
aliasToColumnBuilder,
);
qb.select({
[column.column_name]: selectQb.builder,
[column.title]: selectQb.builder,
});
} catch {
continue;
@ -2136,7 +2156,7 @@ class BaseModelSqlv2 {
break;
default: {
qb.select({
[column.column_name]: barcodeValueColumn.column_name,
[column.title]: barcodeValueColumn.column_name,
});
break;
}
@ -2601,7 +2621,20 @@ class BaseModelSqlv2 {
}
const ai = this.model.columns.find((c) => c.ai);
if (
let ag: Column;
if (!ai) ag = this.model.columns.find((c) => c.meta?.ag);
// handle if autogenerated primary key is used
if (ag) {
if (!response) await this.execAndParse(query);
response = await this.readByPk(
data[ag.title],
false,
{},
{ ignoreView: true, getHiddenColumn: true },
);
} else if (
!response ||
(typeof response?.[0] !== 'object' && response?.[0] !== null)
) {
@ -2660,12 +2693,14 @@ class BaseModelSqlv2 {
await Promise.all(postInsertOps.map((f) => f(rowId)));
response = await this.readByPk(
rowId,
false,
{},
{ ignoreView: true, getHiddenColumn: true },
);
if (rowId !== null && rowId !== undefined) {
response = await this.readByPk(
rowId,
false,
{},
{ ignoreView: true, getHiddenColumn: true },
);
}
await this.afterInsert(response, this.dbDriver, cookie);
@ -2971,7 +3006,7 @@ class BaseModelSqlv2 {
}
} else {
response =
this.isPg || this.isMssql
!raw && (this.isPg || this.isMssql)
? await trx
.batchInsert(this.tnPath, insertDatas, chunkSize)
.returning({

2
packages/nocodb/src/db/sql-client/lib/pg/PgClient.ts

@ -891,7 +891,7 @@ class PGClient extends KnexClient {
// column['unique'] = response.rows[i]['cst'].indexOf('UNIQUE') === -1 ? false : true;
column.cdf = response.rows[i].cdf
? response.rows[i].cdf.replace(/::\w+$/, '').replace(/^'|'$/g, '')
? response.rows[i].cdf.replace(/::[\w ]+$/, '').replace(/^'|'$/g, '')
: response.rows[i].cdf;
// todo : need to find column comment

5
packages/nocodb/src/helpers/columnHelpers.ts

@ -35,6 +35,7 @@ export async function createHmAndBtColumn(
isSystemCol = false,
columnMeta = null,
isLinks = false,
colExtra?: any,
) {
// save bt column
{
@ -59,6 +60,7 @@ export async function createHmAndBtColumn(
system: isSystemCol || parent.id === child.id,
fk_col_name: fkColName,
fk_index_name: fkColName,
...(type === 'bt' ? colExtra : {}),
});
}
// save hm column
@ -85,6 +87,7 @@ export async function createHmAndBtColumn(
fk_col_name: fkColName,
fk_index_name: fkColName,
meta,
...(type === 'hm' ? colExtra : {}),
});
}
}
@ -264,7 +267,7 @@ export async function populateRollupForLTAR({
}
export const sanitizeColumnName = (name: string) => {
const columnName = name.replace(/\W+/g, '_').trim();
const columnName = name.replace(/\W/g, '_');
// if column name only contains _ then return as 'field'
if (/^_+$/.test(columnName)) return 'field';

1
packages/nocodb/src/interface/Jobs.ts

@ -3,6 +3,7 @@ export const JOBS_QUEUE = 'jobs';
export enum JobTypes {
DuplicateBase = 'duplicate-base',
DuplicateModel = 'duplicate-model',
DuplicateColumn = 'duplicate-column',
AtImport = 'at-import',
MetaSync = 'meta-sync',
SourceCreate = 'source-create',

3
packages/nocodb/src/models/Model.ts

@ -381,12 +381,13 @@ export default class Model implements TableType {
viewId?: string;
dbDriver: XKnex;
model?: Model;
extractDefaultView?: boolean;
},
ncMeta = Noco.ncMeta,
): Promise<BaseModelSqlv2> {
const model = args?.model || (await this.get(args.id, ncMeta));
if (!args?.viewId) {
if (!args?.viewId && args.extractDefaultView) {
const view = await View.getDefaultView(model.id, ncMeta);
args.viewId = view.id;
}

4
packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts

@ -68,6 +68,10 @@ export class QueueService {
this: this.duplicateProcessor,
fn: this.duplicateProcessor.duplicateModel,
},
[JobTypes.DuplicateColumn]: {
this: this.duplicateProcessor,
fn: this.duplicateProcessor.duplicateColumn,
},
[JobTypes.AtImport]: {
this: this.atImportProcessor,
fn: this.atImportProcessor.job,

58
packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts

@ -13,7 +13,7 @@ import { ProjectStatus } from 'nocodb-sdk';
import { GlobalGuard } from '~/guards/global/global.guard';
import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware';
import { BasesService } from '~/services/bases.service';
import { Base, Model, Source } from '~/models';
import { Base, Column, Model, Source } from '~/models';
import { generateUniqueName } from '~/helpers/exportImportHelpers';
import { JobTypes } from '~/interface/Jobs';
import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard';
@ -217,4 +217,60 @@ export class DuplicateController {
return { id: job.id };
}
@Post([
'/api/v1/db/meta/duplicate/:baseId/column/:columnId',
'/api/v2/meta/duplicate/:baseId/column/:columnId',
])
@HttpCode(200)
@Acl('duplicateColumn')
async duplicateColumn(
@Req() req: Request,
@Param('baseId') baseId: string,
@Param('columnId') columnId?: string,
@Body()
body?: {
options?: {
excludeData?: boolean;
};
extra?: any;
},
) {
const base = await Base.get(baseId);
if (!base) {
throw new Error(`Base not found for id '${baseId}'`);
}
const column = await Column.get({
source_id: base.id,
colId: columnId,
});
if (!column) {
throw new Error(`Column not found!`);
}
const model = await Model.get(column.fk_model_id);
if (!model) {
throw new Error(`Model not found!`);
}
const job = await this.jobsService.add(JobTypes.DuplicateColumn, {
baseId: base.id,
sourceId: column.source_id,
modelId: model.id,
columnId: column.id,
options: body.options || {},
extra: body.extra || {},
req: {
user: req.user,
clientIp: req.clientIp,
headers: req.headers,
},
});
return { id: job.id };
}
}

209
packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@ -6,8 +6,12 @@ import debug from 'debug';
import { isLinksOrLTAR } from 'nocodb-sdk';
import { Base, Column, Model, Source } from '~/models';
import { BasesService } from '~/services/bases.service';
import { findWithIdentifier } from '~/helpers/exportImportHelpers';
import {
findWithIdentifier,
generateUniqueName,
} from '~/helpers/exportImportHelpers';
import { BulkDataAliasService } from '~/services/bulk-data-alias.service';
import { ColumnsService } from '~/services/columns.service';
import { JOBS_QUEUE, JobTypes } from '~/interface/Jobs';
import { elapsedTime, initTime } from '~/modules/jobs/helpers';
import { ExportService } from '~/modules/jobs/jobs/export-import/export.service';
@ -22,6 +26,7 @@ export class DuplicateProcessor {
private readonly importService: ImportService,
private readonly projectsService: BasesService,
private readonly bulkDataService: BulkDataAliasService,
private readonly columnsService: ColumnsService,
) {}
@Process(JobTypes.DuplicateBase)
@ -227,6 +232,164 @@ export class DuplicateProcessor {
return await Model.get(findWithIdentifier(idMap, sourceModel.id));
}
@Process(JobTypes.DuplicateColumn)
async duplicateColumn(job: Job) {
this.debugLog(`job started for ${job.id} (${JobTypes.DuplicateColumn})`);
const hrTime = initTime();
const { baseId, sourceId, columnId, extra, req, options } = job.data;
const excludeData = options?.excludeData || false;
const base = await Base.get(baseId);
const sourceColumn = await Column.get({
source_id: sourceId,
colId: columnId,
});
const user = (req as any).user;
const source = await Source.get(sourceColumn.source_id);
const models = (await source.getModels()).filter(
(m) => !m.mm && m.type === 'table',
);
const sourceModel = models.find((m) => m.id === sourceColumn.fk_model_id);
const columns = await sourceModel.getColumns();
const title = generateUniqueName(
`${sourceColumn.title} copy`,
columns.map((p) => p.title),
);
const relatedModelIds = [sourceColumn]
.filter((col) => isLinksOrLTAR(col))
.map((col) => col.colOptions.fk_related_model_id)
.filter((id) => id);
const relatedModels = models.filter((m) => relatedModelIds.includes(m.id));
const exportedModel = (
await this.exportService.serializeModels({
modelIds: [sourceModel.id],
excludeData,
excludeHooks: true,
excludeViews: true,
})
)[0];
elapsedTime(
hrTime,
`serialize model schema for ${sourceModel.id}`,
'duplicateColumn',
);
if (!exportedModel) {
throw new Error(`Export failed for model '${sourceModel.id}'`);
}
exportedModel.model.columns = exportedModel.model.columns.filter((c) =>
c.id.includes(columnId),
);
if (exportedModel.model.columns.length !== 1) {
throw new Error(`There was an error duplicating column!`);
}
const replacedColumn = exportedModel.model.columns.find((c) =>
c.id.includes(columnId),
);
// save old default value
const oldCdf = replacedColumn.cdf;
replacedColumn.title = title;
replacedColumn.column_name = title.toLowerCase().replace(/ /g, '_');
// remove default value to avoid filling existing empty rows
replacedColumn.cdf = null;
Object.assign(replacedColumn, extra);
const idMap = await this.importService.importModels({
baseId,
sourceId: source.id,
data: [exportedModel],
user,
req,
externalModels: relatedModels,
existingModel: sourceModel,
});
elapsedTime(hrTime, 'import model schema', 'duplicateColumn');
if (!idMap) {
throw new Error(`Import failed for model '${sourceModel.id}'`);
}
if (!excludeData) {
const fields: Record<string, string[]> = {};
fields[sourceModel.id] = [sourceModel.primaryKey.id];
fields[sourceModel.id].push(columnId);
for (const md of relatedModels) {
const bts = md.columns
.filter(
(c) =>
isLinksOrLTAR(c) &&
c.colOptions.type === 'bt' &&
c.colOptions.fk_related_model_id === sourceModel.id,
)
.map((c) => c.id);
if (bts.length > 0) {
fields[md.id] = [md.primaryKey.id];
fields[md.id].push(...bts);
}
}
await this.importModelsData({
idMap,
sourceProject: base,
sourceModels: [],
destProject: base,
destBase: source,
hrTime,
modelFieldIds: fields,
externalModels: [sourceModel, ...relatedModels],
});
elapsedTime(hrTime, 'import model data', 'duplicateColumn');
}
const destColumn = await Column.get({
source_id: base.id,
colId: findWithIdentifier(idMap, sourceColumn.id),
});
// update cdf
await this.columnsService.columnUpdate({
columnId: findWithIdentifier(idMap, sourceColumn.id),
column: {
...destColumn,
cdf: oldCdf,
},
user: req.user,
});
this.debugLog(`job completed for ${job.id} (${JobTypes.DuplicateModel})`);
return await Column.get({
source_id: base.id,
colId: findWithIdentifier(idMap, sourceColumn.id),
});
}
async importModelsData(param: {
idMap: Map<string, string>;
sourceProject: Base;
@ -395,13 +558,17 @@ export class DuplicateProcessor {
if (chunk.length > 1000) {
parser.pause();
try {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
// remove empty rows (only pk is present)
chunk = chunk.filter((r) => Object.keys(r).length > 1);
if (chunk.length > 0) {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
}
} catch (e) {
this.debugLog(e);
}
@ -414,13 +581,17 @@ export class DuplicateProcessor {
complete: async () => {
if (chunk.length > 0) {
try {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
// remove empty rows (only pk is present)
chunk = chunk.filter((r) => Object.keys(r).length > 1);
if (chunk.length > 0) {
await this.bulkDataService.bulkDataUpdate({
baseName: destProject.id,
tableName: model.id,
body: chunk,
cookie: null,
raw: true,
});
}
} catch (e) {
this.debugLog(e);
}
@ -433,6 +604,14 @@ export class DuplicateProcessor {
if (error) throw error;
handledLinks = await this.importService.importLinkFromCsvStream({
idMap,
linkStream,
destProject,
destBase,
handledLinks,
});
elapsedTime(
hrTime,
`map existing links to ${model.title}`,

25
packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@ -143,6 +143,19 @@ export class ExportService {
}
}
}
// pg default value fix
if (source.type === 'pg') {
if (column.cdf) {
// check if column.cdf has unmatched single quotes
const matches = column.cdf.match(/'/g);
if (matches && matches.length % 2 !== 0) {
// if so remove after last single quote
const lastQuoteIndex = column.cdf.lastIndexOf("'");
column.cdf = column.cdf.substring(0, lastQuoteIndex);
}
}
}
}
for (const view of model.views) {
@ -395,9 +408,13 @@ export class ExportService {
.map((c) => c.title)
.join(',');
const mmColumns = model.columns.filter(
(col) => isLinksOrLTAR(col) && col.colOptions?.type === 'mm',
);
const mmColumns = param._fieldIds
? model.columns
.filter((c) => param._fieldIds?.includes(c.id))
.filter((col) => isLinksOrLTAR(col) && col.colOptions?.type === 'mm')
: model.columns.filter(
(col) => isLinksOrLTAR(col) && col.colOptions?.type === 'mm',
);
const hasLink = mmColumns.length > 0;
@ -564,6 +581,7 @@ export class ExportService {
view,
query: { limit, offset, fields },
baseModel,
ignoreViewFilterAndSort: true,
})
.then((result) => {
try {
@ -612,6 +630,7 @@ export class ExportService {
view,
query: { limit, offset, fields },
baseModel,
ignoreViewFilterAndSort: true,
})
.then((result) => {
try {

103
packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@ -67,6 +67,7 @@ export class ImportService {
| { model: any; views: any[]; hooks?: any[] }[];
req: NcRequest;
externalModels?: Model[];
existingModel?: Model;
}) {
const hrTime = initTime();
@ -93,6 +94,10 @@ export class ImportService {
param.data = Array.isArray(param.data) ? param.data : param.data.models;
// allow existing model to be linked
if (param.existingModel)
param.externalModels = [param.existingModel, ...param.externalModels];
// allow existing models to be linked
if (param.externalModels) {
for (const model of param.externalModels) {
@ -131,43 +136,69 @@ export class ImportService {
);
// create table with static columns
const table = await this.tablesService.tableCreate({
baseId: base.id,
sourceId: source.id,
user: param.user,
table: withoutId({
...modelData,
columns: reducedColumnSet.map((a) => withoutId(a)),
}),
});
const table =
param.existingModel ||
(await this.tablesService.tableCreate({
baseId: base.id,
sourceId: source.id,
user: param.user,
table: withoutId({
...modelData,
columns: reducedColumnSet.map((a) => withoutId(a)),
}),
}));
idMap.set(modelData.id, table.id);
// map column id's with new created column id's
for (const col of table.columns) {
const colRef = modelData.columns.find(
(a) => sanitizeColumnName(a.column_name) === col.column_name,
);
idMap.set(colRef.id, col.id);
// setval for auto increment column in pg
if (source.type === 'pg') {
if (modelData.pgSerialLastVal) {
if (col.ai) {
const baseModel = await Model.getBaseModelSQL({
id: table.id,
viewId: null,
dbDriver: await NcConnectionMgrv2.get(source),
});
const sqlClient = await NcConnectionMgrv2.getSqlClient(source);
await sqlClient.raw(
`SELECT setval(pg_get_serial_sequence('??', ?), ?);`,
[
baseModel.getTnPath(table.table_name),
col.column_name,
modelData.pgSerialLastVal,
],
);
if (param.existingModel) {
if (reducedColumnSet.length) {
for (const col of reducedColumnSet) {
const freshModelData = await this.columnsService.columnAdd({
tableId: getIdOrExternalId(getParentIdentifier(col.id)),
column: withoutId({
...col,
}) as any,
req: param.req,
user: param.user,
});
for (const nColumn of freshModelData.columns) {
if (nColumn.title === col.title) {
idMap.set(col.id, nColumn.id);
break;
}
}
}
}
} else {
// map column id's with new created column id's
for (const col of table.columns) {
const colRef = modelData.columns.find(
(a) =>
a.column_name &&
sanitizeColumnName(a.column_name) === col.column_name,
);
idMap.set(colRef.id, col.id);
// setval for auto increment column in pg
if (source.type === 'pg') {
if (modelData.pgSerialLastVal) {
if (col.ai) {
const baseModel = await Model.getBaseModelSQL({
id: table.id,
viewId: null,
dbDriver: await NcConnectionMgrv2.get(source),
});
const sqlClient = await NcConnectionMgrv2.getSqlClient(source);
await sqlClient.raw(
`SELECT setval(pg_get_serial_sequence('??', ?), ?);`,
[
baseModel.getTnPath(table.table_name),
col.column_name,
modelData.pgSerialLastVal,
],
);
}
}
}
}
@ -883,6 +914,7 @@ export class ImportService {
}
}
} else if (col.uidt === UITypes.Barcode) {
flatCol.validate = null;
const freshModelData = await this.columnsService.columnAdd({
tableId: getIdOrExternalId(getParentIdentifier(col.id)),
column: withoutId({
@ -910,6 +942,8 @@ export class ImportService {
// create views
for (const data of param.data) {
if (param.existingModel) break;
const modelData = data.model;
const viewsData = data.views;
@ -1043,6 +1077,7 @@ export class ImportService {
// create hooks
for (const data of param.data) {
if (param.existingModel) break;
if (!data?.hooks) break;
const modelData = data.model;
const hookData = data.hooks;

90
packages/nocodb/src/schema/swagger.json

@ -4589,6 +4589,96 @@
]
}
},
"/api/v1/db/meta/duplicate/{baseId}/column/{columnId}": {
"post": {
"summary": "Duplicate Column",
"operationId": "duplicate-column",
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"id": {
"type": "string"
}
}
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest"
}
},
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"excludeData": {
"type": "boolean",
"required": false
}
}
},
"extra": {
"type": "object"
}
}
},
"examples": {
"Example 1": {
"value": {
"excludeData": true
}
}
}
}
}
},
"tags": [
"DB Table"
],
"description": "Duplicate a column",
"parameters": [
{
"$ref": "#/components/parameters/xc-auth"
},
{
"schema": {
"$ref": "#/components/schemas/Id",
"example": "p_124hhlkbeasewh",
"type": "string"
},
"name": "baseId",
"in": "path",
"required": true,
"description": "Unique Base ID"
},
{
"schema": {
"$ref": "#/components/schemas/Id",
"example": "md_124hhlkbeasewh",
"type": "string"
},
"name": "columnId",
"in": "path",
"required": true,
"description": "Unique Column ID"
}
]
}
},
"/api/v1/db/meta/projects/{baseId}/{sourceId}/tables": {
"parameters": [
{

49
packages/nocodb/src/services/columns.service.ts

@ -124,6 +124,11 @@ export class ColumnsService {
param.column.column_name = sanitizeColumnName(param.column.column_name);
}
// trim leading and trailing spaces from column title as knex trim them by default
if (param.column.title) {
param.column.title = param.column.title.trim();
}
if (param.column.column_name) {
// - 5 is a buffer for suffix
let colName = param.column.column_name.slice(0, mxColumnLength - 5);
@ -351,12 +356,10 @@ export class ColumnsService {
}),
);
const data = await baseModel.execAndParse(
sqlClient.raw('SELECT DISTINCT ?? FROM ??', [
column.column_name,
baseModel.getTnPath(table.table_name),
]),
);
const data = await sqlClient.raw('SELECT DISTINCT ?? FROM ??', [
column.column_name,
baseModel.getTnPath(table.table_name),
]);
if (data.length) {
const existingOptions = colBody.colOptions.options.map(
@ -1068,6 +1071,11 @@ export class ColumnsService {
param.column.column_name = sanitizeColumnName(param.column.column_name);
}
// trim leading and trailing spaces from column title as knex trim them by default
if (param.column.title) {
param.column.title = param.column.title.trim();
}
if (param.column.column_name) {
// - 5 is a buffer for suffix
let colName = param.column.column_name.slice(0, mxColumnLength - 5);
@ -1119,6 +1127,12 @@ export class ColumnsService {
}
let colBody: any = param.column;
const colExtra = {
view_id: colBody.view_id,
column_order: colBody.column_order,
};
switch (colBody.uidt) {
case UITypes.Rollup:
{
@ -1143,7 +1157,13 @@ export class ColumnsService {
case UITypes.Links:
case UITypes.LinkToAnotherRecord:
await this.createLTARColumn({ ...param, source, base, reuse });
await this.createLTARColumn({
...param,
source,
base,
reuse,
colExtra,
});
this.appHooksService.emit(AppEvents.RELATION_DELETE, {
column: {
@ -1810,6 +1830,7 @@ export class ColumnsService {
source: Source;
base: Base;
reuse?: ReusableParams;
colExtra?: any;
}) {
validateParams(['parentId', 'childId', 'type'], param.column);
@ -1829,7 +1850,9 @@ export class ColumnsService {
id: param.source.base_id,
}),
);
const isLinks = param.column.uidt === UITypes.Links;
const isLinks =
param.column.uidt === UITypes.Links ||
(param.column as LinkToAnotherColumnReqType).type === 'bt';
// if xcdb base then treat as virtual relation to avoid creating foreign key
if (param.source.isMeta()) {
@ -1933,6 +1956,7 @@ export class ColumnsService {
null,
param.column['meta'],
isLinks,
param.colExtra,
);
} else if ((param.column as LinkToAnotherColumnReqType).type === 'mm') {
const aTn = `${param.base?.prefix ?? ''}_nc_m2m_${randomID()}`;
@ -2038,6 +2062,9 @@ export class ColumnsService {
foreignKeyName1,
(param.column as LinkToAnotherColumnReqType).virtual,
true,
null,
false,
param.colExtra,
);
await createHmAndBtColumn(
assocModel,
@ -2048,6 +2075,9 @@ export class ColumnsService {
foreignKeyName2,
(param.column as LinkToAnotherColumnReqType).virtual,
true,
null,
false,
param.colExtra,
);
await Column.insert({
@ -2100,6 +2130,9 @@ export class ColumnsService {
plural: param.column['meta']?.plural || pluralize(child.title),
singular: param.column['meta']?.singular || singularize(child.title),
},
// column_order and view_id if provided
...param.colExtra,
});
// todo: create index for virtual relations as well

15
packages/nocodb/src/services/datas.service.ts

@ -134,8 +134,9 @@ export class DatasService {
query: any;
baseModel?: BaseModelSqlv2;
throwErrorIfInvalidParams?: boolean;
ignoreViewFilterAndSort?: boolean;
}) {
const { model, view, query = {} } = param;
const { model, view, query = {}, ignoreViewFilterAndSort = false } = param;
const source = await Source.get(model.source_id);
@ -163,18 +164,16 @@ export class DatasService {
} catch (e) {}
const [count, data] = await Promise.all([
baseModel.count(listArgs),
baseModel.count(listArgs, false, param.throwErrorIfInvalidParams),
(async () => {
let data = [];
try {
data = await nocoExecute(
ast,
await baseModel.list(
listArgs,
false,
false,
param.throwErrorIfInvalidParams,
),
await baseModel.list(listArgs, {
ignoreViewFilterAndSort,
throwErrorIfInvalidParams: param.throwErrorIfInvalidParams,
}),
{},
listArgs,
);

1
packages/nocodb/src/utils/acl.ts

@ -116,6 +116,7 @@ const permissionScopes = {
'bulkDataDeleteAll',
'relationDataRemove',
'relationDataAdd',
'duplicateColumn',
// Base API Tokens
'baseApiTokenList',

2
packages/nocodb/tests/unit/factory/row.ts

@ -232,7 +232,7 @@ const listRow = async ({
const ignorePagination = !options;
return await baseModel.list(options, ignorePagination);
return await baseModel.list(options, { ignorePagination });
};
const getOneRow = async (

6
packages/nocodb/tests/unit/rest/tests/groupby.test.ts

@ -7,7 +7,7 @@ import { listRow } from '../../factory/row';
import { getTable } from '../../factory/table';
import { getView, updateView } from '../../factory/view';
import init from '../../init';
import type { Column, Model, Base, View } from '../../../../src/models';
import type { Base, Column, Model, View } from '../../../../src/models';
import 'mocha';
function groupByTests() {
@ -226,7 +226,9 @@ function groupByTests() {
],
});
const response = await request(context.app)
.get(`/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/groupby`)
.get(
`/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/views/${filmView.id}/groupby`,
)
.set('xc-auth', context.token)
.query({
column_name: _lengthColumn.column_name,

6
tests/playwright/pages/Dashboard/Grid/Column/index.ts

@ -319,12 +319,14 @@ export class ColumnPageObject extends BasePage {
await this.rootPage.locator('.nc-more-options').click();
}
async duplicateColumn({ title, expectedTitle = `${title}_copy` }: { title: string; expectedTitle?: string }) {
async duplicateColumn({ title, expectedTitle = `${title} copy` }: { title: string; expectedTitle?: string }) {
await this.grid.get().locator(`th[data-title="${title}"] .nc-ui-dt-dropdown`).click();
await this.rootPage.locator('li[role="menuitem"]:has-text("Duplicate"):visible').click();
await this.rootPage.locator('.nc-modal-column-duplicate .nc-button:has-text("Confirm"):visible').click();
// await this.verifyToast({ message: 'Column duplicated successfully' });
await this.grid.get().locator(`th[data-title="${expectedTitle}"]`).waitFor({ state: 'visible' });
await this.grid.get().locator(`th[data-title="${expectedTitle}"]`).waitFor({ state: 'visible', timeout: 10000 });
}
async hideColumn({ title, isDisplayValue = false }: { title: string; isDisplayValue?: boolean }) {

2
tests/playwright/tests/db/columns/columnMenuOperations.spec.ts

@ -70,7 +70,7 @@ test.describe('Column menu operations', () => {
});
await dashboard.grid.column.duplicateColumn({
title,
expectedTitle: `${title}_copy_1`,
expectedTitle: `${title} copy_1`,
});
}
await dashboard.closeTab({ title: 'Film' });

Loading…
Cancel
Save