Browse Source

fix: use xc table for at import (#8605)

* fix: use generic id for at imported bases

Signed-off-by: mertmit <mertmit99@gmail.com>

* fix: restrict editing system columns

Signed-off-by: mertmit <mertmit99@gmail.com>

* fix: grid read only handling

Signed-off-by: mertmit <mertmit99@gmail.com>

* fix: avoid clear & update for readonly columns on UI

Signed-off-by: mertmit <mertmit99@gmail.com>

* fix: required field missing error & avoid clearing read only cells

Signed-off-by: mertmit <mertmit99@gmail.com>

* fix: bypass system column restriction for at import

Signed-off-by: mertmit <mertmit99@gmail.com>

* fix: handle clashing columns with system columns for at import

Signed-off-by: mertmit <mertmit99@gmail.com>

* fix: import data for clashing columns

Signed-off-by: mertmit <mertmit99@gmail.com>

* fix: handle stringify at function level

Signed-off-by: mertmit <mertmit99@gmail.com>

* fix: compilation errors

Signed-off-by: mertmit <mertmit99@gmail.com>

---------

Signed-off-by: mertmit <mertmit99@gmail.com>
nc-oss/f4dcddf8
Mert E 3 weeks ago committed by GitHub
parent
commit
f4dcddf88c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 38
      packages/nc-gui/components/dlg/AirtableImport.vue
  2. 81
      packages/nc-gui/components/smartsheet/grid/Table.vue
  3. 1
      packages/nocodb-sdk/src/lib/globals.ts
  4. 64
      packages/nocodb/src/db/BaseModelSqlv2.ts
  5. 28
      packages/nocodb/src/helpers/catchError.ts
  6. 57
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
  7. 4
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts
  8. 2
      packages/nocodb/src/services/bulk-data-alias.service.ts

38
packages/nc-gui/components/dlg/AirtableImport.vue

@ -54,7 +54,7 @@ const syncSource = ref({
syncLookup: true,
syncFormula: false,
syncAttachment: true,
syncUsers: true,
syncUsers: false,
},
},
})
@ -210,7 +210,7 @@ async function loadSyncSrc() {
syncLookup: true,
syncFormula: false,
syncAttachment: true,
syncUsers: true,
syncUsers: false,
},
},
}
@ -403,22 +403,30 @@ function downloadLogs(filename: string) {
</a-checkbox>
</div>
<!-- Import Users Columns -->
<!-- Import Formula Columns -->
<div class="my-2">
<a-checkbox v-model:checked="syncSource.details.options.syncUsers">
{{ $t('labels.importUsers') }}
</a-checkbox>
<a-tooltip placement="top">
<template #title>
<span>{{ $t('title.comingSoon') }}</span>
</template>
<a-checkbox v-model:checked="syncSource.details.options.syncFormula" disabled>
{{ $t('labels.importFormulaColumns') }}
</a-checkbox>
</a-tooltip>
</div>
<!-- Import Formula Columns -->
<a-tooltip placement="top">
<template #title>
<span>{{ $t('title.comingSoon') }}</span>
</template>
<a-checkbox v-model:checked="syncSource.details.options.syncFormula" disabled>
{{ $t('labels.importFormulaColumns') }}
</a-checkbox>
</a-tooltip>
<!-- Invite Users
<div class="my-2">
<a-tooltip placement="top">
<template #title>
<span>{{ $t('title.comingSoon') }}</span>
</template>
<a-checkbox v-model:checked="syncSource.details.options.syncUsers" disabled>
{{ $t('labels.importUsers') }}
</a-checkbox>
</a-tooltip>
</div>
-->
</a-form>
<a-divider />

81
packages/nc-gui/components/smartsheet/grid/Table.vue

@ -234,6 +234,9 @@ const isKeyDown = ref(false)
async function clearCell(ctx: { row: number; col: number } | null, skipUpdate = false) {
if (!ctx || !hasEditPermission.value || (!isLinksOrLTAR(fields.value[ctx.col]) && isVirtualCol(fields.value[ctx.col]))) return
// eslint-disable-next-line @typescript-eslint/no-use-before-define
if (colMeta.value[ctx.col].isReadonly) return
const rowObj = dataRef.value[ctx.row]
const columnObj = fields.value[ctx.col]
@ -424,15 +427,23 @@ const cellMeta = computed(() => {
})
})
const isReadonly = (col: ColumnType) => {
return (
isSystemColumn(col) ||
isLookup(col) ||
isRollup(col) ||
isFormula(col) ||
isVirtualCol(col) ||
isCreatedOrLastModifiedTimeCol(col) ||
isCreatedOrLastModifiedByCol(col)
)
}
const colMeta = computed(() => {
return fields.value.map((col) => {
return {
isLookup: isLookup(col),
isRollup: isRollup(col),
isFormula: isFormula(col),
isCreatedOrLastModifiedTimeCol: isCreatedOrLastModifiedTimeCol(col),
isCreatedOrLastModifiedByCol: isCreatedOrLastModifiedByCol(col),
isVirtualCol: isVirtualCol(col),
isReadonly: isReadonly(col),
}
})
})
@ -932,6 +943,9 @@ async function clearSelectedRangeOfCells() {
continue
}
// skip readonly columns
if (isReadonly(col)) continue
row.row[col.title] = null
props.push(col.title)
}
@ -1251,6 +1265,16 @@ const refreshFillHandle = () => {
})
}
const selectedReadonly = computed(
() =>
// if all the selected columns are not readonly
(selectedRange.isEmpty() && activeCell.col && colMeta.value[activeCell.col].isReadonly) ||
(!selectedRange.isEmpty() &&
Array.from({ length: selectedRange.end.col - selectedRange.start.col + 1 }).every(
(_, i) => colMeta.value[selectedRange.start.col + i].isReadonly,
)),
)
const showFillHandle = computed(
() =>
!readOnly.value &&
@ -1259,16 +1283,10 @@ const showFillHandle = computed(
!dataRef.value[(isNaN(selectedRange.end.row) ? activeCell.row : selectedRange.end.row) ?? -1]?.rowMeta?.new &&
activeCell.col !== null &&
fields.value[activeCell.col] &&
!(
isLookup(fields.value[activeCell.col]) ||
isRollup(fields.value[activeCell.col]) ||
isFormula(fields.value[activeCell.col]) ||
isCreatedOrLastModifiedTimeCol(fields.value[activeCell.col]) ||
isCreatedOrLastModifiedByCol(fields.value[activeCell.col])
) &&
!isViewDataLoading.value &&
!isPaginationLoading.value &&
dataRef.value.length,
dataRef.value.length &&
!selectedReadonly.value,
)
watch(
@ -1660,7 +1678,7 @@ onKeyStroke('ArrowDown', onDown)
</div>
</th>
<th
v-if="fields[0]"
v-if="fields[0] && fields[0].id"
v-xc-ver-resize
:data-col="fields[0].id"
:data-title="fields[0].title"
@ -1693,9 +1711,9 @@ onKeyStroke('ArrowDown', onDown)
<LazySmartsheetHeaderVirtualCell
v-if="fields[0] && colMeta[0].isVirtualCol"
:column="fields[0]"
:hide-menu="readOnly || isMobileMode"
:hide-menu="readOnly || !!isMobileMode"
/>
<LazySmartsheetHeaderCell v-else :column="fields[0]" :hide-menu="readOnly || isMobileMode" />
<LazySmartsheetHeaderCell v-else :column="fields[0]" :hide-menu="readOnly || !!isMobileMode" />
</div>
</th>
<th
@ -1728,9 +1746,9 @@ onKeyStroke('ArrowDown', onDown)
<LazySmartsheetHeaderVirtualCell
v-if="colMeta[index].isVirtualCol"
:column="col"
:hide-menu="readOnly || isMobileMode"
:hide-menu="readOnly || !!isMobileMode"
/>
<LazySmartsheetHeaderCell v-else :column="col" :hide-menu="readOnly || isMobileMode" />
<LazySmartsheetHeaderCell v-else :column="col" :hide-menu="readOnly || !!isMobileMode" />
</div>
</th>
<th
@ -1979,14 +1997,7 @@ onKeyStroke('ArrowDown', onDown)
'align-middle': !rowHeight || rowHeight === 1,
'align-top': rowHeight && rowHeight !== 1,
'filling': fillRangeMap[`${rowIndex}-0`],
'readonly':
(colMeta[0].isLookup ||
colMeta[0].isRollup ||
colMeta[0].isFormula ||
colMeta[0].isCreatedOrLastModifiedTimeCol ||
colMeta[0].isCreatedOrLastModifiedByCol) &&
hasEditPermission &&
selectRangeMap[`${rowIndex}-0`],
'readonly': colMeta[0].isReadonly && hasEditPermission && selectRangeMap[`${rowIndex}-0`],
'!border-r-blue-400 !border-r-3': toBeDroppedColId === fields[0].id,
}"
:style="{
@ -2054,13 +2065,7 @@ onKeyStroke('ArrowDown', onDown)
'align-top': rowHeight && rowHeight !== 1,
'filling': fillRangeMap[`${rowIndex}-${colIndex}`],
'readonly':
(colMeta[colIndex].isLookup ||
colMeta[colIndex].isRollup ||
colMeta[colIndex].isFormula ||
colMeta[colIndex].isCreatedOrLastModifiedTimeCol ||
colMeta[colIndex].isCreatedOrLastModifiedByCol) &&
hasEditPermission &&
selectRangeMap[`${rowIndex}-${colIndex}`],
colMeta[colIndex].isReadonly && hasEditPermission && selectRangeMap[`${rowIndex}-${colIndex}`],
'!border-r-blue-400 !border-r-3': toBeDroppedColId === columnObj.id,
}"
:style="{
@ -2222,7 +2227,7 @@ onKeyStroke('ArrowDown', onDown)
v-if="contextMenuTarget && hasEditPermission"
class="nc-base-menu-item"
data-testid="context-menu-item-paste"
:disabled="isSystemColumn(fields[contextMenuTarget.col])"
:disabled="selectedReadonly"
@click="paste"
>
<div v-e="['a:row:paste']" class="flex gap-2 items-center">
@ -2241,7 +2246,7 @@ onKeyStroke('ArrowDown', onDown)
(isLinksOrLTAR(fields[contextMenuTarget.col]) || !cellMeta[0]?.[contextMenuTarget.col].isVirtualCol)
"
class="nc-base-menu-item"
:disabled="isSystemColumn(fields[contextMenuTarget.col])"
:disabled="selectedReadonly"
data-testid="context-menu-item-clear"
@click="clearCell(contextMenuTarget)"
>
@ -2255,7 +2260,7 @@ onKeyStroke('ArrowDown', onDown)
<NcMenuItem
v-else-if="contextMenuTarget && hasEditPermission"
class="nc-base-menu-item"
:disabled="isSystemColumn(fields[contextMenuTarget.col])"
:disabled="selectedReadonly"
data-testid="context-menu-item-clear"
@click="clearSelectedRangeOfCells()"
>
@ -2306,8 +2311,8 @@ onKeyStroke('ArrowDown', onDown)
</div>
<LazySmartsheetPagination
v-if="headerOnly !== true"
:key="isMobileMode"
v-if="headerOnly !== true && paginationDataRef"
:key="`nc-pagination-${isMobileMode}`"
v-model:pagination-data="paginationDataRef"
:show-api-timing="!isGroupBy"
align-count-on-right

1
packages/nocodb-sdk/src/lib/globals.ts

@ -135,6 +135,7 @@ export enum NcErrorType {
FIELD_NOT_FOUND = 'FIELD_NOT_FOUND',
RECORD_NOT_FOUND = 'RECORD_NOT_FOUND',
GENERIC_NOT_FOUND = 'GENERIC_NOT_FOUND',
REQUIRED_FIELD_MISSING = 'REQUIRED_FIELD_MISSING',
ERROR_DUPLICATE_RECORD = 'ERROR_DUPLICATE_RECORD',
USER_NOT_FOUND = 'USER_NOT_FOUND',
INVALID_OFFSET_VALUE = 'INVALID_OFFSET_VALUE',

64
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -3394,6 +3394,10 @@ class BaseModelSqlv2 {
{ ignoreView: true, getHiddenColumn: true },
);
if (!prevData) {
NcError.recordNotFound(id);
}
const query = this.dbDriver(this.tnPath)
.update(updateObj)
.where(await this._wherePk(id, true));
@ -3825,6 +3829,7 @@ class BaseModelSqlv2 {
raw = false,
insertOneByOneAsFallback = false,
isSingleRecordInsertion = false,
allowSystemColumn = false,
}: {
chunkSize?: number;
cookie?: any;
@ -3833,6 +3838,7 @@ class BaseModelSqlv2 {
raw?: boolean;
insertOneByOneAsFallback?: boolean;
isSingleRecordInsertion?: boolean;
allowSystemColumn?: boolean;
} = {},
) {
let trx;
@ -3856,14 +3862,21 @@ class BaseModelSqlv2 {
for (let i = 0; i < this.model.columns.length; ++i) {
const col = this.model.columns[i];
if (
col.title in d &&
(isCreatedOrLastModifiedTimeCol(col) ||
isCreatedOrLastModifiedByCol(col))
) {
NcError.badRequest(
`Column "${col.title}" is auto generated and cannot be updated`,
);
if (col.title in d) {
if (
isCreatedOrLastModifiedTimeCol(col) ||
isCreatedOrLastModifiedByCol(col)
) {
NcError.badRequest(
`Column "${col.title}" is auto generated and cannot be updated`,
);
}
if (col.system && !allowSystemColumn) {
NcError.badRequest(
`Column "${col.title}" is system column and cannot be updated`,
);
}
}
// populate pk columns
@ -4171,7 +4184,7 @@ class BaseModelSqlv2 {
if (!pkValues) {
// throw or skip if no pk provided
if (throwExceptionIfNotExist) {
NcError.recordNotFound(JSON.stringify(pkValues));
NcError.recordNotFound(pkValues);
}
continue;
}
@ -4208,7 +4221,7 @@ class BaseModelSqlv2 {
if (!oldRecord) {
// throw or skip if no record found
if (throwExceptionIfNotExist) {
NcError.recordNotFound(JSON.stringify(recordPk));
NcError.recordNotFound(recordPk);
}
continue;
}
@ -4407,7 +4420,7 @@ class BaseModelSqlv2 {
if (!pkValues) {
// throw or skip if no pk provided
if (throwExceptionIfNotExist) {
NcError.recordNotFound(JSON.stringify(pkValues));
NcError.recordNotFound(pkValues);
}
continue;
}
@ -4437,7 +4450,7 @@ class BaseModelSqlv2 {
if (!oldRecord) {
// throw or skip if no record found
if (throwExceptionIfNotExist) {
NcError.recordNotFound(JSON.stringify(pk));
NcError.recordNotFound(pk);
}
continue;
}
@ -4926,14 +4939,21 @@ class BaseModelSqlv2 {
for (let i = 0; i < cols.length; ++i) {
const column = this.model.columns[i];
if (
column.title in data &&
(isCreatedOrLastModifiedTimeCol(column) ||
isCreatedOrLastModifiedByCol(column))
) {
NcError.badRequest(
`Column "${column.title}" is auto generated and cannot be updated`,
);
if (column.title in data) {
if (
isCreatedOrLastModifiedTimeCol(column) ||
isCreatedOrLastModifiedByCol(column)
) {
NcError.badRequest(
`Column "${column.title}" is auto generated and cannot be updated`,
);
}
if (column.system) {
NcError.badRequest(
`Column "${column.title}" is system column and cannot be updated`,
);
}
}
await this.validateOptions(column, data);
// Validates the constraints on the data based on the column definitions
@ -7480,6 +7500,10 @@ export function _wherePk(
export function getCompositePkValue(primaryKeys: Column[], row) {
if (typeof row !== 'object') return row;
if (row === null)
NcError.requiredFieldMissing(primaryKeys.map((c) => c.title).join(','));
return primaryKeys.map((c) => row[c.title] ?? row[c.column_name]).join('___');
}

28
packages/nocodb/src/helpers/catchError.ts

@ -502,6 +502,10 @@ const errorHelpers: {
message: (resource: string, id: string) => `${resource} '${id}' not found`,
code: 404,
},
[NcErrorType.REQUIRED_FIELD_MISSING]: {
message: (field: string) => `Field '${field}' is required`,
code: 422,
},
[NcErrorType.ERROR_DUPLICATE_RECORD]: {
message: (...ids: string[]) => {
const isMultiple = Array.isArray(ids) && ids.length > 1;
@ -662,7 +666,22 @@ export class NcError {
});
}
static recordNotFound(id: string | string[], args?: NcErrorArgs) {
static recordNotFound(
id: string | string[] | Record<string, string> | Record<string, string>[],
args?: NcErrorArgs,
) {
if (typeof id === 'string') {
id = [id];
} else if (Array.isArray(id)) {
if (id.every((i) => typeof i === 'string')) {
id = id as string[];
} else {
id = id.map((i) => Object.values(i).join('___'));
}
} else {
id = Object.values(id).join('___');
}
throw new NcBaseErrorv2(NcErrorType.RECORD_NOT_FOUND, {
params: id,
...args,
@ -676,6 +695,13 @@ export class NcError {
});
}
static requiredFieldMissing(field: string, args?: NcErrorArgs) {
throw new NcBaseErrorv2(NcErrorType.REQUIRED_FIELD_MISSING, {
params: field,
...args,
});
}
static duplicateRecord(id: string | string[], args?: NcErrorArgs) {
throw new NcBaseErrorv2(NcErrorType.ERROR_DUPLICATE_RECORD, {
params: id,

57
packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@ -1,5 +1,5 @@
import moment from 'moment';
import { UITypes } from 'nocodb-sdk';
import { SqlUiFactory, UITypes } from 'nocodb-sdk';
import Airtable from 'airtable';
import hash from 'object-hash';
import dayjs from 'dayjs';
@ -200,6 +200,19 @@ export class AtImportProcessor {
};
} = {};
const atFieldAliasToNcFieldAlias = {};
const addFieldAlias = (ncTableTitle, atFieldAlias, ncFieldAlias) => {
if (!atFieldAliasToNcFieldAlias[ncTableTitle]) {
atFieldAliasToNcFieldAlias[ncTableTitle] = {};
}
atFieldAliasToNcFieldAlias[ncTableTitle][atFieldAlias] = ncFieldAlias;
};
const getNcFieldAlias = (ncTableTitle, atFieldAlias) => {
return atFieldAliasToNcFieldAlias[ncTableTitle][atFieldAlias];
};
const uniqueTableNameGen = getUniqueNameGenerator('sheet');
// run time counter (statistics)
@ -313,7 +326,7 @@ export class AtImportProcessor {
rollup: UITypes.Rollup,
count: UITypes.Rollup,
lookup: UITypes.Lookup,
autoNumber: UITypes.AutoNumber,
autoNumber: UITypes.Decimal,
barcode: UITypes.SingleLineText,
button: UITypes.Button,
};
@ -519,14 +532,18 @@ export class AtImportProcessor {
table.table_name = uniqueTableNameGen(sanitizedName);
table.columns = [];
const source = await Source.get(context, syncDB.sourceId);
const sqlUi = SqlUiFactory.create({ client: source.type });
const sysColumns = [
...sqlUi.getNewTableColumns().filter((c) => c.column_name === 'id'),
{
title: ncSysFields.id,
column_name: ncSysFields.id,
uidt: UITypes.ID,
meta: {
ag: 'nc',
},
uidt: UITypes.SingleLineText,
system: true,
},
{
title: ncSysFields.hash,
@ -536,6 +553,14 @@ export class AtImportProcessor {
},
];
table.columns.push(...sysColumns);
for (const sysCol of sysColumns) {
// call to avoid clash with system columns
nc_getSanitizedColumnName(sysCol.title, table.table_name);
addFieldAlias(table.title, sysCol.title, sysCol.title);
}
for (let j = 0; j < tblSchema[i].columns.length; j++) {
const col = tblSchema[i].columns[j];
@ -549,6 +574,9 @@ export class AtImportProcessor {
col.name,
table.table_name,
);
addFieldAlias(table.title, col.name, ncName.title);
const ncCol: any = {
// Enable to use aTbl identifiers as is: id: col.id,
title: ncName.title,
@ -605,8 +633,6 @@ export class AtImportProcessor {
}
table.columns.push(ncCol);
}
table.columns.push(sysColumns[0]);
table.columns.push(sysColumns[1]);
tables.push(table);
}
@ -639,7 +665,10 @@ export class AtImportProcessor {
for (let colIdx = 0; colIdx < table.columns.length; colIdx++) {
const aId = aTblSchema[idx].columns.find(
(x) =>
x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title,
getNcFieldAlias(
table.title,
x.name.trim().replace(/\./g, '_'),
) === table.columns[colIdx].title,
)?.id;
if (aId)
await sMap.addToMappingTbl(
@ -1398,7 +1427,10 @@ export class AtImportProcessor {
// trim spaces on either side of column name
// leads to error in NocoDB
Object.keys(rec).forEach((key) => {
const replacedKey = key.trim().replace(/\./g, '_');
const replacedKey = getNcFieldAlias(
table.title,
key.trim().replace(/\./g, '_'),
);
if (key !== replacedKey) {
rec[replacedKey] = rec[key];
delete rec[key];
@ -2556,7 +2588,6 @@ export class AtImportProcessor {
data: { error: e.message },
});
logger.log(e);
throw new Error(e.message);
}
throw e;
}
@ -2573,10 +2604,10 @@ const getUniqueNameGenerator = (defaultName = 'name', context = 'default') => {
return (initName: string = defaultName): string => {
let name = initName === '_' ? defaultName : initName;
let c = 0;
while (name in namesRef[finalContext]) {
while (name.toLowerCase() in namesRef[finalContext]) {
name = `${initName}_${++c}`;
}
namesRef[finalContext][name] = true;
namesRef[finalContext][name.toLowerCase()] = true;
return name;
};
};

4
packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts

@ -229,6 +229,7 @@ export async function importData(
cookie: {},
skip_hooks: true,
foreign_key_checks: !!source.isMeta(),
allowSystemColumn: true,
});
logBasic(
@ -275,6 +276,7 @@ export async function importData(
cookie: {},
skip_hooks: true,
foreign_key_checks: !!source.isMeta(),
allowSystemColumn: true,
});
logBasic(
@ -448,6 +450,7 @@ export async function importLTARData(
cookie: {},
skip_hooks: true,
foreign_key_checks: !!source.isMeta(),
allowSystemColumn: true,
});
insertArray = [];
@ -491,6 +494,7 @@ export async function importLTARData(
cookie: {},
skip_hooks: true,
foreign_key_checks: !!source.isMeta(),
allowSystemColumn: true,
});
importedCount += assocTableData[assocMeta.modelMeta.id].length;

2
packages/nocodb/src/services/bulk-data-alias.service.ts

@ -48,6 +48,7 @@ export class BulkDataAliasService {
foreign_key_checks?: boolean;
skip_hooks?: boolean;
raw?: boolean;
allowSystemColumn?: boolean;
},
) {
return await this.executeBulkOperation(context, {
@ -60,6 +61,7 @@ export class BulkDataAliasService {
foreign_key_checks: param.foreign_key_checks,
skip_hooks: param.skip_hooks,
raw: param.raw,
allowSystemColumn: param.allowSystemColumn,
},
],
});

Loading…
Cancel
Save