github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Nc feature: Import CSV - skip field mapping during import (#8915) 

* fix(nc-gui): import csv changes

* fix(nc-gui): tooltip alignment issue

* fix(nc-gui): duplicate table name issue

* fix(nc-gui): uncheck auto select field type checkbox

* fix(nc-gui): skip field mapping during csv import

* chore(nc-gui): lint

* fix(nc-gui): small changes

* fix(nc-gui): remove auto mapping option

* fix(test): update import excel test cases

* fix(nc-gui): review changes

* fix(nc-gui): quick import modal width issue

* fix(nc-gui): w.replace is not a function
pull/8952/head
Ramesh Mane 5 months ago committed by GitHub
parent
b56c585572
commit
5cfe0f781b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 142 additions and 162 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 58
      packages/nc-gui/components/dlg/QuickImport.vue
  2. 215
      packages/nc-gui/components/template/Editor.vue
  3. 11
      packages/nc-gui/components/template/utils.ts
  4. 10
      packages/nc-gui/helpers/parsers/ExcelTemplateAdapter.ts
  5. 5
      packages/nc-gui/lang/en.json
  6. 5
      tests/playwright/tests/db/features/import.spec.ts

58
packages/nc-gui/components/dlg/QuickImport.vue
Unescape View File

@ -70,6 +70,7 @@ const defaultImportState = {
autoSelectFieldTypes: true,
firstRowAsHeaders: true,
shouldImportData: true,
importDataOnly: true,
},
}
const importState = reactive(defaultImportState)
@ -84,7 +85,6 @@ const IsImportTypeExcel = computed(() => importType === 'excel')
const validators = computed(() => ({
url: [fieldRequiredValidator(), importUrlValidator, isImportTypeCsv.value ? importCsvUrlValidator : importExcelUrlValidator],
maxRowsToParse: [fieldRequiredValidator()],
}))
const { validate, validateInfos } = useForm(importState, validators)
@ -152,10 +152,6 @@ const disableImportButton = computed(() => !templateEditorRef.value?.isValid ||
const disableFormatJsonButton = computed(() => !jsonEditorRef.value?.isValid)
const modalWidth = computed(() => {
if (importType === 'excel' && templateEditorModal.value) {
return 'max(90vw, 600px)'
}
return 'max(60vw, 600px)'
})
@ -254,9 +250,10 @@ function formatJson() {
jsonEditorRef.value?.format()
}
function populateUniqueTableName(tn: string) {
function populateUniqueTableName(tn: string, draftTn: string[] = []) {
let c = 1
while (
draftTn.includes(tn) ||
baseTables.value.get(baseId)?.some((t: TableType) => {
const s = t.table_name.split('___')
let target = t.table_name
@ -492,10 +489,13 @@ async function parseAndExtractData(val: UploadFile[] | ArrayBuffer | string) {
if (importDataOnly) importColumns.value = templateGenerator!.getColumns()
else {
// ensure the target table name not exist in current table list
templateData.value.tables = templateData.value.tables.map((table: Record<string, any>) => ({
...table,
table_name: populateUniqueTableName(table.table_name),
}))
const draftTableNames = [] as string[]
templateData.value.tables = templateData.value.tables.map((table: Record<string, any>) => {
const table_name = populateUniqueTableName(table.table_name, draftTableNames)
draftTableNames.push(table_name)
return { ...table, table_name }
})
}
importData.value = templateGenerator!.getData()
}
@ -517,6 +517,11 @@ const onError = () => {
const onChange = () => {
isError.value = false
}
onMounted(() => {
importState.parserConfig.importDataOnly = importDataOnly
importState.parserConfig.autoSelectFieldTypes = importDataOnly
})
</script>
<template>
@ -531,7 +536,12 @@ const onChange = () => {
<div class="px-5">
<div class="prose-xl font-weight-bold my-5">{{ importMeta.header }}</div>
<div class="mt-5">
<div
class="mt-5"
:class="{
'mb-4': templateEditorModal,
}"
>
<LazyTemplateEditor
v-if="templateEditorModal"
ref="templateEditorRef"
@ -582,6 +592,9 @@ const onChange = () => {
<p class="ant-upload-hint">
{{ importMeta.uploadHint }}
</p>
<template #removeIcon>
<component :is="iconMap.deleteListItem" />
</template>
</a-upload-dragger>
</div>
</a-tab-pane>
@ -608,9 +621,9 @@ const onChange = () => {
</template>
<div class="pr-10 pt-5">
<a-form :model="importState" name="quick-import-url-form" layout="vertical" class="mb-0">
<a-form :model="importState" name="quick-import-url-form" layout="vertical" class="mb-0 !ml-0.5">
<a-form-item :label="importMeta.urlInputLabel" v-bind="validateInfos.url">
<a-input v-model:value="importState.url" size="large" />
<a-input v-model:value="importState.url" size="large" class="!rounded-md" />
</a-form-item>
</a-form>
</div>
@ -625,16 +638,6 @@ const onChange = () => {
<!-- Advanced Settings -->
<span class="prose-lg">{{ $t('title.advancedSettings') }}</span>
<a-form-item class="!my-2" :label="t('msg.info.footMsg')" v-bind="validateInfos.maxRowsToParse">
<a-input-number v-model:value="importState.parserConfig.maxRowsToParse" :min="1" :max="50000" />
</a-form-item>
<a-form-item v-if="!importDataOnly" class="!my-2">
<a-checkbox v-model:checked="importState.parserConfig.autoSelectFieldTypes">
<span class="caption">{{ $t('labels.autoSelectFieldTypes') }}</span>
</a-checkbox>
</a-form-item>
<a-form-item v-if="isImportTypeCsv || IsImportTypeExcel" class="!my-2">
<a-checkbox v-model:checked="importState.parserConfig.firstRowAsHeaders">
<span class="caption">{{ $t('labels.firstRowAsHeaders') }}</span>
@ -699,3 +702,12 @@ const onChange = () => {
</template>
</a-modal>
</template>
<style lang="scss" scoped>
:deep(.ant-upload-list-item-thumbnail) {
line-height: 48px;
}
:deep(.ant-upload-list-item-card-actions-btn.ant-btn-icon-only) {
@apply !h-6;
}
</style>

215
packages/nc-gui/components/template/Editor.vue
Unescape View File

@ -101,7 +101,7 @@ const isImporting = ref(false)
const importingTips = ref<Record<string, string>>({})
const checkAllRecord = ref<boolean[]>([])
const checkAllRecord = ref<Record<string, boolean>>({})
const formError = ref()
@ -168,14 +168,20 @@ watch(
let res = true
if (importDataOnly) {
for (const tn of Object.keys(srcDestMapping.value)) {
let flag = false
if (!atLeastOneEnabledValidation(tn)) {
res = false
}
for (const record of srcDestMapping.value[tn]) {
if (!fieldsValidation(record, tn)) {
return false
res = false
flag = true
break
}
}
if (flag) {
break
}
}
} else {
for (const [_, o] of Object.entries(validateInfos)) {
@ -258,20 +264,13 @@ function deleteTable(tableIdx: number) {
function deleteTableColumn(tableIdx: number, columnKey: number) {
const columnIdx = data.tables[tableIdx].columns.findIndex((c: ColumnType & { key: number }) => c.key === columnKey)
data.tables[tableIdx].columns.splice(columnIdx, 1)
}
function addNewColumnRow(tableIdx: number, uidt: string) {
data.tables[tableIdx].columns.push({
key: data.tables[tableIdx].columns.length,
title: `title${data.tables[tableIdx].columns.length + 1}`,
column_name: `title${data.tables[tableIdx].columns.length + 1}`,
uidt,
})
let key = 0
nextTick(() => {
const input = inputRefs.value[data.tables[tableIdx].columns.length - 1]
input.focus()
input.select()
data.tables[tableIdx].columns.forEach((_c: ColumnType & { key: number }, i: number) => {
if (data.tables[tableIdx].columns[i].key !== undefined) {
data.tables[tableIdx].columns[i].key = key
key++
}
})
}
@ -763,7 +762,7 @@ watch(modelRef, async () => {
</template>
<template #extra>
<a-tooltip bottom>
<NcTooltip bottom class="inline-block">
<template #title>
<span>{{ $t('activity.deleteTable') }}</span>
</template>
@ -773,7 +772,7 @@ watch(modelRef, async () => {
class="text-lg mr-8"
@click.stop="deleteTable(tableIdx)"
/>
</a-tooltip>
</NcTooltip>
</template>
<a-table
@ -801,10 +800,10 @@ watch(modelRef, async () => {
<template #bodyCell="{ column, record }">
<template v-if="column.key === 'source_column'">
<NcTooltip class="truncate"
><template #title>{{ record.srcCn }}</template
>{{ record.srcCn }}</NcTooltip
>
<NcTooltip class="truncate inline-block">
<template #title>{{ record.srcCn }}</template>
{{ record.srcCn }}
</NcTooltip>
</template>
<template v-else-if="column.key === 'destination_column'">
@ -815,10 +814,13 @@ watch(modelRef, async () => {
:filter-option="filterOption"
dropdown-class-name="nc-dropdown-filter-field"
>
<template #suffixIcon>
<GeneralIcon icon="arrowDown" class="text-current" />
</template>
<a-select-option v-for="(col, i) of columns" :key="i" :value="col.title">
<div class="flex items-center">
<component :is="getUIDTIcon(col.uidt)" />
<span class="ml-2">{{ col.title }}</span>
<div class="flex items-center gap-2">
<component :is="getUIDTIcon(col.uidt)" class="w-3.5 h-3.5" />
<span>{{ col.title }}</span>
</div>
</a-select-option>
</a-select>
@ -849,13 +851,13 @@ watch(modelRef, async () => {
<a-collapse-panel v-for="(table, tableIdx) of data.tables" :key="tableIdx">
<template #header>
<a-form-item v-bind="validateInfos[`tables.${tableIdx}.table_name`]" no-style>
<div class="flex flex-col w-full">
<div class="flex flex-col w-full mr-2">
<a-input
v-model:value="table.table_name"
class="font-weight-bold text-lg"
class="font-weight-bold text-lg !rounded-md"
size="large"
hide-details
:bordered="false"
:bordered="true"
@click.stop
@blur="handleEditableTnChange(tableIdx)"
@keydown.enter="handleEditableTnChange(tableIdx)"
@ -869,17 +871,17 @@ watch(modelRef, async () => {
</template>
<template #extra>
<a-tooltip bottom>
<NcTooltip bottom class="inline-block mr-8">
<template #title>
<span>{{ $t('activity.deleteTable') }}</span>
</template>
<component
:is="iconMap.delete"
:is="iconMap.deleteListItem"
v-if="data.tables.length > 1"
class="text-lg mr-8"
class="text-lg"
@click.stop="deleteTable(tableIdx)"
/>
</a-tooltip>
</NcTooltip>
</template>
<a-table
v-if="table.columns && table.columns.length"
@ -916,117 +918,71 @@ watch(modelRef, async () => {
<template #bodyCell="{ column, record }">
<template v-if="column.key === 'column_name'">
<a-form-item v-bind="validateInfos[`tables.${tableIdx}.columns.${record.key}.${column.key}`]">
<a-input :ref="(el: HTMLInputElement) => (inputRefs[record.key] = el)" v-model:value="record.title" />
<a-input
:ref="(el: HTMLInputElement) => (inputRefs[record.key] = el)"
v-model:value="record.title"
class="!rounded-md"
/>
</a-form-item>
</template>
<template v-else-if="column.key === 'uidt'">
<a-form-item v-bind="validateInfos[`tables.${tableIdx}.columns.${record.key}.${column.key}`]">
<a-select
v-model:value="record.uidt"
class="w-52"
show-search
:filter-option="filterOption"
dropdown-class-name="nc-dropdown-template-uidt"
@change="handleUIDTChange(record, table)"
>
<a-select-option v-for="(option, i) of uiTypeOptions" :key="i" :value="option.value">
<a-tooltip placement="right">
<template v-if="isSelectDisabled(option.label, table.columns[record.key]?._disableSelect)" #title>
{{
$t('msg.tooLargeFieldEntity', {
entity: option.label,
})
}}
</template>
{{ option.label }}
</a-tooltip>
</a-select-option>
</a-select>
</a-form-item>
</template>
<template v-else-if="column.key === 'dtxp'">
<a-form-item v-if="isSelect(record)">
<a-input v-model:value="record.dtxp" />
<NcTooltip :disabled="importDataOnly">
<template #title>
{{ $t('tooltip.useFieldEditMenuToConfigFieldType') }}
</template>
<a-select
v-model:value="record.uidt"
class="w-52"
show-search
:filter-option="filterOption"
dropdown-class-name="nc-dropdown-template-uidt"
:disabled="!importDataOnly"
@change="handleUIDTChange(record, table)"
>
<template #suffixIcon>
<GeneralIcon icon="arrowDown" class="text-current" />
</template>
<a-select-option v-for="(option, i) of uiTypeOptions" :key="i" :value="option.value">
<div class="flex items-center gap-2">
<component :is="getUIDTIcon(UITypes[option.value])" class="h-3.5 w-3.5" />
<NcTooltip placement="right" :disabled="!importDataOnly" show-on-truncate-only>
<template v-if="isSelectDisabled(option.label, table.columns[record.key]?._disableSelect)" #title>
{{
$t('msg.tooLargeFieldEntity', {
entity: option.label,
})
}}
</template>
{{ option.label }}
</NcTooltip>
</div>
</a-select-option>
</a-select>
</NcTooltip>
</a-form-item>
</template>
<template v-if="column.key === 'action'">
<a-tooltip v-if="record.key === 0">
<template #title>
<span>{{ $t('general.primaryValue') }}</span>
</template>
<div class="flex items-center float-right mr-4">
<mdi-key-star class="text-lg" />
</div>
</a-tooltip>
<a-tooltip v-else>
<NcTooltip class="inline-block">
<template #title>
<span>{{ $t('activity.column.delete') }}</span>
</template>
<a-button type="text" @click="deleteTableColumn(tableIdx, record.key)">
<div class="flex items-center">
<component :is="iconMap.delete" class="text-lg" />
</div>
</a-button>
</a-tooltip>
<NcButton
type="text"
size="small"
:disabled="table.columns.length === 1"
@click="deleteTableColumn(tableIdx, record.key)"
>
<component :is="iconMap.deleteListItem" />
</NcButton>
</NcTooltip>
</template>
</template>
</a-table>
<div class="mt-5 flex gap-2 justify-center">
<a-tooltip bottom>
<template #title>
<span>{{ $t('activity.column.addNumber') }}</span>
</template>
<a-button class="group" @click="addNewColumnRow(tableIdx, 'Number')">
<div class="flex items-center">
<component :is="iconMap.number" class="group-hover:!text-accent flex text-lg" />
</div>
</a-button>
</a-tooltip>
<a-tooltip bottom>
<template #title>
<span>{{ $t('activity.column.addSingleLineText') }}</span>
</template>
<a-button class="group" @click="addNewColumnRow(tableIdx, 'SingleLineText')">
<div class="flex items-center">
<component :is="iconMap.text" class="group-hover:!text-accent text-lg" />
</div>
</a-button>
</a-tooltip>
<a-tooltip bottom>
<template #title>
<span>{{ $t('activity.column.addLongText') }}</span>
</template>
<a-button class="group" @click="addNewColumnRow(tableIdx, 'LongText')">
<div class="flex items-center">
<component :is="iconMap.longText" class="group-hover:!text-accent text-lg" />
</div>
</a-button>
</a-tooltip>
<a-tooltip bottom>
<template #title>
<span>{{ $t('activity.column.addOther') }}</span>
</template>
<a-button class="group" @click="addNewColumnRow(tableIdx, 'SingleLineText')">
<div class="flex items-center gap-1">
<component :is="iconMap.plus" class="group-hover:!text-accent text-lg" />
</div>
</a-button>
</a-tooltip>
</div>
</a-collapse-panel>
</a-collapse>
</a-form>
@ -1051,4 +1007,11 @@ watch(modelRef, async () => {
}
}
}
:deep(.ant-collapse-header) {
@apply !items-center;
& > div {
@apply flex;
}
}
</style>

11
packages/nc-gui/components/template/utils.ts
Unescape View File

@ -13,14 +13,15 @@ export const tableColumns: (Omit<ColumnGroupType<any>, 'children'> & { dataIndex
key: 'uidt',
width: 250,
},
{
name: 'Select Option',
key: 'dtxp',
},
// {
// name: 'Select Option',
// key: 'dtxp',
// },
{
name: 'Action',
key: 'action',
align: 'right',
align: 'center',
width: 40,
},
]

10
packages/nc-gui/helpers/parsers/ExcelTemplateAdapter.ts
Unescape View File

@ -142,9 +142,11 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
r: +this.config.firstRowAsHeaders,
})
const cellProps = ws[cellId] || {}
column.uidt = excelTypeToUidt[cellProps.t] || UITypes.SingleLineText
column.uidt = this.config.importDataOnly
? excelTypeToUidt[cellProps.t] || UITypes.SingleLineText
: UITypes.SingleLineText
if (column.uidt === UITypes.SingleLineText) {
if (column.uidt === UITypes.SingleLineText && this.config.importDataOnly) {
// check for long text
if (isMultiLineTextType(rows, col)) {
column.uidt = UITypes.LongText
@ -238,7 +240,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
for (const row of rows.slice(1)) {
const rowData: Record<string, any> = {}
for (let i = 0; i < table.columns.length; i++) {
if (!this.config.autoSelectFieldTypes) {
if (!this.config.autoSelectFieldTypes || !this.config.importDataOnly) {
// take raw data instead of data parsed by xlsx
const cellId = this.xlsx.utils.encode_cell({
c: range.s.c + i,
@ -257,7 +259,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
const cellObj = ws[cellId]
rowData[table.columns[i].column_name] =
(cellObj && cellObj.w && cellObj.w.replace(/[^\d.]+/g, '')) || row[i]
(cellObj && typeof cellObj?.w === 'string' && cellObj.w.replace(/[^\d.]+/g, '')) || row[i]
} else if (table.columns[i].uidt === UITypes.SingleSelect || table.columns[i].uidt === UITypes.MultiSelect) {
rowData[table.columns[i].column_name] = (row[i] || '').toString().trim() || null
} else if (table.columns[i].uidt === UITypes.Date) {

5
packages/nc-gui/lang/en.json
Unescape View File

@ -483,7 +483,7 @@
"quickImportExcel": "Quick Import - Excel",
"quickImportJSON": "Quick Import - JSON",
"jsonEditor": "JSON Editor",
"comingSoon": "Coming Soon",
"comingSoon": "Coming soon",
"advancedSettings": "Advanced Settings",
"codeSnippet": "Code Snippet",
"keyboardShortcut": "Keyboard Shortcuts",
@ -1149,7 +1149,8 @@
"clientCA": "Select CA file",
"changeIconColour": "Change icon colour",
"preFillFormInfo": "To get a prefilled link, make sure you’ve filled the necessary fields in the form view builder.",
"surveyFormInfo": "Form mode with one field per page"
"surveyFormInfo": "Form mode with one field per page",
"useFieldEditMenuToConfigFieldType": "Use field edit menu for type conversions after file is imported"
},
"placeholder": {
"selectSlackChannels": "Select Slack channels",

5
tests/playwright/tests/db/features/import.spec.ts
Unescape View File

@ -37,9 +37,10 @@ test.describe('Import', () => {
});
test('Excel', async () => {
// Everything will be mapped with `SingleLineText` as we disabled auto column mapping
const col = [
{ type: 'Number', name: 'number' },
{ type: 'Decimal', name: 'float' },
{ type: 'SingleLineText', name: 'number' },
{ type: 'SingleLineText', name: 'float' },
{ type: 'SingleLineText', name: 'text' },
];
const expected = [

Write Preview
Loading…
Cancel
Save