Browse Source

Merge pull request #3811 from nocodb/feat/import-optimization

feat: import optimization
pull/4039/head
աɨռɢӄաօռɢ 2 years ago committed by GitHub
parent
commit
5a0fa932d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 72
      packages/nc-gui/components/template/Editor.vue
  2. 93
      packages/nc-gui/package-lock.json
  3. 2
      packages/nc-gui/package.json
  4. 9
      packages/nc-gui/utils/dateTimeUtils.ts
  5. 45
      packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts

72
packages/nc-gui/components/template/Editor.vue

@ -1,6 +1,8 @@
<script setup lang="ts"> <script setup lang="ts">
import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc'
import type { ColumnType, TableType } from 'nocodb-sdk' import type { ColumnType, TableType } from 'nocodb-sdk'
import { UITypes, isVirtualCol } from 'nocodb-sdk' import { UITypes, isSystemColumn, isVirtualCol } from 'nocodb-sdk'
import { srcDestMappingColumns, tableColumns } from './utils' import { srcDestMappingColumns, tableColumns } from './utils'
import { import {
Empty, Empty,
@ -11,6 +13,7 @@ import {
createEventHook, createEventHook,
extractSdkResponseErrorMsg, extractSdkResponseErrorMsg,
fieldRequiredValidator, fieldRequiredValidator,
getDateFormat,
getUIDTIcon, getUIDTIcon,
inject, inject,
message, message,
@ -30,6 +33,8 @@ const { quickImportType, projectTemplate, importData, importColumns, importOnly,
const emit = defineEmits(['import']) const emit = defineEmits(['import'])
dayjs.extend(utc)
const { t } = useI18n() const { t } = useI18n()
interface Props { interface Props {
@ -89,7 +94,11 @@ const uiTypeOptions = ref<Option[]>(
const srcDestMapping = ref<Record<string, any>[]>([]) const srcDestMapping = ref<Record<string, any>[]>([])
const data = reactive<{ title: string | null; name: string; tables: (TableType & { ref_table_name: string })[] }>({ const data = reactive<{
title: string | null
name: string
tables: (TableType & { ref_table_name: string; columns: (ColumnType & { _disableSelect?: boolean })[] })[]
}>({
title: null, title: null,
name: 'Project Name', name: 'Project Name',
tables: [], tables: [],
@ -217,14 +226,29 @@ function setEditableTn(tableIdx: number, val: boolean) {
} }
function remapColNames(batchData: any[], columns: ColumnType[]) { function remapColNames(batchData: any[], columns: ColumnType[]) {
const dateFormatMap: Record<number, string> = {}
return batchData.map((data) => return batchData.map((data) =>
(columns || []).reduce( (columns || []).reduce((aggObj, col: Record<string, any>) => {
(aggObj, col: Record<string, any>) => ({ let d = data[col.ref_column_name || col.column_name]
if (col.uidt === UITypes.Date && d) {
let dateFormat
if (col.key in dateFormatMap) {
dateFormat = dateFormatMap[col.key]
} else {
dateFormat = getDateFormat(d)
dateFormatMap[col.key] = dateFormat
}
d = dayjs(d).utc().format(dateFormat)
} else if (col.uidt === UITypes.DateTime && d) {
d = dayjs(data[col.ref_column_name || col.column_name])
.utc()
.format('YYYY-MM-DD HH:mm')
}
return {
...aggObj, ...aggObj,
[col.column_name]: data[col.ref_column_name || col.column_name], [col.column_name]: d,
}), }
{}, }, {}),
),
) )
} }
@ -428,16 +452,22 @@ async function importTemplate() {
} }
} }
// set pk & rqd if ID is provided
if (table.columns) { if (table.columns) {
for (const column of table.columns) { for (const column of table.columns) {
// set pk & rqd if ID is provided
if (column.column_name?.toLowerCase() === 'id' && !('pk' in column)) { if (column.column_name?.toLowerCase() === 'id' && !('pk' in column)) {
column.pk = true column.pk = true
column.rqd = true column.rqd = true
break }
if (!isSystemColumn(column) && column.uidt !== UITypes.SingleSelect && column.uidt !== UITypes.MultiSelect) {
// delete dtxp if the final data type is not single & multi select
// e.g. import -> detect as single / multi select -> switch to SingleLineText
// the correct dtxp will be generated during column creation
delete column.dtxp
} }
} }
} }
const tableMeta = await $api.dbTable.create(project?.value?.id as string, { const tableMeta = await $api.dbTable.create(project?.value?.id as string, {
table_name: table.ref_table_name, table_name: table.ref_table_name,
// leave title empty to get a generated one based on ref_table_name // leave title empty to get a generated one based on ref_table_name
@ -537,6 +567,10 @@ function handleEditableTnChange(idx: number) {
} }
setEditableTn(idx, false) setEditableTn(idx, false)
} }
function isSelectDisabled(uidt: string, disableSelect = false) {
return (uidt === UITypes.SingleSelect || uidt === UITypes.MultiSelect) && disableSelect
}
</script> </script>
<template> <template>
@ -649,7 +683,6 @@ function handleEditableTnChange(idx: number) {
<mdi-delete-outline v-if="data.tables.length > 1" class="text-lg mr-8" @click.stop="deleteTable(tableIdx)" /> <mdi-delete-outline v-if="data.tables.length > 1" class="text-lg mr-8" @click.stop="deleteTable(tableIdx)" />
</a-tooltip> </a-tooltip>
</template> </template>
<a-table <a-table
v-if="table.columns && table.columns.length" v-if="table.columns && table.columns.length"
class="template-form" class="template-form"
@ -696,10 +729,23 @@ function handleEditableTnChange(idx: number) {
v-model:value="record.uidt" v-model:value="record.uidt"
class="w-52" class="w-52"
show-search show-search
:options="uiTypeOptions"
:filter-option="filterOption" :filter-option="filterOption"
dropdown-class-name="nc-dropdown-template-uidt" dropdown-class-name="nc-dropdown-template-uidt"
/> >
<a-select-option
v-for="(option, i) of uiTypeOptions"
:key="i"
:value="option.value"
:disabled="isSelectDisabled(option.label, table.columns[record.key]?._disableSelect)"
>
<a-tooltip placement="right">
<template v-if="isSelectDisabled(option.label, table.columns[record.key]?._disableSelect)" #title>
The field is too large to be converted to {{ option.label }}
</template>
{{ option.label }}
</a-tooltip>
</a-select-option>
</a-select>
</a-form-item> </a-form-item>
</template> </template>

93
packages/nc-gui/package-lock.json generated

@ -33,7 +33,7 @@
"vue-github-button": "^3.0.3", "vue-github-button": "^3.0.3",
"vue-i18n": "^9.2.2", "vue-i18n": "^9.2.2",
"vuedraggable": "^4.1.0", "vuedraggable": "^4.1.0",
"xlsx": "^0.17.3" "xlsx": "^0.18.5"
}, },
"devDependencies": { "devDependencies": {
"@antfu/eslint-config": "^0.26.0", "@antfu/eslint-config": "^0.26.0",
@ -4220,16 +4220,9 @@
} }
}, },
"node_modules/adler-32": { "node_modules/adler-32": {
"version": "1.2.0", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.2.0.tgz", "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-/vUqU/UY4MVeFsg+SsK6c+/05RZXIHZMGJA+PX5JyWI0ZRcBpupnRuPLU/NXXoFwMYCPCoxIfElM2eS+DUXCqQ==", "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
"dependencies": {
"exit-on-epipe": "~1.0.1",
"printj": "~1.1.0"
},
"bin": {
"adler32": "bin/adler32.njs"
},
"engines": { "engines": {
"node": ">=0.8" "node": ">=0.8"
} }
@ -4882,14 +4875,6 @@
"node": ">=0.8" "node": ">=0.8"
} }
}, },
"node_modules/cfb/node_modules/adler-32": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/chai": { "node_modules/chai": {
"version": "4.3.6", "version": "4.3.6",
"resolved": "https://registry.npmjs.org/chai/-/chai-4.3.6.tgz", "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.6.tgz",
@ -7964,14 +7949,6 @@
"url": "https://github.com/sindresorhus/execa?sponsor=1" "url": "https://github.com/sindresorhus/execa?sponsor=1"
} }
}, },
"node_modules/exit-on-epipe": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz",
"integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/expand-template": { "node_modules/expand-template": {
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@ -13103,17 +13080,6 @@
"url": "https://github.com/sponsors/sindresorhus" "url": "https://github.com/sponsors/sindresorhus"
} }
}, },
"node_modules/printj": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz",
"integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==",
"bin": {
"printj": "bin/printj.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/process-nextick-args": { "node_modules/process-nextick-args": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
@ -16433,14 +16399,14 @@
} }
}, },
"node_modules/xlsx": { "node_modules/xlsx": {
"version": "0.17.5", "version": "0.18.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.5.tgz", "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
"integrity": "sha512-lXNU0TuYsvElzvtI6O7WIVb9Zar1XYw7Xb3VAx2wn8N/n0whBYrCnHMxtFyIiUU1Wjf09WzmLALDfBO5PqTb1g==", "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
"dependencies": { "dependencies": {
"adler-32": "~1.2.0", "adler-32": "~1.3.0",
"cfb": "^1.1.4", "cfb": "~1.2.1",
"codepage": "~1.15.0", "codepage": "~1.15.0",
"crc-32": "~1.2.0", "crc-32": "~1.2.1",
"ssf": "~0.11.2", "ssf": "~0.11.2",
"wmf": "~1.0.1", "wmf": "~1.0.1",
"word": "~0.3.0" "word": "~0.3.0"
@ -19663,13 +19629,9 @@
"peer": true "peer": true
}, },
"adler-32": { "adler-32": {
"version": "1.2.0", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.2.0.tgz", "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-/vUqU/UY4MVeFsg+SsK6c+/05RZXIHZMGJA+PX5JyWI0ZRcBpupnRuPLU/NXXoFwMYCPCoxIfElM2eS+DUXCqQ==", "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A=="
"requires": {
"exit-on-epipe": "~1.0.1",
"printj": "~1.1.0"
}
}, },
"agent-base": { "agent-base": {
"version": "6.0.2", "version": "6.0.2",
@ -20132,13 +20094,6 @@
"requires": { "requires": {
"adler-32": "~1.3.0", "adler-32": "~1.3.0",
"crc-32": "~1.2.0" "crc-32": "~1.2.0"
},
"dependencies": {
"adler-32": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A=="
}
} }
}, },
"chai": { "chai": {
@ -22346,11 +22301,6 @@
"strip-final-newline": "^2.0.0" "strip-final-newline": "^2.0.0"
} }
}, },
"exit-on-epipe": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz",
"integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw=="
},
"expand-template": { "expand-template": {
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@ -26135,11 +26085,6 @@
"integrity": "sha512-6UqkYefdogmzqAZWzJ7laYeJnaXDy2/J+ZqiiMtS7t7OfpXWTlaeGMwX8U6EFvPV/YWWEKRkS8hKS4k60WHTOg==", "integrity": "sha512-6UqkYefdogmzqAZWzJ7laYeJnaXDy2/J+ZqiiMtS7t7OfpXWTlaeGMwX8U6EFvPV/YWWEKRkS8hKS4k60WHTOg==",
"dev": true "dev": true
}, },
"printj": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz",
"integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ=="
},
"process-nextick-args": { "process-nextick-args": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
@ -28599,14 +28544,14 @@
"requires": {} "requires": {}
}, },
"xlsx": { "xlsx": {
"version": "0.17.5", "version": "0.18.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.5.tgz", "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
"integrity": "sha512-lXNU0TuYsvElzvtI6O7WIVb9Zar1XYw7Xb3VAx2wn8N/n0whBYrCnHMxtFyIiUU1Wjf09WzmLALDfBO5PqTb1g==", "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
"requires": { "requires": {
"adler-32": "~1.2.0", "adler-32": "~1.3.0",
"cfb": "^1.1.4", "cfb": "~1.2.1",
"codepage": "~1.15.0", "codepage": "~1.15.0",
"crc-32": "~1.2.0", "crc-32": "~1.2.1",
"ssf": "~0.11.2", "ssf": "~0.11.2",
"wmf": "~1.0.1", "wmf": "~1.0.1",
"word": "~0.3.0" "word": "~0.3.0"

2
packages/nc-gui/package.json

@ -42,7 +42,7 @@
"vue-github-button": "^3.0.3", "vue-github-button": "^3.0.3",
"vue-i18n": "^9.2.2", "vue-i18n": "^9.2.2",
"vuedraggable": "^4.1.0", "vuedraggable": "^4.1.0",
"xlsx": "^0.17.3" "xlsx": "^0.18.5"
}, },
"devDependencies": { "devDependencies": {
"@antfu/eslint-config": "^0.26.0", "@antfu/eslint-config": "^0.26.0",

9
packages/nc-gui/utils/dateTimeUtils.ts

@ -42,3 +42,12 @@ export function validateDateWithUnknownFormat(v: string) {
} }
return res return res
} }
export function getDateFormat(v: string) {
for (const format of dateFormats) {
if (dayjs(v, format, true).isValid()) {
return format
}
}
return 'YYYY/MM/DD'
}

45
packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts

@ -72,6 +72,9 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
parse() { parse() {
const tableNamePrefixRef: Record<string, any> = {} const tableNamePrefixRef: Record<string, any> = {}
// TODO: find the upper bound / make it configurable
const maxSelectOptionsAllowed = 64
for (let i = 0; i < this.wb.SheetNames.length; i++) { for (let i = 0; i < this.wb.SheetNames.length; i++) {
const columnNamePrefixRef: Record<string, any> = { id: 0 } const columnNamePrefixRef: Record<string, any> = { id: 0 }
const sheet: any = this.wb.SheetNames[i] const sheet: any = this.wb.SheetNames[i]
@ -128,8 +131,6 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
ref_column_name: cn, ref_column_name: cn,
} }
table.columns.push(column)
// const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`; // const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`;
const cellId = this.xlsx.utils.encode_cell({ const cellId = this.xlsx.utils.encode_cell({
c: range.s.c + col, c: range.s.c + col,
@ -153,11 +154,8 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
if (checkboxType.length === 1) { if (checkboxType.length === 1) {
column.uidt = UITypes.Checkbox column.uidt = UITypes.Checkbox
} else { } else {
// todo: optimize
// check column is multi or single select by comparing unique values
// todo:
if (vals.some((v: any) => v && v.toString().includes(','))) { if (vals.some((v: any) => v && v.toString().includes(','))) {
let flattenedVals = vals.flatMap((v: any) => const flattenedVals = vals.flatMap((v: any) =>
v v
? v ? v
.toString() .toString()
@ -165,19 +163,41 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
.split(/\s*,\s*/) .split(/\s*,\s*/)
: [], : [],
) )
const uniqueVals = (flattenedVals = flattenedVals.filter(
(v: any, i: any, arr: any) => i === arr.findIndex((v1: any) => v.toLowerCase() === v1.toLowerCase()), // TODO: handle case sensitive case
)) const uniqueVals = [...new Set(flattenedVals.map((v: any) => v.toString().trim().toLowerCase()))]
if (uniqueVals.length > maxSelectOptionsAllowed) {
// too many options are detected, convert the column to SingleLineText instead
column.uidt = UITypes.SingleLineText
// _disableSelect is used to disable the <a-select-option/> in TemplateEditor
column._disableSelect = true
} else {
// assume the column type is multiple select if there are repeated values
if (flattenedVals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(flattenedVals.length / 2)) { if (flattenedVals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(flattenedVals.length / 2)) {
column.uidt = UITypes.MultiSelect column.uidt = UITypes.MultiSelect
}
// set dtxp here so that users can have the options even they switch the type from other types to MultiSelect
// once it's set, dtxp needs to be reset if the final column type is not MultiSelect
column.dtxp = `'${uniqueVals.join("','")}'` column.dtxp = `'${uniqueVals.join("','")}'`
} }
} else { } else {
const uniqueVals = vals // TODO: handle case sensitive case
.map((v: any) => v.toString().trim()) const uniqueVals = [...new Set(vals.map((v: any) => v.toString().trim().toLowerCase()))]
.filter((v: any, i: any, arr: any) => i === arr.findIndex((v1: any) => v.toLowerCase() === v1.toLowerCase()))
if (uniqueVals.length > maxSelectOptionsAllowed) {
// too many options are detected, convert the column to SingleLineText instead
column.uidt = UITypes.SingleLineText
// _disableSelect is used to disable the <a-select-option/> in TemplateEditor
column._disableSelect = true
} else {
// assume the column type is single select if there are repeated values
// once it's set, dtxp needs to be reset if the final column type is not Single Select
if (vals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(vals.length / 2)) { if (vals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(vals.length / 2)) {
column.uidt = UITypes.SingleSelect column.uidt = UITypes.SingleSelect
}
// set dtxp here so that users can have the options even they switch the type from other types to SingleSelect
// once it's set, dtxp needs to be reset if the final column type is not SingleSelect
column.dtxp = `'${uniqueVals.join("','")}'` column.dtxp = `'${uniqueVals.join("','")}'`
} }
} }
@ -220,6 +240,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
column.uidt = UITypes.Date column.uidt = UITypes.Date
} }
} }
table.columns.push(column)
} }
let rowIndex = 0 let rowIndex = 0

Loading…
Cancel
Save