Browse Source

Merge pull request #3811 from nocodb/feat/import-optimization

feat: import optimization
pull/4039/head
աɨռɢӄաօռɢ 2 years ago committed by GitHub
parent
commit
5a0fa932d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 72
      packages/nc-gui/components/template/Editor.vue
  2. 93
      packages/nc-gui/package-lock.json
  3. 2
      packages/nc-gui/package.json
  4. 9
      packages/nc-gui/utils/dateTimeUtils.ts
  5. 53
      packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts

72
packages/nc-gui/components/template/Editor.vue

@ -1,6 +1,8 @@
<script setup lang="ts">
import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc'
import type { ColumnType, TableType } from 'nocodb-sdk'
import { UITypes, isVirtualCol } from 'nocodb-sdk'
import { UITypes, isSystemColumn, isVirtualCol } from 'nocodb-sdk'
import { srcDestMappingColumns, tableColumns } from './utils'
import {
Empty,
@ -11,6 +13,7 @@ import {
createEventHook,
extractSdkResponseErrorMsg,
fieldRequiredValidator,
getDateFormat,
getUIDTIcon,
inject,
message,
@ -30,6 +33,8 @@ const { quickImportType, projectTemplate, importData, importColumns, importOnly,
const emit = defineEmits(['import'])
dayjs.extend(utc)
const { t } = useI18n()
interface Props {
@ -89,7 +94,11 @@ const uiTypeOptions = ref<Option[]>(
const srcDestMapping = ref<Record<string, any>[]>([])
const data = reactive<{ title: string | null; name: string; tables: (TableType & { ref_table_name: string })[] }>({
const data = reactive<{
title: string | null
name: string
tables: (TableType & { ref_table_name: string; columns: (ColumnType & { _disableSelect?: boolean })[] })[]
}>({
title: null,
name: 'Project Name',
tables: [],
@ -217,14 +226,29 @@ function setEditableTn(tableIdx: number, val: boolean) {
}
function remapColNames(batchData: any[], columns: ColumnType[]) {
const dateFormatMap: Record<number, string> = {}
return batchData.map((data) =>
(columns || []).reduce(
(aggObj, col: Record<string, any>) => ({
(columns || []).reduce((aggObj, col: Record<string, any>) => {
let d = data[col.ref_column_name || col.column_name]
if (col.uidt === UITypes.Date && d) {
let dateFormat
if (col.key in dateFormatMap) {
dateFormat = dateFormatMap[col.key]
} else {
dateFormat = getDateFormat(d)
dateFormatMap[col.key] = dateFormat
}
d = dayjs(d).utc().format(dateFormat)
} else if (col.uidt === UITypes.DateTime && d) {
d = dayjs(data[col.ref_column_name || col.column_name])
.utc()
.format('YYYY-MM-DD HH:mm')
}
return {
...aggObj,
[col.column_name]: data[col.ref_column_name || col.column_name],
}),
{},
),
[col.column_name]: d,
}
}, {}),
)
}
@ -428,16 +452,22 @@ async function importTemplate() {
}
}
// set pk & rqd if ID is provided
if (table.columns) {
for (const column of table.columns) {
// set pk & rqd if ID is provided
if (column.column_name?.toLowerCase() === 'id' && !('pk' in column)) {
column.pk = true
column.rqd = true
break
}
if (!isSystemColumn(column) && column.uidt !== UITypes.SingleSelect && column.uidt !== UITypes.MultiSelect) {
// delete dtxp if the final data type is not single & multi select
// e.g. import -> detect as single / multi select -> switch to SingleLineText
// the correct dtxp will be generated during column creation
delete column.dtxp
}
}
}
const tableMeta = await $api.dbTable.create(project?.value?.id as string, {
table_name: table.ref_table_name,
// leave title empty to get a generated one based on ref_table_name
@ -537,6 +567,10 @@ function handleEditableTnChange(idx: number) {
}
setEditableTn(idx, false)
}
function isSelectDisabled(uidt: string, disableSelect = false) {
return (uidt === UITypes.SingleSelect || uidt === UITypes.MultiSelect) && disableSelect
}
</script>
<template>
@ -649,7 +683,6 @@ function handleEditableTnChange(idx: number) {
<mdi-delete-outline v-if="data.tables.length > 1" class="text-lg mr-8" @click.stop="deleteTable(tableIdx)" />
</a-tooltip>
</template>
<a-table
v-if="table.columns && table.columns.length"
class="template-form"
@ -696,10 +729,23 @@ function handleEditableTnChange(idx: number) {
v-model:value="record.uidt"
class="w-52"
show-search
:options="uiTypeOptions"
:filter-option="filterOption"
dropdown-class-name="nc-dropdown-template-uidt"
/>
>
<a-select-option
v-for="(option, i) of uiTypeOptions"
:key="i"
:value="option.value"
:disabled="isSelectDisabled(option.label, table.columns[record.key]?._disableSelect)"
>
<a-tooltip placement="right">
<template v-if="isSelectDisabled(option.label, table.columns[record.key]?._disableSelect)" #title>
The field is too large to be converted to {{ option.label }}
</template>
{{ option.label }}
</a-tooltip>
</a-select-option>
</a-select>
</a-form-item>
</template>

93
packages/nc-gui/package-lock.json generated

@ -33,7 +33,7 @@
"vue-github-button": "^3.0.3",
"vue-i18n": "^9.2.2",
"vuedraggable": "^4.1.0",
"xlsx": "^0.17.3"
"xlsx": "^0.18.5"
},
"devDependencies": {
"@antfu/eslint-config": "^0.26.0",
@ -4220,16 +4220,9 @@
}
},
"node_modules/adler-32": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.2.0.tgz",
"integrity": "sha512-/vUqU/UY4MVeFsg+SsK6c+/05RZXIHZMGJA+PX5JyWI0ZRcBpupnRuPLU/NXXoFwMYCPCoxIfElM2eS+DUXCqQ==",
"dependencies": {
"exit-on-epipe": "~1.0.1",
"printj": "~1.1.0"
},
"bin": {
"adler32": "bin/adler32.njs"
},
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
"engines": {
"node": ">=0.8"
}
@ -4882,14 +4875,6 @@
"node": ">=0.8"
}
},
"node_modules/cfb/node_modules/adler-32": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/chai": {
"version": "4.3.6",
"resolved": "https://registry.npmjs.org/chai/-/chai-4.3.6.tgz",
@ -7964,14 +7949,6 @@
"url": "https://github.com/sindresorhus/execa?sponsor=1"
}
},
"node_modules/exit-on-epipe": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz",
"integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/expand-template": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@ -13103,17 +13080,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/printj": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz",
"integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==",
"bin": {
"printj": "bin/printj.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
@ -16433,14 +16399,14 @@
}
},
"node_modules/xlsx": {
"version": "0.17.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.5.tgz",
"integrity": "sha512-lXNU0TuYsvElzvtI6O7WIVb9Zar1XYw7Xb3VAx2wn8N/n0whBYrCnHMxtFyIiUU1Wjf09WzmLALDfBO5PqTb1g==",
"version": "0.18.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
"integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
"dependencies": {
"adler-32": "~1.2.0",
"cfb": "^1.1.4",
"adler-32": "~1.3.0",
"cfb": "~1.2.1",
"codepage": "~1.15.0",
"crc-32": "~1.2.0",
"crc-32": "~1.2.1",
"ssf": "~0.11.2",
"wmf": "~1.0.1",
"word": "~0.3.0"
@ -19663,13 +19629,9 @@
"peer": true
},
"adler-32": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.2.0.tgz",
"integrity": "sha512-/vUqU/UY4MVeFsg+SsK6c+/05RZXIHZMGJA+PX5JyWI0ZRcBpupnRuPLU/NXXoFwMYCPCoxIfElM2eS+DUXCqQ==",
"requires": {
"exit-on-epipe": "~1.0.1",
"printj": "~1.1.0"
}
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A=="
},
"agent-base": {
"version": "6.0.2",
@ -20132,13 +20094,6 @@
"requires": {
"adler-32": "~1.3.0",
"crc-32": "~1.2.0"
},
"dependencies": {
"adler-32": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A=="
}
}
},
"chai": {
@ -22346,11 +22301,6 @@
"strip-final-newline": "^2.0.0"
}
},
"exit-on-epipe": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz",
"integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw=="
},
"expand-template": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@ -26135,11 +26085,6 @@
"integrity": "sha512-6UqkYefdogmzqAZWzJ7laYeJnaXDy2/J+ZqiiMtS7t7OfpXWTlaeGMwX8U6EFvPV/YWWEKRkS8hKS4k60WHTOg==",
"dev": true
},
"printj": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz",
"integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ=="
},
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
@ -28599,14 +28544,14 @@
"requires": {}
},
"xlsx": {
"version": "0.17.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.17.5.tgz",
"integrity": "sha512-lXNU0TuYsvElzvtI6O7WIVb9Zar1XYw7Xb3VAx2wn8N/n0whBYrCnHMxtFyIiUU1Wjf09WzmLALDfBO5PqTb1g==",
"version": "0.18.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
"integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
"requires": {
"adler-32": "~1.2.0",
"cfb": "^1.1.4",
"adler-32": "~1.3.0",
"cfb": "~1.2.1",
"codepage": "~1.15.0",
"crc-32": "~1.2.0",
"crc-32": "~1.2.1",
"ssf": "~0.11.2",
"wmf": "~1.0.1",
"word": "~0.3.0"

2
packages/nc-gui/package.json

@ -42,7 +42,7 @@
"vue-github-button": "^3.0.3",
"vue-i18n": "^9.2.2",
"vuedraggable": "^4.1.0",
"xlsx": "^0.17.3"
"xlsx": "^0.18.5"
},
"devDependencies": {
"@antfu/eslint-config": "^0.26.0",

9
packages/nc-gui/utils/dateTimeUtils.ts

@ -42,3 +42,12 @@ export function validateDateWithUnknownFormat(v: string) {
}
return res
}
export function getDateFormat(v: string) {
for (const format of dateFormats) {
if (dayjs(v, format, true).isValid()) {
return format
}
}
return 'YYYY/MM/DD'
}

53
packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts

@ -72,6 +72,9 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
parse() {
const tableNamePrefixRef: Record<string, any> = {}
// TODO: find the upper bound / make it configurable
const maxSelectOptionsAllowed = 64
for (let i = 0; i < this.wb.SheetNames.length; i++) {
const columnNamePrefixRef: Record<string, any> = { id: 0 }
const sheet: any = this.wb.SheetNames[i]
@ -128,8 +131,6 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
ref_column_name: cn,
}
table.columns.push(column)
// const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`;
const cellId = this.xlsx.utils.encode_cell({
c: range.s.c + col,
@ -153,11 +154,8 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
if (checkboxType.length === 1) {
column.uidt = UITypes.Checkbox
} else {
// todo: optimize
// check column is multi or single select by comparing unique values
// todo:
if (vals.some((v: any) => v && v.toString().includes(','))) {
let flattenedVals = vals.flatMap((v: any) =>
const flattenedVals = vals.flatMap((v: any) =>
v
? v
.toString()
@ -165,19 +163,41 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
.split(/\s*,\s*/)
: [],
)
const uniqueVals = (flattenedVals = flattenedVals.filter(
(v: any, i: any, arr: any) => i === arr.findIndex((v1: any) => v.toLowerCase() === v1.toLowerCase()),
))
if (flattenedVals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(flattenedVals.length / 2)) {
column.uidt = UITypes.MultiSelect
// TODO: handle case sensitive case
const uniqueVals = [...new Set(flattenedVals.map((v: any) => v.toString().trim().toLowerCase()))]
if (uniqueVals.length > maxSelectOptionsAllowed) {
// too many options are detected, convert the column to SingleLineText instead
column.uidt = UITypes.SingleLineText
// _disableSelect is used to disable the <a-select-option/> in TemplateEditor
column._disableSelect = true
} else {
// assume the column type is multiple select if there are repeated values
if (flattenedVals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(flattenedVals.length / 2)) {
column.uidt = UITypes.MultiSelect
}
// set dtxp here so that users can have the options even they switch the type from other types to MultiSelect
// once it's set, dtxp needs to be reset if the final column type is not MultiSelect
column.dtxp = `'${uniqueVals.join("','")}'`
}
} else {
const uniqueVals = vals
.map((v: any) => v.toString().trim())
.filter((v: any, i: any, arr: any) => i === arr.findIndex((v1: any) => v.toLowerCase() === v1.toLowerCase()))
if (vals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(vals.length / 2)) {
column.uidt = UITypes.SingleSelect
// TODO: handle case sensitive case
const uniqueVals = [...new Set(vals.map((v: any) => v.toString().trim().toLowerCase()))]
if (uniqueVals.length > maxSelectOptionsAllowed) {
// too many options are detected, convert the column to SingleLineText instead
column.uidt = UITypes.SingleLineText
// _disableSelect is used to disable the <a-select-option/> in TemplateEditor
column._disableSelect = true
} else {
// assume the column type is single select if there are repeated values
// once it's set, dtxp needs to be reset if the final column type is not Single Select
if (vals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(vals.length / 2)) {
column.uidt = UITypes.SingleSelect
}
// set dtxp here so that users can have the options even they switch the type from other types to SingleSelect
// once it's set, dtxp needs to be reset if the final column type is not SingleSelect
column.dtxp = `'${uniqueVals.join("','")}'`
}
}
@ -220,6 +240,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
column.uidt = UITypes.Date
}
}
table.columns.push(column)
}
let rowIndex = 0

Loading…
Cancel
Save