Browse Source

feat: excel parser - handle date, datetime, duplicate column

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/789/head
Pranav C 3 years ago
parent
commit
cc9956df1f
  1. 87
      packages/nc-gui/components/import/excelImport.vue
  2. 46
      packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js
  3. 4
      packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js
  4. 19
      packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue
  5. 2
      packages/nc-gui/helpers/sqlUi/SqliteUi.js
  6. 2
      packages/nocodb/src/lib/sqlUi/SqliteUi.ts
  7. 2
      packages/nocodb/src/lib/sqlUi/index.ts

87
packages/nc-gui/components/import/excelImport.vue

@ -72,11 +72,28 @@
</v-tab-item> </v-tab-item>
</v-tabs> </v-tabs>
<!-- <div class="my-4 text-center grey&#45;&#45;text">--> <div class="px-4 pb-2">
<!-- OR--> <div class="d-flex">
<!-- </div>--> <v-spacer />
<span class="caption pointer grey--text" @click="showMore = !showMore">{{ showMore ? 'Hide' : 'Show' }} more
<v-icon small>mdi-menu-{{ showMore ? 'up':'down' }}</v-icon>
</span>
</div>
<div class="mb-2 pt-2 nc-excel-import-options" :style="{ maxHeight: showMore ? '100px' : '0'}">
<p />
<!-- <drop-or-select-file />--> <v-text-field
v-model="parserConfig.maxRowsToParse"
style="max-width: 250px"
class="caption mx-auto"
dense
persistent-hint
hint="# of rows to parse to infer data type"
outlined
type="number"
/>
</div>
</div>
</v-card> </v-card>
</v-dialog> </v-dialog>
@ -113,8 +130,6 @@
<template #toolbar> <template #toolbar>
<v-spacer /> <v-spacer />
<create-project-from-template-btn <create-project-from-template-btn
:loader-message.sync="loaderMessage"
:progress.sync="progress"
:template-data="templateData" :template-data="templateData"
:import-data="importData" :import-data="importData"
/> />
@ -122,23 +137,6 @@
</template-editor> </template-editor>
</v-card> </v-card>
</v-dialog> </v-dialog>
<v-overlay :value="loaderMessage" z-index="99999" opacity=".9">
<div class="d-flex flex-column align-center">
<v-progress-circular
v-if="progress !== null "
:rotate="360"
:size="100"
:width="15"
:value="progress"
>
{{ progress }}%
</v-progress-circular>
<v-progress-circular v-else indeterminate size="100" width="15" class="mb-10" />
<span class="title">{{ loaderMessage }}</span>
</div>
</v-overlay>
</div> </div>
</template> </template>
@ -163,9 +161,11 @@ export default {
templateData: null, templateData: null,
importData: null, importData: null,
dragOver: false, dragOver: false,
loaderMessage: null, url: '',
progress: null, showMore: false,
url: '' parserConfig: {
maxRowsToParse: 500
}
} }
}, },
computed: { computed: {
@ -194,10 +194,10 @@ export default {
} }
}, },
async _file(file) { async _file(file) {
this.loaderMessage = 'Loading excel file' this.$store.commit('loader/MutMessage', 'Loading excel file')
let i = 0 let i = 0
const int = setInterval(() => { const int = setInterval(() => {
this.loaderMessage = `Loading excel file${'.'.repeat(++i % 4)}` this.$store.commit('loader/MutMessage', `Loading excel file${'.'.repeat(++i % 4)}`)
}, 1000) }, 1000)
this.dropOrUpload = false this.dropOrUpload = false
@ -206,17 +206,18 @@ export default {
reader.onload = async(e) => { reader.onload = async(e) => {
const ab = e.target.result const ab = e.target.result
await this.parseAndExtractData('file', ab, file.name) await this.parseAndExtractData('file', ab, file.name)
this.loaderMessage = null this.$store.commit('loader/MutMessage', null)
clearInterval(int) clearInterval(int)
} }
const handleEvent = (event) => { const handleEvent = (event) => {
this.loaderMessage = `${event.type}: ${event.loaded} bytes transferred` this.$store.commit('loader/MutMessage', `${event.type}: ${event.loaded} bytes transferred`)
} }
reader.addEventListener('progress', handleEvent) reader.addEventListener('progress', handleEvent)
reader.onerror = () => { reader.onerror = () => {
this.loaderMessage = null this.$store.commit('loader/MutClear')
} }
reader.readAsArrayBuffer(file) reader.readAsArrayBuffer(file)
}, },
@ -225,10 +226,10 @@ export default {
let templateGenerator let templateGenerator
switch (type) { switch (type) {
case 'file': case 'file':
templateGenerator = new ExcelTemplateAdapter(name, val) templateGenerator = new ExcelTemplateAdapter(name, val, this.parserConfig)
break break
case 'url': case 'url':
templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store) templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store, this.parserConfig)
break break
} }
await templateGenerator.init() await templateGenerator.init()
@ -264,19 +265,22 @@ export default {
}, },
async loadUrl() { async loadUrl() {
if (!this.$refs.form.validate()) { return } if (!this.$refs.form.validate()) {
return
}
this.$store.commit('loader/MutMessage', 'Loading excel file from url')
this.loaderMessage = 'Loading excel file from url'
let i = 0 let i = 0
const int = setInterval(() => { const int = setInterval(() => {
this.loaderMessage = `Loading excel file${'.'.repeat(++i % 4)}` this.$store.commit('loader/MutMessage', `Loading excel file${'.'.repeat(++i % 4)}`)
}, 1000) }, 1000)
this.dropOrUpload = false this.dropOrUpload = false
await this.parseAndExtractData('url', this.url, '') await this.parseAndExtractData('url', this.url, '')
clearInterval(int) clearInterval(int)
this.loaderMessage = null this.$store.commit('loader/MutClear')
} }
} }
@ -291,11 +295,16 @@ export default {
border: 2px dashed #ddd; border: 2px dashed #ddd;
} }
.nc-excel-import-tab-item{ .nc-excel-import-tab-item {
min-height: 400px; min-height: 400px;
padding: 20px; padding: 20px;
display: flex; display: flex;
align-items: stretch; align-items: stretch;
width:100%; width: 100%;
}
.nc-excel-import-options{
transition:.4s max-height;
overflow: hidden;
} }
</style> </style>

46
packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js

@ -11,8 +11,12 @@ const excelTypeToUidt = {
} }
export default class ExcelTemplateAdapter extends TemplateGenerator { export default class ExcelTemplateAdapter extends TemplateGenerator {
constructor(name, ab) { constructor(name, ab, parserConfig = {}) {
super() super()
this.config = {
maxRowsToParse: 500,
...parserConfig
}
this.name = name this.name = name
this.excelData = ab this.excelData = ab
this.project = { this.project = {
@ -23,22 +27,39 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
} }
async init() { async init() {
this.wb = XLSX.read(new Uint8Array(this.excelData), { type: 'array' }) this.wb = XLSX.read(new Uint8Array(this.excelData), { type: 'array', cellText: true, cellDates: true })
} }
parse() { parse() {
const tableNamePrefixRef = {}
for (let i = 0; i < this.wb.SheetNames.length; i++) { for (let i = 0; i < this.wb.SheetNames.length; i++) {
const columnNamePrefixRef = {}
const sheet = this.wb.SheetNames[i] const sheet = this.wb.SheetNames[i]
const table = { tn: sheet, columns: [] } let tn = sheet
if (tn in tableNamePrefixRef) {
tn = `${tn}${++tableNamePrefixRef[tn]}`
} else {
tableNamePrefixRef[tn] = 0
}
const table = { tn, columns: [] }
this.data[sheet] = [] this.data[sheet] = []
const ws = this.wb.Sheets[sheet] const ws = this.wb.Sheets[sheet]
const range = XLSX.utils.decode_range(ws['!ref']) const range = XLSX.utils.decode_range(ws['!ref'])
const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false }) const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false, cellDates: true })
for (let col = 0; col < rows[0].length; col++) { for (let col = 0; col < rows[0].length; col++) {
let cn = (rows[0][col] ||
`field${col + 1}`).replace(/\./, '_')
if (cn in columnNamePrefixRef) {
cn = `${cn}${++columnNamePrefixRef[cn]}`
} else {
columnNamePrefixRef[cn] = 0
}
const column = { const column = {
cn: (rows[0][col] || cn
`field${col + 1}`).replace(/\./, '_')
} }
// const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`; // const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`;
@ -101,6 +122,19 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
})) { })) {
column.uidt = UITypes.Currency column.uidt = UITypes.Currency
} }
} else if (column.uidt === UITypes.DateTime) {
if (rows.slice(1, 500).every((v, i) => {
const cellId = XLSX.utils.encode_cell({
c: range.s.c + col,
r: i + 2
})
const cellObj = ws[cellId]
return !cellObj || (cellObj.w && cellObj.w.split(' ').length === 1)
})) {
column.uidt = UITypes.Date
}
} }
table.columns.push(column) table.columns.push(column)

4
packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js

@ -1,9 +1,9 @@
import ExcelTemplateAdapter from '~/components/import/templateParsers/ExcelTemplateAdapter' import ExcelTemplateAdapter from '~/components/import/templateParsers/ExcelTemplateAdapter'
export default class ExcelUrlTemplateAdapter extends ExcelTemplateAdapter { export default class ExcelUrlTemplateAdapter extends ExcelTemplateAdapter {
constructor(url, $store) { constructor(url, $store, parserConfig) {
const name = url.split('/').pop() const name = url.split('/').pop()
super(name, null) super(name, null, parserConfig)
this.url = url this.url = url
this.$store = $store this.$store = $store
} }

19
packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue

@ -44,9 +44,7 @@ export default {
props: { props: {
loading: Boolean, loading: Boolean,
templateData: [Array, Object], templateData: [Array, Object],
importData: [Array, Object], importData: [Array, Object]
loaderMessage: String,
progress: Number
}, },
data() { data() {
return { return {
@ -84,7 +82,7 @@ export default {
try { try {
const interv = setInterval(() => { const interv = setInterval(() => {
this.loaderMessagesIndex = this.loaderMessagesIndex < this.loaderMessages.length - 1 ? this.loaderMessagesIndex + 1 : 6 this.loaderMessagesIndex = this.loaderMessagesIndex < this.loaderMessages.length - 1 ? this.loaderMessagesIndex + 1 : 6
this.$emit('update:loaderMessage', this.loaderMessages[this.loaderMessagesIndex]) this.$store.commit('loader/MutMessage', this.loaderMessages[this.loaderMessagesIndex])
}, 1000) }, 1000)
const result = await this.$store.dispatch('sqlMgr/ActSqlOp', [null, 'projectCreateByWebWithXCDB', { const result = await this.$store.dispatch('sqlMgr/ActSqlOp', [null, 'projectCreateByWebWithXCDB', {
@ -97,11 +95,10 @@ export default {
clearInterval(interv) clearInterval(interv)
if (this.importData) { if (this.importData) {
this.$emit('update:loaderMessage', 'Importing excel data to project') this.$store.commit('loader/MutMessage', 'Importing excel data to project')
await this.importDataToProject({ projectId: result.id, projectType, prefix: result.prefix }) await this.importDataToProject({ projectId: result.id, projectType, prefix: result.prefix })
} }
this.$store.commit('loader/MutMessage', null)
this.$emit('update:loaderMessage', null)
this.projectReloading = false this.projectReloading = false
@ -123,7 +120,6 @@ export default {
let total = 0; let progress = 0 let total = 0; let progress = 0
console.log(this.importData)
await Promise.all(Object.entries(this.importData).map(v => (async([table, data]) => { await Promise.all(Object.entries(this.importData).map(v => (async([table, data]) => {
await this.$store.dispatch('meta/ActLoadMeta', { await this.$store.dispatch('meta/ActLoadMeta', {
tn: `${prefix}${table}`, project_id: projectId tn: `${prefix}${table}`, project_id: projectId
@ -136,14 +132,13 @@ export default {
}) })
total += data.length total += data.length
for (let i = 0; i < data.length; i += 500) { for (let i = 0; i < data.length; i += 500) {
this.$emit('update:loaderMessage', `Importing data : ${progress}/${total}`) this.$store.commit('loader/MutMessage', `Importing data : ${progress}/${total}`)
this.$emit('update:progress', Math.round(progress && 100 * progress / total)) this.$store.commit('loader/MutProgress', Math.round(progress && 100 * progress / total))
const batchData = data.slice(i, i + 500) const batchData = data.slice(i, i + 500)
await api.insertBulk(batchData) await api.insertBulk(batchData)
progress += batchData.length progress += batchData.length
} }
this.$store.commit('loader/MutClear')
this.$emit('update:progress', null)
})(v))) })(v)))
} }
} }

2
packages/nc-gui/helpers/sqlUi/SqliteUi.js

@ -776,7 +776,7 @@ export class SqliteUi {
colProp.dt = 'varchar' colProp.dt = 'varchar'
break break
case 'Date': case 'Date':
colProp.dt = 'varchar' colProp.dt = 'date'
break break
case 'Year': case 'Year':

2
packages/nocodb/src/lib/sqlUi/SqliteUi.ts

@ -788,7 +788,7 @@ export class SqliteUi {
colProp.dt = 'varchar'; colProp.dt = 'varchar';
break; break;
case 'Date': case 'Date':
colProp.dt = 'varchar'; colProp.dt = 'date';
break; break;
case 'Year': case 'Year':

2
packages/nocodb/src/lib/sqlUi/index.ts

@ -1,3 +1,5 @@
// todo: move to a common library
export * from './MysqlUi'; export * from './MysqlUi';
export * from './PgUi'; export * from './PgUi';
export * from './MssqlUi'; export * from './MssqlUi';

Loading…
Cancel
Save