diff --git a/packages/nc-gui/components/import/excelImport.vue b/packages/nc-gui/components/import/excelImport.vue index facfe7560c..04712e0fc6 100644 --- a/packages/nc-gui/components/import/excelImport.vue +++ b/packages/nc-gui/components/import/excelImport.vue @@ -72,11 +72,28 @@ - - - +
+
+ + {{ showMore ? 'Hide' : 'Show' }} more + mdi-menu-{{ showMore ? 'up':'down' }} + +
+
+

- + +

+
@@ -113,8 +130,6 @@ @@ -163,9 +161,11 @@ export default { templateData: null, importData: null, dragOver: false, - loaderMessage: null, - progress: null, - url: '' + url: '', + showMore: false, + parserConfig: { + maxRowsToParse: 500 + } } }, computed: { @@ -194,10 +194,10 @@ export default { } }, async _file(file) { - this.loaderMessage = 'Loading excel file' + this.$store.commit('loader/MutMessage', 'Loading excel file') let i = 0 const int = setInterval(() => { - this.loaderMessage = `Loading excel file${'.'.repeat(++i % 4)}` + this.$store.commit('loader/MutMessage', `Loading excel file${'.'.repeat(++i % 4)}`) }, 1000) this.dropOrUpload = false @@ -206,17 +206,18 @@ export default { reader.onload = async(e) => { const ab = e.target.result await this.parseAndExtractData('file', ab, file.name) - this.loaderMessage = null + this.$store.commit('loader/MutMessage', null) + clearInterval(int) } const handleEvent = (event) => { - this.loaderMessage = `${event.type}: ${event.loaded} bytes transferred` + this.$store.commit('loader/MutMessage', `${event.type}: ${event.loaded} bytes transferred`) } reader.addEventListener('progress', handleEvent) reader.onerror = () => { - this.loaderMessage = null + this.$store.commit('loader/MutClear') } reader.readAsArrayBuffer(file) }, @@ -225,10 +226,10 @@ export default { let templateGenerator switch (type) { case 'file': - templateGenerator = new ExcelTemplateAdapter(name, val) + templateGenerator = new ExcelTemplateAdapter(name, val, this.parserConfig) break case 'url': - templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store) + templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store, this.parserConfig) break } await templateGenerator.init() @@ -264,19 +265,22 @@ export default { }, async loadUrl() { - if (!this.$refs.form.validate()) { return } + if (!this.$refs.form.validate()) { + return + } + + this.$store.commit('loader/MutMessage', 'Loading excel file from url') - this.loaderMessage = 'Loading excel file from url' let i = 0 const int = setInterval(() => { - this.loaderMessage = `Loading excel file${'.'.repeat(++i % 4)}` + this.$store.commit('loader/MutMessage', `Loading excel file${'.'.repeat(++i % 4)}`) }, 1000) this.dropOrUpload = false await this.parseAndExtractData('url', this.url, '') clearInterval(int) - this.loaderMessage = null + this.$store.commit('loader/MutClear') } } @@ -291,11 +295,16 @@ export default { border: 2px dashed #ddd; } -.nc-excel-import-tab-item{ +.nc-excel-import-tab-item { min-height: 400px; padding: 20px; display: flex; align-items: stretch; -width:100%; + width: 100%; +} + +.nc-excel-import-options{ + transition:.4s max-height; + overflow: hidden; } diff --git a/packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js b/packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js index faa61749a2..f95ab9f37d 100644 --- a/packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js +++ b/packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js @@ -11,8 +11,12 @@ const excelTypeToUidt = { } export default class ExcelTemplateAdapter extends TemplateGenerator { - constructor(name, ab) { + constructor(name, ab, parserConfig = {}) { super() + this.config = { + maxRowsToParse: 500, + ...parserConfig + } this.name = name this.excelData = ab this.project = { @@ -23,22 +27,39 @@ export default class ExcelTemplateAdapter extends TemplateGenerator { } async init() { - this.wb = XLSX.read(new Uint8Array(this.excelData), { type: 'array' }) + this.wb = XLSX.read(new Uint8Array(this.excelData), { type: 'array', cellText: true, cellDates: true }) } parse() { + const tableNamePrefixRef = {} for (let i = 0; i < this.wb.SheetNames.length; i++) { + const columnNamePrefixRef = {} const sheet = this.wb.SheetNames[i] - const table = { tn: sheet, columns: [] } + let tn = sheet + if (tn in tableNamePrefixRef) { + tn = `${tn}${++tableNamePrefixRef[tn]}` + } else { + tableNamePrefixRef[tn] = 0 + } + + const table = { tn, columns: [] } this.data[sheet] = [] const ws = this.wb.Sheets[sheet] const range = XLSX.utils.decode_range(ws['!ref']) - const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false }) + const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false, cellDates: true }) for (let col = 0; col < rows[0].length; col++) { + let cn = (rows[0][col] || + `field${col + 1}`).replace(/\./, '_') + + if (cn in columnNamePrefixRef) { + cn = `${cn}${++columnNamePrefixRef[cn]}` + } else { + columnNamePrefixRef[cn] = 0 + } + const column = { - cn: (rows[0][col] || - `field${col + 1}`).replace(/\./, '_') + cn } // const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`; @@ -101,6 +122,19 @@ export default class ExcelTemplateAdapter extends TemplateGenerator { })) { column.uidt = UITypes.Currency } + } else if (column.uidt === UITypes.DateTime) { + if (rows.slice(1, 500).every((v, i) => { + const cellId = XLSX.utils.encode_cell({ + c: range.s.c + col, + r: i + 2 + }) + + const cellObj = ws[cellId] + + return !cellObj || (cellObj.w && cellObj.w.split(' ').length === 1) + })) { + column.uidt = UITypes.Date + } } table.columns.push(column) diff --git a/packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js b/packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js index dfbc9987d2..6c8e774f76 100644 --- a/packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js +++ b/packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js @@ -1,9 +1,9 @@ import ExcelTemplateAdapter from '~/components/import/templateParsers/ExcelTemplateAdapter' export default class ExcelUrlTemplateAdapter extends ExcelTemplateAdapter { - constructor(url, $store) { + constructor(url, $store, parserConfig) { const name = url.split('/').pop() - super(name, null) + super(name, null, parserConfig) this.url = url this.$store = $store } diff --git a/packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue b/packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue index d96dfc3ff0..427ee9092a 100644 --- a/packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue +++ b/packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue @@ -44,9 +44,7 @@ export default { props: { loading: Boolean, templateData: [Array, Object], - importData: [Array, Object], - loaderMessage: String, - progress: Number + importData: [Array, Object] }, data() { return { @@ -84,7 +82,7 @@ export default { try { const interv = setInterval(() => { this.loaderMessagesIndex = this.loaderMessagesIndex < this.loaderMessages.length - 1 ? this.loaderMessagesIndex + 1 : 6 - this.$emit('update:loaderMessage', this.loaderMessages[this.loaderMessagesIndex]) + this.$store.commit('loader/MutMessage', this.loaderMessages[this.loaderMessagesIndex]) }, 1000) const result = await this.$store.dispatch('sqlMgr/ActSqlOp', [null, 'projectCreateByWebWithXCDB', { @@ -97,11 +95,10 @@ export default { clearInterval(interv) if (this.importData) { - this.$emit('update:loaderMessage', 'Importing excel data to project') + this.$store.commit('loader/MutMessage', 'Importing excel data to project') await this.importDataToProject({ projectId: result.id, projectType, prefix: result.prefix }) } - - this.$emit('update:loaderMessage', null) + this.$store.commit('loader/MutMessage', null) this.projectReloading = false @@ -123,7 +120,6 @@ export default { let total = 0; let progress = 0 - console.log(this.importData) await Promise.all(Object.entries(this.importData).map(v => (async([table, data]) => { await this.$store.dispatch('meta/ActLoadMeta', { tn: `${prefix}${table}`, project_id: projectId @@ -136,14 +132,13 @@ export default { }) total += data.length for (let i = 0; i < data.length; i += 500) { - this.$emit('update:loaderMessage', `Importing data : ${progress}/${total}`) - this.$emit('update:progress', Math.round(progress && 100 * progress / total)) + this.$store.commit('loader/MutMessage', `Importing data : ${progress}/${total}`) + this.$store.commit('loader/MutProgress', Math.round(progress && 100 * progress / total)) const batchData = data.slice(i, i + 500) await api.insertBulk(batchData) progress += batchData.length } - - this.$emit('update:progress', null) + this.$store.commit('loader/MutClear') })(v))) } } diff --git a/packages/nc-gui/helpers/sqlUi/SqliteUi.js b/packages/nc-gui/helpers/sqlUi/SqliteUi.js index 26fc3dca01..253f2d7fb6 100644 --- a/packages/nc-gui/helpers/sqlUi/SqliteUi.js +++ b/packages/nc-gui/helpers/sqlUi/SqliteUi.js @@ -776,7 +776,7 @@ export class SqliteUi { colProp.dt = 'varchar' break case 'Date': - colProp.dt = 'varchar' + colProp.dt = 'date' break case 'Year': diff --git a/packages/nocodb/src/lib/sqlUi/SqliteUi.ts b/packages/nocodb/src/lib/sqlUi/SqliteUi.ts index 1f39cb17b5..0ced7d0373 100644 --- a/packages/nocodb/src/lib/sqlUi/SqliteUi.ts +++ b/packages/nocodb/src/lib/sqlUi/SqliteUi.ts @@ -788,7 +788,7 @@ export class SqliteUi { colProp.dt = 'varchar'; break; case 'Date': - colProp.dt = 'varchar'; + colProp.dt = 'date'; break; case 'Year': diff --git a/packages/nocodb/src/lib/sqlUi/index.ts b/packages/nocodb/src/lib/sqlUi/index.ts index fcf4440516..57d4af6936 100644 --- a/packages/nocodb/src/lib/sqlUi/index.ts +++ b/packages/nocodb/src/lib/sqlUi/index.ts @@ -1,3 +1,5 @@ +// todo: move to a common library + export * from './MysqlUi'; export * from './PgUi'; export * from './MssqlUi';