feat: excel parser - handle date, datetime, duplicate column

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nc-gui/components/import/excelImport.vue

@@ -72,11 +72,28 @@
           </v-tab-item>
         </v-tabs>
 
-        <!--        <div class="my-4 text-center grey&#45;&#45;text">-->
+        <div class="px-4 pb-2">
-        <!--          OR-->
+          <div class="d-flex">
-        <!--        </div>-->
+            <v-spacer />
+            <span class="caption pointer grey--text" @click="showMore = !showMore">{{ showMore ? 'Hide' : 'Show' }} more
+              <v-icon small>mdi-menu-{{ showMore ? 'up':'down' }}</v-icon>
+            </span>
+          </div>
+          <div class="mb-2 pt-2 nc-excel-import-options" :style="{ maxHeight: showMore ? '100px' : '0'}">
+            <p />
 
-        <!--          <drop-or-select-file />-->
+            <v-text-field
+              v-model="parserConfig.maxRowsToParse"
+              style="max-width: 250px"
+              class="caption mx-auto"
+              dense
+              persistent-hint
+              hint="# of rows to parse to infer data type"
+              outlined
+              type="number"
+            />
+          </div>
+        </div>
       </v-card>
     </v-dialog>
 
@@ -113,8 +130,6 @@
           <template #toolbar>
             <v-spacer />
             <create-project-from-template-btn
-              :loader-message.sync="loaderMessage"
-              :progress.sync="progress"
               :template-data="templateData"
               :import-data="importData"
             />
@@ -122,23 +137,6 @@
         </template-editor>
       </v-card>
     </v-dialog>
-
-    <v-overlay :value="loaderMessage" z-index="99999" opacity=".9">
-      <div class="d-flex flex-column align-center">
-        <v-progress-circular
-          v-if="progress !== null "
-          :rotate="360"
-          :size="100"
-          :width="15"
-          :value="progress"
-        >
-          {{ progress }}%
-        </v-progress-circular>
-
-        <v-progress-circular v-else indeterminate size="100" width="15" class="mb-10" />
-        <span class="title">{{ loaderMessage }}</span>
-      </div>
-    </v-overlay>
   </div>
 </template>
 
@@ -163,9 +161,11 @@ export default {
       templateData: null,
       importData: null,
       dragOver: false,
-      loaderMessage: null,
+      url: '',
-      progress: null,
+      showMore: false,
-      url: ''
+      parserConfig: {
+        maxRowsToParse: 500
+      }
     }
   },
   computed: {
@@ -194,10 +194,10 @@ export default {
       }
     },
     async _file(file) {
-      this.loaderMessage = 'Loading excel file'
+      this.$store.commit('loader/MutMessage', 'Loading excel file')
       let i = 0
       const int = setInterval(() => {
-        this.loaderMessage = `Loading excel file${'.'.repeat(++i % 4)}`
+        this.$store.commit('loader/MutMessage', `Loading excel file${'.'.repeat(++i % 4)}`)
       }, 1000)
 
       this.dropOrUpload = false
@@ -206,17 +206,18 @@ export default {
       reader.onload = async(e) => {
         const ab = e.target.result
         await this.parseAndExtractData('file', ab, file.name)
-        this.loaderMessage = null
+        this.$store.commit('loader/MutMessage', null)
+
         clearInterval(int)
       }
 
       const handleEvent = (event) => {
-        this.loaderMessage = `${event.type}: ${event.loaded} bytes transferred`
+        this.$store.commit('loader/MutMessage', `${event.type}: ${event.loaded} bytes transferred`)
       }
 
       reader.addEventListener('progress', handleEvent)
       reader.onerror = () => {
-        this.loaderMessage = null
+        this.$store.commit('loader/MutClear')
       }
       reader.readAsArrayBuffer(file)
     },
@@ -225,10 +226,10 @@ export default {
       let templateGenerator
       switch (type) {
         case 'file':
-          templateGenerator = new ExcelTemplateAdapter(name, val)
+          templateGenerator = new ExcelTemplateAdapter(name, val, this.parserConfig)
           break
         case 'url':
-          templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store)
+          templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store, this.parserConfig)
           break
       }
       await templateGenerator.init()
@@ -264,19 +265,22 @@ export default {
     },
 
     async loadUrl() {
-      if (!this.$refs.form.validate()) { return }
+      if (!this.$refs.form.validate()) {
+        return
+      }
+
+      this.$store.commit('loader/MutMessage', 'Loading excel file from url')
 
-      this.loaderMessage = 'Loading excel file from url'
       let i = 0
       const int = setInterval(() => {
-        this.loaderMessage = `Loading excel file${'.'.repeat(++i % 4)}`
+        this.$store.commit('loader/MutMessage', `Loading excel file${'.'.repeat(++i % 4)}`)
       }, 1000)
 
       this.dropOrUpload = false
 
       await this.parseAndExtractData('url', this.url, '')
       clearInterval(int)
-      this.loaderMessage = null
+      this.$store.commit('loader/MutClear')
     }
 
   }
@@ -291,11 +295,16 @@ export default {
   border: 2px dashed #ddd;
 }
 
-.nc-excel-import-tab-item{
+.nc-excel-import-tab-item {
   min-height: 400px;
   padding: 20px;
   display: flex;
   align-items: stretch;
-width:100%;
+  width: 100%;
+}
+
+.nc-excel-import-options{
+  transition:.4s max-height;
+  overflow: hidden;
 }
 </style>

packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js

@@ -11,8 +11,12 @@ const excelTypeToUidt = {
 }
 
 export default class ExcelTemplateAdapter extends TemplateGenerator {
-  constructor(name, ab) {
+  constructor(name, ab, parserConfig = {}) {
     super()
+    this.config = {
+      maxRowsToParse: 500,
+      ...parserConfig
+    }
     this.name = name
     this.excelData = ab
     this.project = {
@@ -23,22 +27,39 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
   }
 
   async init() {
-    this.wb = XLSX.read(new Uint8Array(this.excelData), { type: 'array' })
+    this.wb = XLSX.read(new Uint8Array(this.excelData), { type: 'array', cellText: true, cellDates: true })
   }
 
   parse() {
+    const tableNamePrefixRef = {}
     for (let i = 0; i < this.wb.SheetNames.length; i++) {
+      const columnNamePrefixRef = {}
       const sheet = this.wb.SheetNames[i]
-      const table = { tn: sheet, columns: [] }
+      let tn = sheet
+      if (tn in tableNamePrefixRef) {
+        tn = `${tn}${++tableNamePrefixRef[tn]}`
+      } else {
+        tableNamePrefixRef[tn] = 0
+      }
+
+      const table = { tn, columns: [] }
       this.data[sheet] = []
       const ws = this.wb.Sheets[sheet]
       const range = XLSX.utils.decode_range(ws['!ref'])
-      const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false })
+      const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false, cellDates: true })
 
       for (let col = 0; col < rows[0].length; col++) {
+        let cn = (rows[0][col] ||
+          `field${col + 1}`).replace(/\./, '_')
+
+        if (cn in columnNamePrefixRef) {
+          cn = `${cn}${++columnNamePrefixRef[cn]}`
+        } else {
+          columnNamePrefixRef[cn] = 0
+        }
+
         const column = {
-          cn: (rows[0][col] ||
+          cn
-            `field${col + 1}`).replace(/\./, '_')
         }
 
         // const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`;
@@ -101,6 +122,19 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
           })) {
             column.uidt = UITypes.Currency
           }
+        } else if (column.uidt === UITypes.DateTime) {
+          if (rows.slice(1, 500).every((v, i) => {
+            const cellId = XLSX.utils.encode_cell({
+              c: range.s.c + col,
+              r: i + 2
+            })
+
+            const cellObj = ws[cellId]
+
+            return !cellObj || (cellObj.w && cellObj.w.split(' ').length === 1)
+          })) {
+            column.uidt = UITypes.Date
+          }
         }
 
         table.columns.push(column)

packages/nc-gui/components/import/templateParsers/ExcelUrlTemplateAdapter.js

@@ -1,9 +1,9 @@
 import ExcelTemplateAdapter from '~/components/import/templateParsers/ExcelTemplateAdapter'
 
 export default class ExcelUrlTemplateAdapter extends ExcelTemplateAdapter {
-  constructor(url, $store) {
+  constructor(url, $store, parserConfig) {
     const name = url.split('/').pop()
-    super(name, null)
+    super(name, null, parserConfig)
     this.url = url
     this.$store = $store
   }

packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue

@@ -44,9 +44,7 @@ export default {
   props: {
     loading: Boolean,
     templateData: [Array, Object],
-    importData: [Array, Object],
+    importData: [Array, Object]
-    loaderMessage: String,
-    progress: Number
   },
   data() {
     return {
@@ -84,7 +82,7 @@ export default {
       try {
         const interv = setInterval(() => {
           this.loaderMessagesIndex = this.loaderMessagesIndex < this.loaderMessages.length - 1 ? this.loaderMessagesIndex + 1 : 6
-          this.$emit('update:loaderMessage', this.loaderMessages[this.loaderMessagesIndex])
+          this.$store.commit('loader/MutMessage', this.loaderMessages[this.loaderMessagesIndex])
         }, 1000)
 
         const result = await this.$store.dispatch('sqlMgr/ActSqlOp', [null, 'projectCreateByWebWithXCDB', {
@@ -97,11 +95,10 @@ export default {
 
         clearInterval(interv)
         if (this.importData) {
-          this.$emit('update:loaderMessage', 'Importing excel data to project')
+          this.$store.commit('loader/MutMessage', 'Importing excel data to project')
           await this.importDataToProject({ projectId: result.id, projectType, prefix: result.prefix })
         }
-
+        this.$store.commit('loader/MutMessage', null)
-        this.$emit('update:loaderMessage', null)
 
         this.projectReloading = false
 
@@ -123,7 +120,6 @@ export default {
 
       let total = 0; let progress = 0
 
-      console.log(this.importData)
       await Promise.all(Object.entries(this.importData).map(v => (async([table, data]) => {
         await this.$store.dispatch('meta/ActLoadMeta', {
           tn: `${prefix}${table}`, project_id: projectId
@@ -136,14 +132,13 @@ export default {
         })
         total += data.length
         for (let i = 0; i < data.length; i += 500) {
-          this.$emit('update:loaderMessage', `Importing data : ${progress}/${total}`)
+          this.$store.commit('loader/MutMessage', `Importing data : ${progress}/${total}`)
-          this.$emit('update:progress', Math.round(progress && 100 * progress / total))
+          this.$store.commit('loader/MutProgress', Math.round(progress && 100 * progress / total))
           const batchData = data.slice(i, i + 500)
           await api.insertBulk(batchData)
           progress += batchData.length
         }
-
+        this.$store.commit('loader/MutClear')
-        this.$emit('update:progress', null)
       })(v)))
     }
   }

packages/nc-gui/helpers/sqlUi/SqliteUi.js

@@ -776,7 +776,7 @@ export class SqliteUi {
         colProp.dt = 'varchar'
         break
       case 'Date':
-        colProp.dt = 'varchar'
+        colProp.dt = 'date'
 
         break
       case 'Year':

packages/nocodb/src/lib/sqlUi/SqliteUi.ts

@@ -788,7 +788,7 @@ export class SqliteUi {
         colProp.dt = 'varchar';
         break;
       case 'Date':
-        colProp.dt = 'varchar';
+        colProp.dt = 'date';
 
         break;
       case 'Year':

packages/nocodb/src/lib/sqlUi/index.ts

@@ -1,3 +1,5 @@
+// todo: move to a common library
+
 export * from './MysqlUi';
 export * from './PgUi';
 export * from './MssqlUi';