feat: excel datatype parsing(in progress)

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js

@@ -1,6 +1,7 @@
 import XLSX from 'xlsx'
 import { UITypes } from '~/components/project/spreadsheet/helpers/uiTypes'
 import TemplateGenerator from '~/components/import/templateParsers/TemplateGenerator'
+import { getCheckboxValue, isCheckboxType } from '~/components/import/templateParsers/parserHelpers'
 
 const excelTypeToUidt = {
   d: UITypes.DateTime,
@@ -31,41 +32,75 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
       const table = { tn: sheet, columns: [] }
       this.data[sheet] = []
       const ws = this.wb.Sheets[sheet]
-      const rows = XLSX.utils.sheet_to_json(ws, { header: 1 })
+      const range = XLSX.utils.decode_range(ws['!ref'])
+      const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false })
 
       for (let col = 0; col < rows[0].length; col++) {
         const column = {
-          cn: (rows[0][col] || `field${col + 1}`).replace(/\./, '_')
+          cn: (rows[0][col] ||
+            `field${col + 1}`).replace(/\./, '_')
         }
 
-        const cellProps = ws[`${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`] || {}
+        // const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`;
-
+        const cellId = XLSX.utils.encode_cell({
+          c: range.s.c + col,
+          r: 1
+        })
+        const cellProps = ws[cellId] || {}
         column.uidt = excelTypeToUidt[cellProps.t] || UITypes.SingleLineText
 
         // todo: optimize
         if (column.uidt === UITypes.SingleLineText) {
           // check for long text
-          if (rows.some(r => (r[col] || '').toString().length > 255)) {
+          if (rows.some(r =>
+            (r[col] || '').toString().match(/[\r\n]/) ||
+            (r[col] || '').toString().length > 255)
+          ) {
             column.uidt = UITypes.LongText
           } else {
-            const vals = rows.slice(1).map(r => r[col]).filter(v => v !== null && v !== undefined)
+            let vals = rows.slice(1).map(r => r[col])
-
+
-            // check column is multi or single select by comparing unique values
+            const checkboxType = isCheckboxType(vals)
-            if (vals.some(v => v && v.toString().includes(','))) {
+            if (checkboxType.length === 1) {
-              const flattenedVals = vals.flatMap(v => v ? v.toString().split(',') : [])
+              column.uidt = UITypes.Checkbox
-              const uniqueVals = new Set(flattenedVals)
-              if (flattenedVals.length > uniqueVals.size && uniqueVals.size <= flattenedVals.length / 10) {
-                column.uidt = UITypes.MultiSelect
-                column.dtxp = [...uniqueVals].join(',')
-              }
             } else {
-              const uniqueVals = new Set(vals)
+              vals = vals.filter(v => v !== null && v !== undefined)
-              if (vals.length > uniqueVals.size && uniqueVals.size <= vals.length / 10) {
+
-                column.uidt = UITypes.SingleSelect
+              // check column is multi or single select by comparing unique values
-                column.dtxp = [...uniqueVals].join(',')
+              if (vals.some(v => v && v.toString().includes(','))) {
+                const flattenedVals = vals.flatMap(v => v ? v.toString().split(',') : [])
+                const uniqueVals = new Set(flattenedVals)
+                if (flattenedVals.length > uniqueVals.size && uniqueVals.size <= flattenedVals.length / 10) {
+                  column.uidt = UITypes.MultiSelect
+                  column.dtxp = [...uniqueVals].join(',')
+                }
+              } else {
+                const uniqueVals = new Set(vals)
+                if (vals.length > uniqueVals.size && uniqueVals.size <= vals.length / 10) {
+                  column.uidt = UITypes.SingleSelect
+                  column.dtxp = [...uniqueVals].join(',')
+                }
               }
             }
           }
+        } else if (column.uidt === UITypes.Number) {
+          if (rows.slice(1, 500).some((v) => {
+            return v && v[col] && parseInt(+v[col]) !== +v[col]
+          })) {
+            column.uidt = UITypes.Decimal
+          }
+          if (rows.slice(1, 500).every((v, i) => {
+            const cellId = XLSX.utils.encode_cell({
+              c: range.s.c + col,
+              r: i + 2
+            })
+
+            const cellObj = ws[cellId]
+
+            return !cellObj || (cellObj.w && cellObj.w.startsWith('$'))
+          })) {
+            column.uidt = UITypes.Currency
+          }
         }
 
         table.columns.push(column)
@@ -74,8 +109,12 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
       for (const row of rows.slice(1)) {
         const rowData = {}
         for (let i = 0; i < table.columns.length; i++) {
-          // toto: do parsing if necessary based on type
+          if (table.columns[i].uidt === UITypes.Checkbox) {
-          rowData[table.columns[i].cn] = row[i]
+            rowData[table.columns[i].cn] = getCheckboxValue(row[i])
+          } else {
+            // toto: do parsing if necessary based on type
+            rowData[table.columns[i].cn] = row[i]
+          }
         }
         this.data[sheet].push(rowData)
       }

packages/nc-gui/components/import/templateParsers/parserHelpers.js

@@ -0,0 +1,31 @@
+const booleanOptions = [
+  { checked: true, unchecked: false },
+  { x: true, '': false },
+  { yes: true, no: false },
+  { y: true, n: false },
+  { 1: true, 0: false },
+  { '[x]': true, '[]': false, '[ ]': false },
+  { '☑': true, '': false },
+  { '✅': true, '': false },
+  { '✓': true, '': false },
+  { '✔': true, '': false },
+  { enabled: true, disabled: false },
+  { on: true, off: false },
+  { done: true, '': false }
+]
+const aggBooleanOptions = booleanOptions.reduce((obj, o) => ({ ...obj, ...o }), {})
+export const isCheckboxType = (values, col = '') => {
+  let options = booleanOptions
+  for (let i = 0; i < values.length; i++) {
+    let val = col ? values[i][col] : values[i]
+    val = val === null || val === undefined ? '' : val
+    options = options.filter(v => val in v)
+    if (!options.length) {
+      return false
+    }
+  }
+  return options
+}
+export const getCheckboxValue = (value) => {
+  return value && aggBooleanOptions[value]
+}