feat(gui-v2): ExcelTemplateAdapter

packages/nc-gui-v2/utils/parsers/ExcelTemplateAdapter.ts

@@ -0,0 +1,240 @@
+import XLSX from 'xlsx'
+import { UITypes } from 'nocodb-sdk'
+import TemplateGenerator from './TemplateGenerator'
+import { getCheckboxValue, isCheckboxType } from './parserHelpers'
+
+const excelTypeToUidt: Record<any, any> = {
+  d: UITypes.DateTime,
+  b: UITypes.Checkbox,
+  n: UITypes.Number,
+  s: UITypes.SingleLineText,
+}
+
+export default class ExcelTemplateAdapter extends TemplateGenerator {
+  config: Record<string, any>
+  name: string
+  excelData: any
+  project: Record<string, any>
+  data: Record<string, any>
+  wb: any
+  constructor(name = '', data = {}, parserConfig = {}) {
+    super()
+    this.config = {
+      maxRowsToParse: 500,
+      ...parserConfig,
+    }
+    this.name = name
+    this.excelData = data
+    this.project = {
+      title: this.name,
+      tables: [],
+    }
+    this.data = {}
+  }
+
+  async init() {
+    const options: Record<any, boolean> = {
+      cellText: true,
+      cellDates: true,
+    }
+    if (this.name.slice(-3) === 'csv') {
+      this.wb = XLSX.read(new TextDecoder().decode(new Uint8Array(this.excelData)), {
+        type: 'string',
+        ...options,
+      })
+    } else {
+      this.wb = XLSX.read(new Uint8Array(this.excelData), {
+        type: 'array',
+        ...options,
+      })
+    }
+  }
+
+  parse() {
+    const tableNamePrefixRef: any = {}
+    for (let i = 0; i < this.wb.SheetNames.length; i++) {
+      const columnNamePrefixRef: Record<any, any> = { id: 0 }
+      const sheet: any = this.wb.SheetNames[i]
+      let tn: string = (sheet || 'table').replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_').trim()
+
+      while (tn in tableNamePrefixRef) {
+        tn = `${tn}${++tableNamePrefixRef[tn]}`
+      }
+      tableNamePrefixRef[tn] = 0
+
+      const table: Record<string, any> = { table_name: tn, ref_table_name: tn, columns: [] }
+      this.data[tn] = []
+      const ws: any = this.wb.Sheets[sheet]
+      const range = XLSX.utils.decode_range(ws['!ref'])
+      let rows: any = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false, defval: null })
+
+      if (this.name.slice(-3) !== 'csv') {
+        // fix precision bug & timezone offset issues introduced by xlsx
+        const basedate = new Date(1899, 11, 30, 0, 0, 0)
+        // number of milliseconds since base date
+        const dnthresh = basedate.getTime() + (new Date().getTimezoneOffset() - basedate.getTimezoneOffset()) * 60000
+        // number of milliseconds in a day
+        const day_ms = 24 * 60 * 60 * 1000
+        // handle date1904 property
+        const fixImportedDate = (date: Date) => {
+          const parsed = XLSX.SSF.parse_date_code((date.getTime() - dnthresh) / day_ms, {
+            date1904: this.wb.Workbook.WBProps.date1904,
+          })
+          return new Date(parsed.y, parsed.m, parsed.d, parsed.H, parsed.M, parsed.S)
+        }
+        // fix imported date
+        rows = rows.map((r: any) =>
+          r.map((v: any) => {
+            return v instanceof Date ? fixImportedDate(v) : v
+          }),
+        )
+      }
+
+      const columnNameRowExist = +rows[0].every((v: any) => v === null || typeof v === 'string')
+
+      // const colLen = Math.max()
+      for (let col = 0; col < rows[0].length; col++) {
+        let cn: string = ((columnNameRowExist && rows[0] && rows[0][col] && rows[0][col].toString().trim()) || `field_${col + 1}`)
+          .replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_')
+          .trim()
+
+        while (cn in columnNamePrefixRef) {
+          cn = `${cn}${++columnNamePrefixRef[cn]}`
+        }
+        columnNamePrefixRef[cn] = 0
+
+        const column: Record<string, any> = {
+          column_name: cn,
+          ref_column_name: cn,
+        }
+
+        table.columns.push(column)
+
+        // const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`;
+        const cellId = XLSX.utils.encode_cell({
+          c: range.s.c + col,
+          r: columnNameRowExist,
+        })
+        const cellProps = ws[cellId] || {}
+        column.uidt = excelTypeToUidt[cellProps.t] || UITypes.SingleLineText
+
+        // todo: optimize
+        if (column.uidt === UITypes.SingleLineText) {
+          // check for long text
+          if (rows.some((r: any) => (r[col] || '').toString().match(/[\r\n]/) || (r[col] || '').toString().length > 255)) {
+            column.uidt = UITypes.LongText
+          } else {
+            const vals = rows
+              .slice(columnNameRowExist ? 1 : 0)
+              .map((r: any) => r[col])
+              .filter((v: any) => v !== null && v !== undefined && v.toString().trim() !== '')
+
+            const checkboxType = isCheckboxType(vals)
+            if (checkboxType.length === 1) {
+              column.uidt = UITypes.Checkbox
+            } else {
+              // todo: optimize
+              // check column is multi or single select by comparing unique values
+              // todo:
+              if (vals.some((v: any) => v && v.toString().includes(','))) {
+                let flattenedVals = vals.flatMap((v: any) =>
+                  v
+                    ? v
+                        .toString()
+                        .trim()
+                        .split(/\s*,\s*/)
+                    : [],
+                )
+                const uniqueVals = (flattenedVals = flattenedVals.filter(
+                  (v: any, i: any, arr: any) => i === arr.findIndex((v1: any) => v.toLowerCase() === v1.toLowerCase()),
+                ))
+                if (flattenedVals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(flattenedVals.length / 2)) {
+                  column.uidt = UITypes.MultiSelect
+                  column.dtxp = `'${uniqueVals.join("','")}'`
+                }
+              } else {
+                const uniqueVals = vals
+                  .map((v: any) => v.toString().trim())
+                  .filter((v: any, i: any, arr: any) => i === arr.findIndex((v1: any) => v.toLowerCase() === v1.toLowerCase()))
+                if (vals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(vals.length / 2)) {
+                  column.uidt = UITypes.SingleSelect
+                  column.dtxp = `'${uniqueVals.join("','")}'`
+                }
+              }
+            }
+          }
+        } else if (column.uidt === UITypes.Number) {
+          if (
+            rows.slice(1, this.config.maxRowsToParse).some((v: any) => {
+              return v && v[col] && parseInt(v[col]) !== +v[col]
+            })
+          ) {
+            column.uidt = UITypes.Decimal
+          }
+          if (
+            rows.slice(1, this.config.maxRowsToParse).every((v: any, i: any) => {
+              const cellId = XLSX.utils.encode_cell({
+                c: range.s.c + col,
+                r: i + columnNameRowExist,
+              })
+
+              const cellObj = ws[cellId]
+
+              return !cellObj || (cellObj.w && cellObj.w.startsWith('$'))
+            })
+          ) {
+            column.uidt = UITypes.Currency
+          }
+        } else if (column.uidt === UITypes.DateTime) {
+          if (
+            rows.slice(1, this.config.maxRowsToParse).every((v: any, i: any) => {
+              const cellId = XLSX.utils.encode_cell({
+                c: range.s.c + col,
+                r: i + columnNameRowExist,
+              })
+
+              const cellObj = ws[cellId]
+              return !cellObj || (cellObj.w && cellObj.w.split(' ').length === 1)
+            })
+          ) {
+            column.uidt = UITypes.Date
+          }
+        }
+      }
+
+      let rowIndex = 0
+      for (const row of rows.slice(1)) {
+        const rowData: Record<string, any> = {}
+        for (let i = 0; i < table.columns.length; i++) {
+          if (table.columns[i].uidt === UITypes.Checkbox) {
+            rowData[table.columns[i].column_name] = getCheckboxValue(row[i])
+          } else if (table.columns[i].uidt === UITypes.Currency) {
+            const cellId = XLSX.utils.encode_cell({
+              c: range.s.c + i,
+              r: rowIndex + columnNameRowExist,
+            })
+
+            const cellObj = ws[cellId]
+            rowData[table.columns[i].column_name] = (cellObj && cellObj.w && cellObj.w.replace(/[^\d.]+/g, '')) || row[i]
+          } else if (table.columns[i].uidt === UITypes.SingleSelect || table.columns[i].uidt === UITypes.MultiSelect) {
+            rowData[table.columns[i].column_name] = (row[i] || '').toString().trim() || null
+          } else {
+            // toto: do parsing if necessary based on type
+            rowData[table.columns[i].column_name] = row[i]
+          }
+        }
+        this.data[tn].push(rowData)
+        rowIndex++
+      }
+      this.project.tables.push(table)
+    }
+  }
+
+  getTemplate() {
+    return this.project
+  }
+
+  getData() {
+    return this.data
+  }
+}