From 8e15a8bb88187cbdb66e9164e61f6add6dc20d41 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 17 Nov 2021 22:27:46 +0530 Subject: [PATCH] feat: excel datatype parsing(in progress) Signed-off-by: Pranav C --- .../templateParsers/ExcelTemplateAdapter.js | 81 ++++++++++++++----- .../import/templateParsers/parserHelpers.js | 31 +++++++ 2 files changed, 91 insertions(+), 21 deletions(-) create mode 100644 packages/nc-gui/components/import/templateParsers/parserHelpers.js diff --git a/packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js b/packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js index 2c14dcc3bf..faa61749a2 100644 --- a/packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js +++ b/packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js @@ -1,6 +1,7 @@ import XLSX from 'xlsx' import { UITypes } from '~/components/project/spreadsheet/helpers/uiTypes' import TemplateGenerator from '~/components/import/templateParsers/TemplateGenerator' +import { getCheckboxValue, isCheckboxType } from '~/components/import/templateParsers/parserHelpers' const excelTypeToUidt = { d: UITypes.DateTime, @@ -31,41 +32,75 @@ export default class ExcelTemplateAdapter extends TemplateGenerator { const table = { tn: sheet, columns: [] } this.data[sheet] = [] const ws = this.wb.Sheets[sheet] - const rows = XLSX.utils.sheet_to_json(ws, { header: 1 }) + const range = XLSX.utils.decode_range(ws['!ref']) + const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false }) for (let col = 0; col < rows[0].length; col++) { const column = { - cn: (rows[0][col] || `field${col + 1}`).replace(/\./, '_') + cn: (rows[0][col] || + `field${col + 1}`).replace(/\./, '_') } - const cellProps = ws[`${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`] || {} - + // const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`; + const cellId = XLSX.utils.encode_cell({ + c: range.s.c + col, + r: 1 + }) + const cellProps = ws[cellId] || {} column.uidt = excelTypeToUidt[cellProps.t] || UITypes.SingleLineText // todo: optimize if (column.uidt === UITypes.SingleLineText) { // check for long text - if (rows.some(r => (r[col] || '').toString().length > 255)) { + if (rows.some(r => + (r[col] || '').toString().match(/[\r\n]/) || + (r[col] || '').toString().length > 255) + ) { column.uidt = UITypes.LongText } else { - const vals = rows.slice(1).map(r => r[col]).filter(v => v !== null && v !== undefined) - - // check column is multi or single select by comparing unique values - if (vals.some(v => v && v.toString().includes(','))) { - const flattenedVals = vals.flatMap(v => v ? v.toString().split(',') : []) - const uniqueVals = new Set(flattenedVals) - if (flattenedVals.length > uniqueVals.size && uniqueVals.size <= flattenedVals.length / 10) { - column.uidt = UITypes.MultiSelect - column.dtxp = [...uniqueVals].join(',') - } + let vals = rows.slice(1).map(r => r[col]) + + const checkboxType = isCheckboxType(vals) + if (checkboxType.length === 1) { + column.uidt = UITypes.Checkbox } else { - const uniqueVals = new Set(vals) - if (vals.length > uniqueVals.size && uniqueVals.size <= vals.length / 10) { - column.uidt = UITypes.SingleSelect - column.dtxp = [...uniqueVals].join(',') + vals = vals.filter(v => v !== null && v !== undefined) + + // check column is multi or single select by comparing unique values + if (vals.some(v => v && v.toString().includes(','))) { + const flattenedVals = vals.flatMap(v => v ? v.toString().split(',') : []) + const uniqueVals = new Set(flattenedVals) + if (flattenedVals.length > uniqueVals.size && uniqueVals.size <= flattenedVals.length / 10) { + column.uidt = UITypes.MultiSelect + column.dtxp = [...uniqueVals].join(',') + } + } else { + const uniqueVals = new Set(vals) + if (vals.length > uniqueVals.size && uniqueVals.size <= vals.length / 10) { + column.uidt = UITypes.SingleSelect + column.dtxp = [...uniqueVals].join(',') + } } } } + } else if (column.uidt === UITypes.Number) { + if (rows.slice(1, 500).some((v) => { + return v && v[col] && parseInt(+v[col]) !== +v[col] + })) { + column.uidt = UITypes.Decimal + } + if (rows.slice(1, 500).every((v, i) => { + const cellId = XLSX.utils.encode_cell({ + c: range.s.c + col, + r: i + 2 + }) + + const cellObj = ws[cellId] + + return !cellObj || (cellObj.w && cellObj.w.startsWith('$')) + })) { + column.uidt = UITypes.Currency + } } table.columns.push(column) @@ -74,8 +109,12 @@ export default class ExcelTemplateAdapter extends TemplateGenerator { for (const row of rows.slice(1)) { const rowData = {} for (let i = 0; i < table.columns.length; i++) { - // toto: do parsing if necessary based on type - rowData[table.columns[i].cn] = row[i] + if (table.columns[i].uidt === UITypes.Checkbox) { + rowData[table.columns[i].cn] = getCheckboxValue(row[i]) + } else { + // toto: do parsing if necessary based on type + rowData[table.columns[i].cn] = row[i] + } } this.data[sheet].push(rowData) } diff --git a/packages/nc-gui/components/import/templateParsers/parserHelpers.js b/packages/nc-gui/components/import/templateParsers/parserHelpers.js new file mode 100644 index 0000000000..75fd80ead7 --- /dev/null +++ b/packages/nc-gui/components/import/templateParsers/parserHelpers.js @@ -0,0 +1,31 @@ +const booleanOptions = [ + { checked: true, unchecked: false }, + { x: true, '': false }, + { yes: true, no: false }, + { y: true, n: false }, + { 1: true, 0: false }, + { '[x]': true, '[]': false, '[ ]': false }, + { '☑': true, '': false }, + { '✅': true, '': false }, + { '✓': true, '': false }, + { '✔': true, '': false }, + { enabled: true, disabled: false }, + { on: true, off: false }, + { done: true, '': false } +] +const aggBooleanOptions = booleanOptions.reduce((obj, o) => ({ ...obj, ...o }), {}) +export const isCheckboxType = (values, col = '') => { + let options = booleanOptions + for (let i = 0; i < values.length; i++) { + let val = col ? values[i][col] : values[i] + val = val === null || val === undefined ? '' : val + options = options.filter(v => val in v) + if (!options.length) { + return false + } + } + return options +} +export const getCheckboxValue = (value) => { + return value && aggBooleanOptions[value] +}