diff --git a/packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts b/packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts index bc3ae723a5..53e7a05e00 100644 --- a/packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts +++ b/packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts @@ -3,12 +3,12 @@ import TemplateGenerator from './TemplateGenerator' import { extractMultiOrSingleSelectProps, getCheckboxValue, + getDateFormat, isCheckboxType, isEmailType, isMultiLineTextType, isUrlType, -} from './parserHelpers' -import { getDateFormat } from '~/utils' +} from '#imports' const excelTypeToUidt: Record = { d: UITypes.DateTime, @@ -58,200 +58,192 @@ export default class ExcelTemplateAdapter extends TemplateGenerator { cellDates: true, } - // TODO(import): remove later - // if (this.name.slice(-3) === 'csv') { - // this.wb = this.xlsx.read(new TextDecoder().decode(new Uint8Array(this.excelData)), { - // type: 'string', - // ...options, - // }) - // } else { - // this.wb = this.xlsx.read(new Uint8Array(this.excelData), { - // type: 'array', - // ...options, - // }) - // } this.wb = this.xlsx.read(new Uint8Array(this.excelData), { type: 'array', ...options, }) } - parse() { + parse(callback: Function) { const tableNamePrefixRef: Record = {} - - for (let i = 0; i < this.wb.SheetNames.length; i++) { - const columnNamePrefixRef: Record = { id: 0 } - const sheet: any = this.wb.SheetNames[i] - let tn: string = (sheet || 'table').replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_').trim() - - while (tn in tableNamePrefixRef) { - tn = `${tn}${++tableNamePrefixRef[tn]}` - } - tableNamePrefixRef[tn] = 0 - - const table = { table_name: tn, ref_table_name: tn, columns: [] as any[] } - this.data[tn] = [] - const ws: any = this.wb.Sheets[sheet] - const range = this.xlsx.utils.decode_range(ws['!ref']) - const rows: any = this.xlsx.utils.sheet_to_json(ws, { header: 1, blankrows: false, defval: null }) - - // TODO(import): remove later - // if (this.name.slice(-3) !== 'csv') { - // // fix precision bug & timezone offset issues introduced by xlsx - // const basedate = new Date(1899, 11, 30, 0, 0, 0) - // // number of milliseconds since base date - // const dnthresh = basedate.getTime() + (new Date().getTimezoneOffset() - basedate.getTimezoneOffset()) * 60000 - // // number of milliseconds in a day - // const day_ms = 24 * 60 * 60 * 1000 - // // handle date1904 property - // const fixImportedDate = (date: Date) => { - // const parsed = this.xlsx.SSF.parse_date_code((date.getTime() - dnthresh) / day_ms, { - // date1904: this.wb.Workbook.WBProps.date1904, - // }) - // return new Date(parsed.y, parsed.m, parsed.d, parsed.H, parsed.M, parsed.S) - // } - // // fix imported date - // rows = rows.map((r: any) => - // r.map((v: any) => { - // return v instanceof Date ? fixImportedDate(v) : v - // }), - // ) - // } - - const columnNameRowExist = +rows[0].every((v: any) => v === null || typeof v === 'string') - - for (let col = 0; col < rows[0].length; col++) { - let cn: string = ((columnNameRowExist && rows[0] && rows[0][col] && rows[0][col].toString().trim()) || `field_${col + 1}`) - .replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_') - .trim() - - while (cn in columnNamePrefixRef) { - cn = `${cn}${++columnNamePrefixRef[cn]}` - } - columnNamePrefixRef[cn] = 0 - - const column: Record = { - column_name: cn, - ref_column_name: cn, - meta: {}, - } - - const cellId = this.xlsx.utils.encode_cell({ - c: range.s.c + col, - r: columnNameRowExist, - }) - const cellProps = ws[cellId] || {} - column.uidt = excelTypeToUidt[cellProps.t] || UITypes.SingleLineText - - if (column.uidt === UITypes.SingleLineText) { - // check for long text - if (isMultiLineTextType(rows)) { - column.uidt = UITypes.LongText - } - - if (isEmailType(rows)) { - column.uidt = UITypes.Email - } - - if (isUrlType(rows)) { - column.uidt = UITypes.URL - } else { - const vals = rows - .slice(columnNameRowExist ? 1 : 0) - .map((r: any) => r[col]) - .filter((v: any) => v !== null && v !== undefined && v.toString().trim() !== '') - - const checkboxType = isCheckboxType(vals) - if (checkboxType.length === 1) { - column.uidt = UITypes.Checkbox - } else { - // Single Select / Multi Select - Object.assign(column, extractMultiOrSingleSelectProps(vals)) + this.wb.SheetNames.reduce((acc: any, sheet: any) => { + return acc.then( + () => + new Promise((resolve) => { + const columnNamePrefixRef: Record = { id: 0 } + let tn: string = (sheet || 'table').replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_').trim() + + while (tn in tableNamePrefixRef) { + tn = `${tn}${++tableNamePrefixRef[tn]}` } - } - } else if (column.uidt === UITypes.Number) { - if ( - rows.slice(1, this.config.maxRowsToParse).some((v: any) => { - return v && v[col] && parseInt(v[col]) !== +v[col] - }) - ) { - column.uidt = UITypes.Decimal - } - if ( - rows.slice(1, this.config.maxRowsToParse).every((v: any, i: any) => { - const cellId = this.xlsx.utils.encode_cell({ - c: range.s.c + col, - r: i + columnNameRowExist, + tableNamePrefixRef[tn] = 0 + + const table = { table_name: tn, ref_table_name: tn, columns: [] as any[] } + this.data[tn] = [] + const ws: any = this.wb.Sheets[sheet] + const range = this.xlsx.utils.decode_range(ws['!ref']) + let rows: any = this.xlsx.utils.sheet_to_json(ws, { header: 1, blankrows: false, defval: null }) + + // fix precision bug & timezone offset issues introduced by xlsx + const basedate = new Date(1899, 11, 30, 0, 0, 0) + // number of milliseconds since base date + const dnthresh = basedate.getTime() + (new Date().getTimezoneOffset() - basedate.getTimezoneOffset()) * 60000 + // number of milliseconds in a day + const day_ms = 24 * 60 * 60 * 1000 + // handle date1904 property + const fixImportedDate = (date: Date) => { + const parsed = this.xlsx.SSF.parse_date_code((date.getTime() - dnthresh) / day_ms, { + date1904: this.wb.Workbook.WBProps.date1904, }) + return new Date(parsed.y, parsed.m, parsed.d, parsed.H, parsed.M, parsed.S) + } + // fix imported date + rows = rows.map((r: any) => + r.map((v: any) => { + return v instanceof Date ? fixImportedDate(v) : v + }), + ) + + const columnNameRowExist = +rows[0].every((v: any) => v === null || typeof v === 'string') + + for (let col = 0; col < rows[0].length; col++) { + let cn: string = ( + (columnNameRowExist && rows[0] && rows[0][col] && rows[0][col].toString().trim()) || + `field_${col + 1}` + ) + .replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_') + .trim() + + while (cn in columnNamePrefixRef) { + cn = `${cn}${++columnNamePrefixRef[cn]}` + } + columnNamePrefixRef[cn] = 0 + + const column: Record = { + column_name: cn, + ref_column_name: cn, + meta: {}, + } - const cellObj = ws[cellId] - - return !cellObj || (cellObj.w && cellObj.w.startsWith('$')) - }) - ) { - column.uidt = UITypes.Currency - } - } else if (column.uidt === UITypes.DateTime) { - // TODO(import): centralise - // hold the possible date format found in the date - const dateFormat: Record = {} - if ( - rows.slice(1, this.config.maxRowsToParse).every((v: any, i: any) => { const cellId = this.xlsx.utils.encode_cell({ c: range.s.c + col, - r: i + columnNameRowExist, + r: columnNameRowExist, }) + const cellProps = ws[cellId] || {} + column.uidt = excelTypeToUidt[cellProps.t] || UITypes.SingleLineText + + if (column.uidt === UITypes.SingleLineText) { + // check for long text + if (isMultiLineTextType(rows)) { + column.uidt = UITypes.LongText + } + + if (isEmailType(rows)) { + column.uidt = UITypes.Email + } + + if (isUrlType(rows)) { + column.uidt = UITypes.URL + } else { + const vals = rows + .slice(columnNameRowExist ? 1 : 0) + .map((r: any) => r[col]) + .filter((v: any) => v !== null && v !== undefined && v.toString().trim() !== '') + + const checkboxType = isCheckboxType(vals) + if (checkboxType.length === 1) { + column.uidt = UITypes.Checkbox + } else { + // Single Select / Multi Select + Object.assign(column, extractMultiOrSingleSelectProps(vals)) + } + } + } else if (column.uidt === UITypes.Number) { + if ( + rows.slice(1, this.config.maxRowsToParse).some((v: any) => { + return v && v[col] && parseInt(v[col]) !== +v[col] + }) + ) { + column.uidt = UITypes.Decimal + } + if ( + rows.slice(1, this.config.maxRowsToParse).every((v: any, i: any) => { + const cellId = this.xlsx.utils.encode_cell({ + c: range.s.c + col, + r: i + columnNameRowExist, + }) + + const cellObj = ws[cellId] + + return !cellObj || (cellObj.w && cellObj.w.startsWith('$')) + }) + ) { + column.uidt = UITypes.Currency + } + } else if (column.uidt === UITypes.DateTime) { + // TODO(import): centralise + // hold the possible date format found in the date + const dateFormat: Record = {} + if ( + rows.slice(1, this.config.maxRowsToParse).every((v: any, i: any) => { + const cellId = this.xlsx.utils.encode_cell({ + c: range.s.c + col, + r: i + columnNameRowExist, + }) + + const cellObj = ws[cellId] + const isDate = !cellObj || (cellObj.w && cellObj.w.split(' ').length === 1) + if (isDate && cellObj) { + dateFormat[getDateFormat(cellObj.w)] = (dateFormat[getDateFormat(cellObj.w)] || 0) + 1 + } + return isDate + }) + ) { + column.uidt = UITypes.Date + // take the date format with the max occurrence + column.meta.date_format = + Object.keys(dateFormat).reduce((x, y) => (dateFormat[x] > dateFormat[y] ? x : y)) || 'YYYY/MM/DD' + } + } + table.columns.push(column) + } - const cellObj = ws[cellId] - const isDate = !cellObj || (cellObj.w && cellObj.w.split(' ').length === 1) - if (isDate && cellObj) { - dateFormat[getDateFormat(cellObj.w)] = (dateFormat[getDateFormat(cellObj.w)] || 0) + 1 + let rowIndex = 0 + for (const row of rows.slice(1)) { + const rowData: Record = {} + for (let i = 0; i < table.columns.length; i++) { + if (table.columns[i].uidt === UITypes.Checkbox) { + rowData[table.columns[i].column_name] = getCheckboxValue(row[i]) + } else if (table.columns[i].uidt === UITypes.Currency) { + const cellId = this.xlsx.utils.encode_cell({ + c: range.s.c + i, + r: rowIndex + columnNameRowExist, + }) + + const cellObj = ws[cellId] + rowData[table.columns[i].column_name] = (cellObj && cellObj.w && cellObj.w.replace(/[^\d.]+/g, '')) || row[i] + } else if (table.columns[i].uidt === UITypes.SingleSelect || table.columns[i].uidt === UITypes.MultiSelect) { + rowData[table.columns[i].column_name] = (row[i] || '').toString().trim() || null + } else if (table.columns[i].uidt === UITypes.Date) { + const cellId = this.xlsx.utils.encode_cell({ + c: range.s.c + i, + r: rowIndex + columnNameRowExist, + }) + const cellObj = ws[cellId] + rowData[table.columns[i].column_name] = (cellObj && cellObj.w) || row[i] + } else { + // toto: do parsing if necessary based on type + rowData[table.columns[i].column_name] = row[i] + } } - return isDate - }) - ) { - column.uidt = UITypes.Date - // take the date format with the max occurrence - column.meta.date_format = - Object.keys(dateFormat).reduce((x, y) => (dateFormat[x] > dateFormat[y] ? x : y)) || 'YYYY/MM/DD' - } - } - table.columns.push(column) - } - - let rowIndex = 0 - for (const row of rows.slice(1)) { - const rowData: Record = {} - for (let i = 0; i < table.columns.length; i++) { - if (table.columns[i].uidt === UITypes.Checkbox) { - rowData[table.columns[i].column_name] = getCheckboxValue(row[i]) - } else if (table.columns[i].uidt === UITypes.Currency) { - const cellId = this.xlsx.utils.encode_cell({ - c: range.s.c + i, - r: rowIndex + columnNameRowExist, - }) - - const cellObj = ws[cellId] - rowData[table.columns[i].column_name] = (cellObj && cellObj.w && cellObj.w.replace(/[^\d.]+/g, '')) || row[i] - } else if (table.columns[i].uidt === UITypes.SingleSelect || table.columns[i].uidt === UITypes.MultiSelect) { - rowData[table.columns[i].column_name] = (row[i] || '').toString().trim() || null - } else if (table.columns[i].uidt === UITypes.Date) { - const cellId = this.xlsx.utils.encode_cell({ - c: range.s.c + i, - r: rowIndex + columnNameRowExist, - }) - const cellObj = ws[cellId] - rowData[table.columns[i].column_name] = (cellObj && cellObj.w) || row[i] - } else { - // toto: do parsing if necessary based on type - rowData[table.columns[i].column_name] = row[i] - } - } - this.data[tn].push(rowData) - rowIndex++ - } - this.project.tables.push(table) - } + this.data[tn].push(rowData) + rowIndex++ + } + this.project.tables.push(table) + resolve(true) + }), + ) + }, Promise.resolve()).then(callback) } getTemplate() {