From 7068fbcdd9cca45298464595b023ba7b2c7d9336 Mon Sep 17 00:00:00 2001 From: Wing-Kam Wong Date: Fri, 21 Oct 2022 18:08:51 +0800 Subject: [PATCH] feat(nc-gui): import url logic --- .../nc-gui/components/dlg/QuickImport.vue | 27 ++-- .../utils/parsers/CSVTemplateAdapter.ts | 119 +++++++++++------- .../utils/parsers/ExcelTemplateAdapter.ts | 4 +- 3 files changed, 89 insertions(+), 61 deletions(-) diff --git a/packages/nc-gui/components/dlg/QuickImport.vue b/packages/nc-gui/components/dlg/QuickImport.vue index 159d3e4ab4..4326db70e5 100644 --- a/packages/nc-gui/components/dlg/QuickImport.vue +++ b/packages/nc-gui/components/dlg/QuickImport.vue @@ -197,14 +197,14 @@ async function parseAndExtractData(val: UploadFile[] | ArrayBuffer | string) { await templateGenerator.init() - templateGenerator.parse(() => { - templateData.value = templateGenerator!.getTemplate() - if (importDataOnly) importColumns.value = templateGenerator!.getColumns() - importData.value = templateGenerator!.getData() - templateEditorModal.value = true - isParsingData.value = false - preImportLoading.value = false - }) + await templateGenerator.parse() + + templateData.value = templateGenerator!.getTemplate() + if (importDataOnly) importColumns.value = templateGenerator!.getColumns() + importData.value = templateGenerator!.getData() + templateEditorModal.value = true + isParsingData.value = false + preImportLoading.value = false } catch (e: any) { message.error(await extractSdkResponseErrorMsg(e)) } @@ -273,10 +273,15 @@ function getAdapter(val: any) { if (isImportTypeCsv.value) { switch (activeKey.value) { case 'uploadTab': - return new CSVTemplateAdapter(val, importState.parserConfig) + return new CSVTemplateAdapter(val, { + ...importState.parserConfig, + importFromURL: false, + }) case 'urlTab': - // TODO(import): implement one for CSV - return new ExcelUrlTemplateAdapter(val, importState.parserConfig) + return new CSVTemplateAdapter(val, { + ...importState.parserConfig, + importFromURL: true, + }) } } else if (IsImportTypeExcel.value) { switch (activeKey.value) { diff --git a/packages/nc-gui/utils/parsers/CSVTemplateAdapter.ts b/packages/nc-gui/utils/parsers/CSVTemplateAdapter.ts index 68877002d2..89e71cf368 100644 --- a/packages/nc-gui/utils/parsers/CSVTemplateAdapter.ts +++ b/packages/nc-gui/utils/parsers/CSVTemplateAdapter.ts @@ -15,7 +15,7 @@ import { export default class CSVTemplateAdapter { config: Record - files: UploadFile[] + source: UploadFile[] | string detectedColumnTypes: Record> distinctValues: Record> headers: Record @@ -26,9 +26,9 @@ export default class CSVTemplateAdapter { data: Record = {} columnValues: Record - constructor(files: UploadFile[], parserConfig = {}) { + constructor(source: UploadFile[] | string, parserConfig = {}) { this.config = parserConfig - this.files = files + this.source = source this.project = { tables: [], } @@ -199,15 +199,59 @@ export default class CSVTemplateAdapter { } } - parse(callback: Function) { - const that = this - for (const [tableIdx, file] of this.files.entries()) { + async _parseTableData(tableIdx: number, source: UploadFile | string, tn: string) { + return new Promise((resolve) => { + const that = this let steppers = 0 - const tn = file.name.replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_').trim() - this.data[tn] = [] + if (that.config.shouldImportData) { + steppers = 0 + const parseSource = (this.config.importFromURL ? (source as string) : (source as UploadFile).originFileObj)! + parse(parseSource, { + download: that.config.importFromURL, + worker: true, + skipEmptyLines: 'greedy', + step(row) { + steppers += 1 + if (row && steppers >= +that.config.firstRowAsHeaders + 1) { + const rowData: Record = {} + for (let columnIdx = 0; columnIdx < that.headers[tableIdx].length; columnIdx++) { + const column = that.project.tables[tableIdx].columns[columnIdx] + const data = (row.data as [])[columnIdx] === '' ? null : (row.data as [])[columnIdx] + if (column.uidt === UITypes.Checkbox) { + rowData[column.column_name] = getCheckboxValue(data) + rowData[column.column_name] = data + } else if (column.uidt === UITypes.SingleSelect || column.uidt === UITypes.MultiSelect) { + rowData[column.column_name] = (data || '').toString().trim() || null + } else { + // TODO(import): do parsing if necessary based on type + rowData[column.column_name] = data + } + } + that.data[tn].push(rowData) + } + }, + complete() { + console.log('getData(): complete') + console.log(`getData(): steppers: ${steppers}`) + resolve(true) + }, + // TODO(import): add error + }) + } + }) + } - // parse column meta - parse(file.originFileObj as File, { + async _parseTableMeta(tableIdx: number, source: UploadFile | string) { + return new Promise((resolve) => { + const that = this + let steppers = 0 + const tn = ((this.config.importFromURL ? (source as string).split('/').pop() : (source as UploadFile).name) as string) + .replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_') + .trim()! + this.data[tn] = [] + const parseSource = (this.config.importFromURL ? (source as string) : (source as UploadFile).originFileObj)! + parse(parseSource, { + download: that.config.importFromURL, worker: true, skipEmptyLines: 'greedy', step(row) { @@ -237,49 +281,28 @@ export default class CSVTemplateAdapter { } } }, - complete() { + async complete() { console.log('complete') console.log(`steppers: ${steppers}`) that.updateTemplate(tableIdx) - - // parse table data - if (that.config.shouldImportData) { - steppers = 0 - parse(file.originFileObj as File, { - worker: true, - skipEmptyLines: 'greedy', - step(row) { - steppers += 1 - if (row && steppers >= +that.config.firstRowAsHeaders + 1) { - const rowData: Record = {} - for (let columnIdx = 0; columnIdx < that.headers[tableIdx].length; columnIdx++) { - const column = that.project.tables[tableIdx].columns[columnIdx] - const data = (row.data as [])[columnIdx] === '' ? null : (row.data as [])[columnIdx] - if (column.uidt === UITypes.Checkbox) { - rowData[column.column_name] = getCheckboxValue(data) - rowData[column.column_name] = data - } else if (column.uidt === UITypes.SingleSelect || column.uidt === UITypes.MultiSelect) { - rowData[column.column_name] = (data || '').toString().trim() || null - } else { - // TODO(import): do parsing if necessary based on type - rowData[column.column_name] = data - } - } - that.data[tn].push(rowData) - } - }, - complete() { - console.log('getData(): complete') - console.log(`getData(): steppers: ${steppers}`) - callback() - }, - // TODO(import): add error - }) - } else { - callback() - } + await that._parseTableData(tableIdx, source, tn) + resolve(true) }, }) + }) + } + + async parse() { + if (this.config.importFromURL) { + await this._parseTableMeta(0, this.source as string) + } else { + await Promise.all( + (this.source as UploadFile[]).map((file: UploadFile, tableIdx: number) => + (async (f, idx) => { + await this._parseTableMeta(idx, f) + })(file, tableIdx), + ), + ) } } diff --git a/packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts b/packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts index c74db91162..ff4650d851 100644 --- a/packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts +++ b/packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts @@ -56,7 +56,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator { }) } - parse(callback: Function) { + async parse() { const tableNamePrefixRef: Record = {} this.wb.SheetNames.reduce((acc: any, sheet: any) => { return acc.then( @@ -253,7 +253,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator { resolve(true) }), ) - }, Promise.resolve()).then(callback) + }, Promise.resolve()) } getTemplate() {