feat(nc-gui): import url logic

packages/nc-gui/components/dlg/QuickImport.vue

@@ -197,14 +197,14 @@ async function parseAndExtractData(val: UploadFile[] | ArrayBuffer | string) {
 
     await templateGenerator.init()
 
-    templateGenerator.parse(() => {
+    await templateGenerator.parse()
-      templateData.value = templateGenerator!.getTemplate()
+
-      if (importDataOnly) importColumns.value = templateGenerator!.getColumns()
+    templateData.value = templateGenerator!.getTemplate()
-      importData.value = templateGenerator!.getData()
+    if (importDataOnly) importColumns.value = templateGenerator!.getColumns()
-      templateEditorModal.value = true
+    importData.value = templateGenerator!.getData()
-      isParsingData.value = false
+    templateEditorModal.value = true
-      preImportLoading.value = false
+    isParsingData.value = false
-    })
+    preImportLoading.value = false
   } catch (e: any) {
     message.error(await extractSdkResponseErrorMsg(e))
   }
@@ -273,10 +273,15 @@ function getAdapter(val: any) {
   if (isImportTypeCsv.value) {
     switch (activeKey.value) {
       case 'uploadTab':
-        return new CSVTemplateAdapter(val, importState.parserConfig)
+        return new CSVTemplateAdapter(val, {
+          ...importState.parserConfig,
+          importFromURL: false,
+        })
       case 'urlTab':
-        // TODO(import): implement one for CSV
+        return new CSVTemplateAdapter(val, {
-        return new ExcelUrlTemplateAdapter(val, importState.parserConfig)
+          ...importState.parserConfig,
+          importFromURL: true,
+        })
     }
   } else if (IsImportTypeExcel.value) {
     switch (activeKey.value) {

packages/nc-gui/utils/parsers/CSVTemplateAdapter.ts

@@ -15,7 +15,7 @@ import {
 
 export default class CSVTemplateAdapter {
   config: Record<string, any>
-  files: UploadFile[]
+  source: UploadFile[] | string
   detectedColumnTypes: Record<number, Record<string, number>>
   distinctValues: Record<number, Set<string>>
   headers: Record<number, string[]>
@@ -26,9 +26,9 @@ export default class CSVTemplateAdapter {
   data: Record<string, any> = {}
   columnValues: Record<number, []>
 
-  constructor(files: UploadFile[], parserConfig = {}) {
+  constructor(source: UploadFile[] | string, parserConfig = {}) {
     this.config = parserConfig
-    this.files = files
+    this.source = source
     this.project = {
       tables: [],
     }
@@ -199,15 +199,59 @@ export default class CSVTemplateAdapter {
     }
   }
 
-  parse(callback: Function) {
+  async _parseTableData(tableIdx: number, source: UploadFile | string, tn: string) {
-    const that = this
+    return new Promise((resolve) => {
-    for (const [tableIdx, file] of this.files.entries()) {
+      const that = this
       let steppers = 0
-      const tn = file.name.replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_').trim()
+      if (that.config.shouldImportData) {
-      this.data[tn] = []
+        steppers = 0
+        const parseSource = (this.config.importFromURL ? (source as string) : (source as UploadFile).originFileObj)!
+        parse(parseSource, {
+          download: that.config.importFromURL,
+          worker: true,
+          skipEmptyLines: 'greedy',
+          step(row) {
+            steppers += 1
+            if (row && steppers >= +that.config.firstRowAsHeaders + 1) {
+              const rowData: Record<string, any> = {}
+              for (let columnIdx = 0; columnIdx < that.headers[tableIdx].length; columnIdx++) {
+                const column = that.project.tables[tableIdx].columns[columnIdx]
+                const data = (row.data as [])[columnIdx] === '' ? null : (row.data as [])[columnIdx]
+                if (column.uidt === UITypes.Checkbox) {
+                  rowData[column.column_name] = getCheckboxValue(data)
+                  rowData[column.column_name] = data
+                } else if (column.uidt === UITypes.SingleSelect || column.uidt === UITypes.MultiSelect) {
+                  rowData[column.column_name] = (data || '').toString().trim() || null
+                } else {
+                  // TODO(import): do parsing if necessary based on type
+                  rowData[column.column_name] = data
+                }
+              }
+              that.data[tn].push(rowData)
+            }
+          },
+          complete() {
+            console.log('getData(): complete')
+            console.log(`getData(): steppers: ${steppers}`)
+            resolve(true)
+          },
+          // TODO(import): add error
+        })
+      }
+    })
+  }
 
-      // parse column meta
+  async _parseTableMeta(tableIdx: number, source: UploadFile | string) {
-      parse(file.originFileObj as File, {
+    return new Promise((resolve) => {
+      const that = this
+      let steppers = 0
+      const tn = ((this.config.importFromURL ? (source as string).split('/').pop() : (source as UploadFile).name) as string)
+        .replace(/[` ~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/g, '_')
+        .trim()!
+      this.data[tn] = []
+      const parseSource = (this.config.importFromURL ? (source as string) : (source as UploadFile).originFileObj)!
+      parse(parseSource, {
+        download: that.config.importFromURL,
         worker: true,
         skipEmptyLines: 'greedy',
         step(row) {
@@ -237,49 +281,28 @@ export default class CSVTemplateAdapter {
             }
           }
         },
-        complete() {
+        async complete() {
           console.log('complete')
           console.log(`steppers: ${steppers}`)
           that.updateTemplate(tableIdx)
-
+          await that._parseTableData(tableIdx, source, tn)
-          // parse table data
+          resolve(true)
-          if (that.config.shouldImportData) {
-            steppers = 0
-            parse(file.originFileObj as File, {
-              worker: true,
-              skipEmptyLines: 'greedy',
-              step(row) {
-                steppers += 1
-                if (row && steppers >= +that.config.firstRowAsHeaders + 1) {
-                  const rowData: Record<string, any> = {}
-                  for (let columnIdx = 0; columnIdx < that.headers[tableIdx].length; columnIdx++) {
-                    const column = that.project.tables[tableIdx].columns[columnIdx]
-                    const data = (row.data as [])[columnIdx] === '' ? null : (row.data as [])[columnIdx]
-                    if (column.uidt === UITypes.Checkbox) {
-                      rowData[column.column_name] = getCheckboxValue(data)
-                      rowData[column.column_name] = data
-                    } else if (column.uidt === UITypes.SingleSelect || column.uidt === UITypes.MultiSelect) {
-                      rowData[column.column_name] = (data || '').toString().trim() || null
-                    } else {
-                      // TODO(import): do parsing if necessary based on type
-                      rowData[column.column_name] = data
-                    }
-                  }
-                  that.data[tn].push(rowData)
-                }
-              },
-              complete() {
-                console.log('getData(): complete')
-                console.log(`getData(): steppers: ${steppers}`)
-                callback()
-              },
-              // TODO(import): add error
-            })
-          } else {
-            callback()
-          }
         },
       })
+    })
+  }
+
+  async parse() {
+    if (this.config.importFromURL) {
+      await this._parseTableMeta(0, this.source as string)
+    } else {
+      await Promise.all(
+        (this.source as UploadFile[]).map((file: UploadFile, tableIdx: number) =>
+          (async (f, idx) => {
+            await this._parseTableMeta(idx, f)
+          })(file, tableIdx),
+        ),
+      )
     }
   }
 

packages/nc-gui/utils/parsers/ExcelTemplateAdapter.ts

@@ -56,7 +56,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
     })
   }
 
-  parse(callback: Function) {
+  async parse() {
     const tableNamePrefixRef: Record<string, any> = {}
     this.wb.SheetNames.reduce((acc: any, sheet: any) => {
       return acc.then(
@@ -253,7 +253,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
             resolve(true)
           }),
       )
-    }, Promise.resolve()).then(callback)
+    }, Promise.resolve())
   }
 
   getTemplate() {