From 94f72c80f737a679e1ac7d0f67a9a951e5b9e629 Mon Sep 17 00:00:00 2001
From: mertmit <mertmit99@gmail.com>
Date: Thu, 18 Apr 2024 11:33:18 +0000
Subject: [PATCH] feat: use papaparse with worker mode

---
 .../nc-gui/extensions/csv-import/index.vue    | 67 ++++++++++++-------
 1 file changed, 43 insertions(+), 24 deletions(-)

diff --git a/packages/nc-gui/extensions/csv-import/index.vue b/packages/nc-gui/extensions/csv-import/index.vue
index 9dd915a871..72747e5c17 100644
--- a/packages/nc-gui/extensions/csv-import/index.vue
+++ b/packages/nc-gui/extensions/csv-import/index.vue
@@ -3,6 +3,23 @@ import type { UploadFile } from 'ant-design-vue'
 import { type ColumnType, UITypes } from 'nocodb-sdk'
 import papaparse from 'papaparse'
 
+const CHUNK_SIZE = 100
+
+const GENERATED_COLUMN_TYPES = [
+  UITypes.Links,
+  UITypes.LinkToAnotherRecord,
+  UITypes.Barcode,
+  UITypes.QrCode,
+  UITypes.AutoNumber,
+  UITypes.CreatedBy,
+  UITypes.CreatedTime,
+  UITypes.LastModifiedBy,
+  UITypes.LastModifiedTime,
+  UITypes.Formula,
+  UITypes.Lookup,
+  UITypes.Rollup,
+]
+
 const { fullscreen, extension, tables, insertData, upsertData, getTableMeta, reloadData } = useExtensionHelperOrThrow()
 
 const fileList = ref<UploadFile[]>([])
@@ -13,33 +30,20 @@ const fileList = ref<UploadFile[]>([])
 // step 3: stats
 const step = ref(0)
 
-const CHUNK_SIZE = 100
-
 const stats = ref<{ inserted: number; updated: number; error?: { title?: string; message: string } }>({
   inserted: 0,
   updated: 0,
 })
 
+const processingFile = ref(false)
+
 const totalRecords = ref(0)
 
 const processedRecords = ref(0)
 
 const parsedData = ref<any>()
 
-const generatedColumnTypes = [
-  UITypes.Links,
-  UITypes.LinkToAnotherRecord,
-  UITypes.Barcode,
-  UITypes.QrCode,
-  UITypes.AutoNumber,
-  UITypes.CreatedBy,
-  UITypes.CreatedTime,
-  UITypes.LastModifiedBy,
-  UITypes.LastModifiedTime,
-  UITypes.Formula,
-  UITypes.Lookup,
-  UITypes.Rollup,
-]
+const columns = ref<Record<string, ColumnType>>({})
 
 const tableList = computed(() => {
   return tables.value.map((table) => {
@@ -86,8 +90,6 @@ const importPayload = computed(() => {
   return savedPayloads.value[savedPayloads.value.length - 1]
 })
 
-const columns = ref<Record<string, ColumnType>>({})
-
 const updateHistory = async () => {
   // update last used
   importPayload.value.lastUsed = Date.now()
@@ -115,7 +117,7 @@ const headers = computed(() => {
 const tableColumns = computed(() => {
   return importPayload.value.importColumns.reduce((acc, importColumn) => {
     const column = columns.value[importColumn.columnId]
-    if (!column.id || !column.title || generatedColumnTypes.includes(column.uidt as UITypes)) return acc
+    if (!column.id || !column.title || GENERATED_COLUMN_TYPES.includes(column.uidt as UITypes)) return acc
     acc.push({
       label: column.title,
       value: column.id,
@@ -145,7 +147,7 @@ const onTableSelect = async () => {
 
       importPayload.value.importColumns.push(
         ...tableMeta.columns.reduce((acc, column) => {
-          if (!column.id || column.system || generatedColumnTypes.includes(column.uidt as UITypes)) return acc
+          if (!column.id || column.system || GENERATED_COLUMN_TYPES.includes(column.uidt as UITypes)) return acc
           if (importPayload.value.importColumns.find((m) => m.columnId === column.id)) return acc
           acc.push({ enabled: false, mapIndex: '', columnId: column.id })
           return acc
@@ -158,6 +160,7 @@ const onTableSelect = async () => {
 }
 
 const handleChange = (info: { file: UploadFile }) => {
+  processingFile.value = true
   const reader = new FileReader()
   reader.onload = (e) => {
     const text = e.target?.result
@@ -165,8 +168,19 @@ const handleChange = (info: { file: UploadFile }) => {
       fileList.value = []
       return
     }
-    parsedData.value = papaparse.parse(text.trim())
-    step.value = 1
+    papaparse.parse(text.trim(), {
+      worker: true,
+      complete: (results) => {
+        parsedData.value = results
+        step.value = 1
+        processingFile.value = false
+      },
+      error: () => {
+        fileList.value = []
+        processingFile.value = false
+        message.error('There was an error parsing the file. Please check the file and try again.')
+      },
+    })
 
     if (importPayload.value.tableId) {
       onTableSelect()
@@ -235,7 +249,7 @@ useProvideSmartsheetRowStore({} as any, {} as any)
 const previewColumns = computed(() => {
   return importPayload.value.importColumns.reduce((acc, importMeta) => {
     const column = columns.value[importMeta.columnId]
-    if (!column.id || !column.title || generatedColumnTypes.includes(column.uidt as UITypes)) return acc
+    if (!column.id || !column.title || GENERATED_COLUMN_TYPES.includes(column.uidt as UITypes)) return acc
     acc[column.title] = column
     return acc
   }, {} as Record<string, ColumnType>)
@@ -325,7 +339,12 @@ onMounted(async () => {
 
 <template>
   <template v-if="step === 0">
-    <a-upload-dragger v-model:fileList="fileList" name="file" accept=".csv" :multiple="false" @change="handleChange">
+    <template v-if="processingFile">
+      <div class="flex flex-col h-full items-center justify-center p-4">
+        <GeneralLoader size="xlarge" />
+      </div>
+    </template>
+    <a-upload-dragger v-else v-model:fileList="fileList" name="file" accept=".csv" :multiple="false" @change="handleChange">
       <GeneralIcon class="text-[30px]" icon="inbox" />
       <p class="ant-upload-text">Drag and drop a CSV file</p>
       <p class="ant-upload-hint">Or click to select a file</p>