From be8f4059238c2022245875bb279592792366a8c3 Mon Sep 17 00:00:00 2001
From: mertmit <mertmit99@gmail.com>
Date: Tue, 18 Apr 2023 16:09:24 +0300
Subject: [PATCH] feat: raw bulk insert for import

Signed-off-by: mertmit <mertmit99@gmail.com>
---
 .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 13 +++++++++----
 .../src/lib/services/dbData/bulkData.ts       |  3 ++-
 .../lib/services/exportImport/import.svc.ts   | 19 +++++++++++--------
 3 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts
index 5569bd4bab..1c02aeff06 100644
--- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts
+++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts
@@ -2067,14 +2067,17 @@ class BaseModelSqlv2 {
       chunkSize: _chunkSize = 100,
       cookie,
       foreign_key_checks = true,
+      raw = false,
     }: {
       chunkSize?: number;
       cookie?: any;
       foreign_key_checks?: boolean;
+      raw?: boolean;
     } = {}
   ) {
     try {
-      const insertDatas = await Promise.all(
+      // TODO: ag column handling for raw bulk insert
+      const insertDatas = raw ? datas : await Promise.all(
         datas.map(async (d) => {
           await populatePk(this.model, d);
           return this.model.mapAliasToColumn(d);
@@ -2083,8 +2086,10 @@ class BaseModelSqlv2 {
 
       // await this.beforeInsertb(insertDatas, null);
 
-      for (const data of datas) {
-        await this.validate(data);
+      if (!raw) {
+        for (const data of datas) {
+          await this.validate(data);
+        } 
       }
 
       // fallbacks to `10` if database client is sqlite
@@ -2123,7 +2128,7 @@ class BaseModelSqlv2 {
 
       await trx.commit();
 
-      await this.afterBulkInsert(insertDatas, this.dbDriver, cookie);
+      if (!raw) await this.afterBulkInsert(insertDatas, this.dbDriver, cookie);
 
       return response;
     } catch (e) {
diff --git a/packages/nocodb/src/lib/services/dbData/bulkData.ts b/packages/nocodb/src/lib/services/dbData/bulkData.ts
index f67edab921..e8c86e063e 100644
--- a/packages/nocodb/src/lib/services/dbData/bulkData.ts
+++ b/packages/nocodb/src/lib/services/dbData/bulkData.ts
@@ -40,12 +40,13 @@ export async function bulkDataInsert(
     cookie: any;
     chunkSize?: number;
     foreign_key_checks?: boolean;
+    raw?: boolean;
   }
 ) {
   return await executeBulkOperation({
     ...param,
     operation: 'bulkInsert',
-    options: [param.body, { cookie: param.cookie, foreign_key_checks: param.foreign_key_checks, chunkSize: param.chunkSize }],
+    options: [param.body, { cookie: param.cookie, foreign_key_checks: param.foreign_key_checks, chunkSize: param.chunkSize, raw: param.raw }],
   });
 }
 
diff --git a/packages/nocodb/src/lib/services/exportImport/import.svc.ts b/packages/nocodb/src/lib/services/exportImport/import.svc.ts
index c5a25b54b2..e6c9972822 100644
--- a/packages/nocodb/src/lib/services/exportImport/import.svc.ts
+++ b/packages/nocodb/src/lib/services/exportImport/import.svc.ts
@@ -653,9 +653,9 @@ export async function importBase(param: {
                             base_id: baseId,
                             colId: col.colOptions.fk_child_column_id,
                           });
-                          headers.push(childCol.title);
+                          headers.push(childCol.column_name);
                         } else {
-                          headers.push(col.title);
+                          headers.push(col.column_name);
                         }
                       } else {
                         debugLog(header);
@@ -671,7 +671,7 @@ export async function importBase(param: {
                         }
                       }
                       chunk.push(row);
-                      if (chunk.length > 1000) {
+                      if (chunk.length > 100) {
                         parser.pause();
                         elapsedTime('before import chunk');
                         try {
@@ -681,7 +681,8 @@ export async function importBase(param: {
                             body: chunk,
                             cookie: null,
                             chunkSize: chunk.length + 1,
-                            foreign_key_checks: false
+                            foreign_key_checks: false,
+                            raw: true,
                           });
                         } catch (e) {
                           debugLog(`${model.title} import throwed an error!`);
@@ -704,7 +705,8 @@ export async function importBase(param: {
                         body: chunk,
                         cookie: null,
                         chunkSize: chunk.length + 1,
-                        foreign_key_checks: false
+                        foreign_key_checks: false,
+                        raw: true,
                       });
                     } catch (e) {
                       debugLog(chunk);
@@ -779,8 +781,8 @@ export async function importBase(param: {
                             const vParentCol = await colOptions.getMMParentColumn();
 
                             mmParentChild[col.colOptions.fk_mm_model_id] = {
-                              parent: vParentCol.title,
-                              child: vChildCol.title,
+                              parent: vParentCol.column_name,
+                              child: vChildCol.column_name,
                             }
 
                             handledLinks.push(col.colOptions.fk_mm_model_id);
@@ -816,7 +818,8 @@ export async function importBase(param: {
                         body: v,
                         cookie: null,
                         chunkSize: 1000,
-                        foreign_key_checks: false
+                        foreign_key_checks: false,
+                        raw: true,
                       });
                       elapsedTime('insert link chunk');
                     } catch (e) {