feat: raw bulk insert for import

Signed-off-by: mertmit <mertmit99@gmail.com>

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts

@@ -2067,14 +2067,17 @@ class BaseModelSqlv2 {
       chunkSize: _chunkSize = 100,
       cookie,
       foreign_key_checks = true,
+      raw = false,
     }: {
       chunkSize?: number;
       cookie?: any;
       foreign_key_checks?: boolean;
+      raw?: boolean;
     } = {}
   ) {
     try {
-      const insertDatas = await Promise.all(
+      // TODO: ag column handling for raw bulk insert
+      const insertDatas = raw ? datas : await Promise.all(
         datas.map(async (d) => {
           await populatePk(this.model, d);
           return this.model.mapAliasToColumn(d);
@@ -2083,9 +2086,11 @@ class BaseModelSqlv2 {
 
       // await this.beforeInsertb(insertDatas, null);
 
+      if (!raw) {
         for (const data of datas) {
           await this.validate(data);
         } 
+      }
 
       // fallbacks to `10` if database client is sqlite
       // to avoid `too many SQL variables` error
@@ -2123,7 +2128,7 @@ class BaseModelSqlv2 {
 
       await trx.commit();
 
-      await this.afterBulkInsert(insertDatas, this.dbDriver, cookie);
+      if (!raw) await this.afterBulkInsert(insertDatas, this.dbDriver, cookie);
 
       return response;
     } catch (e) {

packages/nocodb/src/lib/services/dbData/bulkData.ts

@@ -40,12 +40,13 @@ export async function bulkDataInsert(
     cookie: any;
     chunkSize?: number;
     foreign_key_checks?: boolean;
+    raw?: boolean;
   }
 ) {
   return await executeBulkOperation({
     ...param,
     operation: 'bulkInsert',
-    options: [param.body, { cookie: param.cookie, foreign_key_checks: param.foreign_key_checks, chunkSize: param.chunkSize }],
+    options: [param.body, { cookie: param.cookie, foreign_key_checks: param.foreign_key_checks, chunkSize: param.chunkSize, raw: param.raw }],
   });
 }
 

packages/nocodb/src/lib/services/exportImport/import.svc.ts

@@ -653,9 +653,9 @@ export async function importBase(param: {
                             base_id: baseId,
                             colId: col.colOptions.fk_child_column_id,
                           });
-                          headers.push(childCol.title);
+                          headers.push(childCol.column_name);
                         } else {
-                          headers.push(col.title);
+                          headers.push(col.column_name);
                         }
                       } else {
                         debugLog(header);
@@ -671,7 +671,7 @@ export async function importBase(param: {
                         }
                       }
                       chunk.push(row);
-                      if (chunk.length > 1000) {
+                      if (chunk.length > 100) {
                         parser.pause();
                         elapsedTime('before import chunk');
                         try {
@@ -681,7 +681,8 @@ export async function importBase(param: {
                             body: chunk,
                             cookie: null,
                             chunkSize: chunk.length + 1,
-                            foreign_key_checks: false
+                            foreign_key_checks: false,
+                            raw: true,
                           });
                         } catch (e) {
                           debugLog(`${model.title} import throwed an error!`);
@@ -704,7 +705,8 @@ export async function importBase(param: {
                         body: chunk,
                         cookie: null,
                         chunkSize: chunk.length + 1,
-                        foreign_key_checks: false
+                        foreign_key_checks: false,
+                        raw: true,
                       });
                     } catch (e) {
                       debugLog(chunk);
@@ -779,8 +781,8 @@ export async function importBase(param: {
                             const vParentCol = await colOptions.getMMParentColumn();
 
                             mmParentChild[col.colOptions.fk_mm_model_id] = {
-                              parent: vParentCol.title,
-                              child: vChildCol.title,
+                              parent: vParentCol.column_name,
+                              child: vChildCol.column_name,
                             }
 
                             handledLinks.push(col.colOptions.fk_mm_model_id);
@@ -816,7 +818,8 @@ export async function importBase(param: {
                         body: v,
                         cookie: null,
                         chunkSize: 1000,
-                        foreign_key_checks: false
+                        foreign_key_checks: false,
+                        raw: true,
                       });
                       elapsedTime('insert link chunk');
                     } catch (e) {