From fe23cf31a1955403b7034277bd9646d9dfea2f86 Mon Sep 17 00:00:00 2001
From: mertmit <mertmit99@gmail.com>
Date: Tue, 18 Apr 2023 12:39:07 +0300
Subject: [PATCH] feat: debugLog for import/export

Signed-off-by: mertmit <mertmit99@gmail.com>
---
 .../lib/services/exportImport/import.svc.ts   | 85 ++++++++++++-------
 1 file changed, 53 insertions(+), 32 deletions(-)

diff --git a/packages/nocodb/src/lib/services/exportImport/import.svc.ts b/packages/nocodb/src/lib/services/exportImport/import.svc.ts
index 31fd4b32e6..c5a25b54b2 100644
--- a/packages/nocodb/src/lib/services/exportImport/import.svc.ts
+++ b/packages/nocodb/src/lib/services/exportImport/import.svc.ts
@@ -573,13 +573,20 @@ export async function importBase(param: {
 }) {
   const { user, projectId, baseId, src, req } = param;
 
+  const debug = req.params.debug === 'true';
+
+  const debugLog = (...args: any[]) => {
+    if (!debug) return;
+    console.log(...args);
+  }
+
   let start = process.hrtime();
 
-  let elapsed_time = function(label: string){
-      const elapsedS = (process.hrtime(start)[0]).toFixed(3);
-      const elapsedMs = process.hrtime(start)[1] / 1000000;
-      console.log(`${label}: ${elapsedS}s ${elapsedMs}ms`);
-      start = process.hrtime();
+  let elapsedTime = function(label?: string){
+    const elapsedS = (process.hrtime(start)[0]).toFixed(3);
+    const elapsedMs = process.hrtime(start)[1] / 1000000;
+    if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
+    start = process.hrtime();
   }
 
   switch (src.type) {
@@ -591,7 +598,7 @@ export async function importBase(param: {
       try {
         const schema = JSON.parse(await storageAdapter.fileRead(`${path}/schema.json`));
 
-        elapsed_time('read schema');
+        elapsedTime('read schema');
 
         // store fk_mm_model_id (mm) to link once
         const handledLinks = [];
@@ -604,7 +611,7 @@ export async function importBase(param: {
           req,
         });
 
-        elapsed_time('import models');
+        elapsedTime('import models');
 
         if (idMap) {
           const files = await storageAdapter.getDirectoryList(`${path}/data`);
@@ -626,7 +633,7 @@ export async function importBase(param: {
 
             const model = await Model.get(modelId);
 
-            console.log(`Importing ${model.title}...`);
+            debugLog(`Importing ${model.title}...`);
 
             await new Promise(async (resolve) => {
               papaparse.parse(readStream, {
@@ -650,9 +657,8 @@ export async function importBase(param: {
                         } else {
                           headers.push(col.title);
                         }
-                        
                       } else {
-                          console.log(header);
+                        debugLog(header);
                       }
                     }
                     parser.resume();
@@ -667,17 +673,22 @@ export async function importBase(param: {
                       chunk.push(row);
                       if (chunk.length > 1000) {
                         parser.pause();
-                        elapsed_time('before chunk');
-                        await bulkDataService.bulkDataInsert({
-                          projectName: projectId,
-                          tableName: modelId,
-                          body: chunk,
-                          cookie: null,
-                          chunkSize: 1000,
-                          foreign_key_checks: false
-                        });
+                        elapsedTime('before import chunk');
+                        try {
+                          await bulkDataService.bulkDataInsert({
+                            projectName: projectId,
+                            tableName: modelId,
+                            body: chunk,
+                            cookie: null,
+                            chunkSize: chunk.length + 1,
+                            foreign_key_checks: false
+                          });
+                        } catch (e) {
+                          debugLog(`${model.title} import throwed an error!`);
+                          console.log(e);
+                        }
                         chunk = [];
-                        elapsed_time('after chunk');
+                        elapsedTime('after import chunk');
                         parser.resume();
                       }
                     }
@@ -685,16 +696,22 @@ export async function importBase(param: {
                 },
                 complete: async function () {
                   if (chunk.length > 0) {
-                    elapsed_time('before chunk');
-                    await bulkDataService.bulkDataInsert({
-                      projectName: projectId,
-                      tableName: modelId,
-                      body: chunk,
-                      cookie: null,
-                      foreign_key_checks: false
-                    });
+                    elapsedTime('before import chunk');
+                    try {
+                      await bulkDataService.bulkDataInsert({
+                        projectName: projectId,
+                        tableName: modelId,
+                        body: chunk,
+                        cookie: null,
+                        chunkSize: chunk.length + 1,
+                        foreign_key_checks: false
+                      });
+                    } catch (e) {
+                      debugLog(chunk);
+                      console.log(e);
+                    }
                     chunk = [];
-                    elapsed_time('after chunk');
+                    elapsedTime('after import chunk');
                   }
                   resolve(null);
                 },
@@ -702,6 +719,9 @@ export async function importBase(param: {
             });
           }
 
+          // reset timer
+          elapsedTime();
+
           for (const file of linkFiles) {
             const readStream = await storageAdapter.fileReadByStream(
               `${path}/data/${file}`
@@ -719,7 +739,7 @@ export async function importBase(param: {
             
             let pkIndex = -1;
 
-            console.log(`Linking ${model.title}...`);
+            debugLog(`Linking ${model.title}...`);
 
             await new Promise(async (resolve) => {
               papaparse.parse(readStream, {
@@ -788,7 +808,8 @@ export async function importBase(param: {
                 },
                 complete: async function () {
                   for (const [k, v] of Object.entries(chunk)) {
-                    try {                      
+                    try {
+                      elapsedTime('prepare link chunk');
                       await bulkDataService.bulkDataInsert({
                         projectName: projectId,
                         tableName: k,
@@ -797,8 +818,8 @@ export async function importBase(param: {
                         chunkSize: 1000,
                         foreign_key_checks: false
                       });
+                      elapsedTime('insert link chunk');
                     } catch (e) {
-                      console.log('linkError');
                       console.log(e);
                     }
                   }