From 771c1c543bb2842e13686caa2c19df50ee38ad68 Mon Sep 17 00:00:00 2001
From: Pranav C <pranavxc@gmail.com>
Date: Tue, 31 May 2022 12:32:48 +0530
Subject: [PATCH] refactor: add option to pass chunk size in bulk insert method

Signed-off-by: Pranav C <pranavxc@gmail.com>
---
 .../src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts      | 11 +++++++++--
 .../noco/meta/api/sync/helpers/readAndProcessData.ts  | 10 +++++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts
index 0e6a2c073d..9c49608848 100644
--- a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts
+++ b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts
@@ -1513,7 +1513,14 @@ class BaseModelSqlv2 {
     }
   }
 
-  async bulkInsert(datas: any[]) {
+  async bulkInsert(
+    datas: any[],
+    {
+      chunkSize: _chunkSize = 100
+    }: {
+      chunkSize?: number;
+    } = {}
+  ) {
     try {
       const insertDatas = await Promise.all(
         datas.map(async d => {
@@ -1536,7 +1543,7 @@ class BaseModelSqlv2 {
       // fallbacks to `10` if database client is sqlite
       // to avoid `too many SQL variables` error
       // refer : https://www.sqlite.org/limits.html
-      const chunkSize = this.isSqlite ? 10 : 50;
+      const chunkSize = this.isSqlite ? 10 : _chunkSize;
 
       const response = await this.dbDriver
         .batchInsert(this.model.table_name, insertDatas, chunkSize)
diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts
index b255be6827..a15f5556f1 100644
--- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts
+++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts
@@ -126,7 +126,7 @@ export async function importLTARData({
   logDetailed: (string) => void;
   logBasic: (string) => void;
   api: Api<any>;
-  insertedAssocRef: { [assocId: string]: boolean };
+  insertedAssocRef: { [assocTableId: string]: boolean };
 }) {
   const assocTableMetas: Array<{
     modelMeta: { id?: string; title?: string };
@@ -139,6 +139,7 @@ export async function importLTARData({
   const modelMeta: any = await api.dbTable.read(table.id);
 
   for (const colMeta of modelMeta.columns) {
+    // skip columns which are not LTAR and Many to many
     if (
       colMeta.uidt !== UITypes.LinkToAnotherRecord ||
       colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY
@@ -146,14 +147,17 @@ export async function importLTARData({
       continue;
     }
 
+    // skip if already inserted
     if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue;
 
+    // mark as inserted
     insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true;
 
     const assocModelMeta: TableType = (await api.dbTable.read(
       colMeta.colOptions.fk_mm_model_id
     )) as any;
 
+    // extract associative table and columns meta
     assocTableMetas.push({
       modelMeta: assocModelMeta,
       colMeta,
@@ -166,8 +170,11 @@ export async function importLTARData({
     });
   }
 
+  // Iterate over all related M2M associative  table
   for (const assocMeta of assocTableMetas) {
     const insertData = [];
+
+    //  extract insert data from records
     for (const record of allData) {
       const rec = record.fields;
 
@@ -180,6 +187,7 @@ export async function importLTARData({
       );
     }
 
+    // Insert datas as chunks of size `ASSOC_BULK_DATA_BATCH_SIZE`
     for (
       let i = 0;
       i < insertData.length / ASSOC_BULK_DATA_BATCH_SIZE;