fix: dynamic batch based on byte size

Signed-off-by: mertmit <mertmit99@gmail.com>

packages/nocodb/package-lock.json

@@ -86,6 +86,7 @@
         "nocodb-sdk": "file:../nocodb-sdk",
         "nodemailer": "^6.4.10",
         "object-hash": "^3.0.0",
+        "object-sizeof": "^2.6.1",
         "os-locale": "^6.0.2",
         "p-queue": "^6.6.2",
         "papaparse": "^5.3.1",
@@ -13568,6 +13569,37 @@
       "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-0.4.0.tgz",
       "integrity": "sha512-ncrLw+X55z7bkl5PnUvHwFK9FcGuFYo9gtjws2XtSzL+aZ8tm830P60WJ0dSmFVaSalWieW5MD7kEdnXda9yJw=="
     },
+    "node_modules/object-sizeof": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/object-sizeof/-/object-sizeof-2.6.1.tgz",
+      "integrity": "sha512-a7VJ1Zx7ZuHceKwjgfsSqzV/X0PVGvpZz7ho3Dn4Cs0LLcR5e5WuV+gsbizmplD8s0nAXMJmckKB2rkSiPm/Gg==",
+      "dependencies": {
+        "buffer": "^6.0.3"
+      }
+    },
+    "node_modules/object-sizeof/node_modules/buffer": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
+      "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.2.1"
+      }
+    },
     "node_modules/object.assign": {
       "version": "4.1.4",
       "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz",
@@ -28765,6 +28797,25 @@
       "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-0.4.0.tgz",
       "integrity": "sha512-ncrLw+X55z7bkl5PnUvHwFK9FcGuFYo9gtjws2XtSzL+aZ8tm830P60WJ0dSmFVaSalWieW5MD7kEdnXda9yJw=="
     },
+    "object-sizeof": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/object-sizeof/-/object-sizeof-2.6.1.tgz",
+      "integrity": "sha512-a7VJ1Zx7ZuHceKwjgfsSqzV/X0PVGvpZz7ho3Dn4Cs0LLcR5e5WuV+gsbizmplD8s0nAXMJmckKB2rkSiPm/Gg==",
+      "requires": {
+        "buffer": "^6.0.3"
+      },
+      "dependencies": {
+        "buffer": {
+          "version": "6.0.3",
+          "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
+          "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
+          "requires": {
+            "base64-js": "^1.3.1",
+            "ieee754": "^1.2.1"
+          }
+        }
+      }
+    },
     "object.assign": {
       "version": "4.1.4",
       "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz",

packages/nocodb/package.json

@@ -119,6 +119,7 @@
     "nocodb-sdk": "file:../nocodb-sdk",
     "nodemailer": "^6.4.10",
     "object-hash": "^3.0.0",
+    "object-sizeof": "^2.6.1",
     "os-locale": "^6.0.2",
     "p-queue": "^6.6.2",
     "papaparse": "^5.3.1",

packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts

@@ -1,5 +1,6 @@
 /* eslint-disable no-async-promise-executor */
 import { RelationTypes, UITypes } from 'nocodb-sdk';
+import sizeof from 'object-sizeof';
 import EntityMap from './EntityMap';
 import type { BulkDataAliasService } from '../../../../../services/bulk-data-alias.service';
 import type { TablesService } from '../../../../../services/tables.service';
@@ -7,8 +8,8 @@ import type { TablesService } from '../../../../../services/tables.service';
 import type { AirtableBase } from 'airtable/lib/airtable_base';
 import type { TableType } from 'nocodb-sdk';
 
-const BULK_DATA_BATCH_SIZE = 500;
+const BULK_DATA_BATCH_COUNT = 100; // check size for every 100 records
-const ASSOC_BULK_DATA_BATCH_SIZE = 1000;
+const BULK_DATA_BATCH_SIZE = 102400; // in bytes
 const BULK_PARALLEL_PROCESS = 5;
 
 interface AirtableImportContext {
@@ -122,27 +123,28 @@ export async function importData({
               fields,
             });
             tempData.push(r);
-
+            if (tempData.length % BULK_DATA_BATCH_COUNT === 0) {
-            if (tempData.length >= BULK_DATA_BATCH_SIZE) {
+              if (sizeof(tempData) >= BULK_DATA_BATCH_SIZE) {
-              let insertArray = tempData.splice(0, tempData.length);
+                let insertArray = tempData.splice(0, tempData.length);
-
+
-              await services.bulkDataService.bulkDataInsert({
+                await services.bulkDataService.bulkDataInsert({
-                projectName,
+                  projectName,
-                tableName: table.title,
+                  tableName: table.title,
-                body: insertArray,
+                  body: insertArray,
-                cookie: {},
+                  cookie: {},
-              });
+                });
-
+
-              logBasic(
+                logBasic(
-                `:: Importing '${
+                  `:: Importing '${
-                  table.title
+                    table.title
-                }' data :: ${importedCount} - ${Math.min(
+                  }' data :: ${importedCount} - ${Math.min(
-                  importedCount + BULK_DATA_BATCH_SIZE,
+                    importedCount + insertArray.length,
-                  allRecordsCount,
+                    allRecordsCount,
-                )}`,
+                  )}`,
-              );
+                );
-              importedCount += insertArray.length;
+                importedCount += insertArray.length;
-              insertArray = [];
+                insertArray = [];
+              }
             }
             activeProcess--;
             if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume();
@@ -164,7 +166,7 @@ export async function importData({
             `:: Importing '${
               table.title
             }' data :: ${importedCount} - ${Math.min(
-              importedCount + BULK_DATA_BATCH_SIZE,
+              importedCount + tempData.length,
               allRecordsCount,
             )}`,
           );
@@ -300,26 +302,31 @@ export async function importLTARData({
               })),
             );
 
-            if (assocTableData.length >= ASSOC_BULK_DATA_BATCH_SIZE) {
+            if (assocTableData.length % BULK_DATA_BATCH_COUNT === 0) {
-              let insertArray = assocTableData.splice(0, assocTableData.length);
+              if (sizeof(assocTableData) >= BULK_DATA_BATCH_SIZE) {
-              logBasic(
+                let insertArray = assocTableData.splice(
-                `:: Importing '${
+                  0,
-                  table.title
+                  assocTableData.length,
-                }' LTAR data :: ${importedCount} - ${Math.min(
+                );
-                  importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
+                logBasic(
-                  insertArray.length,
+                  `:: Importing '${
-                )}`,
+                    table.title
-              );
+                  }' LTAR data :: ${importedCount} - ${Math.min(
-
+                    importedCount + insertArray.length,
-              await services.bulkDataService.bulkDataInsert({
+                    insertArray.length,
-                projectName,
+                  )}`,
-                tableName: assocMeta.modelMeta.title,
+                );
-                body: insertArray,
+
-                cookie: {},
+                await services.bulkDataService.bulkDataInsert({
-              });
+                  projectName,
-
+                  tableName: assocMeta.modelMeta.title,
-              importedCount += insertArray.length;
+                  body: insertArray,
-              insertArray = [];
+                  cookie: {},
+                });
+
+                importedCount += insertArray.length;
+                insertArray = [];
+              }
             }
             activeProcess--;
             if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume();
@@ -334,7 +341,7 @@ export async function importLTARData({
             `:: Importing '${
               table.title
             }' LTAR data :: ${importedCount} - ${Math.min(
-              importedCount + ASSOC_BULK_DATA_BATCH_SIZE,
+              importedCount + assocTableData.length,
               assocTableData.length,
             )}`,
           );