From 92138fbc3d419c7fb6591510da6cf9d7d77e0ee0 Mon Sep 17 00:00:00 2001
From: Raju Udava <86527202+dstala@users.noreply.github.com>
Date: Fri, 27 May 2022 21:46:20 +0530
Subject: [PATCH] feat: skip virtual col during data read, pageSize made
 configurable

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>
---
 .../src/lib/noco/meta/api/sync/helpers/job.ts | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts
index ffbe417402..a904542185 100644
--- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts
+++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts
@@ -82,6 +82,7 @@ export default async (
   const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' };
   const storeLinks = false;
   const ncLinkDataStore: any = {};
+  const pageSize = 100;
 
   const uniqueTableNameGen = getUniqueNameGenerator('sheet');
 
@@ -1375,14 +1376,20 @@ export default async (
     return rec;
   }
 
-  async function nocoReadData(sDB, table) {
+  async function nocoReadData(sDB, table, aTbl) {
     ncLinkDataStore[table.title] = {};
     const insertJobs: Promise<any>[] = [];
 
+    // skip virtual columns
+    const fieldsArray = aTbl.columns
+      .filter(a => !['formula', 'lookup', 'rollup'].includes(a.type))
+      .map(a => a.name);
+
     return new Promise((resolve, reject) => {
       base(table.title)
         .select({
-          pageSize: 100
+          pageSize: pageSize,
+          fields: fieldsArray
           // maxRecords: 1,
         })
         .eachPage(
@@ -1391,7 +1398,8 @@ export default async (
 
             // This function (`page`) will get called for each page of records.
             logBasic(
-              `:: ${table.title} : ${recordCnt + 1} ~ ${(recordCnt += 100)}`
+              `:: ${table.title} : ${recordCnt +
+                1} ~ ${(recordCnt += pageSize)}`
             );
 
             // await Promise.all(
@@ -1444,7 +1452,7 @@ export default async (
     return new Promise((resolve, reject) => {
       base(table.title)
         .select({
-          pageSize: 100,
+          pageSize: pageSize,
           // maxRecords: 100,
           fields: fields
         })
@@ -1456,7 +1464,7 @@ export default async (
             // records.forEach(record => callback(table, record));
             logBasic(
               `:: ${table.title} / ${fields} : ${recordCnt +
-                1} ~ ${(recordCnt += 100)}`
+                1} ~ ${(recordCnt += pageSize)}`
             );
             await Promise.all(
               records.map(r => callback(projName, table, r, fields))
@@ -2193,11 +2201,11 @@ export default async (
           recordPerfStats(_perfStart, 'dbTable.read');
 
           // not a migrated table, skip
-          if (undefined === aTblSchema.find(x => x.name === ncTbl.title))
-            continue;
+          const aTbl = aTblSchema.find(x => x.name === ncTbl.title);
+          if (undefined === aTbl) continue;
 
           recordCnt = 0;
-          await nocoReadData(syncDB, ncTbl);
+          await nocoReadData(syncDB, ncTbl, aTbl);
           logDetailed(`Data inserted from ${ncTbl.title}`);
         }