feat: local cache (wip)

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts

@@ -1397,11 +1397,7 @@ export default async (
     return rec;
   }
 
-  async function nocoReadData(sDB, table, aTbl) {
+  async function processRecords(records, sDB, table, insertJobs) {
-    ncLinkDataStore[table.title] = {};
-    const insertJobs: Promise<any>[] = [];
-
-    async function processRecords(records) {
     // This function (`page`) will get called for each page of records.
     logBasic(
       `:: ${table.title} : ${recordCnt + 1} ~ ${(recordCnt += pageSize)}`
@@ -1431,22 +1427,27 @@ export default async (
     recordPerfStats(_perfStart, 'dbTableRow.bulkCreate');
   }
 
+  async function nocoReadData(sDB, table, aTbl) {
+    ncLinkDataStore[table.title] = {};
+    const insertJobs: Promise<any>[] = [];
+
     // skip virtual columns
     const fieldsArray = aTbl.columns
       .filter(a => !['formula', 'lookup', 'rollup'].includes(a.type))
       .map(a => a.name);
 
-    if (enableLocalCache) {
+    if (
+      enableLocalCache &&
+      fs2.existsSync(`./migrationCache/data/${table.title}_0.json`)
+    ) {
       // check if cached file exists for this table
-      const f = `./migrationCache/data/${table.title}_${recordCnt /
+      let f = `./migrationCache/data/${table.title}_${recordCnt /
         pageSize}.json`;
 
       while (fs2.existsSync(f)) {
         const records = jsonfile.readFileSync(f);
-        logBasic(
+        await processRecords(records, sDB, table, insertJobs);
-          `:: ${table.title} : ${recordCnt + 1} ~ ${(recordCnt += pageSize)}`
+        f = `./migrationCache/data/${table.title}_${recordCnt / pageSize}.json`;
-        );
-        await processRecords(records);
       }
 
       // scenarios to handle
@@ -1476,7 +1477,7 @@ export default async (
               );
             // console.log(JSON.stringify(records, null, 2));
 
-            await processRecords(records);
+            await processRecords(records, sDB, table, insertJobs);
 
             // To fetch the next page of records, call `fetchNextPage`.
             // If there are more records, `page` will get called again.