From 509a697a4af3899ca046d301c627f3b8b7c9af53 Mon Sep 17 00:00:00 2001
From: mertmit <mertmit99@gmail.com>
Date: Thu, 19 Sep 2024 11:54:49 +0000
Subject: [PATCH] feat: use stream-json to filter response

---
 .../jobs/jobs/at-import/helpers/fetchAT.ts    | 56 +++++++++++++++++--
 1 file changed, 52 insertions(+), 4 deletions(-)

diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
index ac0fa7fcd0..e685c511cf 100644
--- a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
+++ b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
@@ -1,5 +1,8 @@
 import { Logger } from '@nestjs/common';
 import axios from 'axios';
+import { streamObject } from 'stream-json/streamers/StreamObject';
+import { parser } from 'stream-json/Parser';
+import { ignore } from 'stream-json/filters/Ignore';
 
 const logger = new Logger('FetchAT');
 
@@ -126,11 +129,33 @@ async function read() {
         referrerPolicy: 'no-referrer',
         body: null,
         method: 'GET',
+        responseType: 'stream',
       });
 
-      if (resreq?.data?.data) {
+      const data: any = await new Promise((resolve, reject) => {
+        const jsonStream = resreq.data
+          .pipe(parser())
+          .pipe(ignore({ filter: 'data.tableDatas' }))
+          .pipe(streamObject());
+
+        const fullObject = {};
+
+        jsonStream.on('data', (chunk) => {
+          if (chunk.key) fullObject[chunk.key] = chunk.value;
+        });
+
+        jsonStream.on('error', (err) => {
+          reject(err);
+        });
+
+        jsonStream.on('end', () => {
+          resolve(fullObject);
+        });
+      });
+
+      if (data?.data) {
         return {
-          schema: resreq.data.data,
+          schema: data?.data,
           baseId: info.baseId,
           baseInfo: info.baseInfo,
         };
@@ -194,11 +219,34 @@ async function readView(viewId) {
           referrerPolicy: 'no-referrer',
           body: null,
           method: 'GET',
+          responseType: 'stream',
         },
       );
 
-      if (resreq?.data?.data) {
-        return { view: resreq.data.data };
+      const data: any = await new Promise((resolve, reject) => {
+        const jsonStream = resreq.data
+          .pipe(parser())
+          .pipe(ignore({ filter: 'data.rowOrder' }))
+          .pipe(streamObject());
+
+        const fullObject = {};
+
+        jsonStream.on('data', (chunk) => {
+          if (chunk.key) fullObject[chunk.key] = chunk.value;
+        });
+
+        jsonStream.on('error', (err) => {
+          reject(err);
+        });
+
+        jsonStream.on('end', () => {
+          resolve(fullObject);
+        });
+      });
+      
+
+      if (data?.data) {
+        return { view: data.data };
       } else {
         throw new Error('Error Reading :: Data missing');
       }