fix: lower parallel process and migrate request to axios for local

Signed-off-by: mertmit <mertmit99@gmail.com>

packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts

@@ -4,7 +4,7 @@ import EntityMap from './EntityMap';
 
 const BULK_DATA_BATCH_SIZE = 500;
 const ASSOC_BULK_DATA_BATCH_SIZE = 1000;
-const BULK_PARALLEL_PROCESS = 100;
+const BULK_PARALLEL_PROCESS = 5;
 
 async function readAllData({
   table,

packages/nocodb/src/lib/v1-legacy/plugins/adapters/storage/Local.ts

@@ -6,7 +6,7 @@ import mkdirp from 'mkdirp';
 import { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import NcConfigFactory from '../../../../utils/NcConfigFactory';
 
-import request from 'request';
+import axios from 'axios';
 
 export default class Local implements IStorageAdapterV2 {
   constructor() {}
@@ -27,37 +27,36 @@ export default class Local implements IStorageAdapterV2 {
   async fileCreateByUrl(key: string, url: string): Promise<any> {
     const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
     return new Promise((resolve, reject) => {
-      mkdirp.sync(path.dirname(destPath));
+      axios.get((url), { responseType: "stream", headers: {
-      const file = fs.createWriteStream(destPath);
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
-      const sendReq = request.get(url);
+        "accept-language": "en-US,en;q=0.9",
-
+        "cache-control": "no-cache",
-      // verify response code
+        "pragma": "no-cache",
-      sendReq.on('response', (response) => {
+        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
-        if (response.statusCode !== 200) {
+        "origin": "https://www.airtable.com/",
-          return reject('Response status was ' + response.statusCode);
+      } })
-        }
+      .then(response => {
-
+        mkdirp.sync(path.dirname(destPath));
-        sendReq.pipe(file);
+        const file = fs.createWriteStream(destPath);
-      });
+        // close() is async, call cb after close completes
-
+        file.on('finish', () => {
-      // close() is async, call cb after close completes
+          file.close((err) => {
-      file.on('finish', () => {
+            if (err) {
-        file.close((err) => {
+              return reject(err);
-          if (err) {
+            }
-            return reject(err);
+            resolve(null);
-          }
+          });
-          resolve(null);
         });
-      });
 
-      // check for request errors
+        file.on('error', (err) => {
-      sendReq.on('error', (err) => {
+          // Handle errors
-        fs.unlink(destPath, () => reject(err.message)); // delete the (partial) file and then return the error
+          fs.unlink(destPath, () => reject(err.message)); // delete the (partial) file and then return the error
-      });
+        });
 
-      file.on('error', (err) => {
+        response.data.pipe(file);
-        // Handle errors
+      })
-        fs.unlink(destPath, () => reject(err.message)); // delete the (partial) file and then return the error
+      .catch((err) => {
+        reject(err.message)
       });
     });
   }