From fe394e61750967868231f1cd541b53108818902c Mon Sep 17 00:00:00 2001 From: mertmit Date: Sun, 27 Nov 2022 18:48:05 +0300 Subject: [PATCH] fix: lower parallel process and migrate request to axios for local Signed-off-by: mertmit --- .../api/sync/helpers/readAndProcessData.ts | 2 +- .../plugins/adapters/storage/Local.ts | 57 +++++++++---------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts index 5fe7f76b5d..ad004205d1 100644 --- a/packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts @@ -4,7 +4,7 @@ import EntityMap from './EntityMap'; const BULK_DATA_BATCH_SIZE = 500; const ASSOC_BULK_DATA_BATCH_SIZE = 1000; -const BULK_PARALLEL_PROCESS = 100; +const BULK_PARALLEL_PROCESS = 5; async function readAllData({ table, diff --git a/packages/nocodb/src/lib/v1-legacy/plugins/adapters/storage/Local.ts b/packages/nocodb/src/lib/v1-legacy/plugins/adapters/storage/Local.ts index edcc20b385..48b695b878 100644 --- a/packages/nocodb/src/lib/v1-legacy/plugins/adapters/storage/Local.ts +++ b/packages/nocodb/src/lib/v1-legacy/plugins/adapters/storage/Local.ts @@ -6,7 +6,7 @@ import mkdirp from 'mkdirp'; import { IStorageAdapterV2, XcFile } from 'nc-plugin'; import NcConfigFactory from '../../../../utils/NcConfigFactory'; -import request from 'request'; +import axios from 'axios'; export default class Local implements IStorageAdapterV2 { constructor() {} @@ -27,37 +27,36 @@ export default class Local implements IStorageAdapterV2 { async fileCreateByUrl(key: string, url: string): Promise { const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/')); return new Promise((resolve, reject) => { - mkdirp.sync(path.dirname(destPath)); - const file = fs.createWriteStream(destPath); - const sendReq = request.get(url); - - // verify response code - sendReq.on('response', (response) => { - if (response.statusCode !== 200) { - return reject('Response status was ' + response.statusCode); - } - - sendReq.pipe(file); - }); - - // close() is async, call cb after close completes - file.on('finish', () => { - file.close((err) => { - if (err) { - return reject(err); - } - resolve(null); + axios.get((url), { responseType: "stream", headers: { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "accept-language": "en-US,en;q=0.9", + "cache-control": "no-cache", + "pragma": "no-cache", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", + "origin": "https://www.airtable.com/", + } }) + .then(response => { + mkdirp.sync(path.dirname(destPath)); + const file = fs.createWriteStream(destPath); + // close() is async, call cb after close completes + file.on('finish', () => { + file.close((err) => { + if (err) { + return reject(err); + } + resolve(null); + }); }); - }); - // check for request errors - sendReq.on('error', (err) => { - fs.unlink(destPath, () => reject(err.message)); // delete the (partial) file and then return the error - }); + file.on('error', (err) => { + // Handle errors + fs.unlink(destPath, () => reject(err.message)); // delete the (partial) file and then return the error + }); - file.on('error', (err) => { - // Handle errors - fs.unlink(destPath, () => reject(err.message)); // delete the (partial) file and then return the error + response.data.pipe(file); + }) + .catch((err) => { + reject(err.message) }); }); }