Browse Source

fix: lower parallel process and migrate request to axios for local

Signed-off-by: mertmit <mertmit99@gmail.com>
pull/4486/head
mertmit 2 years ago
parent
commit
fe394e6175
  1. 2
      packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts
  2. 33
      packages/nocodb/src/lib/v1-legacy/plugins/adapters/storage/Local.ts

2
packages/nocodb/src/lib/meta/api/sync/helpers/readAndProcessData.ts

@ -4,7 +4,7 @@ import EntityMap from './EntityMap';
const BULK_DATA_BATCH_SIZE = 500;
const ASSOC_BULK_DATA_BATCH_SIZE = 1000;
const BULK_PARALLEL_PROCESS = 100;
const BULK_PARALLEL_PROCESS = 5;
async function readAllData({
table,

33
packages/nocodb/src/lib/v1-legacy/plugins/adapters/storage/Local.ts

@ -6,7 +6,7 @@ import mkdirp from 'mkdirp';
import { IStorageAdapterV2, XcFile } from 'nc-plugin';
import NcConfigFactory from '../../../../utils/NcConfigFactory';
import request from 'request';
import axios from 'axios';
export default class Local implements IStorageAdapterV2 {
constructor() {}
@ -27,19 +27,17 @@ export default class Local implements IStorageAdapterV2 {
async fileCreateByUrl(key: string, url: string): Promise<any> {
const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
return new Promise((resolve, reject) => {
axios.get((url), { responseType: "stream", headers: {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept-language": "en-US,en;q=0.9",
"cache-control": "no-cache",
"pragma": "no-cache",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
"origin": "https://www.airtable.com/",
} })
.then(response => {
mkdirp.sync(path.dirname(destPath));
const file = fs.createWriteStream(destPath);
const sendReq = request.get(url);
// verify response code
sendReq.on('response', (response) => {
if (response.statusCode !== 200) {
return reject('Response status was ' + response.statusCode);
}
sendReq.pipe(file);
});
// close() is async, call cb after close completes
file.on('finish', () => {
file.close((err) => {
@ -50,15 +48,16 @@ export default class Local implements IStorageAdapterV2 {
});
});
// check for request errors
sendReq.on('error', (err) => {
fs.unlink(destPath, () => reject(err.message)); // delete the (partial) file and then return the error
});
file.on('error', (err) => {
// Handle errors
fs.unlink(destPath, () => reject(err.message)); // delete the (partial) file and then return the error
});
response.data.pipe(file);
})
.catch((err) => {
reject(err.message)
});
});
}

Loading…
Cancel
Save