From 0cf66771d427a2d55bd8817081f0a1eb4ca2768b Mon Sep 17 00:00:00 2001 From: mertmit Date: Thu, 25 Jan 2024 16:11:11 +0000 Subject: [PATCH] feat: use env variables for at import settings --- .../jobs/at-import/helpers/readAndProcessData.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts index 6dc5ea7849..e06c322340 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts @@ -11,12 +11,15 @@ import type { Source } from '~/models'; const logger = new Logger('BaseModelSqlv2'); -const BULK_DATA_BATCH_COUNT = 20; // check size for every 20 records -const BULK_DATA_BATCH_SIZE = 20 * 1024; // in bytes -const BULK_LINK_BATCH_COUNT = 200; // process 200 links at a time -const BULK_PARALLEL_PROCESS = 5; -const STREAM_BUFFER_LIMIT = 200; -const QUEUE_BUFFER_LIMIT = 50; +const BULK_DATA_BATCH_COUNT = + +process.env.AT_IMPORT_BULK_DATA_BATCH_COUNT || 10; // check size for every N records +const BULK_DATA_BATCH_SIZE = + +process.env.AT_IMPORT_BULK_DATA_BATCH_SIZE || 20 * 1024; // import N bytes at a time +const BULK_LINK_BATCH_COUNT = + +process.env.AT_IMPORT_BULK_LINK_BATCH_COUNT || 200; // import N links at a time +const BULK_PARALLEL_PROCESS = +process.env.AT_IMPORT_BULK_PARALLEL_PROCESS || 2; // process N records at a time +const STREAM_BUFFER_LIMIT = +process.env.AT_IMPORT_STREAM_BUFFER_LIMIT || 100; // pause reading if we have more than N records to avoid backpressure +const QUEUE_BUFFER_LIMIT = +process.env.AT_IMPORT_QUEUE_BUFFER_LIMIT || 20; // pause streaming if we have more than N records in the queue interface AirtableImportContext { bulkDataService: BulkDataAliasService;