From f1bebdd2b19656cc748e6139da52083fe0f5c534 Mon Sep 17 00:00:00 2001 From: mertmit Date: Mon, 22 Jan 2024 09:28:29 +0000 Subject: [PATCH 1/4] fix: improved readAndProcessData error handling --- .../at-import/helpers/readAndProcessData.ts | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts index 0cfd47c6e0..25a402652c 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts @@ -1,6 +1,7 @@ /* eslint-disable no-async-promise-executor */ import { isLinksOrLTAR, RelationTypes } from 'nocodb-sdk'; import sizeof from 'object-sizeof'; +import { Logger } from '@nestjs/common'; import EntityMap from './EntityMap'; import type { BulkDataAliasService } from '../../../../../services/bulk-data-alias.service'; import type { TablesService } from '../../../../../services/tables.service'; @@ -8,6 +9,8 @@ import type { TablesService } from '../../../../../services/tables.service'; import type { AirtableBase } from 'airtable/lib/airtable_base'; import type { TableType } from 'nocodb-sdk'; +const logger = new Logger('BaseModelSqlv2'); + const BULK_DATA_BATCH_COUNT = 20; // check size for every 100 records const BULK_DATA_BATCH_SIZE = 50 * 1024; // in bytes const BULK_LINK_BATCH_COUNT = 1000; // process 1000 records at a time @@ -30,7 +33,7 @@ async function readAllData({ logBasic?: (string) => void; logDetailed?: (string) => void; }): Promise { - return new Promise((resolve, reject) => { + return new Promise((resolve) => { let data = null; const selectParams: any = { @@ -74,8 +77,10 @@ async function readAllData({ }, async function done(err) { if (err) { - console.error(err); - return reject(err); + logger.error(err); + logBasic( + `:: There were errors on reading '${table.title}' data :: ${err}`, + ); } resolve(data); }, @@ -112,7 +117,7 @@ export async function importData({ logBasic, }); - await new Promise(async (resolve, reject) => { + await new Promise(async (resolve) => { const readable = records.getStream(); const allRecordsCount = await records.getCount(); const promises = []; @@ -126,7 +131,7 @@ export async function importData({ readable.on('data', async (record) => { promises.push( - new Promise(async (resolve, reject) => { + new Promise(async (resolve) => { try { activeProcess++; if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause(); @@ -173,7 +178,12 @@ export async function importData({ if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume(); resolve(true); } catch (e) { - reject(e); + logger.error(e); + logBasic( + `:: There were errors on importing '${table.title}' data :: ${e}`, + ); + readable.resume(); + resolve(true); } }), ); @@ -206,7 +216,11 @@ export async function importData({ } resolve(true); } catch (e) { - return reject(e); + logger.error(e); + logBasic( + `:: There were errors on importing '${table.title}' data :: ${e}`, + ); + resolve(true); } }); }); @@ -313,7 +327,7 @@ export async function importLTARData({ // Iterate over all related M2M associative table for (const assocMeta of assocTableMetas) { // extract link data from records - await new Promise((resolve, reject) => { + await new Promise((resolve) => { const promises = []; const readable = allData.getStream(); @@ -364,7 +378,12 @@ export async function importLTARData({ } resolve(true); } catch (e) { - reject(e); + logger.error(e); + logBasic( + `:: There were errors on importing '${table.title}' LTAR data :: ${e}`, + ); + readable.resume(); + resolve(true); } }), ); @@ -398,7 +417,11 @@ export async function importLTARData({ resolve(true); } catch (e) { - reject(e); + logger.error(e); + logBasic( + `:: There were errors on importing '${table.title}' LTAR data :: ${e}`, + ); + resolve(true); } }); }); From 8be23ff184a4892ea358c62f4d3837f8be0a66bd Mon Sep 17 00:00:00 2001 From: mertmit Date: Mon, 22 Jan 2024 09:28:29 +0000 Subject: [PATCH 2/4] fix: remove null character not supported by pg --- .../src/modules/jobs/jobs/at-import/at-import.processor.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index e053b2b195..03ab1ecc9a 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -1528,6 +1528,11 @@ export class AtImportProcessor { } break; + case UITypes.LongText: + // eslint-disable-next-line no-control-regex + rec[key] = value.replace(/\u0000/g, ''); + break; + default: break; } From d1b2f6ce0fee4b4fcd7c1a3fdb6baea241586a14 Mon Sep 17 00:00:00 2001 From: mertmit Date: Mon, 22 Jan 2024 09:28:29 +0000 Subject: [PATCH 3/4] chore: improved debug mode for at import --- .../jobs/jobs/at-import/at-import.processor.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index 03ab1ecc9a..9dde95ba2d 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -2357,6 +2357,22 @@ export class AtImportProcessor { try { logBasic('SDK initialized'); + // clear all tables if debug mode + if (debugMode) { + const tables = await this.tablesService.getAccessibleTables({ + baseId: syncDB.baseId, + sourceId: syncDB.sourceId, + roles: { ...userRole, owner: true }, + }); + for (const table of tables) { + await this.tablesService.tableDelete({ + tableId: table.id, + user: syncDB.user, + forceDeleteRelations: true, + }); + } + } + logDetailed('Base initialization started'); logDetailed('Base initialized'); From ac76353a0dbdb3999787e8fd872682dfb28f387b Mon Sep 17 00:00:00 2001 From: mertmit Date: Mon, 22 Jan 2024 09:28:30 +0000 Subject: [PATCH 4/4] refactor: use logWarning for errors --- .../jobs/at-import/at-import.processor.ts | 10 +++--- .../at-import/helpers/readAndProcessData.ts | 36 +++++++++++-------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index 9dde95ba2d..73924d3ac3 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -2483,14 +2483,15 @@ export class AtImportProcessor { baseName: syncDB.baseId, table: ncTbl, atBase, - logBasic, nocoBaseDataProcessing_v2, sDB: syncDB, - logDetailed, services: { tableService: this.tablesService, bulkDataService: this.bulkDataAliasService, }, + logBasic, + logDetailed, + logWarning, }); rtc.data.records += await recordsMap[ncTbl.id].getCount(); @@ -2518,9 +2519,7 @@ export class AtImportProcessor { baseName: syncDB.baseId, atBase, fields: null, //Object.values(tblLinkGroup).flat(), - logBasic, insertedAssocRef, - logDetailed, records: recordsMap[ncTbl.id], atNcAliasRef, ncLinkMappingTable, @@ -2529,6 +2528,9 @@ export class AtImportProcessor { tableService: this.tablesService, bulkDataService: this.bulkDataAliasService, }, + logBasic, + logDetailed, + logWarning, }); } } catch (error) { diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts index 25a402652c..41bce5e203 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts @@ -26,12 +26,14 @@ async function readAllData({ fields, atBase, logBasic = (_str) => {}, + logWarning = (_str) => {}, }: { table: { title?: string }; fields?; atBase: AirtableBase; logBasic?: (string) => void; logDetailed?: (string) => void; + logWarning?: (string) => void; }): Promise { return new Promise((resolve) => { let data = null; @@ -78,8 +80,8 @@ async function readAllData({ async function done(err) { if (err) { logger.error(err); - logBasic( - `:: There were errors on reading '${table.title}' data :: ${err}`, + logWarning( + `There were errors on reading '${table.title}' data :: ${err}`, ); } resolve(data); @@ -94,8 +96,9 @@ export async function importData({ atBase, nocoBaseDataProcessing_v2, sDB, - logDetailed = (_str) => {}, logBasic = (_str) => {}, + logDetailed = (_str) => {}, + logWarning = (_str) => {}, services, }: { baseName: string; @@ -104,6 +107,7 @@ export async function importData({ atBase: AirtableBase; logBasic: (string) => void; logDetailed: (string) => void; + logWarning: (string) => void; nocoBaseDataProcessing_v2; sDB; services: AirtableImportContext; @@ -179,8 +183,8 @@ export async function importData({ resolve(true); } catch (e) { logger.error(e); - logBasic( - `:: There were errors on importing '${table.title}' data :: ${e}`, + logWarning( + `There were errors on importing '${table.title}' data :: ${e}`, ); readable.resume(); resolve(true); @@ -217,8 +221,8 @@ export async function importData({ resolve(true); } catch (e) { logger.error(e); - logBasic( - `:: There were errors on importing '${table.title}' data :: ${e}`, + logWarning( + `There were errors on importing '${table.title}' data :: ${e}`, ); resolve(true); } @@ -237,20 +241,19 @@ export async function importLTARData({ atBase, baseName, insertedAssocRef = {}, - logDetailed = (_str) => {}, - logBasic = (_str) => {}, records, atNcAliasRef, ncLinkMappingTable, syncDB, services, + logBasic = (_str) => {}, + logDetailed = (_str) => {}, + logWarning = (_str) => {}, }: { baseName: string; table: { title?: string; id?: string }; fields; atBase: AirtableBase; - logDetailed: (string) => void; - logBasic: (string) => void; insertedAssocRef: { [assocTableId: string]: boolean }; records?: EntityMap; atNcAliasRef: { @@ -261,6 +264,9 @@ export async function importLTARData({ ncLinkMappingTable: Record>[]; syncDB; services: AirtableImportContext; + logBasic: (string) => void; + logDetailed: (string) => void; + logWarning: (string) => void; }) { const assocTableMetas: Array<{ modelMeta: { id?: string; title?: string }; @@ -379,8 +385,8 @@ export async function importLTARData({ resolve(true); } catch (e) { logger.error(e); - logBasic( - `:: There were errors on importing '${table.title}' LTAR data :: ${e}`, + logWarning( + `There were errors on importing '${table.title}' LTAR data :: ${e}`, ); readable.resume(); resolve(true); @@ -418,8 +424,8 @@ export async function importLTARData({ resolve(true); } catch (e) { logger.error(e); - logBasic( - `:: There were errors on importing '${table.title}' LTAR data :: ${e}`, + logWarning( + `There were errors on importing '${table.title}' LTAR data :: ${e}`, ); resolve(true); }