From 6bbfa46f81886dd788f9c09b46e6bbb5370145ba Mon Sep 17 00:00:00 2001 From: mertmit Date: Tue, 9 Jan 2024 00:52:55 +0000 Subject: [PATCH 1/5] fix: delete tables on at import fail --- .../jobs/jobs/at-import/at-import.processor.ts | 1 + packages/nocodb/src/services/tables.service.ts | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index 17a73b81f6..ac8cfc5aa7 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -2498,6 +2498,7 @@ export class AtImportProcessor { await this.tablesService.tableDelete({ tableId: table.id, user: syncDB.user, + forceDeleteRelations: true, }); } if (e.message) { diff --git a/packages/nocodb/src/services/tables.service.ts b/packages/nocodb/src/services/tables.service.ts index 01483208f3..0787cc882d 100644 --- a/packages/nocodb/src/services/tables.service.ts +++ b/packages/nocodb/src/services/tables.service.ts @@ -162,7 +162,12 @@ export class TablesService { return Model.updateOrder(param.tableId, param.order); } - async tableDelete(param: { tableId: string; user: User; req?: any }) { + async tableDelete(param: { + tableId: string; + user: User; + forceDeleteRelations?: boolean; + req?: any; + }) { const table = await Model.getByIdOrName({ id: param.tableId }); await table.getColumns(); @@ -171,7 +176,9 @@ export class TablesService { const relationColumns = table.columns.filter((c) => isLinksOrLTAR(c)); - if (relationColumns?.length && !source.isMeta()) { + const deleteRelations = source.isMeta() || param.forceDeleteRelations; + + if (relationColumns?.length && !deleteRelations) { const referredTables = await Promise.all( relationColumns.map(async (c) => c From 6fa7781a260ce9934c74a04eb586e9766e94aa65 Mon Sep 17 00:00:00 2001 From: mertmit Date: Tue, 9 Jan 2024 00:52:56 +0000 Subject: [PATCH 2/5] fix: at-import symmetric column issue --- .../jobs/jobs/at-import/at-import.processor.ts | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index ac8cfc5aa7..1ba0d3161b 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -865,15 +865,6 @@ export class AtImportProcessor { ); } - // check if already a column exists with this name? - const duplicate = childTblSchema.columns.find( - (x) => x.title === aTblLinkColumns[i].name, - ); - const suffix = duplicate ? '_2' : ''; - if (duplicate) - if (enableErrorLogs) - console.log(`## Duplicate ${aTblLinkColumns[i].name}`); - // rename // note that: current rename API requires us to send all parameters, // not just title being renamed @@ -900,12 +891,12 @@ export class AtImportProcessor { updateNcTblSchema(ncTbl); const ncId = ncTbl.columns.find( - (x) => x.title === aTblLinkColumns[i].name + suffix, + (x) => x.title === ncName.title, )?.id; await sMap.addToMappingTbl( aTblLinkColumns[i].id, ncId, - aTblLinkColumns[i].name + suffix, + ncName.title, ncTbl.id, ); } From d46f7edce8692bd014f55b9906386757c12798b8 Mon Sep 17 00:00:00 2001 From: mertmit Date: Tue, 9 Jan 2024 00:52:56 +0000 Subject: [PATCH 3/5] fix: better error for private at bases --- .../modules/jobs/jobs/at-import/helpers/fetchAT.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts index 6eb677fcec..471dd9eae8 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts @@ -50,12 +50,24 @@ async function initialize(shareId, appId?: string) { }; }); + const headers = hreq.match(/(?<=var headers =)(.*)(?=;)/g); + const link = hreq.match(/(?<=fetch\(")(\\.*)(?=")/g); + + if (!headers || !link) { + throw { + message: + 'Please ensure www.airtable.com/ is available for public access. Refer https://bit.ly/3x0OdXI for details', + }; + } + info.headers = JSON.parse( hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(), ); + info.link = unicodeToChar( hreq.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(), ); + info.baseInfo = decodeURIComponent(info.link) .match(/{(.*)}/g)[0] .split('&') From 4403d80d45a5e21175162ced535ef9dcd6cf2532 Mon Sep 17 00:00:00 2001 From: mertmit Date: Tue, 9 Jan 2024 00:52:56 +0000 Subject: [PATCH 4/5] fix: various errors for at --- .../jobs/at-import/at-import.processor.ts | 96 +++++++++++-------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index 1ba0d3161b..1e11159082 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -146,6 +146,11 @@ export class AtImportProcessor { this.debugLog(log); }; + const logWarning = (log) => { + this.jobsLogService.sendLog(job, { message: `WARNING: ${log}` }); + this.debugLog(log); + }; + const logDetailed = (log) => { if (debugMode) this.jobsLogService.sendLog(job, { message: log }); this.debugLog(log); @@ -291,7 +296,7 @@ export class AtImportProcessor { rating: UITypes.Rating, formula: UITypes.Formula, rollup: UITypes.Rollup, - count: UITypes.Count, + count: UITypes.Rollup, lookup: UITypes.Lookup, autoNumber: UITypes.AutoNumber, barcode: UITypes.SingleLineText, @@ -1091,7 +1096,7 @@ export class AtImportProcessor { ARRAYCOMPACT: '', ARRAYJOIN: '', ARRAYUNIQUE: '', - AVERAGE: 'average', + AVERAGE: 'avg', CONCATENATE: '', COUNT: 'count', COUNTA: '', @@ -1197,35 +1202,41 @@ export class AtImportProcessor { logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); const _perfStart = recordPerfStart(); - const ncTbl: any = await this.columnsService.columnAdd({ - tableId: srcTableId, - column: { - uidt: UITypes.Rollup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_rollup_column_id: ncRollupColumnId, - rollup_function: ncRollupFn, - }, - req: { - user: syncDB.user.email, - clientIp: '', - }, - user: syncDB.user, - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); + try { + const ncTbl: any = await this.columnsService.columnAdd({ + tableId: srcTableId, + column: { + uidt: UITypes.Rollup, + title: ncName.title, + column_name: ncName.column_name, + fk_relation_column_id: ncRelationColumnId, + fk_rollup_column_id: ncRollupColumnId, + rollup_function: ncRollupFn, + }, + req: { + user: syncDB.user.email, + clientIp: '', + }, + user: syncDB.user, + }); + recordPerfStats(_perfStart, 'dbTableColumn.create'); - updateNcTblSchema(ncTbl); + updateNcTblSchema(ncTbl); - const ncId = ncTbl.columns.find( - (x) => x.title === aTblColumns[i].name, - )?.id; - await sMap.addToMappingTbl( - aTblColumns[i].id, - ncId, - aTblColumns[i].name, - ncTbl.id, - ); + const ncId = ncTbl.columns.find( + (x) => x.title === aTblColumns[i].name, + )?.id; + await sMap.addToMappingTbl( + aTblColumns[i].id, + ncId, + aTblColumns[i].name, + ncTbl.id, + ); + } catch (e) { + logWarning( + `Skipped creating rollup column ${aTblColumns[i].name} :: ${e.message}`, + ); + } } } } @@ -1803,9 +1814,14 @@ export class AtImportProcessor { }, req: { user: syncDB.user, clientIp: '' }, }) - .catch((e) => - e.message ? logBasic(`NOTICE: ${e.message}`) : console.log(e), - ), + .catch((e) => { + if (e.message) { + // TODO enable after fixing user invite role issue + // logWarning(e.message); + } else { + console.log(e); + } + }), ); recordPerfStats(_perfStart, 'auth.baseUserAdd'); } @@ -2097,12 +2113,16 @@ export class AtImportProcessor { // insert filters for (let i = 0; i < ncFilters.length; i++) { const _perfStart = recordPerfStart(); - await this.filtersService.filterCreate({ - viewId: viewId, - filter: ncFilters[i], - user: syncDB.user, - req: {}, - }); + try { + await this.filtersService.filterCreate({ + viewId: viewId, + filter: ncFilters[i], + user: syncDB.user, + req: {}, + }); + } catch (e) { + logWarning(`Skipped creating filter for ${viewId} :: ${e.message}`); + } recordPerfStats(_perfStart, 'dbTableFilter.create'); rtc.filter++; From 8c798de887546a5ce5062d74f6bd50ddc87fd10a Mon Sep 17 00:00:00 2001 From: mertmit Date: Tue, 9 Jan 2024 00:52:56 +0000 Subject: [PATCH 5/5] fix: skip duplicate options for at import --- .../jobs/jobs/at-import/at-import.processor.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index 1e11159082..9619ac6618 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -435,19 +435,14 @@ export class AtImportProcessor { if ((value as any).name === '') { (value as any).name = 'nc_empty'; } - // enumerate duplicates (we don't allow them) - // TODO fix record mapping (this causes every record to map first option, - // we can't handle them using data api as they don't provide option id - // within data we might instead get the correct mapping from schema file ) - let dupNo = 1; - const defaultName = (value as any).name; - while ( + // skip duplicates (we don't allow them) + if ( options.find( (el) => el.title.toLowerCase() === (value as any).name.toLowerCase(), ) ) { - (value as any).name = `${defaultName}_${dupNo++}`; + continue; } options.push({ order: order++,