Merge pull request #7391 from nocodb/nc-fix/dg-at-import

fix: various at import issues

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@@ -146,6 +146,11 @@ export class AtImportProcessor {
       this.debugLog(log);
     };
 
+    const logWarning = (log) => {
+      this.jobsLogService.sendLog(job, { message: `WARNING: ${log}` });
+      this.debugLog(log);
+    };
+
     const logDetailed = (log) => {
       if (debugMode) this.jobsLogService.sendLog(job, { message: log });
       this.debugLog(log);
@@ -291,7 +296,7 @@ export class AtImportProcessor {
       rating: UITypes.Rating,
       formula: UITypes.Formula,
       rollup: UITypes.Rollup,
-      count: UITypes.Count,
+      count: UITypes.Rollup,
       lookup: UITypes.Lookup,
       autoNumber: UITypes.AutoNumber,
       barcode: UITypes.SingleLineText,
@@ -430,19 +435,14 @@ export class AtImportProcessor {
             if ((value as any).name === '') {
               (value as any).name = 'nc_empty';
             }
-            // enumerate duplicates (we don't allow them)
+            // skip duplicates (we don't allow them)
-            // TODO fix record mapping (this causes every record to map first option,
+            if (
-            //  we can't handle them using data api as they don't provide option id
-            //  within data we might instead get the correct mapping from schema file )
-            let dupNo = 1;
-            const defaultName = (value as any).name;
-            while (
               options.find(
                 (el) =>
                   el.title.toLowerCase() === (value as any).name.toLowerCase(),
               )
             ) {
-              (value as any).name = `${defaultName}_${dupNo++}`;
+              continue;
             }
             options.push({
               order: order++,
@@ -865,15 +865,6 @@ export class AtImportProcessor {
                 );
               }
 
-              // check if already a column exists with this name?
-              const duplicate = childTblSchema.columns.find(
-                (x) => x.title === aTblLinkColumns[i].name,
-              );
-              const suffix = duplicate ? '_2' : '';
-              if (duplicate)
-                if (enableErrorLogs)
-                  console.log(`## Duplicate ${aTblLinkColumns[i].name}`);
-
               // rename
               // note that: current rename API requires us to send all parameters,
               // not just title being renamed
@@ -900,12 +891,12 @@ export class AtImportProcessor {
               updateNcTblSchema(ncTbl);
 
               const ncId = ncTbl.columns.find(
-                (x) => x.title === aTblLinkColumns[i].name + suffix,
+                (x) => x.title === ncName.title,
               )?.id;
               await sMap.addToMappingTbl(
                 aTblLinkColumns[i].id,
                 ncId,
-                aTblLinkColumns[i].name + suffix,
+                ncName.title,
                 ncTbl.id,
               );
             }
@@ -1100,7 +1091,7 @@ export class AtImportProcessor {
         ARRAYCOMPACT: '',
         ARRAYJOIN: '',
         ARRAYUNIQUE: '',
-        AVERAGE: 'average',
+        AVERAGE: 'avg',
         CONCATENATE: '',
         COUNT: 'count',
         COUNTA: '',
@@ -1206,35 +1197,41 @@ export class AtImportProcessor {
 
             logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`);
             const _perfStart = recordPerfStart();
-            const ncTbl: any = await this.columnsService.columnAdd({
+            try {
-              tableId: srcTableId,
+              const ncTbl: any = await this.columnsService.columnAdd({
-              column: {
+                tableId: srcTableId,
-                uidt: UITypes.Rollup,
+                column: {
-                title: ncName.title,
+                  uidt: UITypes.Rollup,
-                column_name: ncName.column_name,
+                  title: ncName.title,
-                fk_relation_column_id: ncRelationColumnId,
+                  column_name: ncName.column_name,
-                fk_rollup_column_id: ncRollupColumnId,
+                  fk_relation_column_id: ncRelationColumnId,
-                rollup_function: ncRollupFn,
+                  fk_rollup_column_id: ncRollupColumnId,
-              },
+                  rollup_function: ncRollupFn,
-              req: {
+                },
-                user: syncDB.user.email,
+                req: {
-                clientIp: '',
+                  user: syncDB.user.email,
-              },
+                  clientIp: '',
-              user: syncDB.user,
+                },
-            });
+                user: syncDB.user,
-            recordPerfStats(_perfStart, 'dbTableColumn.create');
+              });
+              recordPerfStats(_perfStart, 'dbTableColumn.create');
 
-            updateNcTblSchema(ncTbl);
+              updateNcTblSchema(ncTbl);
 
-            const ncId = ncTbl.columns.find(
+              const ncId = ncTbl.columns.find(
-              (x) => x.title === aTblColumns[i].name,
+                (x) => x.title === aTblColumns[i].name,
-            )?.id;
+              )?.id;
-            await sMap.addToMappingTbl(
+              await sMap.addToMappingTbl(
-              aTblColumns[i].id,
+                aTblColumns[i].id,
-              ncId,
+                ncId,
-              aTblColumns[i].name,
+                aTblColumns[i].name,
-              ncTbl.id,
+                ncTbl.id,
-            );
+              );
+            } catch (e) {
+              logWarning(
+                `Skipped creating rollup column ${aTblColumns[i].name} :: ${e.message}`,
+              );
+            }
           }
         }
       }
@@ -1812,9 +1809,14 @@ export class AtImportProcessor {
               },
               req: { user: syncDB.user, clientIp: '' },
             })
-            .catch((e) =>
+            .catch((e) => {
-              e.message ? logBasic(`NOTICE: ${e.message}`) : console.log(e),
+              if (e.message) {
-            ),
+                // TODO enable after fixing user invite role issue
+                // logWarning(e.message);
+              } else {
+                console.log(e);
+              }
+            }),
         );
         recordPerfStats(_perfStart, 'auth.baseUserAdd');
       }
@@ -2106,12 +2108,16 @@ export class AtImportProcessor {
         // insert filters
         for (let i = 0; i < ncFilters.length; i++) {
           const _perfStart = recordPerfStart();
-          await this.filtersService.filterCreate({
+          try {
-            viewId: viewId,
+            await this.filtersService.filterCreate({
-            filter: ncFilters[i],
+              viewId: viewId,
-            user: syncDB.user,
+              filter: ncFilters[i],
-            req: {},
+              user: syncDB.user,
-          });
+              req: {},
+            });
+          } catch (e) {
+            logWarning(`Skipped creating filter for ${viewId} :: ${e.message}`);
+          }
           recordPerfStats(_perfStart, 'dbTableFilter.create');
 
           rtc.filter++;
@@ -2498,6 +2504,7 @@ export class AtImportProcessor {
         await this.tablesService.tableDelete({
           tableId: table.id,
           user: syncDB.user,
+          forceDeleteRelations: true,
         });
       }
       if (e.message) {

packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts

@@ -50,12 +50,24 @@ async function initialize(shareId, appId?: string) {
         };
       });
 
+    const headers = hreq.match(/(?<=var headers =)(.*)(?=;)/g);
+    const link = hreq.match(/(?<=fetch\(")(\\.*)(?=")/g);
+
+    if (!headers || !link) {
+      throw {
+        message:
+          'Please ensure www.airtable.com/<SharedBaseID> is available for public access. Refer https://bit.ly/3x0OdXI for details',
+      };
+    }
+
     info.headers = JSON.parse(
       hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
     );
+
     info.link = unicodeToChar(
       hreq.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(),
     );
+
     info.baseInfo = decodeURIComponent(info.link)
       .match(/{(.*)}/g)[0]
       .split('&')

packages/nocodb/src/services/tables.service.ts

@@ -162,7 +162,12 @@ export class TablesService {
     return Model.updateOrder(param.tableId, param.order);
   }
 
-  async tableDelete(param: { tableId: string; user: User; req?: any }) {
+  async tableDelete(param: {
+    tableId: string;
+    user: User;
+    forceDeleteRelations?: boolean;
+    req?: any;
+  }) {
     const table = await Model.getByIdOrName({ id: param.tableId });
     await table.getColumns();
 
@@ -171,7 +176,9 @@ export class TablesService {
 
     const relationColumns = table.columns.filter((c) => isLinksOrLTAR(c));
 
-    if (relationColumns?.length && !source.isMeta()) {
+    const deleteRelations = source.isMeta() || param.forceDeleteRelations;
+
+    if (relationColumns?.length && !deleteRelations) {
       const referredTables = await Promise.all(
         relationColumns.map(async (c) =>
           c