Merge pull request #9517 from nocodb/nc-fix/at-stall

fix: at import stall issue

packages/nocodb/src/models/Column.ts

@@ -816,6 +816,7 @@ export default class Column<T = any> implements ColumnType {
 
         if (button.type === 'url') {
           if (
+            button.formula &&
             addFormulaErrorIfMissingColumn({
               formula: button,
               columnId: id,
@@ -858,6 +859,7 @@ export default class Column<T = any> implements ColumnType {
           formulaCol,
         ).getColOptions<FormulaColumn>(context, ncMeta);
         if (
+          formula.formula &&
           addFormulaErrorIfMissingColumn({
             formula,
             columnId: id,

packages/nocodb/src/modules/jobs/jobs.module.ts

@@ -54,6 +54,7 @@ export const JobsModuleMetadata = {
             name: JOBS_QUEUE,
             defaultJobOptions: {
               removeOnComplete: true,
+              attempts: 1,
             },
           }),
         ]

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@@ -469,7 +469,7 @@ export class AtImportProcessor {
             if (
               options.find(
                 (el) =>
-                  el.title.toLowerCase() === (value as any).name.toLowerCase(),
+                  el.title === (value as any).name,
               )
             ) {
               logWarning(

packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts

@@ -1,5 +1,8 @@
 import { Logger } from '@nestjs/common';
 import axios from 'axios';
+import { streamObject } from 'stream-json/streamers/StreamObject';
+import { parser } from 'stream-json/Parser';
+import { ignore } from 'stream-json/filters/Ignore';
 
 const logger = new Logger('FetchAT');
 
@@ -17,8 +20,7 @@ async function initialize(shareId, appId?: string) {
   const url = `https://airtable.com/${appId ? `${appId}/` : ''}${shareId}`;
 
   try {
-    const hreq = await axios
-      .get(url, {
+    const hreq = await axios.get(url, {
       headers: {
         accept:
           'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
@@ -39,23 +41,16 @@ async function initialize(shareId, appId?: string) {
       referrerPolicy: 'strict-origin-when-cross-origin',
       body: null,
       method: 'GET',
-      })
-      .then((response) => {
-        for (const ck of response.headers['set-cookie']) {
+    });
+
+    for (const ck of hreq.headers['set-cookie']) {
       info.cookie += ck.split(';')[0] + '; ';
     }
-        return response.data;
-      })
-      .catch((e) => {
-        logger.log(e);
-        throw {
-          message:
-            'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
-        };
-      });
 
-    const headers = hreq.match(/(?<=var headers =)(.*)(?=;)/g);
-    const link = hreq.match(/(?<=fetch\(")(\\.*)(?=")/g);
+    const data = hreq.data;
+
+    const headers = data.match(/(?<=var headers =)(.*)(?=;)/g);
+    const link = data.match(/(?<=fetch\(")(\\.*)(?=")/g);
 
     if (!headers || !link) {
       throw {
@@ -65,11 +60,11 @@ async function initialize(shareId, appId?: string) {
     }
 
     info.headers = JSON.parse(
-      hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
+      data.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
     );
 
     info.link = unicodeToChar(
-      hreq.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(),
+      data.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(),
     );
 
     info.link = info.link.replace(
@@ -112,6 +107,7 @@ async function initialize(shareId, appId?: string) {
 
 async function read() {
   if (info.initialized) {
+    try {
       const resreq = await axios('https://airtable.com' + info.link, {
         headers: {
           accept: '*/*',
@@ -133,23 +129,46 @@ async function read() {
         referrerPolicy: 'no-referrer',
         body: null,
         method: 'GET',
-    })
-      .then((response) => {
-        return response.data;
-      })
-      .catch((e) => {
-        logger.log(e);
-        throw {
-          message:
-            'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
-        };
+        responseType: 'stream',
+      });
+
+      const data: any = await new Promise((resolve, reject) => {
+        const jsonStream = resreq.data
+          .pipe(parser())
+          .pipe(ignore({ filter: 'data.tableDatas' }))
+          .pipe(streamObject());
+
+        const fullObject = {};
+
+        jsonStream.on('data', (chunk) => {
+          if (chunk.key) fullObject[chunk.key] = chunk.value;
+        });
+
+        jsonStream.on('error', (err) => {
+          reject(err);
+        });
+
+        jsonStream.on('end', () => {
+          resolve(fullObject);
+        });
       });
 
+      if (data?.data) {
         return {
-      schema: resreq.data,
+          schema: data?.data,
           baseId: info.baseId,
           baseInfo: info.baseInfo,
         };
+      } else {
+        throw new Error('Error Reading :: Data missing');
+      }
+    } catch (e) {
+      logger.log(e);
+      throw {
+        message:
+          'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+      };
+    }
   } else {
     throw {
       message: 'Error Initializing :: please try again !!',
@@ -159,6 +178,7 @@ async function read() {
 
 async function readView(viewId) {
   if (info.initialized) {
+    try {
       const resreq = await axios(
         `https://airtable.com/v0.3/view/${viewId}/readData?` +
           `stringifiedObjectParams=${encodeURIComponent(
@@ -199,19 +219,44 @@ async function readView(viewId) {
           referrerPolicy: 'no-referrer',
           body: null,
           method: 'GET',
+          responseType: 'stream',
         },
-    )
-      .then((response) => {
-        return response.data;
-      })
-      .catch((e) => {
+      );
+
+      const data: any = await new Promise((resolve, reject) => {
+        const jsonStream = resreq.data
+          .pipe(parser())
+          .pipe(ignore({ filter: 'data.rowOrder' }))
+          .pipe(streamObject());
+
+        const fullObject = {};
+
+        jsonStream.on('data', (chunk) => {
+          if (chunk.key) fullObject[chunk.key] = chunk.value;
+        });
+
+        jsonStream.on('error', (err) => {
+          reject(err);
+        });
+
+        jsonStream.on('end', () => {
+          resolve(fullObject);
+        });
+      });
+      
+
+      if (data?.data) {
+        return { view: data.data };
+      } else {
+        throw new Error('Error Reading :: Data missing');
+      }
+    } catch (e) {
       logger.log(e);
       throw {
         message:
           'Error Reading View :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
       };
-      });
-    return { view: resreq.data };
+    }
   } else {
     throw {
       message: 'Error Initializing :: please try again !!',
@@ -223,6 +268,8 @@ async function readTemplate(templateId) {
   if (!info.initialized) {
     await initialize('shrO8aYf3ybwSdDKn');
   }
+
+  try {
     const resreq = await axios(
       `https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
       {
@@ -250,18 +297,20 @@ async function readTemplate(templateId) {
         mode: 'cors',
         credentials: 'include',
       },
-  )
-    .then((response) => {
-      return response.data;
-    })
-    .catch((e) => {
+    );
+
+    if (resreq?.data) {
+      return { template: resreq.data };
+    } else {
+      throw new Error('Error Reading :: Data missing');
+    }
+  } catch (e) {
     logger.log(e);
     throw {
       message:
         'Error Fetching :: Ensure www.airtable.com/templates/featured/<TemplateID> is accessible.',
     };
-    });
-  return { template: resreq };
+  }
 }
 
 function unicodeToChar(text) {