Merge pull request #9517 from nocodb/nc-fix/at-stall

fix: at import stall issue

packages/nocodb/src/models/Column.ts

@@ -816,6 +816,7 @@ export default class Column<T = any> implements ColumnType {
 
         if (button.type === 'url') {
           if (
+            button.formula &&
             addFormulaErrorIfMissingColumn({
               formula: button,
               columnId: id,
@@ -858,6 +859,7 @@ export default class Column<T = any> implements ColumnType {
           formulaCol,
         ).getColOptions<FormulaColumn>(context, ncMeta);
         if (
+          formula.formula &&
           addFormulaErrorIfMissingColumn({
             formula,
             columnId: id,

packages/nocodb/src/modules/jobs/jobs.module.ts

@@ -54,6 +54,7 @@ export const JobsModuleMetadata = {
             name: JOBS_QUEUE,
             defaultJobOptions: {
               removeOnComplete: true,
+              attempts: 1,
             },
           }),
         ]

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@@ -469,7 +469,7 @@ export class AtImportProcessor {
             if (
               options.find(
                 (el) =>
-                  el.title.toLowerCase() === (value as any).name.toLowerCase(),
+                  el.title === (value as any).name,
               )
             ) {
               logWarning(

packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts

@@ -1,5 +1,8 @@
 import { Logger } from '@nestjs/common';
 import axios from 'axios';
+import { streamObject } from 'stream-json/streamers/StreamObject';
+import { parser } from 'stream-json/Parser';
+import { ignore } from 'stream-json/filters/Ignore';
 
 const logger = new Logger('FetchAT');
 
@@ -17,45 +20,37 @@ async function initialize(shareId, appId?: string) {
   const url = `https://airtable.com/${appId ? `${appId}/` : ''}${shareId}`;
 
   try {
-    const hreq = await axios
+    const hreq = await axios.get(url, {
-      .get(url, {
+      headers: {
-        headers: {
+        accept:
-          accept:
+          'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-            'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+        'accept-language': 'en-US,en;q=0.9',
-          'accept-language': 'en-US,en;q=0.9',
+        'sec-ch-ua':
-          'sec-ch-ua':
+          '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
-            '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+        'sec-ch-ua-mobile': '?0',
-          'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua-platform': '"Linux"',
-          'sec-ch-ua-platform': '"Linux"',
+        'sec-fetch-dest': 'document',
-          'sec-fetch-dest': 'document',
+        'sec-fetch-mode': 'navigate',
-          'sec-fetch-mode': 'navigate',
+        'sec-fetch-site': 'none',
-          'sec-fetch-site': 'none',
+        'sec-fetch-user': '?1',
-          'sec-fetch-user': '?1',
+        'upgrade-insecure-requests': '1',
-          'upgrade-insecure-requests': '1',
+        'User-Agent':
-          'User-Agent':
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+      },
-        },
+      // @ts-ignore
-        // @ts-ignore
+      referrerPolicy: 'strict-origin-when-cross-origin',
-        referrerPolicy: 'strict-origin-when-cross-origin',
+      body: null,
-        body: null,
+      method: 'GET',
-        method: 'GET',
+    });
-      })
+
-      .then((response) => {
+    for (const ck of hreq.headers['set-cookie']) {
-        for (const ck of response.headers['set-cookie']) {
+      info.cookie += ck.split(';')[0] + '; ';
-          info.cookie += ck.split(';')[0] + '; ';
+    }
-        }
+
-        return response.data;
+    const data = hreq.data;
-      })
-      .catch((e) => {
-        logger.log(e);
-        throw {
-          message:
-            'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
-        };
-      });
 
-    const headers = hreq.match(/(?<=var headers =)(.*)(?=;)/g);
+    const headers = data.match(/(?<=var headers =)(.*)(?=;)/g);
-    const link = hreq.match(/(?<=fetch\(")(\\.*)(?=")/g);
+    const link = data.match(/(?<=fetch\(")(\\.*)(?=")/g);
 
     if (!headers || !link) {
       throw {
@@ -65,11 +60,11 @@ async function initialize(shareId, appId?: string) {
     }
 
     info.headers = JSON.parse(
-      hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
+      data.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
     );
 
     info.link = unicodeToChar(
-      hreq.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(),
+      data.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(),
     );
 
     info.link = info.link.replace(
@@ -112,44 +107,68 @@ async function initialize(shareId, appId?: string) {
 
 async function read() {
   if (info.initialized) {
-    const resreq = await axios('https://airtable.com' + info.link, {
+    try {
-      headers: {
+      const resreq = await axios('https://airtable.com' + info.link, {
-        accept: '*/*',
+        headers: {
-        'accept-language': 'en-US,en;q=0.9',
+          accept: '*/*',
-        'sec-ch-ua':
+          'accept-language': 'en-US,en;q=0.9',
-          '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+          'sec-ch-ua':
-        'sec-ch-ua-mobile': '?0',
+            '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
-        'sec-ch-ua-platform': '"Linux"',
+          'sec-ch-ua-mobile': '?0',
-        'sec-fetch-dest': 'empty',
+          'sec-ch-ua-platform': '"Linux"',
-        'sec-fetch-mode': 'cors',
+          'sec-fetch-dest': 'empty',
-        'sec-fetch-site': 'same-origin',
+          'sec-fetch-mode': 'cors',
-        'User-Agent':
+          'sec-fetch-site': 'same-origin',
-          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+          'User-Agent':
-        'x-time-zone': 'Europe/Berlin',
+            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
-        cookie: info.cookie,
+          'x-time-zone': 'Europe/Berlin',
-        ...info.headers,
+          cookie: info.cookie,
-      },
+          ...info.headers,
-      // @ts-ignore
+        },
-      referrerPolicy: 'no-referrer',
+        // @ts-ignore
-      body: null,
+        referrerPolicy: 'no-referrer',
-      method: 'GET',
+        body: null,
-    })
+        method: 'GET',
-      .then((response) => {
+        responseType: 'stream',
-        return response.data;
-      })
-      .catch((e) => {
-        logger.log(e);
-        throw {
-          message:
-            'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
-        };
       });
 
-    return {
+      const data: any = await new Promise((resolve, reject) => {
-      schema: resreq.data,
+        const jsonStream = resreq.data
-      baseId: info.baseId,
+          .pipe(parser())
-      baseInfo: info.baseInfo,
+          .pipe(ignore({ filter: 'data.tableDatas' }))
-    };
+          .pipe(streamObject());
+
+        const fullObject = {};
+
+        jsonStream.on('data', (chunk) => {
+          if (chunk.key) fullObject[chunk.key] = chunk.value;
+        });
+
+        jsonStream.on('error', (err) => {
+          reject(err);
+        });
+
+        jsonStream.on('end', () => {
+          resolve(fullObject);
+        });
+      });
+
+      if (data?.data) {
+        return {
+          schema: data?.data,
+          baseId: info.baseId,
+          baseInfo: info.baseInfo,
+        };
+      } else {
+        throw new Error('Error Reading :: Data missing');
+      }
+    } catch (e) {
+      logger.log(e);
+      throw {
+        message:
+          'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+      };
+    }
   } else {
     throw {
       message: 'Error Initializing :: please try again !!',
@@ -159,25 +178,100 @@ async function read() {
 
 async function readView(viewId) {
   if (info.initialized) {
+    try {
+      const resreq = await axios(
+        `https://airtable.com/v0.3/view/${viewId}/readData?` +
+          `stringifiedObjectParams=${encodeURIComponent(
+            JSON.stringify({
+              mayOnlyIncludeRowAndCellDataForIncludedViews: true,
+              mayExcludeCellDataForLargeViews: true,
+            }),
+          )}&requestId=${
+            info.baseInfo.requestId
+          }&accessPolicy=${encodeURIComponent(
+            JSON.stringify({
+              allowedActions: info.baseInfo.allowedActions,
+              shareId: info.baseInfo.shareId,
+              applicationId: info.baseInfo.applicationId,
+              generationNumber: info.baseInfo.generationNumber,
+              expires: info.baseInfo.expires,
+              signature: info.baseInfo.signature,
+            }),
+          )}`,
+        {
+          headers: {
+            accept: '*/*',
+            'accept-language': 'en-US,en;q=0.9',
+            'sec-ch-ua':
+              '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-platform': '"Linux"',
+            'sec-fetch-dest': 'empty',
+            'sec-fetch-mode': 'cors',
+            'sec-fetch-site': 'same-origin',
+            'User-Agent':
+              'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+            'x-time-zone': 'Europe/Berlin',
+            cookie: info.cookie,
+            ...info.headers,
+          },
+          // @ts-ignore
+          referrerPolicy: 'no-referrer',
+          body: null,
+          method: 'GET',
+          responseType: 'stream',
+        },
+      );
+
+      const data: any = await new Promise((resolve, reject) => {
+        const jsonStream = resreq.data
+          .pipe(parser())
+          .pipe(ignore({ filter: 'data.rowOrder' }))
+          .pipe(streamObject());
+
+        const fullObject = {};
+
+        jsonStream.on('data', (chunk) => {
+          if (chunk.key) fullObject[chunk.key] = chunk.value;
+        });
+
+        jsonStream.on('error', (err) => {
+          reject(err);
+        });
+
+        jsonStream.on('end', () => {
+          resolve(fullObject);
+        });
+      });
+      
+
+      if (data?.data) {
+        return { view: data.data };
+      } else {
+        throw new Error('Error Reading :: Data missing');
+      }
+    } catch (e) {
+      logger.log(e);
+      throw {
+        message:
+          'Error Reading View :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+      };
+    }
+  } else {
+    throw {
+      message: 'Error Initializing :: please try again !!',
+    };
+  }
+}
+
+async function readTemplate(templateId) {
+  if (!info.initialized) {
+    await initialize('shrO8aYf3ybwSdDKn');
+  }
+
+  try {
     const resreq = await axios(
-      `https://airtable.com/v0.3/view/${viewId}/readData?` +
+      `https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
-        `stringifiedObjectParams=${encodeURIComponent(
-          JSON.stringify({
-            mayOnlyIncludeRowAndCellDataForIncludedViews: true,
-            mayExcludeCellDataForLargeViews: true,
-          }),
-        )}&requestId=${
-          info.baseInfo.requestId
-        }&accessPolicy=${encodeURIComponent(
-          JSON.stringify({
-            allowedActions: info.baseInfo.allowedActions,
-            shareId: info.baseInfo.shareId,
-            applicationId: info.baseInfo.applicationId,
-            generationNumber: info.baseInfo.generationNumber,
-            expires: info.baseInfo.expires,
-            signature: info.baseInfo.signature,
-          }),
-        )}`,
       {
         headers: {
           accept: '*/*',
@@ -196,74 +290,29 @@ async function readView(viewId) {
           ...info.headers,
         },
         // @ts-ignore
-        referrerPolicy: 'no-referrer',
+        referrer: 'https://www.airtable.com/',
+        referrerPolicy: 'same-origin',
         body: null,
         method: 'GET',
+        mode: 'cors',
+        credentials: 'include',
       },
-    )
+    );
-      .then((response) => {
+
-        return response.data;
+    if (resreq?.data) {
-      })
+      return { template: resreq.data };
-      .catch((e) => {
+    } else {
-        logger.log(e);
+      throw new Error('Error Reading :: Data missing');
-        throw {
+    }
-          message:
+  } catch (e) {
-            'Error Reading View :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
+    logger.log(e);
-        };
-      });
-    return { view: resreq.data };
-  } else {
     throw {
-      message: 'Error Initializing :: please try again !!',
+      message:
+        'Error Fetching :: Ensure www.airtable.com/templates/featured/<TemplateID> is accessible.',
     };
   }
 }
 
-async function readTemplate(templateId) {
-  if (!info.initialized) {
-    await initialize('shrO8aYf3ybwSdDKn');
-  }
-  const resreq = await axios(
-    `https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
-    {
-      headers: {
-        accept: '*/*',
-        'accept-language': 'en-US,en;q=0.9',
-        'sec-ch-ua':
-          '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
-        'sec-ch-ua-mobile': '?0',
-        'sec-ch-ua-platform': '"Linux"',
-        'sec-fetch-dest': 'empty',
-        'sec-fetch-mode': 'cors',
-        'sec-fetch-site': 'same-origin',
-        'User-Agent':
-          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
-        'x-time-zone': 'Europe/Berlin',
-        cookie: info.cookie,
-        ...info.headers,
-      },
-      // @ts-ignore
-      referrer: 'https://www.airtable.com/',
-      referrerPolicy: 'same-origin',
-      body: null,
-      method: 'GET',
-      mode: 'cors',
-      credentials: 'include',
-    },
-  )
-    .then((response) => {
-      return response.data;
-    })
-    .catch((e) => {
-      logger.log(e);
-      throw {
-        message:
-          'Error Fetching :: Ensure www.airtable.com/templates/featured/<TemplateID> is accessible.',
-      };
-    });
-  return { template: resreq };
-}
-
 function unicodeToChar(text) {
   return text.replace(/\\u[\dA-F]{4}/gi, function (match) {
     return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16));