diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts index c659db7a45..406e17885f 100644 --- a/packages/nocodb/src/models/Column.ts +++ b/packages/nocodb/src/models/Column.ts @@ -816,6 +816,7 @@ export default class Column implements ColumnType { if (button.type === 'url') { if ( + button.formula && addFormulaErrorIfMissingColumn({ formula: button, columnId: id, @@ -858,6 +859,7 @@ export default class Column implements ColumnType { formulaCol, ).getColOptions(context, ncMeta); if ( + formula.formula && addFormulaErrorIfMissingColumn({ formula, columnId: id, diff --git a/packages/nocodb/src/modules/jobs/jobs.module.ts b/packages/nocodb/src/modules/jobs/jobs.module.ts index 47397dfd69..8f9079956e 100644 --- a/packages/nocodb/src/modules/jobs/jobs.module.ts +++ b/packages/nocodb/src/modules/jobs/jobs.module.ts @@ -54,6 +54,7 @@ export const JobsModuleMetadata = { name: JOBS_QUEUE, defaultJobOptions: { removeOnComplete: true, + attempts: 1, }, }), ] diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index 0725743064..c79cb42792 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -469,7 +469,7 @@ export class AtImportProcessor { if ( options.find( (el) => - el.title.toLowerCase() === (value as any).name.toLowerCase(), + el.title === (value as any).name, ) ) { logWarning( diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts index 7da9c4d827..e685c511cf 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts @@ -1,5 +1,8 @@ import { Logger } from '@nestjs/common'; import axios from 'axios'; +import { streamObject } from 'stream-json/streamers/StreamObject'; +import { parser } from 'stream-json/Parser'; +import { ignore } from 'stream-json/filters/Ignore'; const logger = new Logger('FetchAT'); @@ -17,45 +20,37 @@ async function initialize(shareId, appId?: string) { const url = `https://airtable.com/${appId ? `${appId}/` : ''}${shareId}`; try { - const hreq = await axios - .get(url, { - headers: { - accept: - 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'document', - 'sec-fetch-mode': 'navigate', - 'sec-fetch-site': 'none', - 'sec-fetch-user': '?1', - 'upgrade-insecure-requests': '1', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - }, - // @ts-ignore - referrerPolicy: 'strict-origin-when-cross-origin', - body: null, - method: 'GET', - }) - .then((response) => { - for (const ck of response.headers['set-cookie']) { - info.cookie += ck.split(';')[0] + '; '; - } - return response.data; - }) - .catch((e) => { - logger.log(e); - throw { - message: - 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - }); + const hreq = await axios.get(url, { + headers: { + accept: + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'document', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-user': '?1', + 'upgrade-insecure-requests': '1', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + }, + // @ts-ignore + referrerPolicy: 'strict-origin-when-cross-origin', + body: null, + method: 'GET', + }); + + for (const ck of hreq.headers['set-cookie']) { + info.cookie += ck.split(';')[0] + '; '; + } + + const data = hreq.data; - const headers = hreq.match(/(?<=var headers =)(.*)(?=;)/g); - const link = hreq.match(/(?<=fetch\(")(\\.*)(?=")/g); + const headers = data.match(/(?<=var headers =)(.*)(?=;)/g); + const link = data.match(/(?<=fetch\(")(\\.*)(?=")/g); if (!headers || !link) { throw { @@ -65,11 +60,11 @@ async function initialize(shareId, appId?: string) { } info.headers = JSON.parse( - hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(), + data.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(), ); info.link = unicodeToChar( - hreq.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(), + data.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(), ); info.link = info.link.replace( @@ -112,44 +107,68 @@ async function initialize(shareId, appId?: string) { async function read() { if (info.initialized) { - const resreq = await axios('https://airtable.com' + info.link, { - headers: { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'x-time-zone': 'Europe/Berlin', - cookie: info.cookie, - ...info.headers, - }, - // @ts-ignore - referrerPolicy: 'no-referrer', - body: null, - method: 'GET', - }) - .then((response) => { - return response.data; - }) - .catch((e) => { - logger.log(e); - throw { - message: - 'Error Reading :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; + try { + const resreq = await axios('https://airtable.com' + info.link, { + headers: { + accept: '*/*', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + 'x-time-zone': 'Europe/Berlin', + cookie: info.cookie, + ...info.headers, + }, + // @ts-ignore + referrerPolicy: 'no-referrer', + body: null, + method: 'GET', + responseType: 'stream', }); - return { - schema: resreq.data, - baseId: info.baseId, - baseInfo: info.baseInfo, - }; + const data: any = await new Promise((resolve, reject) => { + const jsonStream = resreq.data + .pipe(parser()) + .pipe(ignore({ filter: 'data.tableDatas' })) + .pipe(streamObject()); + + const fullObject = {}; + + jsonStream.on('data', (chunk) => { + if (chunk.key) fullObject[chunk.key] = chunk.value; + }); + + jsonStream.on('error', (err) => { + reject(err); + }); + + jsonStream.on('end', () => { + resolve(fullObject); + }); + }); + + if (data?.data) { + return { + schema: data?.data, + baseId: info.baseId, + baseInfo: info.baseInfo, + }; + } else { + throw new Error('Error Reading :: Data missing'); + } + } catch (e) { + logger.log(e); + throw { + message: + 'Error Reading :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + } } else { throw { message: 'Error Initializing :: please try again !!', @@ -159,25 +178,100 @@ async function read() { async function readView(viewId) { if (info.initialized) { + try { + const resreq = await axios( + `https://airtable.com/v0.3/view/${viewId}/readData?` + + `stringifiedObjectParams=${encodeURIComponent( + JSON.stringify({ + mayOnlyIncludeRowAndCellDataForIncludedViews: true, + mayExcludeCellDataForLargeViews: true, + }), + )}&requestId=${ + info.baseInfo.requestId + }&accessPolicy=${encodeURIComponent( + JSON.stringify({ + allowedActions: info.baseInfo.allowedActions, + shareId: info.baseInfo.shareId, + applicationId: info.baseInfo.applicationId, + generationNumber: info.baseInfo.generationNumber, + expires: info.baseInfo.expires, + signature: info.baseInfo.signature, + }), + )}`, + { + headers: { + accept: '*/*', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': + '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Linux"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', + 'x-time-zone': 'Europe/Berlin', + cookie: info.cookie, + ...info.headers, + }, + // @ts-ignore + referrerPolicy: 'no-referrer', + body: null, + method: 'GET', + responseType: 'stream', + }, + ); + + const data: any = await new Promise((resolve, reject) => { + const jsonStream = resreq.data + .pipe(parser()) + .pipe(ignore({ filter: 'data.rowOrder' })) + .pipe(streamObject()); + + const fullObject = {}; + + jsonStream.on('data', (chunk) => { + if (chunk.key) fullObject[chunk.key] = chunk.value; + }); + + jsonStream.on('error', (err) => { + reject(err); + }); + + jsonStream.on('end', () => { + resolve(fullObject); + }); + }); + + + if (data?.data) { + return { view: data.data }; + } else { + throw new Error('Error Reading :: Data missing'); + } + } catch (e) { + logger.log(e); + throw { + message: + 'Error Reading View :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', + }; + } + } else { + throw { + message: 'Error Initializing :: please try again !!', + }; + } +} + +async function readTemplate(templateId) { + if (!info.initialized) { + await initialize('shrO8aYf3ybwSdDKn'); + } + + try { const resreq = await axios( - `https://airtable.com/v0.3/view/${viewId}/readData?` + - `stringifiedObjectParams=${encodeURIComponent( - JSON.stringify({ - mayOnlyIncludeRowAndCellDataForIncludedViews: true, - mayExcludeCellDataForLargeViews: true, - }), - )}&requestId=${ - info.baseInfo.requestId - }&accessPolicy=${encodeURIComponent( - JSON.stringify({ - allowedActions: info.baseInfo.allowedActions, - shareId: info.baseInfo.shareId, - applicationId: info.baseInfo.applicationId, - generationNumber: info.baseInfo.generationNumber, - expires: info.baseInfo.expires, - signature: info.baseInfo.signature, - }), - )}`, + `https://www.airtable.com/v0.3/exploreApplications/${templateId}`, { headers: { accept: '*/*', @@ -196,74 +290,29 @@ async function readView(viewId) { ...info.headers, }, // @ts-ignore - referrerPolicy: 'no-referrer', + referrer: 'https://www.airtable.com/', + referrerPolicy: 'same-origin', body: null, method: 'GET', + mode: 'cors', + credentials: 'include', }, - ) - .then((response) => { - return response.data; - }) - .catch((e) => { - logger.log(e); - throw { - message: - 'Error Reading View :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - }); - return { view: resreq.data }; - } else { + ); + + if (resreq?.data) { + return { template: resreq.data }; + } else { + throw new Error('Error Reading :: Data missing'); + } + } catch (e) { + logger.log(e); throw { - message: 'Error Initializing :: please try again !!', + message: + 'Error Fetching :: Ensure www.airtable.com/templates/featured/ is accessible.', }; } } -async function readTemplate(templateId) { - if (!info.initialized) { - await initialize('shrO8aYf3ybwSdDKn'); - } - const resreq = await axios( - `https://www.airtable.com/v0.3/exploreApplications/${templateId}`, - { - headers: { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'x-time-zone': 'Europe/Berlin', - cookie: info.cookie, - ...info.headers, - }, - // @ts-ignore - referrer: 'https://www.airtable.com/', - referrerPolicy: 'same-origin', - body: null, - method: 'GET', - mode: 'cors', - credentials: 'include', - }, - ) - .then((response) => { - return response.data; - }) - .catch((e) => { - logger.log(e); - throw { - message: - 'Error Fetching :: Ensure www.airtable.com/templates/featured/ is accessible.', - }; - }); - return { template: resreq }; -} - function unicodeToChar(text) { return text.replace(/\\u[\dA-F]{4}/gi, function (match) { return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16));