github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Merge pull request #9517 from nocodb/nc-fix/at-stall 

fix: at import stall issue
pull/9534/head
Pranav C 2 months ago committed by GitHub
parent
e8093293c4 37575f4094
commit
25352e96cc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 207 additions and 155 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      packages/nocodb/src/models/Column.ts
  2. 1
      packages/nocodb/src/modules/jobs/jobs.module.ts
  3. 2
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
  4. 357
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts

2
packages/nocodb/src/models/Column.ts
Unescape View File

@ -816,6 +816,7 @@ export default class Column<T = any> implements ColumnType {
if (button.type === 'url') {
if (
button.formula &&
addFormulaErrorIfMissingColumn({
formula: button,
columnId: id,
@ -858,6 +859,7 @@ export default class Column<T = any> implements ColumnType {
formulaCol,
).getColOptions<FormulaColumn>(context, ncMeta);
if (
formula.formula &&
addFormulaErrorIfMissingColumn({
formula,
columnId: id,

1
packages/nocodb/src/modules/jobs/jobs.module.ts
Unescape View File

@ -54,6 +54,7 @@ export const JobsModuleMetadata = {
name: JOBS_QUEUE,
defaultJobOptions: {
removeOnComplete: true,
attempts: 1,
},
}),
]

2
packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
Unescape View File

@ -469,7 +469,7 @@ export class AtImportProcessor {
if (
options.find(
(el) =>
el.title.toLowerCase() === (value as any).name.toLowerCase(),
el.title === (value as any).name,
)
) {
logWarning(

357
packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
Unescape View File

@ -1,5 +1,8 @@
import { Logger } from '@nestjs/common';
import axios from 'axios';
import { streamObject } from 'stream-json/streamers/StreamObject';
import { parser } from 'stream-json/Parser';
import { ignore } from 'stream-json/filters/Ignore';
const logger = new Logger('FetchAT');
@ -17,45 +20,37 @@ async function initialize(shareId, appId?: string) {
const url = `https://airtable.com/${appId ? `${appId}/` : ''}${shareId}`;
try {
const hreq = await axios
.get(url, {
headers: {
accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
},
// @ts-ignore
referrerPolicy: 'strict-origin-when-cross-origin',
body: null,
method: 'GET',
})
.then((response) => {
for (const ck of response.headers['set-cookie']) {
info.cookie += ck.split(';')[0] + '; ';
}
return response.data;
})
.catch((e) => {
logger.log(e);
throw {
message:
'Invalid Shared Base ID :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
};
});
const hreq = await axios.get(url, {
headers: {
accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
},
// @ts-ignore
referrerPolicy: 'strict-origin-when-cross-origin',
body: null,
method: 'GET',
});
for (const ck of hreq.headers['set-cookie']) {
info.cookie += ck.split(';')[0] + '; ';
}
const data = hreq.data;
const headers = hreq.match(/(?<=var headers =)(.*)(?=;)/g);
const link = hreq.match(/(?<=fetch\(")(\\.*)(?=")/g);
const headers = data.match(/(?<=var headers =)(.*)(?=;)/g);
const link = data.match(/(?<=fetch\(")(\\.*)(?=")/g);
if (!headers || !link) {
throw {
@ -65,11 +60,11 @@ async function initialize(shareId, appId?: string) {
}
info.headers = JSON.parse(
hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
data.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim(),
);
info.link = unicodeToChar(
hreq.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(),
data.match(/(?<=fetch\(")(\\.*)(?=")/g)[0].trim(),
);
info.link = info.link.replace(
@ -112,44 +107,68 @@ async function initialize(shareId, appId?: string) {
async function read() {
if (info.initialized) {
const resreq = await axios('https://airtable.com' + info.link, {
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
// @ts-ignore
referrerPolicy: 'no-referrer',
body: null,
method: 'GET',
})
.then((response) => {
return response.data;
})
.catch((e) => {
logger.log(e);
throw {
message:
'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
};
try {
const resreq = await axios('https://airtable.com' + info.link, {
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
// @ts-ignore
referrerPolicy: 'no-referrer',
body: null,
method: 'GET',
responseType: 'stream',
});
return {
schema: resreq.data,
baseId: info.baseId,
baseInfo: info.baseInfo,
};
const data: any = await new Promise((resolve, reject) => {
const jsonStream = resreq.data
.pipe(parser())
.pipe(ignore({ filter: 'data.tableDatas' }))
.pipe(streamObject());
const fullObject = {};
jsonStream.on('data', (chunk) => {
if (chunk.key) fullObject[chunk.key] = chunk.value;
});
jsonStream.on('error', (err) => {
reject(err);
});
jsonStream.on('end', () => {
resolve(fullObject);
});
});
if (data?.data) {
return {
schema: data?.data,
baseId: info.baseId,
baseInfo: info.baseInfo,
};
} else {
throw new Error('Error Reading :: Data missing');
}
} catch (e) {
logger.log(e);
throw {
message:
'Error Reading :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
};
}
} else {
throw {
message: 'Error Initializing :: please try again !!',
@ -159,25 +178,100 @@ async function read() {
async function readView(viewId) {
if (info.initialized) {
try {
const resreq = await axios(
`https://airtable.com/v0.3/view/${viewId}/readData?` +
`stringifiedObjectParams=${encodeURIComponent(
JSON.stringify({
mayOnlyIncludeRowAndCellDataForIncludedViews: true,
mayExcludeCellDataForLargeViews: true,
}),
)}&requestId=${
info.baseInfo.requestId
}&accessPolicy=${encodeURIComponent(
JSON.stringify({
allowedActions: info.baseInfo.allowedActions,
shareId: info.baseInfo.shareId,
applicationId: info.baseInfo.applicationId,
generationNumber: info.baseInfo.generationNumber,
expires: info.baseInfo.expires,
signature: info.baseInfo.signature,
}),
)}`,
{
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
// @ts-ignore
referrerPolicy: 'no-referrer',
body: null,
method: 'GET',
responseType: 'stream',
},
);
const data: any = await new Promise((resolve, reject) => {
const jsonStream = resreq.data
.pipe(parser())
.pipe(ignore({ filter: 'data.rowOrder' }))
.pipe(streamObject());
const fullObject = {};
jsonStream.on('data', (chunk) => {
if (chunk.key) fullObject[chunk.key] = chunk.value;
});
jsonStream.on('error', (err) => {
reject(err);
});
jsonStream.on('end', () => {
resolve(fullObject);
});
});
if (data?.data) {
return { view: data.data };
} else {
throw new Error('Error Reading :: Data missing');
}
} catch (e) {
logger.log(e);
throw {
message:
'Error Reading View :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
};
}
} else {
throw {
message: 'Error Initializing :: please try again !!',
};
}
}
async function readTemplate(templateId) {
if (!info.initialized) {
await initialize('shrO8aYf3ybwSdDKn');
}
try {
const resreq = await axios(
`https://airtable.com/v0.3/view/${viewId}/readData?` +
`stringifiedObjectParams=${encodeURIComponent(
JSON.stringify({
mayOnlyIncludeRowAndCellDataForIncludedViews: true,
mayExcludeCellDataForLargeViews: true,
}),
)}&requestId=${
info.baseInfo.requestId
}&accessPolicy=${encodeURIComponent(
JSON.stringify({
allowedActions: info.baseInfo.allowedActions,
shareId: info.baseInfo.shareId,
applicationId: info.baseInfo.applicationId,
generationNumber: info.baseInfo.generationNumber,
expires: info.baseInfo.expires,
signature: info.baseInfo.signature,
}),
)}`,
`https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
{
headers: {
accept: '*/*',
@ -196,74 +290,29 @@ async function readView(viewId) {
...info.headers,
},
// @ts-ignore
referrerPolicy: 'no-referrer',
referrer: 'https://www.airtable.com/',
referrerPolicy: 'same-origin',
body: null,
method: 'GET',
mode: 'cors',
credentials: 'include',
},
)
.then((response) => {
return response.data;
})
.catch((e) => {
logger.log(e);
throw {
message:
'Error Reading View :: Ensure www.airtable.com/<SharedBaseID> is accessible. Refer https://bit.ly/3x0OdXI for details',
};
});
return { view: resreq.data };
} else {
);
if (resreq?.data) {
return { template: resreq.data };
} else {
throw new Error('Error Reading :: Data missing');
}
} catch (e) {
logger.log(e);
throw {
message: 'Error Initializing :: please try again !!',
message:
'Error Fetching :: Ensure www.airtable.com/templates/featured/<TemplateID> is accessible.',
};
}
}
async function readTemplate(templateId) {
if (!info.initialized) {
await initialize('shrO8aYf3ybwSdDKn');
}
const resreq = await axios(
`https://www.airtable.com/v0.3/exploreApplications/${templateId}`,
{
headers: {
accept: '*/*',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua':
'" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
'x-time-zone': 'Europe/Berlin',
cookie: info.cookie,
...info.headers,
},
// @ts-ignore
referrer: 'https://www.airtable.com/',
referrerPolicy: 'same-origin',
body: null,
method: 'GET',
mode: 'cors',
credentials: 'include',
},
)
.then((response) => {
return response.data;
})
.catch((e) => {
logger.log(e);
throw {
message:
'Error Fetching :: Ensure www.airtable.com/templates/featured/<TemplateID> is accessible.',
};
});
return { template: resreq };
}
function unicodeToChar(text) {
return text.replace(/\\u[\dA-F]{4}/gi, function (match) {
return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16));

Write Preview
Loading…
Cancel
Save