From 098e9fd8ed110cfd1e49189dd61dd37294bedcce Mon Sep 17 00:00:00 2001 From: mertmit Date: Sat, 30 Apr 2022 03:21:47 +0300 Subject: [PATCH] feat: fetch schema from AT Signed-off-by: mertmit --- packages/nocodb/tests/sync/config.json | 7 +-- packages/nocodb/tests/sync/fetchAT.js | 78 ++++++++++++++++++++++++++ packages/nocodb/tests/sync/sync.js | 18 +++--- 3 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 packages/nocodb/tests/sync/fetchAT.js diff --git a/packages/nocodb/tests/sync/config.json b/packages/nocodb/tests/sync/config.json index cbbfd53dc0..2c3acedb8c 100644 --- a/packages/nocodb/tests/sync/config.json +++ b/packages/nocodb/tests/sync/config.json @@ -1,11 +1,10 @@ { "airtable": { - "apiKey": "keyeZla3k0desT8fU", - "baseId": "appNGAcKwq7eq0xuY", - "schemaJson": "ltar.json" + "apiKey": "keyxxxxxxxxxxxxxx", + "shareId": "shrxxxxxxxxxxxxxxx" }, "projectName": "sample", "baseURL": "http://localhost:8080", "authToken": - "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InVzZXJAbm9jb2RiLmNvbSIsImZpcnN0bmFtZSI6bnVsbCwibGFzdG5hbWUiOm51bGwsImlkIjoidXNfdGQ2emowdTB0Nmx0cDIiLCJyb2xlcyI6InVzZXIsc3VwZXIiLCJpYXQiOjE2NTExMjYzMzl9.9CZsRHi8A8NhI6Unq3hIBMvB8NatlBHPfqRjgEbrogY" + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InRlc3RAdGVzdC50ZXN0IiwiZmlyc3RuYW1lIjpudWxsLCJsYXN0bmFtZSI6bnVsbCwiaWQiOiJ1c185cTVnYW93bDQxMGEzaSIsInJvbGVzIjoidXNlcixzdXBlciIsImlhdCI6MTY1MTI3MTI2NX0.-9xBRg4zj4yj7pKwXx0XvDqcaON3NPvZuMAeIfOVgu0" } \ No newline at end of file diff --git a/packages/nocodb/tests/sync/fetchAT.js b/packages/nocodb/tests/sync/fetchAT.js new file mode 100644 index 0000000000..1b42147763 --- /dev/null +++ b/packages/nocodb/tests/sync/fetchAT.js @@ -0,0 +1,78 @@ +const axios = require('axios').default; + +module.exports = async function main(shareId) { + var Cookie = ""; + const hreq = await axios.get(`https://airtable.com/${shareId}`, { + "headers": { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "accept-language": "en-US,en;q=0.9", + "sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Linux\"", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "none", + "sec-fetch-user": "?1", + "upgrade-insecure-requests": "1", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36" + }, + "referrerPolicy": "strict-origin-when-cross-origin", + "body": null, + "method": "GET" + }) + .then(response => { + for(const ck of response.headers['set-cookie']) { + Cookie += ck.split(';')[0] + '; ' + } + return response.data + }) + + var headers = JSON.parse(hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim()) + var lnk = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim()) + var baseInfo = decodeURIComponent(lnk).match(/{(.*)}/g)[0].split('&').reduce((result, el) => { + try { + return Object.assign(result, JSON.parse((el.includes('=')?el.split('=')[1]:el))) + } catch(e) { + if(el.includes('=')) { + return Object.assign(result, { [el.split('=')[0]]: el.split('=')[1]}) + } + } + }, {}) + var baseId = baseInfo.applicationId; + + const resreq = await axios("https://airtable.com" + lnk, { + "headers": { + "accept": "*/*", + "accept-language": "en-US,en;q=0.9", + "sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Linux\"", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-origin", + "x-time-zone": "Europe/Istanbul", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36", + "accept": "application/json, text/javascript, */*; q=0.01", + "cookie": Cookie, + ...headers + }, + "referrerPolicy": "no-referrer", + "body": null, + "method": "GET" + }) + .then(response => { + return response.data + }) + .catch(error => { + throw "Error while fetching" + }) + + return {schema: resreq.data, baseId, baseInfo} +} + +function unicodeToChar(text) { + return text.replace(/\\u[\dA-F]{4}/gi, + function (match) { + return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16)); + }); +} \ No newline at end of file diff --git a/packages/nocodb/tests/sync/sync.js b/packages/nocodb/tests/sync/sync.js index 7fcc4e6cf7..42e2d23c64 100644 --- a/packages/nocodb/tests/sync/sync.js +++ b/packages/nocodb/tests/sync/sync.js @@ -3,6 +3,9 @@ const jsonfile = require('jsonfile'); const { UITypes } = require('nocodb-sdk'); const axios = require('axios').default; const FormData = require('form-data'); +const FetchAT = require('./fetchAT'); + +var base, baseId; function syncLog(log) { // console.log(log) @@ -122,10 +125,14 @@ const api = new Api({ // global schema store let aTblSchema = {}; -function getAtableSchema() { +async function getAtableSchema() { // let file = jsonfile.readFileSync('./t0v0.json'); - let file = jsonfile.readFileSync(syncDB.airtable.schemaJson); - + let ft = await FetchAT(syncDB.airtable.shareId); + let file = ft.schema; + baseId = ft.baseId; + base = new Airtable({ apiKey: syncDB.airtable.apiKey }).base( + baseId + ); // store copy of atbl schema globally aTblSchema = file.tableSchemas; return file; @@ -776,9 +783,6 @@ async function nocoReconfigureFields(aTblSchema) { // https://www.airtable.com/app1ivUy7ba82jOPn/api/docs#javascript/metadata let Airtable = require('airtable'); -let base = new Airtable({ apiKey: syncDB.airtable.apiKey }).base( - syncDB.airtable.baseId -); let aTblDataLinks = []; @@ -1016,7 +1020,7 @@ async function nc_migrateATbl() { await init() // read schema file - const schema = getAtableSchema(); + const schema = await getAtableSchema(); let aTblSchema = schema.tableSchemas; // create empty project