feat: fetch schema from AT

Signed-off-by: mertmit <mertmit99@gmail.com>

packages/nocodb/tests/sync/config.json

@@ -1,11 +1,10 @@
 {
   "airtable": {
-    "apiKey": "keyeZla3k0desT8fU",
-    "baseId": "appNGAcKwq7eq0xuY",
-    "schemaJson": "ltar.json"
+    "apiKey": "keyxxxxxxxxxxxxxx",
+    "shareId": "shrxxxxxxxxxxxxxxx"
   },
   "projectName": "sample",
   "baseURL": "http://localhost:8080",
   "authToken":
-  "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InVzZXJAbm9jb2RiLmNvbSIsImZpcnN0bmFtZSI6bnVsbCwibGFzdG5hbWUiOm51bGwsImlkIjoidXNfdGQ2emowdTB0Nmx0cDIiLCJyb2xlcyI6InVzZXIsc3VwZXIiLCJpYXQiOjE2NTExMjYzMzl9.9CZsRHi8A8NhI6Unq3hIBMvB8NatlBHPfqRjgEbrogY"
+  "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InRlc3RAdGVzdC50ZXN0IiwiZmlyc3RuYW1lIjpudWxsLCJsYXN0bmFtZSI6bnVsbCwiaWQiOiJ1c185cTVnYW93bDQxMGEzaSIsInJvbGVzIjoidXNlcixzdXBlciIsImlhdCI6MTY1MTI3MTI2NX0.-9xBRg4zj4yj7pKwXx0XvDqcaON3NPvZuMAeIfOVgu0"
 }

packages/nocodb/tests/sync/fetchAT.js

@@ -0,0 +1,78 @@
+const axios = require('axios').default;
+
+module.exports = async function main(shareId) {
+  var Cookie = "";
+  const hreq = await axios.get(`https://airtable.com/${shareId}`, {
+    "headers": {
+      "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+      "accept-language": "en-US,en;q=0.9",
+      "sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"",
+      "sec-ch-ua-mobile": "?0",
+      "sec-ch-ua-platform": "\"Linux\"",
+      "sec-fetch-dest": "document",
+      "sec-fetch-mode": "navigate",
+      "sec-fetch-site": "none",
+      "sec-fetch-user": "?1",
+      "upgrade-insecure-requests": "1",
+      "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36"
+    },
+    "referrerPolicy": "strict-origin-when-cross-origin",
+    "body": null,
+    "method": "GET"
+  })
+  .then(response => {
+    for(const ck of response.headers['set-cookie']) {
+      Cookie += ck.split(';')[0] + '; '
+    }
+    return response.data
+  })
+  
+  var headers = JSON.parse(hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim())
+  var lnk = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim())
+  var baseInfo = decodeURIComponent(lnk).match(/{(.*)}/g)[0].split('&').reduce((result, el) => {
+    try {
+      return Object.assign(result, JSON.parse((el.includes('=')?el.split('=')[1]:el)))
+    } catch(e) {
+      if(el.includes('=')) {
+        return Object.assign(result, { [el.split('=')[0]]: el.split('=')[1]})
+      }
+    }
+  }, {})
+  var baseId = baseInfo.applicationId;
+
+  const resreq = await axios("https://airtable.com" + lnk, {
+    "headers": {
+      "accept": "*/*",
+      "accept-language": "en-US,en;q=0.9",
+      "sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"",
+      "sec-ch-ua-mobile": "?0",
+      "sec-ch-ua-platform": "\"Linux\"",
+      "sec-fetch-dest": "empty",
+      "sec-fetch-mode": "cors",
+      "sec-fetch-site": "same-origin",
+      "x-time-zone": "Europe/Istanbul",
+      "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36",
+      "accept": "application/json, text/javascript, */*; q=0.01",
+      "cookie": Cookie,
+      ...headers
+    },
+    "referrerPolicy": "no-referrer",
+    "body": null,
+    "method": "GET"
+  })
+  .then(response => {
+    return response.data
+  })
+  .catch(error => {
+    throw "Error while fetching"
+  })
+
+  return {schema: resreq.data, baseId, baseInfo}
+}
+
+function unicodeToChar(text) {
+  return text.replace(/\\u[\dA-F]{4}/gi, 
+    function (match) {
+        return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16));
+    });
+}

packages/nocodb/tests/sync/sync.js

@@ -3,6 +3,9 @@ const jsonfile = require('jsonfile');
 const { UITypes } = require('nocodb-sdk');
 const axios = require('axios').default;
 const FormData = require('form-data');
+const FetchAT = require('./fetchAT');
+
+var base, baseId;
 
 function syncLog(log) {
   // console.log(log)
@@ -122,10 +125,14 @@ const api = new Api({
 // global schema store
 let aTblSchema = {};
 
-function getAtableSchema() {
+async function getAtableSchema() {
   // let file = jsonfile.readFileSync('./t0v0.json');
-  let file = jsonfile.readFileSync(syncDB.airtable.schemaJson);
-
+  let ft = await FetchAT(syncDB.airtable.shareId);
+  let file = ft.schema;
+  baseId = ft.baseId;
+  base = new Airtable({ apiKey: syncDB.airtable.apiKey }).base(
+    baseId
+  );
   // store copy of atbl schema globally
   aTblSchema = file.tableSchemas;
   return file;
@@ -776,9 +783,6 @@ async function nocoReconfigureFields(aTblSchema) {
 
 // https://www.airtable.com/app1ivUy7ba82jOPn/api/docs#javascript/metadata
 let Airtable = require('airtable');
-let base = new Airtable({ apiKey: syncDB.airtable.apiKey }).base(
-  syncDB.airtable.baseId
-);
 
 let aTblDataLinks = [];
 
@@ -1016,7 +1020,7 @@ async function nc_migrateATbl() {
   await init()
 
   // read schema file
-  const schema = getAtableSchema();
+  const schema = await getAtableSchema();
   let aTblSchema = schema.tableSchemas;
 
   // create empty project