From 201dd5ce32c3c66566d68737d234fc878b5e1505 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Mon, 2 May 2022 12:11:19 +0530 Subject: [PATCH] refactor: sync trigger function Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- packages/nocodb/tests/sync/config.json | 10 -- packages/nocodb/tests/sync/source.js | 51 --------- packages/nocodb/tests/sync/sync.js | 120 +++++++++------------ packages/nocodb/tests/sync/syncStart.js | 10 ++ packages/nocodb/tests/sync/testConfig.json | 10 ++ 5 files changed, 70 insertions(+), 131 deletions(-) delete mode 100644 packages/nocodb/tests/sync/config.json delete mode 100644 packages/nocodb/tests/sync/source.js create mode 100644 packages/nocodb/tests/sync/syncStart.js create mode 100644 packages/nocodb/tests/sync/testConfig.json diff --git a/packages/nocodb/tests/sync/config.json b/packages/nocodb/tests/sync/config.json deleted file mode 100644 index 2c3acedb8c..0000000000 --- a/packages/nocodb/tests/sync/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "airtable": { - "apiKey": "keyxxxxxxxxxxxxxx", - "shareId": "shrxxxxxxxxxxxxxxx" - }, - "projectName": "sample", - "baseURL": "http://localhost:8080", - "authToken": - "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InRlc3RAdGVzdC50ZXN0IiwiZmlyc3RuYW1lIjpudWxsLCJsYXN0bmFtZSI6bnVsbCwiaWQiOiJ1c185cTVnYW93bDQxMGEzaSIsInJvbGVzIjoidXNlcixzdXBlciIsImlhdCI6MTY1MTI3MTI2NX0.-9xBRg4zj4yj7pKwXx0XvDqcaON3NPvZuMAeIfOVgu0" -} \ No newline at end of file diff --git a/packages/nocodb/tests/sync/source.js b/packages/nocodb/tests/sync/source.js deleted file mode 100644 index 92fd8abbcc..0000000000 --- a/packages/nocodb/tests/sync/source.js +++ /dev/null @@ -1,51 +0,0 @@ -module.exports = { - - title: 'source_type', - - version: '', - - init: function (src, dest) { - - }, - - TablesGet: function () { - - }, - - ColumnsGet: function () { - - }, - - RelationalColumnsGet: function () { - - }, - - VirtualColumnsGet: function () { - - }, - - SchemaGet: function () { - - }, - - SchemaCreate: function () { - - }, - - DataExport: function () { - - }, - - DataWatch: function () { - /** - * Fresh load = schema + data - * Reload = clean the data + reload data - * Real time sync - * airtable to nocodb - * nocodb to airtable - */ - } - - - -} \ No newline at end of file diff --git a/packages/nocodb/tests/sync/sync.js b/packages/nocodb/tests/sync/sync.js index 88aac54b81..ce383e60ed 100644 --- a/packages/nocodb/tests/sync/sync.js +++ b/packages/nocodb/tests/sync/sync.js @@ -1,29 +1,35 @@ const Api = require('nocodb-sdk').Api; -const jsonfile = require('jsonfile'); const { UITypes } = require('nocodb-sdk'); const axios = require('axios').default; const FormData = require('form-data'); const FetchAT = require('./fetchAT'); +let Airtable = require('airtable'); var base, baseId; - -function syncLog(log) { - // console.log(log) -} const start = Date.now(); - let enableErrorLogs = false let process_aTblData = false let generate_migrationStats = true let debugMode = true let aTblNcMappingTbl = {} +let api; +let g_aTblSchema = {}; +let ncCreatedProjectSchema = []; +let ncLinkMappingTable = []; +let aTblDataLinks = []; +let nestedLookupTbl = [] +let nestedRollupTbl = [] + +function syncLog(log) { + // console.log(log) +} // mapping table // // static mapping records between aTblId && ncId async function addToMappingTbl(aTblId, ncId, ncName) { - aTblNcMappingTbl[`${aTblId}`] = { + aTblNcMappingTbl[aTblId] = { ncId: ncId, // name added to assist in quick debug @@ -42,26 +48,11 @@ function getNcNameFromAtId(aId) { } /////////////////////////////////////////////////////////////////////////////// -// read configurations -// -const syncDB = jsonfile.readFileSync('./config.json'); - -const api = new Api({ - baseURL: syncDB.baseURL, - headers: { - 'xc-auth': syncDB.authToken - } -}); - -// global schema store -let g_aTblSchema = {}; - -async function getAtableSchema() { - // let file = jsonfile.readFileSync('./t0v0.json'); - let ft = await FetchAT(syncDB.airtable.shareId); +async function getAtableSchema(sDB) { + let ft = await FetchAT(sDB.airtable.shareId); let file = ft.schema; baseId = ft.baseId; - base = new Airtable({ apiKey: syncDB.airtable.apiKey }).base( + base = new Airtable({ apiKey: sDB.airtable.apiKey }).base( baseId ); // store copy of atbl schema globally @@ -69,8 +60,8 @@ async function getAtableSchema() { return file; } -async function getViewData(tblId, viewId) { - let ft = await FetchAT(syncDB.airtable.shareId, tblId, viewId); +async function getViewData(shareId, tblId, viewId) { + let ft = await FetchAT(shareId, tblId, viewId); return ft.schema?.tableDatas[0]?.viewDatas[0] } @@ -161,13 +152,12 @@ async function nc_getTableSchema(tableName) { } // delete project if already exists -async function init() { - console.log(syncDB) +async function init(projName) { // delete 'sample' project if already exists let x = await api.project.list() - let sampleProj = x.list.find(a => a.title === syncDB.projectName) + let sampleProj = x.list.find(a => a.title === projName) if(sampleProj) { await api.project.delete(sampleProj.id) } @@ -511,8 +501,6 @@ async function nocoCreateLinkToAnotherRecord(aTblSchema) { } } -let nestedLookupTbl = [] - async function nocoCreateLookups(aTblSchema) { // LookUps for (let idx = 0; idx < aTblSchema.length; idx++) { @@ -600,7 +588,6 @@ async function nocoCreateLookups(aTblSchema) { } } -let nestedRollupTbl = [] async function nocoCreateRollups(aTblSchema) { // Rollups for (let idx = 0; idx < aTblSchema.length; idx++) { @@ -734,11 +721,7 @@ async function nocoReconfigureFields(aTblSchema) { ////////// Data processing -// https://www.airtable.com/app1ivUy7ba82jOPn/api/docs#javascript/metadata -let Airtable = require('airtable'); -let aTblDataLinks = []; - -function nocoLinkProcessing(table, record, field) { +function nocoLinkProcessing(projName, table, record, field) { (async () => { let rec = record.fields; @@ -752,7 +735,7 @@ function nocoLinkProcessing(table, record, field) { await api.dbTableRow.nestedAdd( 'noco', - syncDB.projectName, + projName, table.id, `${record.id}`, 'mm', // fix me @@ -769,7 +752,7 @@ function nocoLinkProcessing(table, record, field) { // fix me: // instead of skipping data after retrieval, use select fields option in airtable API -function nocoBaseDataProcessing(table, record) { +function nocoBaseDataProcessing(sDB, table, record) { (async () => { let rec = record.fields; @@ -807,7 +790,6 @@ function nocoBaseDataProcessing(table, record) { // these will be automatically populated depending on schema configuration if (dt === 'Lookup') delete rec[key]; if (dt === 'Rollup') delete rec[key]; - // if (dt === 'Attachment') delete rec[key]; if (dt === 'Collaborator') { rec[key] = `${value?.name} <${value?.email}>` @@ -837,13 +819,13 @@ function nocoBaseDataProcessing(table, record) { }); const rs = await axios - .post(syncDB.baseURL + '/api/v1/db/storage/upload', imageFile, { + .post(sDB.baseURL + '/api/v1/db/storage/upload', imageFile, { params: { - path: `noco/${syncDB.projectName}/${table.title}/${key}` + path: `noco/${sDB.projectName}/${table.title}/${key}` }, headers: { 'Content-Type': `multipart/form-data; boundary=${imageFile._boundary}`, - 'xc-auth': syncDB.authToken + 'xc-auth': sDB.authToken } }) .then(response => { @@ -870,7 +852,7 @@ function nocoBaseDataProcessing(table, record) { // bulk Insert let returnValue = await api.dbTableRow.bulkCreate( 'nc', - syncDB.projectName, + sDB.projectName, table.id, // encodeURIComponent(table.title), [rec] ); @@ -880,7 +862,7 @@ function nocoBaseDataProcessing(table, record) { }); } -async function nocoReadData(table, callback) { +async function nocoReadData(sDB, table, callback) { return new Promise((resolve, reject) => { base(table.title) .select({ @@ -892,7 +874,7 @@ async function nocoReadData(table, callback) { // console.log(JSON.stringify(records, null, 2)); // This function (`page`) will get called for each page of records. - records.forEach(record => callback(table, record)); + records.forEach(record => callback(sDB, table, record)); // To fetch the next page of records, call `fetchNextPage`. // If there are more records, `page` will get called again. @@ -910,7 +892,7 @@ async function nocoReadData(table, callback) { }) } -async function nocoReadDataSelected(table, callback, fields) { +async function nocoReadDataSelected(projName, table, callback, fields) { return new Promise((resolve, reject) => { base(table.title) @@ -926,7 +908,7 @@ async function nocoReadDataSelected(table, callback, fields) { // This function (`page`) will get called for each page of records. // records.forEach(record => callback(table, record)); for(let i=0; i x.type === 'grid'); @@ -977,7 +956,7 @@ async function nocoConfigureGridView(aTblSchema) { for(let i=0; i x.id === gridViews[i].id)?.name @@ -1031,17 +1010,24 @@ async function nocoConfigureGridView(aTblSchema) { } // start function -async function nc_migrateATbl() { +module.exports = async function nc_migrateATbl(syncDB) { + + api = new Api({ + baseURL: syncDB.baseURL, + headers: { + 'xc-auth': syncDB.authToken + } + }); // delete project if already exists - if(debugMode) await init() + if(debugMode) await init(syncDB.projectName) // read schema file - const schema = await getAtableSchema(); + const schema = await getAtableSchema(syncDB); let aTblSchema = schema.tableSchemas; // create empty project - await nocoCreateProject() + await nocoCreateProject(syncDB.projectName) // prepare table schema (base) await nocoCreateBaseSchema(aTblSchema); @@ -1065,21 +1051,21 @@ async function nc_migrateATbl() { await nocoReconfigureFields(aTblSchema); // configure grid views - await nocoConfigureGridView(aTblSchema) + await nocoConfigureGridView(syncDB, aTblSchema) if(process_aTblData) { // await nc_DumpTableSchema(); let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); for (let i = 0; i < ncTblList.list.length; i++) { let ncTbl = await api.dbTable.read(ncTblList.list[i].id); - await nocoReadData(ncTbl, nocoBaseDataProcessing); + await nocoReadData(syncDB, ncTbl, nocoBaseDataProcessing); } // Configure link @ Data row's for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { let x = ncLinkMappingTable[idx]; let ncTbl = await nc_getTableSchema(aTbl_getTableName(x.aTbl.tblId).tn); - await nocoReadDataSelected(ncTbl, nocoLinkProcessing, x.aTbl.name); + await nocoReadDataSelected(syncDB.projectName, ncTbl, nocoLinkProcessing, x.aTbl.name); } } @@ -1088,12 +1074,6 @@ async function nc_migrateATbl() { } } -nc_migrateATbl().catch(e => { - // console.log(e?.config?.url); - console.log(e) -}); - - /////////////////////// // statistics diff --git a/packages/nocodb/tests/sync/syncStart.js b/packages/nocodb/tests/sync/syncStart.js new file mode 100644 index 0000000000..bed013c38b --- /dev/null +++ b/packages/nocodb/tests/sync/syncStart.js @@ -0,0 +1,10 @@ +const jsonfile = require("jsonfile"); +const nc_migrate = require('./sync') +// read configurations +// +const syncDB = jsonfile.readFileSync('./testConfig.json'); + +nc_migrate(syncDB).catch(e => { + // console.log(e?.config?.url); + console.log(e) +}); diff --git a/packages/nocodb/tests/sync/testConfig.json b/packages/nocodb/tests/sync/testConfig.json new file mode 100644 index 0000000000..a93d60d402 --- /dev/null +++ b/packages/nocodb/tests/sync/testConfig.json @@ -0,0 +1,10 @@ +{ + "airtable": { + "apiKey": "keyeZla3k0desT8fU", + "shareId": "shrmXIJn1oEp059Kb" + }, + "projectName": "sample", + "baseURL": "http://localhost:8080", + "authToken": + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InVzZXJAbm9jb2RiLmNvbSIsImZpcnN0bmFtZSI6bnVsbCwibGFzdG5hbWUiOm51bGwsImlkIjoidXNfYWR0YnA4NjNwbGN4d2siLCJyb2xlcyI6InVzZXIsc3VwZXIiLCJpYXQiOjE2NTEzODAyMTZ9.xjsdk06xilkgVEViTM2HSFuCbwbZQs5LlZUl-510rZs" +}