github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

refactor: remove hard-coded values, reorganize functions 

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>
sync
Raju Udava 3 years ago
parent
d534865a13
commit
149c0aa1c3
1 changed files with 152 additions and 141 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 287
      packages/nocodb/tests/sync/sync.js

287
packages/nocodb/tests/sync/sync.js
Unescape View File

@ -7,16 +7,19 @@ const { UITypes } = require('nocodb-sdk');
const syncDB = { const syncDB = {
airtable: { airtable: {
apiKey: 'keyeZla3k0desT8fU', apiKey: 'keyeZla3k0desT8fU',
// baseId: 'appb8CCITtLXZsYqV', baseId: 'appNGAcKwq7eq0xuY',
baseId: 'appNGAcKwq7eq0xuY' schemaJson: './ltar.json'
} },
projectName: 'sample',
baseURL: 'http://localhost:8080',
authToken:
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InVzZXJAbm9jb2RiLmNvbSIsImZpcnN0bmFtZSI6bnVsbCwibGFzdG5hbWUiOm51bGwsImlkIjoidXNfNWVhZDc5NHBkbjRuZTkiLCJyb2xlcyI6InVzZXIsc3VwZXIiLCJpYXQiOjE2NTA5ODA3MTZ9.qlTuirKjUiSCfCwjtW6wVYPdPGQuT-8gfcXVIllIVUc'
}; };
const api = new Api({ const api = new Api({
baseURL: 'http://localhost:8080', baseURL: syncDB.baseURL,
headers: { headers: {
'xc-auth': 'xc-auth': syncDB.authToken
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InVzZXJAbm9jb2RiLmNvbSIsImZpcnN0bmFtZSI6bnVsbCwibGFzdG5hbWUiOm51bGwsImlkIjoidXNfN2NuZmpxMGt5NjczaXkiLCJyb2xlcyI6InVzZXIsc3VwZXIiLCJpYXQiOjE2NTA0NzM3MTl9.VVZKink3FSpajxnfaTVPn2iuCNH3lTjepNQNb4Q8VOE'
} }
}); });
@ -25,7 +28,7 @@ let aTblSchema = {};
function getAtableSchema() { function getAtableSchema() {
// let file = jsonfile.readFileSync('./t0v0.json'); // let file = jsonfile.readFileSync('./t0v0.json');
let file = jsonfile.readFileSync('./ltar.json'); let file = jsonfile.readFileSync(syncDB.airtable.schemaJson);
// store copy of atbl schema globally // store copy of atbl schema globally
aTblSchema = file.tableSchemas; aTblSchema = file.tableSchemas;
@ -85,14 +88,14 @@ function aTbl_getColumnName(colId) {
// nc dump schema // nc dump schema
// //
async function nc_DumpTableSchema() { async function nc_DumpTableSchema() {
console.log('[') console.log('[');
let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id) let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id);
for(let i=0; i<ncTblList.list.length; i++) { for (let i = 0; i < ncTblList.list.length; i++) {
let ncTbl = await api.dbTable.read(ncTblList.list[i].id) let ncTbl = await api.dbTable.read(ncTblList.list[i].id);
console.log(JSON.stringify(ncTbl, null, 2)) console.log(JSON.stringify(ncTbl, null, 2));
console.log(',') console.log(',');
} }
console.log(']') console.log(']');
} }
// retrieve nc column schema from using aTbl field ID as reference // retrieve nc column schema from using aTbl field ID as reference
@ -114,7 +117,6 @@ async function nc_getTableSchema(tableName) {
return ncTbl; return ncTbl;
} }
// map UIDT // map UIDT
// //
function getNocoType(col) { function getNocoType(col) {
@ -172,7 +174,7 @@ function getNocoTypeOptions(col) {
} }
} }
// convert to Nc schema // convert to Nc schema (basic, excluding relations)
// //
function tablesPrepare(tblSchema) { function tablesPrepare(tblSchema) {
let tables = []; let tables = [];
@ -225,123 +227,24 @@ function tablesPrepare(tblSchema) {
return tables; return tables;
} }
async function nocoCreateBaseSchema(srcSchema) {
async function nocoCreateSchema(srcSchema) {
// base schema preparation: exclude // base schema preparation: exclude
let tables = tablesPrepare(srcSchema.tableSchemas); let tables = tablesPrepare(srcSchema.tableSchemas);
// debug
// console.log(JSON.stringify(tables, null, 2));
return tables;
}
////////// Data processing
// https://www.airtable.com/app1ivUy7ba82jOPn/api/docs#javascript/metadata
let Airtable = require('airtable');
let base = new Airtable({ apiKey: syncDB.airtable.apiKey }).base(
syncDB.airtable.baseId
);
async function nocoReadData(table) {
base(table.title)
.select({
pageSize: 25,
// maxRecords: 100,
view: 'Grid view'
})
.eachPage(
function page(records, fetchNextPage) {
console.log(JSON.stringify(records, null, 2));
// This function (`page`) will get called for each page of records.
records.forEach(function(record) {
(async () => {
let rec = record.fields;
// kludge -
// trim spaces on either side of column name
// leads to error in NocoDB
Object.keys(rec).forEach(key => {
let replacedKey = key.trim();
if (key !== replacedKey) {
rec[replacedKey] = rec[key];
delete rec[key];
}
});
// post-processing on the record
for (const [key, value] of Object.entries(rec)) {
// retrieve datatype
let dt = table.columns.find(x => x.title === key).uidt;
// https://www.npmjs.com/package/validator
// default value: digits_after_decimal: [2]
// if currency, set decimal place to 2
//
if (dt === 'Currency') rec[key] = value.toFixed(2);
}
// bulk Insert
let returnValue = await api.dbTableRow.bulkCreate(
'nc',
'sample-4',
table.title,
[rec]
);
})().catch(e => {
console.log(e);
});
});
// To fetch the next page of records, call `fetchNextPage`.
// If there are more records, `page` will get called again.
// If there are no more records, `done` will get called.
fetchNextPage();
},
function done(err) {
if (err) {
console.error(err);
}
}
);
}
//////////
// holds response for all created tables
let ncCreatedProjectSchema = [];
let ncLinkMappingTable = [];
function nc_isLinkExists(atblFieldId) {
if(ncLinkMappingTable.find(x => x.aTbl.typeOptions.symmetricColumnId === atblFieldId))
return true;
return false;
}
// start function
(async () => {
// read schema file
const schema = getAtableSchema();
let aTblSchema = schema.tableSchemas;
// create empty project (XC-DB)
ncCreatedProjectSchema = await api.project.create({
title: 'sample-4'
});
// prepare table schema (basic)
let ncTblSchema = await nocoCreateSchema(schema);
// for each table schema, create nc table // for each table schema, create nc table
for (let idx = 0; idx < ncTblSchema.length; idx++) { for (let idx = 0; idx < tables.length; idx++) {
let table = await api.dbTable.create( let table = await api.dbTable.create(
ncCreatedProjectSchema.id, ncCreatedProjectSchema.id,
ncTblSchema[idx] tables[idx]
); );
} }
// debug
// console.log(JSON.stringify(tables, null, 2));
return tables;
}
async function nocoCreateLinkToAnotherRecord(aTblSchema) {
// Link to another RECORD // Link to another RECORD
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
let aTblLinkColumns = aTblSchema[idx].columns.filter( let aTblLinkColumns = aTblSchema[idx].columns.filter(
@ -352,10 +255,8 @@ function nc_isLinkExists(atblFieldId) {
// //
if (aTblLinkColumns.length) { if (aTblLinkColumns.length) {
for (let i = 0; i < aTblLinkColumns.length; i++) { for (let i = 0; i < aTblLinkColumns.length; i++) {
// check if link already established? // check if link already established?
if(!nc_isLinkExists(aTblLinkColumns[i].id)) { if (!nc_isLinkExists(aTblLinkColumns[i].id)) {
// parent table ID // parent table ID
let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id;
@ -373,7 +274,8 @@ function nc_isLinkExists(atblFieldId) {
title: aTblLinkColumns[i].name, title: aTblLinkColumns[i].name,
parentId: srcTableId, parentId: srcTableId,
childId: childTableId, childId: childTableId,
type: aTblLinkColumns[i].typeOptions.relationship === 'many' type:
aTblLinkColumns[i].typeOptions.relationship === 'many'
? 'mm' ? 'mm'
: 'hm' : 'hm'
}); });
@ -395,7 +297,6 @@ function nc_isLinkExists(atblFieldId) {
ncLinkMappingTable.push(link); ncLinkMappingTable.push(link);
} else { } else {
// if link already exists, we need to change name of linked column // if link already exists, we need to change name of linked column
// to what is represented in airtable // to what is represented in airtable
@ -406,8 +307,7 @@ function nc_isLinkExists(atblFieldId) {
let x = ncLinkMappingTable.findIndex( let x = ncLinkMappingTable.findIndex(
x => x =>
x.aTbl.tblId === aTblLinkColumns[i].typeOptions.foreignTableId && x.aTbl.tblId === aTblLinkColumns[i].typeOptions.foreignTableId &&
x.aTbl.id === x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId
aTblLinkColumns[i].typeOptions.symmetricColumnId
); );
let childTblSchema = await api.dbTable.read( let childTblSchema = await api.dbTable.read(
@ -459,7 +359,9 @@ function nc_isLinkExists(atblFieldId) {
} }
} }
} }
}
async function nocoCreateLookups(aTblSchema) {
// LookUps // LookUps
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
let aTblColumns = aTblSchema[idx].columns.filter(x => x.type === 'lookup'); let aTblColumns = aTblSchema[idx].columns.filter(x => x.type === 'lookup');
@ -486,7 +388,9 @@ function nc_isLinkExists(atblFieldId) {
} }
} }
} }
}
async function nocoCreateRollups(aTblSchema) {
// Rollups // Rollups
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
let aTblColumns = aTblSchema[idx].columns.filter(x => x.type === 'rollup'); let aTblColumns = aTblSchema[idx].columns.filter(x => x.type === 'rollup');
@ -514,17 +418,126 @@ function nc_isLinkExists(atblFieldId) {
} }
} }
} }
}
// await nc_DumpTableSchema(); ////////// Data processing
let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id) // https://www.airtable.com/app1ivUy7ba82jOPn/api/docs#javascript/metadata
for(let i=0; i<ncTblList.list.length; i++) { let Airtable = require('airtable');
let ncTbl = await api.dbTable.read(ncTblList.list[i].id) let base = new Airtable({ apiKey: syncDB.airtable.apiKey }).base(
await nocoReadData(ncTbl) syncDB.airtable.baseId
);
async function nocoReadData(table) {
base(table.title)
.select({
pageSize: 25,
// maxRecords: 100,
view: 'Grid view'
})
.eachPage(
function page(records, fetchNextPage) {
console.log(JSON.stringify(records, null, 2));
// This function (`page`) will get called for each page of records.
records.forEach(function(record) {
(async () => {
let rec = record.fields;
// kludge -
// trim spaces on either side of column name
// leads to error in NocoDB
Object.keys(rec).forEach(key => {
let replacedKey = key.trim();
if (key !== replacedKey) {
rec[replacedKey] = rec[key];
delete rec[key];
}
});
// post-processing on the record
for (const [key, value] of Object.entries(rec)) {
// retrieve datatype
let dt = table.columns.find(x => x.title === key).uidt;
// https://www.npmjs.com/package/validator
// default value: digits_after_decimal: [2]
// if currency, set decimal place to 2
//
if (dt === 'Currency') rec[key] = value.toFixed(2);
} }
// bulk Insert
let returnValue = await api.dbTableRow.bulkCreate(
'nc',
syncDB.projectName,
table.title,
[rec]
);
})().catch(e => { })().catch(e => {
console.log(e); console.log(e);
});
});
// To fetch the next page of records, call `fetchNextPage`.
// If there are more records, `page` will get called again.
// If there are no more records, `done` will get called.
fetchNextPage();
},
function done(err) {
if (err) {
console.error(err);
}
}
);
}
//////////
let ncCreatedProjectSchema = [];
let ncLinkMappingTable = [];
function nc_isLinkExists(atblFieldId) {
if (
ncLinkMappingTable.find(
x => x.aTbl.typeOptions.symmetricColumnId === atblFieldId
)
)
return true;
return false;
}
// start function
(async () => {
// read schema file
const schema = getAtableSchema();
let aTblSchema = schema.tableSchemas;
// create empty project (XC-DB)
ncCreatedProjectSchema = await api.project.create({
title: syncDB.projectName
});
// prepare table schema (base)
await nocoCreateBaseSchema(schema);
// add LTAR
await nocoCreateLinkToAnotherRecord(aTblSchema);
// add lookup's
await nocoCreateLookups(aTblSchema);
// add rollups
await nocoCreateRollups(aTblSchema);
// await nc_DumpTableSchema();
// let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id);
// for (let i = 0; i < ncTblList.list.length; i++) {
// let ncTbl = await api.dbTable.read(ncTblList.list[i].id);
// await nocoReadData(ncTbl);
// }
})().catch(e => {
console.log(e);
}); });
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
@ -538,7 +551,6 @@ function nc_isLinkExists(atblFieldId) {
// Scratch pad // Scratch pad
// await api.dbTableRow.bulkInsert('nc', 'x', 'x', [{Title: 'abc'}, {Title: 'abc'}, {Title: 'abc'}]) // await api.dbTableRow.bulkInsert('nc', 'x', 'x', [{Title: 'abc'}, {Title: 'abc'}, {Title: 'abc'}])
// await api.data.bulkInsert(); // await api.data.bulkInsert();
// let column = await api.meta.columnCreate('md_vnesap07k24lku', { // let column = await api.meta.columnCreate('md_vnesap07k24lku', {
@ -546,7 +558,6 @@ function nc_isLinkExists(atblFieldId) {
// cn: 'col-1', // cn: 'col-1',
// }) // })
// // t0 schema // // t0 schema
// let t0 = { // let t0 = {
// table_name: 't0', // table_name: 't0',

Write Preview
Loading…
Cancel
Save