github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Fix/at sync: store link records (#2156) 

* fix: rollup, conditional inclusion of vColumns

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: reduce at api access/ store links

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: reduce view column list api invocation

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* feat: performance monitor framework

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: use bulk insert per 100 records

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: link error, perf clean up

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: remove duplicate entries from select/ multiselect

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: truncate table name to 50 chars to avoid database specific issue

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: truncate column name to 60 chars to avoid database specific issue

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: disable link-store optimisation, disable parallel link creation

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: invalid date handle, clean-up

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

Co-authored-by: Pranav C <pranavxc@gmail.com>
pull/2175/head
Raju Udava 3 years ago committed by GitHub
parent
5abcdd3e65
commit
f12410aa33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 411 additions and 172 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 583
      packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts

583
packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts
Unescape View File

@ -56,6 +56,17 @@ export default async (
if (debugMode) progress({ level: 1, msg: log }); if (debugMode) progress({ level: 1, msg: log });
} }
const perfStats = [];
function recordPerfStart() {
if (!debugMode) return 0;
return Date.now();
}
function recordPerfStats(start, event) {
if (!debugMode) return;
const duration = Date.now() - start;
perfStats.push({ d: duration, e: event });
}
let base, baseId; let base, baseId;
const start = Date.now(); const start = Date.now();
const enableErrorLogs = false; const enableErrorLogs = false;
@ -69,6 +80,9 @@ export default async (
const nestedLookupTbl: any[] = []; const nestedLookupTbl: any[] = [];
const nestedRollupTbl: any[] = []; const nestedRollupTbl: any[] = [];
const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' }; const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' };
const storeLinks = false;
const skipAttachments = false;
const ncLinkDataStore: any = {};
const uniqueTableNameGen = getUniqueNameGenerator('sheet'); const uniqueTableNameGen = getUniqueNameGenerator('sheet');
@ -173,7 +187,11 @@ export default async (
} }
function nc_getSanitizedColumnName(table, name) { function nc_getSanitizedColumnName(table, name) {
const col_name = nc_sanitizeName(name); let col_name = nc_sanitizeName(name);
// truncate to 60 chars if character if exceeds above 60
col_name = col_name?.slice(0, 60);
// for knex, replace . with _ // for knex, replace . with _
const col_alias = name.trim().replace(/\./g, '_'); const col_alias = name.trim().replace(/\./g, '_');
@ -350,7 +368,8 @@ export default async (
); );
} }
// const csvOpt = "'" + opt.join("','") + "'"; // const csvOpt = "'" + opt.join("','") + "'";
const csvOpt = opt const optSansDuplicate = [...new Set(opt)];
const csvOpt = optSansDuplicate
.map(v => `'${v.replace(/'/g, "\\'").replace(/,/g, '.')}'`) .map(v => `'${v.replace(/'/g, "\\'").replace(/,/g, '.')}'`)
.join(','); .join(',');
return { type: 'select', data: csvOpt }; return { type: 'select', data: csvOpt };
@ -380,7 +399,15 @@ export default async (
// Enable to use aTbl identifiers as is: table.id = tblSchema[i].id; // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id;
table.title = tblSchema[i].name; table.title = tblSchema[i].name;
table.table_name = uniqueTableNameGen(nc_sanitizeName(tblSchema[i].name)); let sanitizedName = nc_sanitizeName(tblSchema[i].name);
// truncate to 50 chars if character if exceeds above 50
// upto 64 should be fine but we are keeping it to 50 since
// meta project adds prefix as well
sanitizedName = sanitizedName?.slice(0, 50);
// check for duplicate and populate a unique name if already exist
table.table_name = uniqueTableNameGen(sanitizedName);
const uniqueColNameGen = getUniqueNameGenerator('field'); const uniqueColNameGen = getUniqueNameGenerator('field');
table.columns = []; table.columns = [];
@ -471,10 +498,14 @@ export default async (
logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`); logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`);
logDetailed(`NC API: dbTable.create ${tables[idx].title}`); logDetailed(`NC API: dbTable.create ${tables[idx].title}`);
let _perfStart = recordPerfStart();
const table: any = await api.dbTable.create( const table: any = await api.dbTable.create(
ncCreatedProjectSchema.id, ncCreatedProjectSchema.id,
tables[idx] tables[idx]
); );
recordPerfStats(_perfStart, 'dbTable.create');
updateNcTblSchema(table); updateNcTblSchema(table);
// update mapping table // update mapping table
@ -494,13 +525,18 @@ export default async (
// update default view name- to match it to airtable view name // update default view name- to match it to airtable view name
logDetailed(`NC API: dbView.list ${table.id}`); logDetailed(`NC API: dbView.list ${table.id}`);
_perfStart = recordPerfStart();
const view = await api.dbView.list(table.id); const view = await api.dbView.list(table.id);
recordPerfStats(_perfStart, 'dbView.list');
const aTbl_grid = aTblSchema[idx].views.find(x => x.type === 'grid'); const aTbl_grid = aTblSchema[idx].views.find(x => x.type === 'grid');
logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`);
_perfStart = recordPerfStart();
await api.dbView.update(view.list[0].id, { await api.dbView.update(view.list[0].id, {
title: aTbl_grid.name title: aTbl_grid.name
}); });
recordPerfStats(_perfStart, 'dbView.update');
await updateNcTblSchemaById(table.id); await updateNcTblSchemaById(table.id);
await sMap.addToMappingTbl( await sMap.addToMappingTbl(
@ -563,7 +599,9 @@ export default async (
} }
// check if already a column exists with this name? // check if already a column exists with this name?
let _perfStart = recordPerfStart();
const srcTbl: any = await api.dbTable.read(srcTableId); const srcTbl: any = await api.dbTable.read(srcTableId);
recordPerfStats(_perfStart, 'dbTable.read');
// create link // create link
const ncName = nc_getSanitizedColumnName( const ncName = nc_getSanitizedColumnName(
@ -574,6 +612,7 @@ export default async (
logDetailed( logDetailed(
`NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}`
); );
_perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, { const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.LinkToAnotherRecord, uidt: UITypes.LinkToAnotherRecord,
title: ncName.title, title: ncName.title,
@ -585,6 +624,8 @@ export default async (
// ? 'mm' // ? 'mm'
// : 'hm' // : 'hm'
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === ncName.title)?.id; const ncId = ncTbl.columns.find(x => x.title === ncName.title)?.id;
@ -599,7 +640,7 @@ export default async (
// this information will be helpful in identifying relation pair // this information will be helpful in identifying relation pair
const link = { const link = {
nc: { nc: {
title: aTblLinkColumns[i].name, title: ncName.title,
parentId: srcTableId, parentId: srcTableId,
childId: childTableId, childId: childTableId,
type: 'mm' type: 'mm'
@ -626,12 +667,17 @@ export default async (
x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId
); );
let _perfStart = recordPerfStart();
const childTblSchema: any = await api.dbTable.read( const childTblSchema: any = await api.dbTable.read(
ncLinkMappingTable[x].nc.childId ncLinkMappingTable[x].nc.childId
); );
recordPerfStats(_perfStart, 'dbTable.read');
_perfStart = recordPerfStart();
const parentTblSchema: any = await api.dbTable.read( const parentTblSchema: any = await api.dbTable.read(
ncLinkMappingTable[x].nc.parentId ncLinkMappingTable[x].nc.parentId
); );
recordPerfStats(_perfStart, 'dbTable.read');
// fix me // fix me
// let childTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.childId] // let childTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.childId]
@ -641,6 +687,16 @@ export default async (
col => col.title === ncLinkMappingTable[x].nc.title col => col.title === ncLinkMappingTable[x].nc.title
); );
if (parentLinkColumn === undefined) {
updateMigrationSkipLog(
parentTblSchema?.title,
ncLinkMappingTable[x].nc.title,
UITypes.LinkToAnotherRecord,
'Link error'
);
continue;
}
// hack // fix me // hack // fix me
if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') { if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') {
parentLinkColumn = parentTblSchema.columns.find( parentLinkColumn = parentTblSchema.columns.find(
@ -698,6 +754,7 @@ export default async (
logDetailed( logDetailed(
`NC API: dbTableColumn.update rename symmetric column ${ncName.title}` `NC API: dbTableColumn.update rename symmetric column ${ncName.title}`
); );
_perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.update( const ncTbl: any = await api.dbTableColumn.update(
childLinkColumn.id, childLinkColumn.id,
{ {
@ -706,6 +763,8 @@ export default async (
column_name: ncName.column_name column_name: ncName.column_name
} }
); );
recordPerfStats(_perfStart, 'dbTableColumn.update');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find( const ncId = ncTbl.columns.find(
@ -768,7 +827,10 @@ export default async (
aTblColumns[i].typeOptions.foreignTableRollupColumnId aTblColumns[i].typeOptions.foreignTableRollupColumnId
); );
if (ncLookupColumnId === undefined) { if (
ncLookupColumnId === undefined ||
ncRelationColumnId === undefined
) {
aTblColumns[i]['srcTableId'] = srcTableId; aTblColumns[i]['srcTableId'] = srcTableId;
nestedLookupTbl.push(aTblColumns[i]); nestedLookupTbl.push(aTblColumns[i]);
continue; continue;
@ -780,6 +842,7 @@ export default async (
); );
logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, { const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Lookup, uidt: UITypes.Lookup,
title: ncName.title, title: ncName.title,
@ -787,6 +850,8 @@ export default async (
fk_relation_column_id: ncRelationColumnId, fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId fk_lookup_column_id: ncLookupColumnId
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name) const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name)
@ -853,6 +918,7 @@ export default async (
); );
logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, { const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Lookup, uidt: UITypes.Lookup,
title: ncName.title, title: ncName.title,
@ -860,6 +926,8 @@ export default async (
fk_relation_column_id: ncRelationColumnId, fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId fk_lookup_column_id: ncLookupColumnId
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find( const ncId = ncTbl.columns.find(
@ -900,6 +968,7 @@ export default async (
return aTbl_ncRollUp[fn]; return aTbl_ncRollUp[fn];
} }
//@ts-ignore
async function nocoCreateRollup(aTblSchema) { async function nocoCreateRollup(aTblSchema) {
// Rollup // Rollup
for (let idx = 0; idx < aTblSchema.length; idx++) { for (let idx = 0; idx < aTblSchema.length; idx++) {
@ -925,8 +994,9 @@ export default async (
const ncRollupFn = getRollupNcFunction( const ncRollupFn = getRollupNcFunction(
aTblColumns[i].typeOptions.formulaTextParsed aTblColumns[i].typeOptions.formulaTextParsed
); );
// const ncRollupFn = '';
if (ncRollupFn === '') { if (ncRollupFn === '' || ncRollupFn === undefined) {
updateMigrationSkipLog( updateMigrationSkipLog(
srcTableSchema.title, srcTableSchema.title,
aTblColumns[i].name, aTblColumns[i].name,
@ -990,6 +1060,7 @@ export default async (
); );
logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`);
const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, { const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Rollup, uidt: UITypes.Rollup,
title: ncName.title, title: ncName.title,
@ -998,6 +1069,8 @@ export default async (
fk_rollup_column_id: ncRollupColumnId, fk_rollup_column_id: ncRollupColumnId,
rollup_function: ncRollupFn rollup_function: ncRollupFn
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name) const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name)
@ -1014,6 +1087,7 @@ export default async (
logDetailed(`Nested rollup: ${nestedRollupTbl.length}`); logDetailed(`Nested rollup: ${nestedRollupTbl.length}`);
} }
//@ts-ignore
async function nocoLookupForRollup() { async function nocoLookupForRollup() {
const nestedCnt = nestedLookupTbl.length; const nestedCnt = nestedLookupTbl.length;
for (let i = 0; i < nestedLookupTbl.length; i++) { for (let i = 0; i < nestedLookupTbl.length; i++) {
@ -1043,6 +1117,7 @@ export default async (
); );
logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, { const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Lookup, uidt: UITypes.Lookup,
title: ncName.title, title: ncName.title,
@ -1050,6 +1125,8 @@ export default async (
fk_relation_column_id: ncRelationColumnId, fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId fk_lookup_column_id: ncLookupColumnId
}); });
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === nestedLookupTbl[0].name) const ncId = ncTbl.columns.find(x => x.title === nestedLookupTbl[0].name)
@ -1080,7 +1157,9 @@ export default async (
// skip primary column configuration if we field not migrated // skip primary column configuration if we field not migrated
if (ncColId) { if (ncColId) {
logDetailed(`NC API: dbTableColumn.primaryColumnSet`); logDetailed(`NC API: dbTableColumn.primaryColumnSet`);
const _perfStart = recordPerfStart();
await api.dbTableColumn.primaryColumnSet(ncColId); await api.dbTableColumn.primaryColumnSet(ncColId);
recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet');
// update schema // update schema
const ncTblId = sMap.getNcIdFromAtId(aTblSchema[idx].id); const ncTblId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
@ -1094,11 +1173,17 @@ export default async (
// retrieve view Info // retrieve view Info
let viewDetails; let viewDetails;
if (viewType === 'form') const _perfStart = recordPerfStart();
if (viewType === 'form') {
viewDetails = (await api.dbView.formRead(viewId)).columns; viewDetails = (await api.dbView.formRead(viewId)).columns;
else if (viewType === 'gallery') recordPerfStats(_perfStart, 'dbView.formRead');
} else if (viewType === 'gallery') {
viewDetails = (await api.dbView.galleryRead(viewId)).columns; viewDetails = (await api.dbView.galleryRead(viewId)).columns;
else viewDetails = await api.dbView.gridColumnsList(viewId); recordPerfStats(_perfStart, 'dbView.galleryRead');
} else {
viewDetails = await api.dbView.gridColumnsList(viewId);
recordPerfStats(_perfStart, 'dbView.gridColumnsList');
}
return viewDetails.find(x => x.fk_column_id === ncColumnId)?.id; return viewDetails.find(x => x.fk_column_id === ncColumnId)?.id;
} }
@ -1107,29 +1192,34 @@ export default async (
async function nocoLinkProcessing(projName, table, record, _field) { async function nocoLinkProcessing(projName, table, record, _field) {
const rec = record.fields; const rec = record.fields;
const refRowIdList: any = Object.values(rec);
const referenceColumnName = Object.keys(rec)[0];
if (refRowIdList.length) { for (const [key, value] of Object.entries(rec)) {
for (let i = 0; i < refRowIdList[0].length; i++) { const refRowIdList: any = value;
logDetailed( const referenceColumnName = key;
`NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}`
);
await api.dbTableRow.nestedAdd( if (refRowIdList.length) {
'noco', for (let i = 0; i < refRowIdList.length; i++) {
projName, logDetailed(
table.id, `NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}`
`${record.id}`, );
'mm', // fix me
encodeURIComponent(referenceColumnName), const _perfStart = recordPerfStart();
`${refRowIdList[0][i]}` await api.dbTableRow.nestedAdd(
); 'noco',
projName,
table.id,
`${record.id}`,
'mm',
encodeURIComponent(referenceColumnName),
`${refRowIdList[i]}`
);
recordPerfStats(_perfStart, 'dbTableRow.nestedAdd');
}
} }
} }
} }
async function nocoBaseDataProcessing(sDB, table, record) { async function nocoBaseDataProcessing_v2(sDB, table, record) {
const recordHash = hash(record); const recordHash = hash(record);
const rec = record.fields; const rec = record.fields;
@ -1149,94 +1239,133 @@ export default async (
// retrieve datatype // retrieve datatype
const dt = table.columns.find(x => x.title === key)?.uidt; const dt = table.columns.find(x => x.title === key)?.uidt;
// https://www.npmjs.com/package/validator switch (dt) {
// default value: digits_after_decimal: [2] // https://www.npmjs.com/package/validator
// if currency, set decimal place to 2 // default value: digits_after_decimal: [2]
// // if currency, set decimal place to 2
if (dt === UITypes.Currency) rec[key] = (+value).toFixed(2); //
case UITypes.Currency:
// we will pick up LTAR once all table data's are in place rec[key] = (+value).toFixed(2);
if (dt === UITypes.LinkToAnotherRecord) { break;
delete rec[key];
} // we will pick up LTAR once all table data's are in place
case UITypes.LinkToAnotherRecord:
// these will be automatically populated depending on schema configuration if (storeLinks) {
if (dt === UITypes.Lookup) delete rec[key]; if (ncLinkDataStore[table.title][record.id] === undefined)
if (dt === UITypes.Rollup) delete rec[key]; ncLinkDataStore[table.title][record.id] = {
id: record.id,
if (dt === UITypes.Collaborator) { fields: {}
// in case of multi-collaborator, this will be an array };
if (Array.isArray(value)) { ncLinkDataStore[table.title][record.id]['fields'][key] = value;
let collaborators = '';
for (let i = 0; i < value.length; i++) {
collaborators += `${value[i]?.name} <${value[i]?.email}>, `;
rec[key] = collaborators;
} }
} else rec[key] = `${value?.name} <${value?.email}>`; delete rec[key];
} break;
if (dt === UITypes.Barcode) rec[key] = value.text; // these will be automatically populated depending on schema configuration
if (dt === UITypes.Button) rec[key] = `${value?.label} <${value?.url}>`; case UITypes.Lookup:
case UITypes.Rollup:
if ( delete rec[key];
dt === UITypes.DateTime || break;
dt === UITypes.CreateTime ||
dt === UITypes.LastModifiedTime case UITypes.Collaborator:
) { // in case of multi-collaborator, this will be an array
const atDateField = dayjs(value); if (Array.isArray(value)) {
rec[key] = atDateField.utc().format('YYYY-MM-DD HH:mm'); let collaborators = '';
} for (let i = 0; i < value.length; i++) {
collaborators += `${value[i]?.name} <${value[i]?.email}>, `;
if (dt === UITypes.SingleSelect) rec[key] = value.replace(/,/g, '.'); rec[key] = collaborators;
}
if (dt === UITypes.MultiSelect) } else rec[key] = `${value?.name} <${value?.email}>`;
rec[key] = value.map(v => `${v.replace(/,/g, '.')}`).join(','); break;
if (dt === UITypes.Attachment) { case UITypes.Barcode:
const tempArr = []; rec[key] = value.text;
for (const v of value) { break;
const binaryImage = await axios
.get(v.url, { case UITypes.Button:
responseType: 'stream', rec[key] = `${value?.label} <${value?.url}>`;
headers: { break;
'Content-Type': v.type
} case UITypes.DateTime:
}) case UITypes.CreateTime:
.then(response => { case UITypes.LastModifiedTime:
return response.data; rec[key] = dayjs(value)
}) .utc()
.catch(error => { .format('YYYY-MM-DD HH:mm');
console.log(error); break;
return false;
}); case UITypes.Date:
if (/\d{5,}/.test(value)) {
const imageFile: any = new FormData(); // skip
imageFile.append('files', binaryImage, { rec[key] = null;
filename: v.filename.includes('?') logBasic(`:: Invalid date ${value}`);
? v.filename.split('?')[0] } else {
: v.filename rec[key] = dayjs(value)
}); .utc()
.format('YYYY-MM-DD');
const rs = await axios }
.post(sDB.baseURL + '/api/v1/db/storage/upload', imageFile, { break;
params: {
path: `noco/${sDB.projectName}/${table.title}/${key}` case UITypes.SingleSelect:
}, rec[key] = value.replace(/,/g, '.');
headers: { break;
'Content-Type': `multipart/form-data; boundary=${imageFile._boundary}`,
'xc-auth': sDB.authToken case UITypes.MultiSelect:
} rec[key] = value.map(v => `${v.replace(/,/g, '.')}`).join(',');
}) break;
.then(response => {
return response.data; case UITypes.Attachment:
}) if (skipAttachments) rec[key] = null;
.catch(e => { else {
console.log(e); const tempArr = [];
}); for (const v of value) {
const binaryImage = await axios
.get(v.url, {
responseType: 'stream',
headers: {
'Content-Type': v.type
}
})
.then(response => {
return response.data;
})
.catch(error => {
console.log(error);
return false;
});
const imageFile: any = new FormData();
imageFile.append('files', binaryImage, {
filename: v.filename.includes('?')
? v.filename.split('?')[0]
: v.filename
});
const rs = await axios
.post(sDB.baseURL + '/api/v1/db/storage/upload', imageFile, {
params: {
path: `noco/${sDB.projectName}/${table.title}/${key}`
},
headers: {
'Content-Type': `multipart/form-data; boundary=${imageFile._boundary}`,
'xc-auth': sDB.authToken
}
})
.then(response => {
return response.data;
})
.catch(e => {
console.log(e);
});
tempArr.push(...rs);
}
rec[key] = JSON.stringify(tempArr);
}
break;
tempArr.push(...rs); default:
} break;
rec[key] = JSON.stringify(tempArr);
} }
} }
@ -1244,17 +1373,13 @@ export default async (
rec[ncSysFields.id] = record.id; rec[ncSysFields.id] = record.id;
rec[ncSysFields.hash] = recordHash; rec[ncSysFields.hash] = recordHash;
// bulk Insert return rec;
logDetailed(`NC API: dbTableRow.bulkCreate ${table.title} [${rec}]`);
await api.dbTableRow.bulkCreate(
'nc',
sDB.projectName,
table.id, // encodeURIComponent(table.title),
[rec]
);
} }
async function nocoReadData(sDB, table, callback) { async function nocoReadData(sDB, table) {
ncLinkDataStore[table.title] = {};
const insertJobs: Promise<any>[] = [];
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
base(table.title) base(table.title)
.select({ .select({
@ -1270,20 +1395,46 @@ export default async (
`:: ${table.title} : ${recordCnt + 1} ~ ${(recordCnt += 100)}` `:: ${table.title} : ${recordCnt + 1} ~ ${(recordCnt += 100)}`
); );
await Promise.all( // await Promise.all(
records.map(record => callback(sDB, table, record)) // records.map(record => _callback(sDB, table, record))
// );
const ncRecords = [];
for (let i = 0; i < records.length; i++) {
const r = await nocoBaseDataProcessing_v2(sDB, table, records[i]);
ncRecords.push(r);
}
// wait for previous job's to finish
await Promise.all(insertJobs);
const _perfStart = recordPerfStart();
insertJobs.push(
api.dbTableRow.bulkCreate(
'nc',
sDB.projectName,
table.id, // encodeURIComponent(table.title),
ncRecords
)
); );
recordPerfStats(_perfStart, 'dbTableRow.bulkCreate');
// To fetch the next page of records, call `fetchNextPage`. // To fetch the next page of records, call `fetchNextPage`.
// If there are more records, `page` will get called again. // If there are more records, `page` will get called again.
// If there are no more records, `done` will get called. // If there are no more records, `done` will get called.
// logBasic(
// `:: ${Date.now()} Awaiting response from Airtable Data API ...`
// );
fetchNextPage(); fetchNextPage();
}, },
function done(err) { async function done(err) {
if (err) { if (err) {
console.error(err); console.error(err);
reject(err); reject(err);
} }
// wait for all jobs to be completed
await Promise.all(insertJobs);
resolve(null); resolve(null);
} }
); );
@ -1296,7 +1447,7 @@ export default async (
.select({ .select({
pageSize: 100, pageSize: 100,
// maxRecords: 100, // maxRecords: 100,
fields: [fields] fields: fields
}) })
.eachPage( .eachPage(
async function page(records, fetchNextPage) { async function page(records, fetchNextPage) {
@ -1339,15 +1490,19 @@ export default async (
async function nocoCreateProject(projName) { async function nocoCreateProject(projName) {
// create empty project (XC-DB) // create empty project (XC-DB)
logDetailed(`Create Project: ${projName}`); logDetailed(`Create Project: ${projName}`);
const _perfStart = recordPerfStart();
ncCreatedProjectSchema = await api.project.create({ ncCreatedProjectSchema = await api.project.create({
title: projName title: projName
}); });
recordPerfStats(_perfStart, 'project.create');
} }
async function nocoGetProject(projId) { async function nocoGetProject(projId) {
// create empty project (XC-DB) // create empty project (XC-DB)
logDetailed(`Getting project meta: ${projId}`); logDetailed(`Getting project meta: ${projId}`);
const _perfStart = recordPerfStart();
ncCreatedProjectSchema = await api.project.read(projId); ncCreatedProjectSchema = await api.project.read(projId);
recordPerfStats(_perfStart, 'project.read');
} }
async function nocoConfigureGalleryView(sDB, aTblSchema) { async function nocoConfigureGalleryView(sDB, aTblSchema) {
@ -1377,7 +1532,10 @@ export default async (
); );
logDetailed(`NC API dbView.galleryCreate :: ${viewName}`); logDetailed(`NC API dbView.galleryCreate :: ${viewName}`);
const _perfStart = recordPerfStart();
await api.dbView.galleryCreate(tblId, { title: viewName }); await api.dbView.galleryCreate(tblId, { title: viewName });
recordPerfStats(_perfStart, 'dbView.galleryCreate');
await updateNcTblSchemaById(tblId); await updateNcTblSchemaById(tblId);
// syncLog(`[${idx+1}/${aTblSchema.length}][Gallery View][${i+1}/${galleryViews.length}] Create ${viewName}`) // syncLog(`[${idx+1}/${aTblSchema.length}][Gallery View][${i+1}/${galleryViews.length}] Create ${viewName}`)
@ -1435,7 +1593,10 @@ export default async (
}; };
logDetailed(`NC API dbView.formCreate :: ${viewName}`); logDetailed(`NC API dbView.formCreate :: ${viewName}`);
const _perfStart = recordPerfStart();
const f = await api.dbView.formCreate(tblId, formData); const f = await api.dbView.formCreate(tblId, formData);
recordPerfStats(_perfStart, 'dbView.formCreate');
logDetailed( logDetailed(
`[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${ `[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${
formViews.length formViews.length
@ -1475,7 +1636,10 @@ export default async (
const viewName = aTblSchema[idx].views.find( const viewName = aTblSchema[idx].views.find(
x => x.id === gridViews[i].id x => x.id === gridViews[i].id
)?.name; )?.name;
const _perfStart = recordPerfStart();
const viewList: any = await api.dbView.list(tblId); const viewList: any = await api.dbView.list(tblId);
recordPerfStats(_perfStart, 'dbView.list');
let ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id; let ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id;
logBasic( logBasic(
@ -1487,9 +1651,12 @@ export default async (
// create view (default already created) // create view (default already created)
if (i > 0) { if (i > 0) {
logDetailed(`NC API dbView.gridCreate :: ${viewName}`); logDetailed(`NC API dbView.gridCreate :: ${viewName}`);
const _perfStart = recordPerfStart();
const viewCreated = await api.dbView.gridCreate(tblId, { const viewCreated = await api.dbView.gridCreate(tblId, {
title: viewName title: viewName
}); });
recordPerfStats(_perfStart, 'dbView.gridCreate');
await updateNcTblSchemaById(tblId); await updateNcTblSchemaById(tblId);
await sMap.addToMappingTbl( await sMap.addToMappingTbl(
gridViews[i].id, gridViews[i].id,
@ -1544,6 +1711,7 @@ export default async (
const userList = aTblSchema.appBlanket.userInfoById; const userList = aTblSchema.appBlanket.userInfoById;
const totalUsers = Object.keys(userList).length; const totalUsers = Object.keys(userList).length;
let cnt = 0; let cnt = 0;
const insertJobs: Promise<any>[] = [];
for (const [, value] of Object.entries( for (const [, value] of Object.entries(
userList as { [key: string]: any } userList as { [key: string]: any }
@ -1551,11 +1719,16 @@ export default async (
logDetailed( logDetailed(
`[${++cnt}/${totalUsers}] NC API auth.projectUserAdd :: ${value.email}` `[${++cnt}/${totalUsers}] NC API auth.projectUserAdd :: ${value.email}`
); );
await api.auth.projectUserAdd(ncCreatedProjectSchema.id, { const _perfStart = recordPerfStart();
email: value.email, insertJobs.push(
roles: userRoles[value.permissionLevel] api.auth.projectUserAdd(ncCreatedProjectSchema.id, {
}); email: value.email,
roles: userRoles[value.permissionLevel]
})
);
recordPerfStats(_perfStart, 'auth.projectUserAdd');
} }
await Promise.all(insertJobs);
} }
function updateNcTblSchema(tblSchema) { function updateNcTblSchema(tblSchema) {
@ -1571,7 +1744,10 @@ export default async (
} }
async function updateNcTblSchemaById(tblId) { async function updateNcTblSchemaById(tblId) {
const _perfStart = recordPerfStart();
const ncTbl = await api.dbTable.read(tblId); const ncTbl = await api.dbTable.read(tblId);
recordPerfStats(_perfStart, 'dbTable.read');
updateNcTblSchema(ncTbl); updateNcTblSchema(ncTbl);
} }
@ -1665,23 +1841,35 @@ export default async (
return accumulator + object.nc.rollup; return accumulator + object.nc.rollup;
}, 0); }, 0);
logDetailed(`Quick Summary:`); logBasic(`Quick Summary:`);
logDetailed(`:: Total Tables: ${aTblSchema.length}`); logBasic(`:: Total Tables: ${aTblSchema.length}`);
logDetailed(`:: Total Columns: ${columnSum}`); logBasic(`:: Total Columns: ${columnSum}`);
logDetailed(`:: Links: ${linkSum}`); logBasic(`:: Links: ${linkSum}`);
logDetailed(`:: Lookup: ${lookupSum}`); logBasic(`:: Lookup: ${lookupSum}`);
logDetailed(`:: Rollup: ${rollupSum}`); logBasic(`:: Rollup: ${rollupSum}`);
logDetailed(`:: Total Filters: ${rtc.filter}`); logBasic(`:: Total Filters: ${rtc.filter}`);
logDetailed(`:: Total Sort: ${rtc.sort}`); logBasic(`:: Total Sort: ${rtc.sort}`);
logDetailed(`:: Total Views: ${rtc.view.total}`); logBasic(`:: Total Views: ${rtc.view.total}`);
logDetailed(`:: Grid: ${rtc.view.grid}`); logBasic(`:: Grid: ${rtc.view.grid}`);
logDetailed(`:: Gallery: ${rtc.view.gallery}`); logBasic(`:: Gallery: ${rtc.view.gallery}`);
logDetailed(`:: Form: ${rtc.view.form}`); logBasic(`:: Form: ${rtc.view.form}`);
const duration = Date.now() - start; const duration = Date.now() - start;
logDetailed(`:: Migration time: ${duration}`); logBasic(`:: Migration time: ${duration}`);
logDetailed(`:: Axios fetch count: ${rtc.fetchAt.count}`); logBasic(`:: Axios fetch count: ${rtc.fetchAt.count}`);
logDetailed(`:: Axios fetch time: ${rtc.fetchAt.time}`); logBasic(`:: Axios fetch time: ${rtc.fetchAt.time}`);
if (debugMode) {
jsonfile.writeFileSync('stats.json', perfStats, { spaces: 2 });
const perflog = [];
for (let i = 0; i < perfStats.length; i++) {
perflog.push(`${perfStats[i].e}, ${perfStats[i].d}`);
}
jsonfile.writeFileSync('stats.csv', perflog, { spaces: 2 });
jsonfile.writeFileSync('skip.txt', rtc.migrationSkipLog.log, {
spaces: 2
});
}
} }
////////////////////////////// //////////////////////////////
@ -1776,9 +1964,12 @@ export default async (
// insert filters // insert filters
for (let i = 0; i < ncFilters.length; i++) { for (let i = 0; i < ncFilters.length; i++) {
const _perfStart = recordPerfStart();
await api.dbTableFilter.create(viewId, { await api.dbTableFilter.create(viewId, {
...ncFilters[i] ...ncFilters[i]
}); });
recordPerfStats(_perfStart, 'dbTableFilter.create');
rtc.filter++; rtc.filter++;
} }
} }
@ -1788,11 +1979,14 @@ export default async (
for (let i = 0; i < s.sortSet.length; i++) { for (let i = 0; i < s.sortSet.length; i++) {
const columnId = (await nc_getColumnSchema(s.sortSet[i].columnId))?.id; const columnId = (await nc_getColumnSchema(s.sortSet[i].columnId))?.id;
if (columnId) if (columnId) {
const _perfStart = recordPerfStart();
await api.dbTableSort.create(viewId, { await api.dbTableSort.create(viewId, {
fk_column_id: columnId, fk_column_id: columnId,
direction: s.sortSet[i].ascending ? 'asc' : 'dsc' direction: s.sortSet[i].ascending ? 'asc' : 'dsc'
}); });
recordPerfStats(_perfStart, 'dbTableSort.create');
}
rtc.sort++; rtc.sort++;
} }
} }
@ -1807,23 +2001,41 @@ export default async (
const ncTbl = await nc_getTableSchema(tblName); const ncTbl = await nc_getTableSchema(tblName);
// retrieve view ID // retrieve view ID
const viewId = ncTbl.views.find(x => x.title === viewName).id; const viewId = ncTbl.views.find(x => x.title === viewName).id;
let viewDetails;
const _perfStart = recordPerfStart();
if (viewType === 'form') {
viewDetails = (await api.dbView.formRead(viewId)).columns;
recordPerfStats(_perfStart, 'dbView.formRead');
} else if (viewType === 'gallery') {
viewDetails = (await api.dbView.galleryRead(viewId)).columns;
recordPerfStats(_perfStart, 'dbView.galleryRead');
} else {
viewDetails = await api.dbView.gridColumnsList(viewId);
recordPerfStats(_perfStart, 'dbView.gridColumnsList');
}
// nc-specific columns; default hide. // nc-specific columns; default hide.
for (let j = 0; j < hiddenColumns.length; j++) { for (let j = 0; j < hiddenColumns.length; j++) {
const ncColumnId = ncTbl.columns.find(x => x.title === hiddenColumns[j]) const ncColumnId = ncTbl.columns.find(x => x.title === hiddenColumns[j])
.id; .id;
const ncViewColumnId = await nc_getViewColumnId( const ncViewColumnId = viewDetails.find(
viewId, x => x.fk_column_id === ncColumnId
viewType, )?.id;
ncColumnId // const ncViewColumnId = await nc_getViewColumnId(
); // viewId,
// viewType,
// ncColumnId
// );
if (ncViewColumnId === undefined) continue; if (ncViewColumnId === undefined) continue;
// first two positions held by record id & record hash // first two positions held by record id & record hash
const _perfStart = recordPerfStart();
await api.dbViewColumn.update(viewId, ncViewColumnId, { await api.dbViewColumn.update(viewId, ncViewColumnId, {
show: false, show: false,
order: j + 1 + c.length order: j + 1 + c.length
}); });
recordPerfStats(_perfStart, 'dbViewColumn.update');
} }
// rest of the columns from airtable- retain order & visibility property // rest of the columns from airtable- retain order & visibility property
@ -1845,10 +2057,14 @@ export default async (
if (x?.title) formData[`label`] = x.title; if (x?.title) formData[`label`] = x.title;
if (x?.required) formData[`required`] = x.required; if (x?.required) formData[`required`] = x.required;
if (x?.description) formData[`description`] = x.description; if (x?.description) formData[`description`] = x.description;
const _perfStart = recordPerfStart();
await api.dbView.formColumnUpdate(ncViewColumnId, formData); await api.dbView.formColumnUpdate(ncViewColumnId, formData);
recordPerfStats(_perfStart, 'dbView.formColumnUpdate');
} }
} }
const _perfStart = recordPerfStart();
await api.dbViewColumn.update(viewId, ncViewColumnId, configData); await api.dbViewColumn.update(viewId, ncViewColumnId, configData);
recordPerfStats(_perfStart, 'dbViewColumn.update');
} }
} }
@ -1935,45 +2151,68 @@ export default async (
if (process_aTblData) { if (process_aTblData) {
try { try {
// await nc_DumpTableSchema(); // await nc_DumpTableSchema();
const _perfStart = recordPerfStart();
const ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); const ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id);
recordPerfStats(_perfStart, 'dbTable.list');
logBasic('Reading Records...'); logBasic('Reading Records...');
for (let i = 0; i < ncTblList.list.length; i++) { for (let i = 0; i < ncTblList.list.length; i++) {
const _perfStart = recordPerfStart();
const ncTbl = await api.dbTable.read(ncTblList.list[i].id); const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
recordPerfStats(_perfStart, 'dbTable.read');
// not a migrated table, skip // not a migrated table, skip
if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) if (undefined === aTblSchema.find(x => x.name === ncTbl.title))
continue; continue;
recordCnt = 0; recordCnt = 0;
await nocoReadData(syncDB, ncTbl, async (sDB, table, record) => { await nocoReadData(syncDB, ncTbl);
await nocoBaseDataProcessing(sDB, table, record);
});
logDetailed(`Data inserted from ${ncTbl.title}`); logDetailed(`Data inserted from ${ncTbl.title}`);
} }
logBasic('Configuring Record Links...'); logBasic('Configuring Record Links...');
// Configure link @ Data row's if (storeLinks) {
for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { // const insertJobs: Promise<any>[] = [];
const x = ncLinkMappingTable[idx]; for (const [pTitle, v] of Object.entries(ncLinkDataStore)) {
const ncTbl = await nc_getTableSchema( logBasic(`:: ${pTitle}`);
aTbl_getTableName(x.aTbl.tblId).tn for (const [, record] of Object.entries(v)) {
); const tbl = ncTblList.list.find(a => a.title === pTitle);
await nocoLinkProcessing(syncDB.projectName, tbl, record, 0);
// insertJobs.push(
// nocoLinkProcessing(syncDB.projectName, tbl, record, 0)
// );
}
}
// await Promise.all(insertJobs);
// await nocoLinkProcessing(syncDB.projectName, 0, 0, 0);
} else {
// create link groups (table: link fields)
const tblLinkGroup = {};
for (let idx = 0; idx < ncLinkMappingTable.length; idx++) {
const x = ncLinkMappingTable[idx];
if (tblLinkGroup[x.aTbl.tblId] === undefined)
tblLinkGroup[x.aTbl.tblId] = [x.aTbl.name];
else tblLinkGroup[x.aTbl.tblId].push(x.aTbl.name);
}
// not a migrated table, skip for (const [k, v] of Object.entries(tblLinkGroup)) {
if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn);
continue;
recordCnt = 0; // not a migrated table, skip
await nocoReadDataSelected( if (undefined === aTblSchema.find(x => x.name === ncTbl.title))
syncDB.projectName, continue;
ncTbl,
async (projName, table, record, _field) => { recordCnt = 0;
await nocoLinkProcessing(projName, table, record, _field); await nocoReadDataSelected(
}, syncDB.projectName,
x.aTbl.name ncTbl,
); async (projName, table, record, _field) => {
logDetailed(`Linked data to ${ncTbl.title}`); await nocoLinkProcessing(projName, table, record, _field);
},
v
);
}
} }
} catch (error) { } catch (error) {
logDetailed( logDetailed(

Write Preview
Loading…
Cancel
Save