Browse Source

Fix/at sync: store link records (#2156)

* fix: rollup, conditional inclusion of vColumns

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: reduce at api access/ store links

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: reduce view column list api invocation

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* feat: performance monitor framework

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: use bulk insert per 100 records

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: link error, perf clean up

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: remove duplicate entries from select/ multiselect

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: truncate table name to 50 chars to avoid database specific issue

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: truncate column name to 60 chars to avoid database specific issue

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: disable link-store optimisation, disable parallel link creation

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

* fix: invalid date handle, clean-up

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>

Co-authored-by: Pranav C <pranavxc@gmail.com>
pull/2175/head
Raju Udava 2 years ago committed by GitHub
parent
commit
f12410aa33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 397
      packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts

397
packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts

@ -56,6 +56,17 @@ export default async (
if (debugMode) progress({ level: 1, msg: log });
}
const perfStats = [];
function recordPerfStart() {
if (!debugMode) return 0;
return Date.now();
}
function recordPerfStats(start, event) {
if (!debugMode) return;
const duration = Date.now() - start;
perfStats.push({ d: duration, e: event });
}
let base, baseId;
const start = Date.now();
const enableErrorLogs = false;
@ -69,6 +80,9 @@ export default async (
const nestedLookupTbl: any[] = [];
const nestedRollupTbl: any[] = [];
const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' };
const storeLinks = false;
const skipAttachments = false;
const ncLinkDataStore: any = {};
const uniqueTableNameGen = getUniqueNameGenerator('sheet');
@ -173,7 +187,11 @@ export default async (
}
function nc_getSanitizedColumnName(table, name) {
const col_name = nc_sanitizeName(name);
let col_name = nc_sanitizeName(name);
// truncate to 60 chars if character if exceeds above 60
col_name = col_name?.slice(0, 60);
// for knex, replace . with _
const col_alias = name.trim().replace(/\./g, '_');
@ -350,7 +368,8 @@ export default async (
);
}
// const csvOpt = "'" + opt.join("','") + "'";
const csvOpt = opt
const optSansDuplicate = [...new Set(opt)];
const csvOpt = optSansDuplicate
.map(v => `'${v.replace(/'/g, "\\'").replace(/,/g, '.')}'`)
.join(',');
return { type: 'select', data: csvOpt };
@ -380,7 +399,15 @@ export default async (
// Enable to use aTbl identifiers as is: table.id = tblSchema[i].id;
table.title = tblSchema[i].name;
table.table_name = uniqueTableNameGen(nc_sanitizeName(tblSchema[i].name));
let sanitizedName = nc_sanitizeName(tblSchema[i].name);
// truncate to 50 chars if character if exceeds above 50
// upto 64 should be fine but we are keeping it to 50 since
// meta project adds prefix as well
sanitizedName = sanitizedName?.slice(0, 50);
// check for duplicate and populate a unique name if already exist
table.table_name = uniqueTableNameGen(sanitizedName);
const uniqueColNameGen = getUniqueNameGenerator('field');
table.columns = [];
@ -471,10 +498,14 @@ export default async (
logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`);
logDetailed(`NC API: dbTable.create ${tables[idx].title}`);
let _perfStart = recordPerfStart();
const table: any = await api.dbTable.create(
ncCreatedProjectSchema.id,
tables[idx]
);
recordPerfStats(_perfStart, 'dbTable.create');
updateNcTblSchema(table);
// update mapping table
@ -494,13 +525,18 @@ export default async (
// update default view name- to match it to airtable view name
logDetailed(`NC API: dbView.list ${table.id}`);
_perfStart = recordPerfStart();
const view = await api.dbView.list(table.id);
recordPerfStats(_perfStart, 'dbView.list');
const aTbl_grid = aTblSchema[idx].views.find(x => x.type === 'grid');
logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`);
_perfStart = recordPerfStart();
await api.dbView.update(view.list[0].id, {
title: aTbl_grid.name
});
recordPerfStats(_perfStart, 'dbView.update');
await updateNcTblSchemaById(table.id);
await sMap.addToMappingTbl(
@ -563,7 +599,9 @@ export default async (
}
// check if already a column exists with this name?
let _perfStart = recordPerfStart();
const srcTbl: any = await api.dbTable.read(srcTableId);
recordPerfStats(_perfStart, 'dbTable.read');
// create link
const ncName = nc_getSanitizedColumnName(
@ -574,6 +612,7 @@ export default async (
logDetailed(
`NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}`
);
_perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.LinkToAnotherRecord,
title: ncName.title,
@ -585,6 +624,8 @@ export default async (
// ? 'mm'
// : 'hm'
});
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === ncName.title)?.id;
@ -599,7 +640,7 @@ export default async (
// this information will be helpful in identifying relation pair
const link = {
nc: {
title: aTblLinkColumns[i].name,
title: ncName.title,
parentId: srcTableId,
childId: childTableId,
type: 'mm'
@ -626,12 +667,17 @@ export default async (
x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId
);
let _perfStart = recordPerfStart();
const childTblSchema: any = await api.dbTable.read(
ncLinkMappingTable[x].nc.childId
);
recordPerfStats(_perfStart, 'dbTable.read');
_perfStart = recordPerfStart();
const parentTblSchema: any = await api.dbTable.read(
ncLinkMappingTable[x].nc.parentId
);
recordPerfStats(_perfStart, 'dbTable.read');
// fix me
// let childTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.childId]
@ -641,6 +687,16 @@ export default async (
col => col.title === ncLinkMappingTable[x].nc.title
);
if (parentLinkColumn === undefined) {
updateMigrationSkipLog(
parentTblSchema?.title,
ncLinkMappingTable[x].nc.title,
UITypes.LinkToAnotherRecord,
'Link error'
);
continue;
}
// hack // fix me
if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') {
parentLinkColumn = parentTblSchema.columns.find(
@ -698,6 +754,7 @@ export default async (
logDetailed(
`NC API: dbTableColumn.update rename symmetric column ${ncName.title}`
);
_perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.update(
childLinkColumn.id,
{
@ -706,6 +763,8 @@ export default async (
column_name: ncName.column_name
}
);
recordPerfStats(_perfStart, 'dbTableColumn.update');
updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(
@ -768,7 +827,10 @@ export default async (
aTblColumns[i].typeOptions.foreignTableRollupColumnId
);
if (ncLookupColumnId === undefined) {
if (
ncLookupColumnId === undefined ||
ncRelationColumnId === undefined
) {
aTblColumns[i]['srcTableId'] = srcTableId;
nestedLookupTbl.push(aTblColumns[i]);
continue;
@ -780,6 +842,7 @@ export default async (
);
logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Lookup,
title: ncName.title,
@ -787,6 +850,8 @@ export default async (
fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId
});
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name)
@ -853,6 +918,7 @@ export default async (
);
logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Lookup,
title: ncName.title,
@ -860,6 +926,8 @@ export default async (
fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId
});
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(
@ -900,6 +968,7 @@ export default async (
return aTbl_ncRollUp[fn];
}
//@ts-ignore
async function nocoCreateRollup(aTblSchema) {
// Rollup
for (let idx = 0; idx < aTblSchema.length; idx++) {
@ -925,8 +994,9 @@ export default async (
const ncRollupFn = getRollupNcFunction(
aTblColumns[i].typeOptions.formulaTextParsed
);
// const ncRollupFn = '';
if (ncRollupFn === '') {
if (ncRollupFn === '' || ncRollupFn === undefined) {
updateMigrationSkipLog(
srcTableSchema.title,
aTblColumns[i].name,
@ -990,6 +1060,7 @@ export default async (
);
logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`);
const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Rollup,
title: ncName.title,
@ -998,6 +1069,8 @@ export default async (
fk_rollup_column_id: ncRollupColumnId,
rollup_function: ncRollupFn
});
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === aTblColumns[i].name)
@ -1014,6 +1087,7 @@ export default async (
logDetailed(`Nested rollup: ${nestedRollupTbl.length}`);
}
//@ts-ignore
async function nocoLookupForRollup() {
const nestedCnt = nestedLookupTbl.length;
for (let i = 0; i < nestedLookupTbl.length; i++) {
@ -1043,6 +1117,7 @@ export default async (
);
logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`);
const _perfStart = recordPerfStart();
const ncTbl: any = await api.dbTableColumn.create(srcTableId, {
uidt: UITypes.Lookup,
title: ncName.title,
@ -1050,6 +1125,8 @@ export default async (
fk_relation_column_id: ncRelationColumnId,
fk_lookup_column_id: ncLookupColumnId
});
recordPerfStats(_perfStart, 'dbTableColumn.create');
updateNcTblSchema(ncTbl);
const ncId = ncTbl.columns.find(x => x.title === nestedLookupTbl[0].name)
@ -1080,7 +1157,9 @@ export default async (
// skip primary column configuration if we field not migrated
if (ncColId) {
logDetailed(`NC API: dbTableColumn.primaryColumnSet`);
const _perfStart = recordPerfStart();
await api.dbTableColumn.primaryColumnSet(ncColId);
recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet');
// update schema
const ncTblId = sMap.getNcIdFromAtId(aTblSchema[idx].id);
@ -1094,11 +1173,17 @@ export default async (
// retrieve view Info
let viewDetails;
if (viewType === 'form')
const _perfStart = recordPerfStart();
if (viewType === 'form') {
viewDetails = (await api.dbView.formRead(viewId)).columns;
else if (viewType === 'gallery')
recordPerfStats(_perfStart, 'dbView.formRead');
} else if (viewType === 'gallery') {
viewDetails = (await api.dbView.galleryRead(viewId)).columns;
else viewDetails = await api.dbView.gridColumnsList(viewId);
recordPerfStats(_perfStart, 'dbView.galleryRead');
} else {
viewDetails = await api.dbView.gridColumnsList(viewId);
recordPerfStats(_perfStart, 'dbView.gridColumnsList');
}
return viewDetails.find(x => x.fk_column_id === ncColumnId)?.id;
}
@ -1107,29 +1192,34 @@ export default async (
async function nocoLinkProcessing(projName, table, record, _field) {
const rec = record.fields;
const refRowIdList: any = Object.values(rec);
const referenceColumnName = Object.keys(rec)[0];
for (const [key, value] of Object.entries(rec)) {
const refRowIdList: any = value;
const referenceColumnName = key;
if (refRowIdList.length) {
for (let i = 0; i < refRowIdList[0].length; i++) {
for (let i = 0; i < refRowIdList.length; i++) {
logDetailed(
`NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}`
);
const _perfStart = recordPerfStart();
await api.dbTableRow.nestedAdd(
'noco',
projName,
table.id,
`${record.id}`,
'mm', // fix me
'mm',
encodeURIComponent(referenceColumnName),
`${refRowIdList[0][i]}`
`${refRowIdList[i]}`
);
recordPerfStats(_perfStart, 'dbTableRow.nestedAdd');
}
}
}
}
async function nocoBaseDataProcessing(sDB, table, record) {
async function nocoBaseDataProcessing_v2(sDB, table, record) {
const recordHash = hash(record);
const rec = record.fields;
@ -1149,22 +1239,35 @@ export default async (
// retrieve datatype
const dt = table.columns.find(x => x.title === key)?.uidt;
switch (dt) {
// https://www.npmjs.com/package/validator
// default value: digits_after_decimal: [2]
// if currency, set decimal place to 2
//
if (dt === UITypes.Currency) rec[key] = (+value).toFixed(2);
case UITypes.Currency:
rec[key] = (+value).toFixed(2);
break;
// we will pick up LTAR once all table data's are in place
if (dt === UITypes.LinkToAnotherRecord) {
delete rec[key];
case UITypes.LinkToAnotherRecord:
if (storeLinks) {
if (ncLinkDataStore[table.title][record.id] === undefined)
ncLinkDataStore[table.title][record.id] = {
id: record.id,
fields: {}
};
ncLinkDataStore[table.title][record.id]['fields'][key] = value;
}
delete rec[key];
break;
// these will be automatically populated depending on schema configuration
if (dt === UITypes.Lookup) delete rec[key];
if (dt === UITypes.Rollup) delete rec[key];
case UITypes.Lookup:
case UITypes.Rollup:
delete rec[key];
break;
if (dt === UITypes.Collaborator) {
case UITypes.Collaborator:
// in case of multi-collaborator, this will be an array
if (Array.isArray(value)) {
let collaborators = '';
@ -1173,26 +1276,47 @@ export default async (
rec[key] = collaborators;
}
} else rec[key] = `${value?.name} <${value?.email}>`;
}
break;
if (dt === UITypes.Barcode) rec[key] = value.text;
if (dt === UITypes.Button) rec[key] = `${value?.label} <${value?.url}>`;
case UITypes.Barcode:
rec[key] = value.text;
break;
if (
dt === UITypes.DateTime ||
dt === UITypes.CreateTime ||
dt === UITypes.LastModifiedTime
) {
const atDateField = dayjs(value);
rec[key] = atDateField.utc().format('YYYY-MM-DD HH:mm');
case UITypes.Button:
rec[key] = `${value?.label} <${value?.url}>`;
break;
case UITypes.DateTime:
case UITypes.CreateTime:
case UITypes.LastModifiedTime:
rec[key] = dayjs(value)
.utc()
.format('YYYY-MM-DD HH:mm');
break;
case UITypes.Date:
if (/\d{5,}/.test(value)) {
// skip
rec[key] = null;
logBasic(`:: Invalid date ${value}`);
} else {
rec[key] = dayjs(value)
.utc()
.format('YYYY-MM-DD');
}
break;
if (dt === UITypes.SingleSelect) rec[key] = value.replace(/,/g, '.');
case UITypes.SingleSelect:
rec[key] = value.replace(/,/g, '.');
break;
if (dt === UITypes.MultiSelect)
case UITypes.MultiSelect:
rec[key] = value.map(v => `${v.replace(/,/g, '.')}`).join(',');
break;
if (dt === UITypes.Attachment) {
case UITypes.Attachment:
if (skipAttachments) rec[key] = null;
else {
const tempArr = [];
for (const v of value) {
const binaryImage = await axios
@ -1238,23 +1362,24 @@ export default async (
}
rec[key] = JSON.stringify(tempArr);
}
break;
default:
break;
}
}
// insert airtable record ID explicitly into each records
rec[ncSysFields.id] = record.id;
rec[ncSysFields.hash] = recordHash;
// bulk Insert
logDetailed(`NC API: dbTableRow.bulkCreate ${table.title} [${rec}]`);
await api.dbTableRow.bulkCreate(
'nc',
sDB.projectName,
table.id, // encodeURIComponent(table.title),
[rec]
);
return rec;
}
async function nocoReadData(sDB, table, callback) {
async function nocoReadData(sDB, table) {
ncLinkDataStore[table.title] = {};
const insertJobs: Promise<any>[] = [];
return new Promise((resolve, reject) => {
base(table.title)
.select({
@ -1270,20 +1395,46 @@ export default async (
`:: ${table.title} : ${recordCnt + 1} ~ ${(recordCnt += 100)}`
);
await Promise.all(
records.map(record => callback(sDB, table, record))
// await Promise.all(
// records.map(record => _callback(sDB, table, record))
// );
const ncRecords = [];
for (let i = 0; i < records.length; i++) {
const r = await nocoBaseDataProcessing_v2(sDB, table, records[i]);
ncRecords.push(r);
}
// wait for previous job's to finish
await Promise.all(insertJobs);
const _perfStart = recordPerfStart();
insertJobs.push(
api.dbTableRow.bulkCreate(
'nc',
sDB.projectName,
table.id, // encodeURIComponent(table.title),
ncRecords
)
);
recordPerfStats(_perfStart, 'dbTableRow.bulkCreate');
// To fetch the next page of records, call `fetchNextPage`.
// If there are more records, `page` will get called again.
// If there are no more records, `done` will get called.
// logBasic(
// `:: ${Date.now()} Awaiting response from Airtable Data API ...`
// );
fetchNextPage();
},
function done(err) {
async function done(err) {
if (err) {
console.error(err);
reject(err);
}
// wait for all jobs to be completed
await Promise.all(insertJobs);
resolve(null);
}
);
@ -1296,7 +1447,7 @@ export default async (
.select({
pageSize: 100,
// maxRecords: 100,
fields: [fields]
fields: fields
})
.eachPage(
async function page(records, fetchNextPage) {
@ -1339,15 +1490,19 @@ export default async (
async function nocoCreateProject(projName) {
// create empty project (XC-DB)
logDetailed(`Create Project: ${projName}`);
const _perfStart = recordPerfStart();
ncCreatedProjectSchema = await api.project.create({
title: projName
});
recordPerfStats(_perfStart, 'project.create');
}
async function nocoGetProject(projId) {
// create empty project (XC-DB)
logDetailed(`Getting project meta: ${projId}`);
const _perfStart = recordPerfStart();
ncCreatedProjectSchema = await api.project.read(projId);
recordPerfStats(_perfStart, 'project.read');
}
async function nocoConfigureGalleryView(sDB, aTblSchema) {
@ -1377,7 +1532,10 @@ export default async (
);
logDetailed(`NC API dbView.galleryCreate :: ${viewName}`);
const _perfStart = recordPerfStart();
await api.dbView.galleryCreate(tblId, { title: viewName });
recordPerfStats(_perfStart, 'dbView.galleryCreate');
await updateNcTblSchemaById(tblId);
// syncLog(`[${idx+1}/${aTblSchema.length}][Gallery View][${i+1}/${galleryViews.length}] Create ${viewName}`)
@ -1435,7 +1593,10 @@ export default async (
};
logDetailed(`NC API dbView.formCreate :: ${viewName}`);
const _perfStart = recordPerfStart();
const f = await api.dbView.formCreate(tblId, formData);
recordPerfStats(_perfStart, 'dbView.formCreate');
logDetailed(
`[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${
formViews.length
@ -1475,7 +1636,10 @@ export default async (
const viewName = aTblSchema[idx].views.find(
x => x.id === gridViews[i].id
)?.name;
const _perfStart = recordPerfStart();
const viewList: any = await api.dbView.list(tblId);
recordPerfStats(_perfStart, 'dbView.list');
let ncViewId = viewList?.list?.find(x => x.tn === viewName)?.id;
logBasic(
@ -1487,9 +1651,12 @@ export default async (
// create view (default already created)
if (i > 0) {
logDetailed(`NC API dbView.gridCreate :: ${viewName}`);
const _perfStart = recordPerfStart();
const viewCreated = await api.dbView.gridCreate(tblId, {
title: viewName
});
recordPerfStats(_perfStart, 'dbView.gridCreate');
await updateNcTblSchemaById(tblId);
await sMap.addToMappingTbl(
gridViews[i].id,
@ -1544,6 +1711,7 @@ export default async (
const userList = aTblSchema.appBlanket.userInfoById;
const totalUsers = Object.keys(userList).length;
let cnt = 0;
const insertJobs: Promise<any>[] = [];
for (const [, value] of Object.entries(
userList as { [key: string]: any }
@ -1551,11 +1719,16 @@ export default async (
logDetailed(
`[${++cnt}/${totalUsers}] NC API auth.projectUserAdd :: ${value.email}`
);
await api.auth.projectUserAdd(ncCreatedProjectSchema.id, {
const _perfStart = recordPerfStart();
insertJobs.push(
api.auth.projectUserAdd(ncCreatedProjectSchema.id, {
email: value.email,
roles: userRoles[value.permissionLevel]
});
})
);
recordPerfStats(_perfStart, 'auth.projectUserAdd');
}
await Promise.all(insertJobs);
}
function updateNcTblSchema(tblSchema) {
@ -1571,7 +1744,10 @@ export default async (
}
async function updateNcTblSchemaById(tblId) {
const _perfStart = recordPerfStart();
const ncTbl = await api.dbTable.read(tblId);
recordPerfStats(_perfStart, 'dbTable.read');
updateNcTblSchema(ncTbl);
}
@ -1665,23 +1841,35 @@ export default async (
return accumulator + object.nc.rollup;
}, 0);
logDetailed(`Quick Summary:`);
logDetailed(`:: Total Tables: ${aTblSchema.length}`);
logDetailed(`:: Total Columns: ${columnSum}`);
logDetailed(`:: Links: ${linkSum}`);
logDetailed(`:: Lookup: ${lookupSum}`);
logDetailed(`:: Rollup: ${rollupSum}`);
logDetailed(`:: Total Filters: ${rtc.filter}`);
logDetailed(`:: Total Sort: ${rtc.sort}`);
logDetailed(`:: Total Views: ${rtc.view.total}`);
logDetailed(`:: Grid: ${rtc.view.grid}`);
logDetailed(`:: Gallery: ${rtc.view.gallery}`);
logDetailed(`:: Form: ${rtc.view.form}`);
logBasic(`Quick Summary:`);
logBasic(`:: Total Tables: ${aTblSchema.length}`);
logBasic(`:: Total Columns: ${columnSum}`);
logBasic(`:: Links: ${linkSum}`);
logBasic(`:: Lookup: ${lookupSum}`);
logBasic(`:: Rollup: ${rollupSum}`);
logBasic(`:: Total Filters: ${rtc.filter}`);
logBasic(`:: Total Sort: ${rtc.sort}`);
logBasic(`:: Total Views: ${rtc.view.total}`);
logBasic(`:: Grid: ${rtc.view.grid}`);
logBasic(`:: Gallery: ${rtc.view.gallery}`);
logBasic(`:: Form: ${rtc.view.form}`);
const duration = Date.now() - start;
logDetailed(`:: Migration time: ${duration}`);
logDetailed(`:: Axios fetch count: ${rtc.fetchAt.count}`);
logDetailed(`:: Axios fetch time: ${rtc.fetchAt.time}`);
logBasic(`:: Migration time: ${duration}`);
logBasic(`:: Axios fetch count: ${rtc.fetchAt.count}`);
logBasic(`:: Axios fetch time: ${rtc.fetchAt.time}`);
if (debugMode) {
jsonfile.writeFileSync('stats.json', perfStats, { spaces: 2 });
const perflog = [];
for (let i = 0; i < perfStats.length; i++) {
perflog.push(`${perfStats[i].e}, ${perfStats[i].d}`);
}
jsonfile.writeFileSync('stats.csv', perflog, { spaces: 2 });
jsonfile.writeFileSync('skip.txt', rtc.migrationSkipLog.log, {
spaces: 2
});
}
}
//////////////////////////////
@ -1776,9 +1964,12 @@ export default async (
// insert filters
for (let i = 0; i < ncFilters.length; i++) {
const _perfStart = recordPerfStart();
await api.dbTableFilter.create(viewId, {
...ncFilters[i]
});
recordPerfStats(_perfStart, 'dbTableFilter.create');
rtc.filter++;
}
}
@ -1788,11 +1979,14 @@ export default async (
for (let i = 0; i < s.sortSet.length; i++) {
const columnId = (await nc_getColumnSchema(s.sortSet[i].columnId))?.id;
if (columnId)
if (columnId) {
const _perfStart = recordPerfStart();
await api.dbTableSort.create(viewId, {
fk_column_id: columnId,
direction: s.sortSet[i].ascending ? 'asc' : 'dsc'
});
recordPerfStats(_perfStart, 'dbTableSort.create');
}
rtc.sort++;
}
}
@ -1807,23 +2001,41 @@ export default async (
const ncTbl = await nc_getTableSchema(tblName);
// retrieve view ID
const viewId = ncTbl.views.find(x => x.title === viewName).id;
let viewDetails;
const _perfStart = recordPerfStart();
if (viewType === 'form') {
viewDetails = (await api.dbView.formRead(viewId)).columns;
recordPerfStats(_perfStart, 'dbView.formRead');
} else if (viewType === 'gallery') {
viewDetails = (await api.dbView.galleryRead(viewId)).columns;
recordPerfStats(_perfStart, 'dbView.galleryRead');
} else {
viewDetails = await api.dbView.gridColumnsList(viewId);
recordPerfStats(_perfStart, 'dbView.gridColumnsList');
}
// nc-specific columns; default hide.
for (let j = 0; j < hiddenColumns.length; j++) {
const ncColumnId = ncTbl.columns.find(x => x.title === hiddenColumns[j])
.id;
const ncViewColumnId = await nc_getViewColumnId(
viewId,
viewType,
ncColumnId
);
const ncViewColumnId = viewDetails.find(
x => x.fk_column_id === ncColumnId
)?.id;
// const ncViewColumnId = await nc_getViewColumnId(
// viewId,
// viewType,
// ncColumnId
// );
if (ncViewColumnId === undefined) continue;
// first two positions held by record id & record hash
const _perfStart = recordPerfStart();
await api.dbViewColumn.update(viewId, ncViewColumnId, {
show: false,
order: j + 1 + c.length
});
recordPerfStats(_perfStart, 'dbViewColumn.update');
}
// rest of the columns from airtable- retain order & visibility property
@ -1845,10 +2057,14 @@ export default async (
if (x?.title) formData[`label`] = x.title;
if (x?.required) formData[`required`] = x.required;
if (x?.description) formData[`description`] = x.description;
const _perfStart = recordPerfStart();
await api.dbView.formColumnUpdate(ncViewColumnId, formData);
recordPerfStats(_perfStart, 'dbView.formColumnUpdate');
}
}
const _perfStart = recordPerfStart();
await api.dbViewColumn.update(viewId, ncViewColumnId, configData);
recordPerfStats(_perfStart, 'dbViewColumn.update');
}
}
@ -1935,30 +2151,53 @@ export default async (
if (process_aTblData) {
try {
// await nc_DumpTableSchema();
const _perfStart = recordPerfStart();
const ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id);
recordPerfStats(_perfStart, 'dbTable.list');
logBasic('Reading Records...');
for (let i = 0; i < ncTblList.list.length; i++) {
const _perfStart = recordPerfStart();
const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
recordPerfStats(_perfStart, 'dbTable.read');
// not a migrated table, skip
if (undefined === aTblSchema.find(x => x.name === ncTbl.title))
continue;
recordCnt = 0;
await nocoReadData(syncDB, ncTbl, async (sDB, table, record) => {
await nocoBaseDataProcessing(sDB, table, record);
});
await nocoReadData(syncDB, ncTbl);
logDetailed(`Data inserted from ${ncTbl.title}`);
}
logBasic('Configuring Record Links...');
// Configure link @ Data row's
if (storeLinks) {
// const insertJobs: Promise<any>[] = [];
for (const [pTitle, v] of Object.entries(ncLinkDataStore)) {
logBasic(`:: ${pTitle}`);
for (const [, record] of Object.entries(v)) {
const tbl = ncTblList.list.find(a => a.title === pTitle);
await nocoLinkProcessing(syncDB.projectName, tbl, record, 0);
// insertJobs.push(
// nocoLinkProcessing(syncDB.projectName, tbl, record, 0)
// );
}
}
// await Promise.all(insertJobs);
// await nocoLinkProcessing(syncDB.projectName, 0, 0, 0);
} else {
// create link groups (table: link fields)
const tblLinkGroup = {};
for (let idx = 0; idx < ncLinkMappingTable.length; idx++) {
const x = ncLinkMappingTable[idx];
const ncTbl = await nc_getTableSchema(
aTbl_getTableName(x.aTbl.tblId).tn
);
if (tblLinkGroup[x.aTbl.tblId] === undefined)
tblLinkGroup[x.aTbl.tblId] = [x.aTbl.name];
else tblLinkGroup[x.aTbl.tblId].push(x.aTbl.name);
}
for (const [k, v] of Object.entries(tblLinkGroup)) {
const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn);
// not a migrated table, skip
if (undefined === aTblSchema.find(x => x.name === ncTbl.title))
@ -1971,9 +2210,9 @@ export default async (
async (projName, table, record, _field) => {
await nocoLinkProcessing(projName, table, record, _field);
},
x.aTbl.name
v
);
logDetailed(`Linked data to ${ncTbl.title}`);
}
}
} catch (error) {
logDetailed(

Loading…
Cancel
Save