From 0b9c7263e8d18aa29bd344e95e2783f5f9b1f82b Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 30 May 2022 20:35:47 +0530 Subject: [PATCH] wip: optimization Signed-off-by: Pranav C --- .../src/lib/noco/meta/api/sync/helpers/job.ts | 152 +++++++++---- .../meta/api/sync/helpers/readAllATData.ts | 201 ++++++++++++++++++ 2 files changed, 316 insertions(+), 37 deletions(-) create mode 100644 packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index fd09a8cd61..1c9243b170 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -13,6 +13,7 @@ import hash from 'object-hash'; import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; +import { importData, importLTARData } from './readAllATData'; dayjs.extend(utc); @@ -82,6 +83,7 @@ export default async ( const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' }; const storeLinks = false; const ncLinkDataStore: any = {}; + const insertedAssocRef: any = {}; const uniqueTableNameGen = getUniqueNameGenerator('sheet'); @@ -208,6 +210,7 @@ export default async ( // aTbl: retrieve table name from table ID // + // @ts-ignore function aTbl_getTableName(tblId) { const sheetObj = g_aTblSchema.find(tbl => tbl.id === tblId); return { @@ -1188,6 +1191,7 @@ export default async ( ////////// Data processing + // @ts-ignore async function nocoLinkProcessing(projName, table, record, _field) { const rec = record.fields; @@ -1216,6 +1220,44 @@ export default async ( } } } + // + // ////////// Data processing + // async function nocoLinkProcessingv2( + // projName, + // table, + // record, + // _field, + // rowsRef = {} + // ) { + // const modelMeta = await api.dbTable.read(table.id); + // + // const rec = record.fields; + // + // for (const [key, value] of Object.entries(rec)) { + // const refRowIdList: any = value; + // const referenceColumnName = key; + // + // if (refRowIdList.length) { + // for (let i = 0; i < refRowIdList.length; i++) { + // logDetailed( + // `NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}` + // ); + // + // const _perfStart = recordPerfStart(); + // await api.dbTableRow.nestedAdd( + // 'noco', + // projName, + // table.id, + // `${record.id}`, + // 'mm',0 + // encodeURIComponent(referenceColumnName), + // `${refRowIdList[i]}` + // ); + // recordPerfStats(_perfStart, 'dbTableRow.nestedAdd'); + // } + // } + // } + // } async function nocoBaseDataProcessing_v2(sDB, table, record) { const recordHash = hash(record); @@ -1374,6 +1416,7 @@ export default async ( return rec; } + // @ts-ignore async function nocoReadData(sDB, table) { ncLinkDataStore[table.title] = {}; const insertJobs: Promise[] = []; @@ -1439,6 +1482,7 @@ export default async ( }); } + // @ts-ignore async function nocoReadDataSelected(projName, table, callback, fields) { return new Promise((resolve, reject) => { base(table.title) @@ -2196,52 +2240,86 @@ export default async ( continue; recordCnt = 0; - await nocoReadData(syncDB, ncTbl); + // await nocoReadData(syncDB, ncTbl); + + await importData({ + projectName: syncDB.projectName, + table: ncTbl, + base, + api, + logBasic, + nocoBaseDataProcessing_v2, + sDB: syncDB + }); + logDetailed(`Data inserted from ${ncTbl.title}`); } logBasic('Configuring Record Links...'); + for (let i = 0; i < ncTblList.list.length; i++) { + const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + await importLTARData({ + table: ncTbl, + projectName: syncDB.projectName, + api, + base, + fields: null, //Object.values(tblLinkGroup).flat(), + logBasic, + insertedAssocRef + }); + } + if (storeLinks) { // const insertJobs: Promise[] = []; - for (const [pTitle, v] of Object.entries(ncLinkDataStore)) { - logBasic(`:: ${pTitle}`); - for (const [, record] of Object.entries(v)) { - const tbl = ncTblList.list.find(a => a.title === pTitle); - await nocoLinkProcessing(syncDB.projectName, tbl, record, 0); - // insertJobs.push( - // nocoLinkProcessing(syncDB.projectName, tbl, record, 0) - // ); - } - } + // for (const [pTitle, v] of Object.entries(ncLinkDataStore)) { + // logBasic(`:: ${pTitle}`); + // for (const [, record] of Object.entries(v)) { + // const tbl = ncTblList.list.find(a => a.title === pTitle); + // await nocoLinkProcessing(syncDB.projectName, tbl, record, 0); + // // insertJobs.push( + // // nocoLinkProcessing(syncDB.projectName, tbl, record, 0) + // // ); + // } + // } // await Promise.all(insertJobs); // await nocoLinkProcessing(syncDB.projectName, 0, 0, 0); } else { - // create link groups (table: link fields) - const tblLinkGroup = {}; - for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { - const x = ncLinkMappingTable[idx]; - if (tblLinkGroup[x.aTbl.tblId] === undefined) - tblLinkGroup[x.aTbl.tblId] = [x.aTbl.name]; - else tblLinkGroup[x.aTbl.tblId].push(x.aTbl.name); - } - - for (const [k, v] of Object.entries(tblLinkGroup)) { - const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); - - // not a migrated table, skip - if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) - continue; - - recordCnt = 0; - await nocoReadDataSelected( - syncDB.projectName, - ncTbl, - async (projName, table, record, _field) => { - await nocoLinkProcessing(projName, table, record, _field); - }, - v - ); - } + // // create link groups (table: link fields) + // // const tblLinkGroup = {}; + // // for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { + // // const x = ncLinkMappingTable[idx]; + // // if (tblLinkGroup[x.aTbl.tblId] === undefined) + // // tblLinkGroup[x.aTbl.tblId] = [x.aTbl.name]; + // // else tblLinkGroup[x.aTbl.tblId].push(x.aTbl.name); + // // } + // // + // // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); + // // + // // await importLTARData({ + // // table: ncTbl, + // // projectName: syncDB.projectName, + // // api, + // // base, + // // fields: Object.values(tblLinkGroup).flat(), + // // logBasic + // // }); + // for (const [k, v] of Object.entries(tblLinkGroup)) { + // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); + // + // // not a migrated table, skip + // if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) + // continue; + // + // recordCnt = 0; + // await nocoReadDataSelected( + // syncDB.projectName, + // ncTbl, + // async (projName, table, record, _field) => { + // await nocoLinkProcessing(projName, table, record, _field); + // }, + // v + // ); + // } } } catch (error) { logDetailed( diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts new file mode 100644 index 0000000000..85392f6959 --- /dev/null +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts @@ -0,0 +1,201 @@ +import { AirtableBase } from 'airtable/lib/airtable_base'; +import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk'; + +async function readAllATData({ + table, + fields, + base +}: // logBasic = _str => () +{ + table: { title?: string }; + fields?; + base: AirtableBase; + logBasic?: (string) => void; +}): Promise> { + return new Promise((resolve, reject) => { + const data = []; + const selectParams: any = { + pageSize: 100 + }; + if (fields) selectParams.fields = fields; + + base(table.title) + .select(selectParams) + .eachPage( + async function page(records, fetchNextPage) { + // console.log(JSON.stringify(records, null, 2)); + + // This function (`page`) will get called for each page of records. + // records.forEach(record => callback(table, record)); + // logBasic( + // `:: ${table.title} / ${fields} : ${recordCnt + + // 1} ~ ${(recordCnt += 100)}` + // ); + data.push(...records); + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + function done(err) { + if (err) { + console.error(err); + reject(err); + } + resolve(data); + } + ); + }); +} + +export async function importData({ + projectName, + table, + base, + api, + nocoBaseDataProcessing_v2, + sDB +}: // logBasic = _str => () +{ + projectName: string; + table: { title?: string; id?: string }; + fields?; + base: AirtableBase; + logBasic: (string) => void; + api: Api; + nocoBaseDataProcessing_v2; + sDB; +}) { + try { + // get all data from a table + const allData = []; + const records = await readAllATData({ + table, + base + }); + + for (let i = 0; i < records.length; i++) { + const r = await nocoBaseDataProcessing_v2(sDB, table, records[i]); + allData.push(r); + } + + for (let i = 0; i < allData.length / 2000; i += 2000) { + await api.dbTableRow.bulkCreate( + 'nc', + projectName, + table.id, // encodeURIComponent(table.title), + allData.slice(i, 2000) + ); + } + } catch (e) { + console.log(e); + } +} + +export async function importLTARData({ + table, + fields, + base, + api, + projectName, + insertedAssocRef = {} +}: // logBasic = _str => () +{ + projectName: string; + table: { title?: string; id?: string }; + fields; + base: AirtableBase; + logBasic: (string) => void; + api: Api; + insertedAssocRef: { [assocId: string]: boolean }; +}) { + const assocTableMetas: Array<{ + modelMeta: { id?: string; title?: string }; + colMeta: { title?: string }; + curCol: { title?: string }; + refCol: { title?: string }; + }> = []; + const allData = await readAllATData({ table, fields, base }); + + const modelMeta: any = await api.dbTable.read(table.id); + + for (const colMeta of modelMeta.columns) { + if ( + colMeta.uidt !== UITypes.LinkToAnotherRecord || + colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY + ) { + continue; + } + + if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue; + + insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; + + const assocModelMeta: TableType = (await api.dbTable.read( + colMeta.colOptions.fk_mm_model_id + )) as any; + + assocTableMetas.push({ + modelMeta: assocModelMeta, + colMeta, + curCol: assocModelMeta.columns.find( + c => c.id === colMeta.colOptions.fk_mm_child_column_id + ), + refCol: assocModelMeta.columns.find( + c => c.id === colMeta.colOptions.fk_mm_parent_column_id + ) + }); + } + + for (const assocMeta of assocTableMetas) { + const insertData = []; + for (const record of allData) { + const rec = record.fields; + + // todo: use actual alias instead of sanitized + insertData.push( + ...(rec?.[assocMeta.colMeta.title] || []).map(id => ({ + [assocMeta.curCol.title]: record.id, + [assocMeta.refCol.title]: id + })) + ); + } + + for (let i = 0; i < insertData.length / 2000; i += 2000) { + await api.dbTableRow.bulkCreate( + 'nc', + projectName, + assocMeta.modelMeta.id, + insertData.slice(i, 2000) + ); + } + } +} + +// for (const [key, value] of Object.entries(rec)) { +// const refRowIdList: any = value; +// const referenceColumnName = key; +// +// if (refRowIdList.length) { +// for (let i = 0; i < refRowIdList.length; i++) { +// logDetailed( +// `NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}` +// ); +// +// const _perfStart = recordPerfStart(); +// await api.dbTableRow.nestedAdd( +// 'noco', +// projName, +// table.id, +// `${record.id}`, +// 'mm', +// encodeURIComponent(referenceColumnName), +// `${refRowIdList[i]}` +// ); +// recordPerfStats(_perfStart, 'dbTableRow.nestedAdd'); +// } +// } +// } +// } +// }