From 0b9c7263e8d18aa29bd344e95e2783f5f9b1f82b Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 30 May 2022 20:35:47 +0530 Subject: [PATCH 01/12] wip: optimization Signed-off-by: Pranav C --- .../src/lib/noco/meta/api/sync/helpers/job.ts | 152 +++++++++---- .../meta/api/sync/helpers/readAllATData.ts | 201 ++++++++++++++++++ 2 files changed, 316 insertions(+), 37 deletions(-) create mode 100644 packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index fd09a8cd61..1c9243b170 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -13,6 +13,7 @@ import hash from 'object-hash'; import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; +import { importData, importLTARData } from './readAllATData'; dayjs.extend(utc); @@ -82,6 +83,7 @@ export default async ( const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' }; const storeLinks = false; const ncLinkDataStore: any = {}; + const insertedAssocRef: any = {}; const uniqueTableNameGen = getUniqueNameGenerator('sheet'); @@ -208,6 +210,7 @@ export default async ( // aTbl: retrieve table name from table ID // + // @ts-ignore function aTbl_getTableName(tblId) { const sheetObj = g_aTblSchema.find(tbl => tbl.id === tblId); return { @@ -1188,6 +1191,7 @@ export default async ( ////////// Data processing + // @ts-ignore async function nocoLinkProcessing(projName, table, record, _field) { const rec = record.fields; @@ -1216,6 +1220,44 @@ export default async ( } } } + // + // ////////// Data processing + // async function nocoLinkProcessingv2( + // projName, + // table, + // record, + // _field, + // rowsRef = {} + // ) { + // const modelMeta = await api.dbTable.read(table.id); + // + // const rec = record.fields; + // + // for (const [key, value] of Object.entries(rec)) { + // const refRowIdList: any = value; + // const referenceColumnName = key; + // + // if (refRowIdList.length) { + // for (let i = 0; i < refRowIdList.length; i++) { + // logDetailed( + // `NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}` + // ); + // + // const _perfStart = recordPerfStart(); + // await api.dbTableRow.nestedAdd( + // 'noco', + // projName, + // table.id, + // `${record.id}`, + // 'mm',0 + // encodeURIComponent(referenceColumnName), + // `${refRowIdList[i]}` + // ); + // recordPerfStats(_perfStart, 'dbTableRow.nestedAdd'); + // } + // } + // } + // } async function nocoBaseDataProcessing_v2(sDB, table, record) { const recordHash = hash(record); @@ -1374,6 +1416,7 @@ export default async ( return rec; } + // @ts-ignore async function nocoReadData(sDB, table) { ncLinkDataStore[table.title] = {}; const insertJobs: Promise[] = []; @@ -1439,6 +1482,7 @@ export default async ( }); } + // @ts-ignore async function nocoReadDataSelected(projName, table, callback, fields) { return new Promise((resolve, reject) => { base(table.title) @@ -2196,52 +2240,86 @@ export default async ( continue; recordCnt = 0; - await nocoReadData(syncDB, ncTbl); + // await nocoReadData(syncDB, ncTbl); + + await importData({ + projectName: syncDB.projectName, + table: ncTbl, + base, + api, + logBasic, + nocoBaseDataProcessing_v2, + sDB: syncDB + }); + logDetailed(`Data inserted from ${ncTbl.title}`); } logBasic('Configuring Record Links...'); + for (let i = 0; i < ncTblList.list.length; i++) { + const ncTbl = await api.dbTable.read(ncTblList.list[i].id); + await importLTARData({ + table: ncTbl, + projectName: syncDB.projectName, + api, + base, + fields: null, //Object.values(tblLinkGroup).flat(), + logBasic, + insertedAssocRef + }); + } + if (storeLinks) { // const insertJobs: Promise[] = []; - for (const [pTitle, v] of Object.entries(ncLinkDataStore)) { - logBasic(`:: ${pTitle}`); - for (const [, record] of Object.entries(v)) { - const tbl = ncTblList.list.find(a => a.title === pTitle); - await nocoLinkProcessing(syncDB.projectName, tbl, record, 0); - // insertJobs.push( - // nocoLinkProcessing(syncDB.projectName, tbl, record, 0) - // ); - } - } + // for (const [pTitle, v] of Object.entries(ncLinkDataStore)) { + // logBasic(`:: ${pTitle}`); + // for (const [, record] of Object.entries(v)) { + // const tbl = ncTblList.list.find(a => a.title === pTitle); + // await nocoLinkProcessing(syncDB.projectName, tbl, record, 0); + // // insertJobs.push( + // // nocoLinkProcessing(syncDB.projectName, tbl, record, 0) + // // ); + // } + // } // await Promise.all(insertJobs); // await nocoLinkProcessing(syncDB.projectName, 0, 0, 0); } else { - // create link groups (table: link fields) - const tblLinkGroup = {}; - for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { - const x = ncLinkMappingTable[idx]; - if (tblLinkGroup[x.aTbl.tblId] === undefined) - tblLinkGroup[x.aTbl.tblId] = [x.aTbl.name]; - else tblLinkGroup[x.aTbl.tblId].push(x.aTbl.name); - } - - for (const [k, v] of Object.entries(tblLinkGroup)) { - const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); - - // not a migrated table, skip - if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) - continue; - - recordCnt = 0; - await nocoReadDataSelected( - syncDB.projectName, - ncTbl, - async (projName, table, record, _field) => { - await nocoLinkProcessing(projName, table, record, _field); - }, - v - ); - } + // // create link groups (table: link fields) + // // const tblLinkGroup = {}; + // // for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { + // // const x = ncLinkMappingTable[idx]; + // // if (tblLinkGroup[x.aTbl.tblId] === undefined) + // // tblLinkGroup[x.aTbl.tblId] = [x.aTbl.name]; + // // else tblLinkGroup[x.aTbl.tblId].push(x.aTbl.name); + // // } + // // + // // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); + // // + // // await importLTARData({ + // // table: ncTbl, + // // projectName: syncDB.projectName, + // // api, + // // base, + // // fields: Object.values(tblLinkGroup).flat(), + // // logBasic + // // }); + // for (const [k, v] of Object.entries(tblLinkGroup)) { + // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); + // + // // not a migrated table, skip + // if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) + // continue; + // + // recordCnt = 0; + // await nocoReadDataSelected( + // syncDB.projectName, + // ncTbl, + // async (projName, table, record, _field) => { + // await nocoLinkProcessing(projName, table, record, _field); + // }, + // v + // ); + // } } } catch (error) { logDetailed( diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts new file mode 100644 index 0000000000..85392f6959 --- /dev/null +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts @@ -0,0 +1,201 @@ +import { AirtableBase } from 'airtable/lib/airtable_base'; +import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk'; + +async function readAllATData({ + table, + fields, + base +}: // logBasic = _str => () +{ + table: { title?: string }; + fields?; + base: AirtableBase; + logBasic?: (string) => void; +}): Promise> { + return new Promise((resolve, reject) => { + const data = []; + const selectParams: any = { + pageSize: 100 + }; + if (fields) selectParams.fields = fields; + + base(table.title) + .select(selectParams) + .eachPage( + async function page(records, fetchNextPage) { + // console.log(JSON.stringify(records, null, 2)); + + // This function (`page`) will get called for each page of records. + // records.forEach(record => callback(table, record)); + // logBasic( + // `:: ${table.title} / ${fields} : ${recordCnt + + // 1} ~ ${(recordCnt += 100)}` + // ); + data.push(...records); + + // To fetch the next page of records, call `fetchNextPage`. + // If there are more records, `page` will get called again. + // If there are no more records, `done` will get called. + fetchNextPage(); + }, + function done(err) { + if (err) { + console.error(err); + reject(err); + } + resolve(data); + } + ); + }); +} + +export async function importData({ + projectName, + table, + base, + api, + nocoBaseDataProcessing_v2, + sDB +}: // logBasic = _str => () +{ + projectName: string; + table: { title?: string; id?: string }; + fields?; + base: AirtableBase; + logBasic: (string) => void; + api: Api; + nocoBaseDataProcessing_v2; + sDB; +}) { + try { + // get all data from a table + const allData = []; + const records = await readAllATData({ + table, + base + }); + + for (let i = 0; i < records.length; i++) { + const r = await nocoBaseDataProcessing_v2(sDB, table, records[i]); + allData.push(r); + } + + for (let i = 0; i < allData.length / 2000; i += 2000) { + await api.dbTableRow.bulkCreate( + 'nc', + projectName, + table.id, // encodeURIComponent(table.title), + allData.slice(i, 2000) + ); + } + } catch (e) { + console.log(e); + } +} + +export async function importLTARData({ + table, + fields, + base, + api, + projectName, + insertedAssocRef = {} +}: // logBasic = _str => () +{ + projectName: string; + table: { title?: string; id?: string }; + fields; + base: AirtableBase; + logBasic: (string) => void; + api: Api; + insertedAssocRef: { [assocId: string]: boolean }; +}) { + const assocTableMetas: Array<{ + modelMeta: { id?: string; title?: string }; + colMeta: { title?: string }; + curCol: { title?: string }; + refCol: { title?: string }; + }> = []; + const allData = await readAllATData({ table, fields, base }); + + const modelMeta: any = await api.dbTable.read(table.id); + + for (const colMeta of modelMeta.columns) { + if ( + colMeta.uidt !== UITypes.LinkToAnotherRecord || + colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY + ) { + continue; + } + + if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue; + + insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; + + const assocModelMeta: TableType = (await api.dbTable.read( + colMeta.colOptions.fk_mm_model_id + )) as any; + + assocTableMetas.push({ + modelMeta: assocModelMeta, + colMeta, + curCol: assocModelMeta.columns.find( + c => c.id === colMeta.colOptions.fk_mm_child_column_id + ), + refCol: assocModelMeta.columns.find( + c => c.id === colMeta.colOptions.fk_mm_parent_column_id + ) + }); + } + + for (const assocMeta of assocTableMetas) { + const insertData = []; + for (const record of allData) { + const rec = record.fields; + + // todo: use actual alias instead of sanitized + insertData.push( + ...(rec?.[assocMeta.colMeta.title] || []).map(id => ({ + [assocMeta.curCol.title]: record.id, + [assocMeta.refCol.title]: id + })) + ); + } + + for (let i = 0; i < insertData.length / 2000; i += 2000) { + await api.dbTableRow.bulkCreate( + 'nc', + projectName, + assocMeta.modelMeta.id, + insertData.slice(i, 2000) + ); + } + } +} + +// for (const [key, value] of Object.entries(rec)) { +// const refRowIdList: any = value; +// const referenceColumnName = key; +// +// if (refRowIdList.length) { +// for (let i = 0; i < refRowIdList.length; i++) { +// logDetailed( +// `NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}` +// ); +// +// const _perfStart = recordPerfStart(); +// await api.dbTableRow.nestedAdd( +// 'noco', +// projName, +// table.id, +// `${record.id}`, +// 'mm', +// encodeURIComponent(referenceColumnName), +// `${refRowIdList[i]}` +// ); +// recordPerfStats(_perfStart, 'dbTableRow.nestedAdd'); +// } +// } +// } +// } +// } From c895b01438d31775675bbf01288d96806a671e4f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 31 May 2022 00:47:12 +0530 Subject: [PATCH 02/12] feat: add log, update batch size Signed-off-by: Pranav C --- .../src/lib/noco/meta/api/sync/helpers/job.ts | 8 +- ...readAllATData.ts => readAndProcessData.ts} | 79 ++++++++++--------- 2 files changed, 45 insertions(+), 42 deletions(-) rename packages/nocodb/src/lib/noco/meta/api/sync/helpers/{readAllATData.ts => readAndProcessData.ts} (77%) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index 1c9243b170..898e4e3ba5 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -13,7 +13,7 @@ import hash from 'object-hash'; import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; -import { importData, importLTARData } from './readAllATData'; +import { importData, importLTARData } from './readAndProcessData'; dayjs.extend(utc); @@ -2249,7 +2249,8 @@ export default async ( api, logBasic, nocoBaseDataProcessing_v2, - sDB: syncDB + sDB: syncDB, + logDetailed }); logDetailed(`Data inserted from ${ncTbl.title}`); @@ -2265,7 +2266,8 @@ export default async ( base, fields: null, //Object.values(tblLinkGroup).flat(), logBasic, - insertedAssocRef + insertedAssocRef, + logDetailed }); } diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts similarity index 77% rename from packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts rename to packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts index 85392f6959..b255be6827 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAllATData.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts @@ -1,16 +1,21 @@ import { AirtableBase } from 'airtable/lib/airtable_base'; import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk'; -async function readAllATData({ +const BULK_DATA_BATCH_SIZE = 500; +const ASSOC_BULK_DATA_BATCH_SIZE = 2000; + +async function readAndProcessData({ table, fields, base -}: // logBasic = _str => () +}: // logDetailed +// logBasic = _str => () { table: { title?: string }; fields?; base: AirtableBase; logBasic?: (string) => void; + logDetailed?: (string) => void; }): Promise> { return new Promise((resolve, reject) => { const data = []; @@ -55,14 +60,15 @@ export async function importData({ base, api, nocoBaseDataProcessing_v2, - sDB -}: // logBasic = _str => () -{ + sDB, + logDetailed = _str => {} +}: { projectName: string; table: { title?: string; id?: string }; fields?; base: AirtableBase; logBasic: (string) => void; + logDetailed: (string) => void; api: Api; nocoBaseDataProcessing_v2; sDB; @@ -70,7 +76,7 @@ export async function importData({ try { // get all data from a table const allData = []; - const records = await readAllATData({ + const records = await readAndProcessData({ table, base }); @@ -80,12 +86,22 @@ export async function importData({ allData.push(r); } - for (let i = 0; i < allData.length / 2000; i += 2000) { + for ( + let i = 0; + i < allData.length / BULK_DATA_BATCH_SIZE; + i += BULK_DATA_BATCH_SIZE + ) { + logDetailed( + `Importing '${table.title}' data :: ${i + 1} - ${Math.min( + i + BULK_DATA_BATCH_SIZE, + records.length + )}` + ); await api.dbTableRow.bulkCreate( 'nc', projectName, table.id, // encodeURIComponent(table.title), - allData.slice(i, 2000) + allData.slice(i, BULK_DATA_BATCH_SIZE) ); } } catch (e) { @@ -99,13 +115,15 @@ export async function importLTARData({ base, api, projectName, - insertedAssocRef = {} + insertedAssocRef = {}, + logDetailed = _str => {} }: // logBasic = _str => () { projectName: string; table: { title?: string; id?: string }; fields; base: AirtableBase; + logDetailed: (string) => void; logBasic: (string) => void; api: Api; insertedAssocRef: { [assocId: string]: boolean }; @@ -116,7 +134,7 @@ export async function importLTARData({ curCol: { title?: string }; refCol: { title?: string }; }> = []; - const allData = await readAllATData({ table, fields, base }); + const allData = await readAndProcessData({ table, fields, base }); const modelMeta: any = await api.dbTable.read(table.id); @@ -162,40 +180,23 @@ export async function importLTARData({ ); } - for (let i = 0; i < insertData.length / 2000; i += 2000) { + for ( + let i = 0; + i < insertData.length / ASSOC_BULK_DATA_BATCH_SIZE; + i += ASSOC_BULK_DATA_BATCH_SIZE + ) { + logDetailed( + `Importing '${table.title}' LTAR data :: ${i + 1} - ${Math.min( + i + ASSOC_BULK_DATA_BATCH_SIZE, + allData.length + )}` + ); await api.dbTableRow.bulkCreate( 'nc', projectName, assocMeta.modelMeta.id, - insertData.slice(i, 2000) + insertData.slice(i, ASSOC_BULK_DATA_BATCH_SIZE) ); } } } - -// for (const [key, value] of Object.entries(rec)) { -// const refRowIdList: any = value; -// const referenceColumnName = key; -// -// if (refRowIdList.length) { -// for (let i = 0; i < refRowIdList.length; i++) { -// logDetailed( -// `NC API: dbTableRow.nestedAdd ${record.id}/mm/${referenceColumnName}/${refRowIdList[0][i]}` -// ); -// -// const _perfStart = recordPerfStart(); -// await api.dbTableRow.nestedAdd( -// 'noco', -// projName, -// table.id, -// `${record.id}`, -// 'mm', -// encodeURIComponent(referenceColumnName), -// `${refRowIdList[i]}` -// ); -// recordPerfStats(_perfStart, 'dbTableRow.nestedAdd'); -// } -// } -// } -// } -// } From 771c1c543bb2842e13686caa2c19df50ee38ad68 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 31 May 2022 12:32:48 +0530 Subject: [PATCH 03/12] refactor: add option to pass chunk size in bulk insert method Signed-off-by: Pranav C --- .../src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts | 11 +++++++++-- .../noco/meta/api/sync/helpers/readAndProcessData.ts | 10 +++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts index 0e6a2c073d..9c49608848 100644 --- a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts @@ -1513,7 +1513,14 @@ class BaseModelSqlv2 { } } - async bulkInsert(datas: any[]) { + async bulkInsert( + datas: any[], + { + chunkSize: _chunkSize = 100 + }: { + chunkSize?: number; + } = {} + ) { try { const insertDatas = await Promise.all( datas.map(async d => { @@ -1536,7 +1543,7 @@ class BaseModelSqlv2 { // fallbacks to `10` if database client is sqlite // to avoid `too many SQL variables` error // refer : https://www.sqlite.org/limits.html - const chunkSize = this.isSqlite ? 10 : 50; + const chunkSize = this.isSqlite ? 10 : _chunkSize; const response = await this.dbDriver .batchInsert(this.model.table_name, insertDatas, chunkSize) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts index b255be6827..a15f5556f1 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts @@ -126,7 +126,7 @@ export async function importLTARData({ logDetailed: (string) => void; logBasic: (string) => void; api: Api; - insertedAssocRef: { [assocId: string]: boolean }; + insertedAssocRef: { [assocTableId: string]: boolean }; }) { const assocTableMetas: Array<{ modelMeta: { id?: string; title?: string }; @@ -139,6 +139,7 @@ export async function importLTARData({ const modelMeta: any = await api.dbTable.read(table.id); for (const colMeta of modelMeta.columns) { + // skip columns which are not LTAR and Many to many if ( colMeta.uidt !== UITypes.LinkToAnotherRecord || colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY @@ -146,14 +147,17 @@ export async function importLTARData({ continue; } + // skip if already inserted if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue; + // mark as inserted insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; const assocModelMeta: TableType = (await api.dbTable.read( colMeta.colOptions.fk_mm_model_id )) as any; + // extract associative table and columns meta assocTableMetas.push({ modelMeta: assocModelMeta, colMeta, @@ -166,8 +170,11 @@ export async function importLTARData({ }); } + // Iterate over all related M2M associative table for (const assocMeta of assocTableMetas) { const insertData = []; + + // extract insert data from records for (const record of allData) { const rec = record.fields; @@ -180,6 +187,7 @@ export async function importLTARData({ ); } + // Insert datas as chunks of size `ASSOC_BULK_DATA_BATCH_SIZE` for ( let i = 0; i < insertData.length / ASSOC_BULK_DATA_BATCH_SIZE; From 392d2c81ba9d0b32c31f59c80e1cd5a603ae15e1 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 31 May 2022 19:03:41 +0530 Subject: [PATCH 04/12] wip: data and LTAR import Signed-off-by: Pranav C --- .../components/import/ImportFromAirtable.vue | 2 +- packages/nocodb/src/lib/noco/Noco.ts | 2 +- .../src/lib/noco/meta/api/sync/helpers/job.ts | 9 +- .../api/sync/helpers/readAndProcessData.ts | 124 +++++++++++------- 4 files changed, 81 insertions(+), 56 deletions(-) diff --git a/packages/nc-gui/components/import/ImportFromAirtable.vue b/packages/nc-gui/components/import/ImportFromAirtable.vue index 1050809630..d22031d35f 100644 --- a/packages/nc-gui/components/import/ImportFromAirtable.vue +++ b/packages/nc-gui/components/import/ImportFromAirtable.vue @@ -169,7 +169,7 @@ mdi-loading mdi-spin - Syncing + Importing diff --git a/packages/nocodb/src/lib/noco/Noco.ts b/packages/nocodb/src/lib/noco/Noco.ts index 5227638c47..5fe7de491d 100644 --- a/packages/nocodb/src/lib/noco/Noco.ts +++ b/packages/nocodb/src/lib/noco/Noco.ts @@ -211,7 +211,7 @@ export default class Noco { this.router.use(cookieParser()); this.router.use( bodyParser.json({ - limit: process.env.NC_REQUEST_BODY_SIZE || 1024 * 1024 + limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' }) ); this.router.use(morgan('tiny')); diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index 898e4e3ba5..999b5e2f71 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -1261,7 +1261,7 @@ export default async ( async function nocoBaseDataProcessing_v2(sDB, table, record) { const recordHash = hash(record); - const rec = record.fields; + const rec = { ...record.fields }; // kludge - // trim spaces on either side of column name @@ -2230,6 +2230,8 @@ export default async ( logBasic('Reading Records...'); + const recordsMap = {}; + for (let i = 0; i < ncTblList.list.length; i++) { const _perfStart = recordPerfStart(); const ncTbl = await api.dbTable.read(ncTblList.list[i].id); @@ -2242,7 +2244,7 @@ export default async ( recordCnt = 0; // await nocoReadData(syncDB, ncTbl); - await importData({ + recordsMap[ncTbl.id] = await importData({ projectName: syncDB.projectName, table: ncTbl, base, @@ -2267,7 +2269,8 @@ export default async ( fields: null, //Object.values(tblLinkGroup).flat(), logBasic, insertedAssocRef, - logDetailed + logDetailed, + records: recordsMap[ncTbl.id] }); } diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts index a15f5556f1..0dfb0dc870 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts @@ -1,52 +1,70 @@ import { AirtableBase } from 'airtable/lib/airtable_base'; import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk'; -const BULK_DATA_BATCH_SIZE = 500; +const BULK_DATA_BATCH_SIZE = 2000; const ASSOC_BULK_DATA_BATCH_SIZE = 2000; -async function readAndProcessData({ +async function readAllData({ table, fields, - base -}: // logDetailed -// logBasic = _str => () -{ + base, + logBasic = _str => {}, + triggerThreshold = BULK_DATA_BATCH_SIZE, + onThreshold = async _rec => {} +}: { table: { title?: string }; fields?; base: AirtableBase; logBasic?: (string) => void; logDetailed?: (string) => void; + triggerThreshold?: number; + onThreshold?: ( + records: Array<{ fields: any; id: string }>, + allRecords?: Array<{ fields: any; id: string }> + ) => Promise; }): Promise> { return new Promise((resolve, reject) => { const data = []; + let thresholdCbkData = []; + const selectParams: any = { pageSize: 100 }; + if (fields) selectParams.fields = fields; base(table.title) .select(selectParams) .eachPage( async function page(records, fetchNextPage) { - // console.log(JSON.stringify(records, null, 2)); - - // This function (`page`) will get called for each page of records. - // records.forEach(record => callback(table, record)); - // logBasic( - // `:: ${table.title} / ${fields} : ${recordCnt + - // 1} ~ ${(recordCnt += 100)}` - // ); data.push(...records); + thresholdCbkData.push(...records); + + logBasic( + `:: Reading '${table.title}' data :: ${Math.max( + 1, + data.length - records.length + )} - ${data.length}` + ); + + if (thresholdCbkData.length >= triggerThreshold) { + await onThreshold(thresholdCbkData, data); + thresholdCbkData = []; + } // To fetch the next page of records, call `fetchNextPage`. // If there are more records, `page` will get called again. // If there are no more records, `done` will get called. fetchNextPage(); }, - function done(err) { + async function done(err) { if (err) { console.error(err); - reject(err); + return reject(err); + } + if (thresholdCbkData.length) { + await onThreshold(thresholdCbkData, data); + thresholdCbkData = []; } resolve(data); } @@ -61,7 +79,8 @@ export async function importData({ api, nocoBaseDataProcessing_v2, sDB, - logDetailed = _str => {} + logDetailed = _str => {}, + logBasic = _str => {} }: { projectName: string; table: { title?: string; id?: string }; @@ -72,38 +91,31 @@ export async function importData({ api: Api; nocoBaseDataProcessing_v2; sDB; -}) { +}): Promise { try { - // get all data from a table - const allData = []; - const records = await readAndProcessData({ + // @ts-ignore + const records = await readAllData({ table, - base - }); + base, + logDetailed, + logBasic, + async onThreshold(records, allRecords) { + const allData = []; + for (let i = 0; i < records.length; i++) { + const r = await nocoBaseDataProcessing_v2(sDB, table, records[i]); + allData.push(r); + } - for (let i = 0; i < records.length; i++) { - const r = await nocoBaseDataProcessing_v2(sDB, table, records[i]); - allData.push(r); - } + logBasic( + `:: Importing '${table.title}' data :: ${allRecords.length - + records.length + + 1} - ${allRecords.length}` + ); + await api.dbTableRow.bulkCreate('nc', projectName, table.id, allData); + } + }); - for ( - let i = 0; - i < allData.length / BULK_DATA_BATCH_SIZE; - i += BULK_DATA_BATCH_SIZE - ) { - logDetailed( - `Importing '${table.title}' data :: ${i + 1} - ${Math.min( - i + BULK_DATA_BATCH_SIZE, - records.length - )}` - ); - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - table.id, // encodeURIComponent(table.title), - allData.slice(i, BULK_DATA_BATCH_SIZE) - ); - } + return records; } catch (e) { console.log(e); } @@ -116,9 +128,10 @@ export async function importLTARData({ api, projectName, insertedAssocRef = {}, - logDetailed = _str => {} -}: // logBasic = _str => () -{ + logDetailed = _str => {}, + logBasic = _str => {}, + records +}: { projectName: string; table: { title?: string; id?: string }; fields; @@ -127,6 +140,7 @@ export async function importLTARData({ logBasic: (string) => void; api: Api; insertedAssocRef: { [assocTableId: string]: boolean }; + records?: Array<{ fields: any; id: string }>; }) { const assocTableMetas: Array<{ modelMeta: { id?: string; title?: string }; @@ -134,7 +148,15 @@ export async function importLTARData({ curCol: { title?: string }; refCol: { title?: string }; }> = []; - const allData = await readAndProcessData({ table, fields, base }); + const allData = + records || + (await readAllData({ + table, + fields, + base, + logDetailed, + logBasic + })); const modelMeta: any = await api.dbTable.read(table.id); @@ -193,8 +215,8 @@ export async function importLTARData({ i < insertData.length / ASSOC_BULK_DATA_BATCH_SIZE; i += ASSOC_BULK_DATA_BATCH_SIZE ) { - logDetailed( - `Importing '${table.title}' LTAR data :: ${i + 1} - ${Math.min( + logBasic( + `:: Importing '${table.title}' LTAR data :: ${i + 1} - ${Math.min( i + ASSOC_BULK_DATA_BATCH_SIZE, allData.length )}` From 392733daeb174177f25849cb20b8fe1f5f702c09 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Tue, 31 May 2022 19:52:45 +0530 Subject: [PATCH 05/12] feat: update record count, error log Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../src/lib/noco/meta/api/sync/helpers/job.ts | 20 +++++++++++++++++-- .../api/sync/helpers/readAndProcessData.ts | 4 ++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index 999b5e2f71..be5a1239dc 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -104,6 +104,10 @@ export default async ( migrationSkipLog: { count: 0, log: [] + }, + data: { + records: 0, + nestedLinks: 0 } }; @@ -119,6 +123,13 @@ export default async ( async function getAirtableSchema(sDB) { const start = Date.now(); + + if (!sDB.shareId) + throw { + message: + 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details' + }; + if (sDB.shareId.startsWith('exp')) { const template = await FetchAT.readTemplate(sDB.shareId); await FetchAT.initialize(template.template.exploreApplication.shareId); @@ -1895,6 +1906,8 @@ export default async ( logBasic(`:: Grid: ${rtc.view.grid}`); logBasic(`:: Gallery: ${rtc.view.gallery}`); logBasic(`:: Form: ${rtc.view.form}`); + logBasic(`:: Total Records: ${rtc.data.records}`); + logBasic(`:: Total Nested Links: ${rtc.data.nestedLinks}`); const duration = Date.now() - start; logBasic(`:: Migration time: ${duration}`); @@ -1934,7 +1947,9 @@ export default async ( axios: { count: rtc.fetchAt.count, time: rtc.fetchAt.time - } + }, + totalRecords: rtc.data.records, + nestedLinks: rtc.data.nestedLinks } } }); @@ -2254,6 +2269,7 @@ export default async ( sDB: syncDB, logDetailed }); + rtc.data.records += recordsMap[ncTbl.id].length; logDetailed(`Data inserted from ${ncTbl.title}`); } @@ -2261,7 +2277,7 @@ export default async ( logBasic('Configuring Record Links...'); for (let i = 0; i < ncTblList.list.length; i++) { const ncTbl = await api.dbTable.read(ncTblList.list[i].id); - await importLTARData({ + rtc.data.nestedLinks += await importLTARData({ table: ncTbl, projectName: syncDB.projectName, api, diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts index 0dfb0dc870..bd6a8c7b70 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts @@ -118,6 +118,7 @@ export async function importData({ return records; } catch (e) { console.log(e); + return 0; } } @@ -192,6 +193,7 @@ export async function importLTARData({ }); } + let nestedLinkCnt = 0; // Iterate over all related M2M associative table for (const assocMeta of assocTableMetas) { const insertData = []; @@ -209,6 +211,7 @@ export async function importLTARData({ ); } + nestedLinkCnt += insertData.length; // Insert datas as chunks of size `ASSOC_BULK_DATA_BATCH_SIZE` for ( let i = 0; @@ -229,4 +232,5 @@ export async function importLTARData({ ); } } + return nestedLinkCnt; } From 41027aafbba8b35e52586c1adb46222c2d8193a0 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Tue, 31 May 2022 22:04:42 +0530 Subject: [PATCH 06/12] feat: continue to read from import data api during bulk insert procedure Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts index bd6a8c7b70..2d69e39b69 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts @@ -32,6 +32,7 @@ async function readAllData({ }; if (fields) selectParams.fields = fields; + const insertJobs: Promise[] = []; base(table.title) .select(selectParams) @@ -48,7 +49,8 @@ async function readAllData({ ); if (thresholdCbkData.length >= triggerThreshold) { - await onThreshold(thresholdCbkData, data); + await Promise.all(insertJobs); + insertJobs.push(onThreshold(thresholdCbkData, data)); thresholdCbkData = []; } @@ -63,6 +65,7 @@ async function readAllData({ return reject(err); } if (thresholdCbkData.length) { + await Promise.all(insertJobs); await onThreshold(thresholdCbkData, data); thresholdCbkData = []; } From fa2fa9c079f5d1713247ba7ce1bf7c4d08d2c970 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Wed, 1 Jun 2022 10:56:33 +0530 Subject: [PATCH 07/12] fix: loop typo correction Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../lib/noco/meta/api/sync/helpers/readAndProcessData.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts index 2d69e39b69..03736cfea5 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts @@ -216,15 +216,11 @@ export async function importLTARData({ nestedLinkCnt += insertData.length; // Insert datas as chunks of size `ASSOC_BULK_DATA_BATCH_SIZE` - for ( - let i = 0; - i < insertData.length / ASSOC_BULK_DATA_BATCH_SIZE; - i += ASSOC_BULK_DATA_BATCH_SIZE - ) { + for (let i = 0; i < insertData.length; i += ASSOC_BULK_DATA_BATCH_SIZE) { logBasic( `:: Importing '${table.title}' LTAR data :: ${i + 1} - ${Math.min( i + ASSOC_BULK_DATA_BATCH_SIZE, - allData.length + insertData.length )}` ); await api.dbTableRow.bulkCreate( From 6c2788e9ae6f16b8b7d1961912c7acfdea3a4acb Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Wed, 1 Jun 2022 11:12:37 +0530 Subject: [PATCH 08/12] fix: attachment conditional inclusion error Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index be5a1239dc..35eba75510 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -1366,7 +1366,7 @@ export default async ( break; case UITypes.Attachment: - if (syncDB.options.syncLookup) rec[key] = null; + if (!syncDB.options.syncAttachment) rec[key] = null; else { const tempArr = []; for (const v of value) { From 603c5a24deae9121f061a888a8c1e1449c51c232 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Jun 2022 11:23:33 +0530 Subject: [PATCH 09/12] fix: LTAR data insertion Signed-off-by: Pranav C --- .../src/lib/noco/meta/api/sync/helpers/job.ts | 2 +- .../api/sync/helpers/readAndProcessData.ts | 23 +++++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts index 35eba75510..486687d5b3 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/job.ts @@ -1409,7 +1409,7 @@ export default async ( console.log(e); }); - tempArr.push(...rs); + tempArr.push(...(rs || [])); } rec[key] = JSON.stringify(tempArr); } diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts index 03736cfea5..fae6b13dbd 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts @@ -2,7 +2,7 @@ import { AirtableBase } from 'airtable/lib/airtable_base'; import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk'; const BULK_DATA_BATCH_SIZE = 2000; -const ASSOC_BULK_DATA_BATCH_SIZE = 2000; +const ASSOC_BULK_DATA_BATCH_SIZE = 5000; async function readAllData({ table, @@ -199,14 +199,14 @@ export async function importLTARData({ let nestedLinkCnt = 0; // Iterate over all related M2M associative table for (const assocMeta of assocTableMetas) { - const insertData = []; + const assocTableData = []; // extract insert data from records for (const record of allData) { const rec = record.fields; // todo: use actual alias instead of sanitized - insertData.push( + assocTableData.push( ...(rec?.[assocMeta.colMeta.title] || []).map(id => ({ [assocMeta.curCol.title]: record.id, [assocMeta.refCol.title]: id @@ -214,20 +214,29 @@ export async function importLTARData({ ); } - nestedLinkCnt += insertData.length; + nestedLinkCnt += assocTableData.length; // Insert datas as chunks of size `ASSOC_BULK_DATA_BATCH_SIZE` - for (let i = 0; i < insertData.length; i += ASSOC_BULK_DATA_BATCH_SIZE) { + for ( + let i = 0; + i < assocTableData.length; + i += ASSOC_BULK_DATA_BATCH_SIZE + ) { logBasic( `:: Importing '${table.title}' LTAR data :: ${i + 1} - ${Math.min( i + ASSOC_BULK_DATA_BATCH_SIZE, - insertData.length + assocTableData.length )}` ); + + console.log( + assocTableData.slice(i, i + ASSOC_BULK_DATA_BATCH_SIZE).length + ); + await api.dbTableRow.bulkCreate( 'nc', projectName, assocMeta.modelMeta.id, - insertData.slice(i, ASSOC_BULK_DATA_BATCH_SIZE) + assocTableData.slice(i, i + ASSOC_BULK_DATA_BATCH_SIZE) ); } } From 7a3a939cd80728ea6bb10af42dbb33a401a2cdc0 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Jun 2022 15:37:05 +0530 Subject: [PATCH 10/12] feat: use new attachment api Signed-off-by: Pranav C --- .../editableCell/EditableAttachmentCell.vue | 10 +- .../project/spreadsheet/helpers/imageExt.js | 4 +- .../src/lib/noco/meta/api/sync/helpers/job.ts | 103 ++++++++++-------- 3 files changed, 66 insertions(+), 51 deletions(-) diff --git a/packages/nc-gui/components/project/spreadsheet/components/editableCell/EditableAttachmentCell.vue b/packages/nc-gui/components/project/spreadsheet/components/editableCell/EditableAttachmentCell.vue index d98e2b4758..3093bbd95c 100644 --- a/packages/nc-gui/components/project/spreadsheet/components/editableCell/EditableAttachmentCell.vue +++ b/packages/nc-gui/components/project/spreadsheet/components/editableCell/EditableAttachmentCell.vue @@ -27,7 +27,7 @@