diff --git a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts index 0e6a2c073d..9c49608848 100644 --- a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts @@ -1513,7 +1513,14 @@ class BaseModelSqlv2 { } } - async bulkInsert(datas: any[]) { + async bulkInsert( + datas: any[], + { + chunkSize: _chunkSize = 100 + }: { + chunkSize?: number; + } = {} + ) { try { const insertDatas = await Promise.all( datas.map(async d => { @@ -1536,7 +1543,7 @@ class BaseModelSqlv2 { // fallbacks to `10` if database client is sqlite // to avoid `too many SQL variables` error // refer : https://www.sqlite.org/limits.html - const chunkSize = this.isSqlite ? 10 : 50; + const chunkSize = this.isSqlite ? 10 : _chunkSize; const response = await this.dbDriver .batchInsert(this.model.table_name, insertDatas, chunkSize) diff --git a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts index b255be6827..a15f5556f1 100644 --- a/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/lib/noco/meta/api/sync/helpers/readAndProcessData.ts @@ -126,7 +126,7 @@ export async function importLTARData({ logDetailed: (string) => void; logBasic: (string) => void; api: Api; - insertedAssocRef: { [assocId: string]: boolean }; + insertedAssocRef: { [assocTableId: string]: boolean }; }) { const assocTableMetas: Array<{ modelMeta: { id?: string; title?: string }; @@ -139,6 +139,7 @@ export async function importLTARData({ const modelMeta: any = await api.dbTable.read(table.id); for (const colMeta of modelMeta.columns) { + // skip columns which are not LTAR and Many to many if ( colMeta.uidt !== UITypes.LinkToAnotherRecord || colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY @@ -146,14 +147,17 @@ export async function importLTARData({ continue; } + // skip if already inserted if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue; + // mark as inserted insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; const assocModelMeta: TableType = (await api.dbTable.read( colMeta.colOptions.fk_mm_model_id )) as any; + // extract associative table and columns meta assocTableMetas.push({ modelMeta: assocModelMeta, colMeta, @@ -166,8 +170,11 @@ export async function importLTARData({ }); } + // Iterate over all related M2M associative table for (const assocMeta of assocTableMetas) { const insertData = []; + + // extract insert data from records for (const record of allData) { const rec = record.fields; @@ -180,6 +187,7 @@ export async function importLTARData({ ); } + // Insert datas as chunks of size `ASSOC_BULK_DATA_BATCH_SIZE` for ( let i = 0; i < insertData.length / ASSOC_BULK_DATA_BATCH_SIZE;