diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index 0b31f370d4..2c649f06df 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -2067,14 +2067,17 @@ class BaseModelSqlv2 { chunkSize: _chunkSize = 100, cookie, foreign_key_checks = true, + raw = false, }: { chunkSize?: number; cookie?: any; foreign_key_checks?: boolean; + raw?: boolean; } = {} ) { try { - const insertDatas = await Promise.all( + // TODO: ag column handling for raw bulk insert + const insertDatas = raw ? datas : await Promise.all( datas.map(async (d) => { await populatePk(this.model, d); return this.model.mapAliasToColumn(d); @@ -2083,8 +2086,10 @@ class BaseModelSqlv2 { // await this.beforeInsertb(insertDatas, null); - for (const data of datas) { - await this.validate(data); + if (!raw) { + for (const data of datas) { + await this.validate(data); + } } // fallbacks to `10` if database client is sqlite @@ -2123,7 +2128,7 @@ class BaseModelSqlv2 { await trx.commit(); - await this.afterBulkInsert(insertDatas, this.dbDriver, cookie); + if (!raw) await this.afterBulkInsert(insertDatas, this.dbDriver, cookie); return response; } catch (e) { diff --git a/packages/nocodb/src/lib/services/dbData/bulkData.ts b/packages/nocodb/src/lib/services/dbData/bulkData.ts index f67edab921..e8c86e063e 100644 --- a/packages/nocodb/src/lib/services/dbData/bulkData.ts +++ b/packages/nocodb/src/lib/services/dbData/bulkData.ts @@ -40,12 +40,13 @@ export async function bulkDataInsert( cookie: any; chunkSize?: number; foreign_key_checks?: boolean; + raw?: boolean; } ) { return await executeBulkOperation({ ...param, operation: 'bulkInsert', - options: [param.body, { cookie: param.cookie, foreign_key_checks: param.foreign_key_checks, chunkSize: param.chunkSize }], + options: [param.body, { cookie: param.cookie, foreign_key_checks: param.foreign_key_checks, chunkSize: param.chunkSize, raw: param.raw }], }); } diff --git a/packages/nocodb/src/lib/services/exportImport/import.svc.ts b/packages/nocodb/src/lib/services/exportImport/import.svc.ts index c5a25b54b2..e6c9972822 100644 --- a/packages/nocodb/src/lib/services/exportImport/import.svc.ts +++ b/packages/nocodb/src/lib/services/exportImport/import.svc.ts @@ -653,9 +653,9 @@ export async function importBase(param: { base_id: baseId, colId: col.colOptions.fk_child_column_id, }); - headers.push(childCol.title); + headers.push(childCol.column_name); } else { - headers.push(col.title); + headers.push(col.column_name); } } else { debugLog(header); @@ -671,7 +671,7 @@ export async function importBase(param: { } } chunk.push(row); - if (chunk.length > 1000) { + if (chunk.length > 100) { parser.pause(); elapsedTime('before import chunk'); try { @@ -681,7 +681,8 @@ export async function importBase(param: { body: chunk, cookie: null, chunkSize: chunk.length + 1, - foreign_key_checks: false + foreign_key_checks: false, + raw: true, }); } catch (e) { debugLog(`${model.title} import throwed an error!`); @@ -704,7 +705,8 @@ export async function importBase(param: { body: chunk, cookie: null, chunkSize: chunk.length + 1, - foreign_key_checks: false + foreign_key_checks: false, + raw: true, }); } catch (e) { debugLog(chunk); @@ -779,8 +781,8 @@ export async function importBase(param: { const vParentCol = await colOptions.getMMParentColumn(); mmParentChild[col.colOptions.fk_mm_model_id] = { - parent: vParentCol.title, - child: vChildCol.title, + parent: vParentCol.column_name, + child: vChildCol.column_name, } handledLinks.push(col.colOptions.fk_mm_model_id); @@ -816,7 +818,8 @@ export async function importBase(param: { body: v, cookie: null, chunkSize: 1000, - foreign_key_checks: false + foreign_key_checks: false, + raw: true, }); elapsedTime('insert link chunk'); } catch (e) {