Browse Source

feat: foreign_key_checks and raw option for bulkInsert operation

Signed-off-by: mertmit <mertmit99@gmail.com>
feat/export-nest-dir-restructure
mertmit 2 years ago committed by starbirdtech383
parent
commit
66aa73939e
  1. 55
      packages/nocodb-nest/src/db/BaseModelSqlv2.ts
  2. 12
      packages/nocodb-nest/src/services/bulk-data-alias.service.ts

55
packages/nocodb-nest/src/db/BaseModelSqlv2.ts

@ -2097,23 +2097,32 @@ class BaseModelSqlv2 {
{ {
chunkSize: _chunkSize = 100, chunkSize: _chunkSize = 100,
cookie, cookie,
foreign_key_checks = true,
raw = false,
}: { }: {
chunkSize?: number; chunkSize?: number;
cookie?: any; cookie?: any;
foreign_key_checks?: boolean;
raw?: boolean;
} = {}, } = {},
) { ) {
try { try {
const insertDatas = await Promise.all( // TODO: ag column handling for raw bulk insert
datas.map(async (d) => { const insertDatas = raw
await populatePk(this.model, d); ? datas
return this.model.mapAliasToColumn(d); : await Promise.all(
}), datas.map(async (d) => {
); await populatePk(this.model, d);
return this.model.mapAliasToColumn(d);
}),
);
// await this.beforeInsertb(insertDatas, null); // await this.beforeInsertb(insertDatas, null);
for (const data of datas) { if (!raw) {
await this.validate(data); for (const data of datas) {
await this.validate(data);
}
} }
// fallbacks to `10` if database client is sqlite // fallbacks to `10` if database client is sqlite
@ -2121,18 +2130,34 @@ class BaseModelSqlv2 {
// refer : https://www.sqlite.org/limits.html // refer : https://www.sqlite.org/limits.html
const chunkSize = this.isSqlite ? 10 : _chunkSize; const chunkSize = this.isSqlite ? 10 : _chunkSize;
const trx = await this.dbDriver.transaction();
if (!foreign_key_checks) {
if (this.isPg) {
await trx.raw('set session_replication_role to replica;');
} else if (this.isMySQL) {
await trx.raw('SET foreign_key_checks = 0;');
}
}
const response = const response =
this.isPg || this.isMssql this.isPg || this.isMssql
? await this.dbDriver ? await trx
.batchInsert(this.tnPath, insertDatas, chunkSize) .batchInsert(this.tnPath, insertDatas, chunkSize)
.returning(this.model.primaryKey?.column_name) .returning(this.model.primaryKey?.column_name)
: await this.dbDriver.batchInsert( : await trx.batchInsert(this.tnPath, insertDatas, chunkSize);
this.tnPath,
insertDatas, if (!foreign_key_checks) {
chunkSize, if (this.isPg) {
); await trx.raw('set session_replication_role to origin;');
} else if (this.isMySQL) {
await trx.raw('SET foreign_key_checks = 1;');
}
}
await trx.commit();
await this.afterBulkInsert(insertDatas, this.dbDriver, cookie); if (!raw) await this.afterBulkInsert(insertDatas, this.dbDriver, cookie);
return response; return response;
} catch (e) { } catch (e) {

12
packages/nocodb-nest/src/services/bulk-data-alias.service.ts

@ -44,12 +44,22 @@ export class BulkDataAliasService {
param: PathParams & { param: PathParams & {
body: any; body: any;
cookie: any; cookie: any;
chunkSize?: number;
foreign_key_checks?: boolean;
raw?: boolean;
}, },
) { ) {
return await this.executeBulkOperation({ return await this.executeBulkOperation({
...param, ...param,
operation: 'bulkInsert', operation: 'bulkInsert',
options: [param.body, { cookie: param.cookie }], options: [
param.body,
{
cookie: param.cookie,
foreign_key_checks: param.foreign_key_checks,
raw: param.raw,
},
],
}); });
} }

Loading…
Cancel
Save