github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Merge pull request #8315 from nocodb/nc-feat/improve-bulk-update 

Nc feat/improve bulk update
pull/8319/head
Pranav C 7 months ago committed by GitHub
parent
e05ca5ea25 f4bbf64109
commit
caef9b0350
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 291 additions and 186 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 10
      packages/nc-gui/composables/useExtensionHelper.ts
  2. 266
      packages/nocodb/src/db/BaseModelSqlv2.ts
  3. 11
      packages/nocodb/src/models/Model.ts

10
packages/nc-gui/composables/useExtensionHelper.ts
Unescape View File

@ -47,7 +47,7 @@ const [useProvideExtensionHelper, useExtensionHelper] = useInjectionState((exten
}) => {
const { tableId, viewId, eachPage, done } = params
let page = 0
let page = 1
const nextPage = async () => {
const { list: records, pageInfo } = await $api.dbViewRow.list(
@ -56,8 +56,8 @@ const [useProvideExtensionHelper, useExtensionHelper] = useInjectionState((exten
tableId,
viewId as string,
{
offset: (page - 1) * 25,
limit: 25,
offset: (page - 1) * 100,
limit: 100,
} as any,
)
@ -175,14 +175,14 @@ const [useProvideExtensionHelper, useExtensionHelper] = useInjectionState((exten
if (insert.length) {
insertCounter += insert.length
for (let i = 0; i < insert.length; i += chunkSize) {
while (insert.length) {
await $api.dbDataTableRow.create(tableId, insert.splice(0, chunkSize))
}
}
if (update.length) {
updateCounter += update.length
for (let i = 0; i < update.length; i += chunkSize) {
while (update.length) {
await $api.dbDataTableRow.update(tableId, update.splice(0, chunkSize))
}
}

266
packages/nocodb/src/db/BaseModelSqlv2.ts
Unescape View File

@ -4,6 +4,7 @@ import DataLoader from 'dataloader';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc.js';
import timezone from 'dayjs/plugin/timezone';
import equal from 'fast-deep-equal';
import { nocoExecute } from 'nc-help';
import {
AuditOperationSubTypes,
@ -260,11 +261,12 @@ class BaseModelSqlv2 {
} = {},
validateFormula = false,
): Promise<any> {
const columns = await this.model.getColumns();
const { where, ...rest } = this._getListArgs(args as any);
const qb = this.dbDriver(this.tnPath);
await this.selectObject({ ...args, qb, validateFormula });
await this.selectObject({ ...args, qb, validateFormula, columns });
const aliasColObjMap = await this.model.getAliasColObjMap();
const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const sorts = extractSortsObject(rest?.sort, aliasColObjMap);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap);
@ -296,8 +298,7 @@ class BaseModelSqlv2 {
try {
data = await this.execAndParse(qb, null, { first: true });
} catch (e) {
if (validateFormula || !haveFormulaColumn(await this.model.getColumns()))
throw e;
if (validateFormula || !haveFormulaColumn(columns)) throw e;
logger.log(e);
return this.findOne(args, true);
}
@ -319,6 +320,7 @@ class BaseModelSqlv2 {
sort?: string | string[];
fieldsSet?: Set<string>;
limitOverride?: number;
pks?: string;
} = {},
options: {
ignoreViewFilterAndSort?: boolean;
@ -336,6 +338,8 @@ class BaseModelSqlv2 {
limitOverride,
} = options;
const columns = await this.model.getColumns();
const { where, fields, ...rest } = this._getListArgs(args as any);
const qb = this.dbDriver(this.tnPath);
@ -345,12 +349,13 @@ class BaseModelSqlv2 {
fieldsSet: args.fieldsSet,
viewId: this.viewId,
validateFormula,
columns,
});
if (+rest?.shuffle) {
await this.shuffle({ qb });
}
const aliasColObjMap = await this.model.getAliasColObjMap();
const aliasColObjMap = await this.model.getAliasColObjMap(columns);
let sorts = extractSortsObject(
rest?.sort,
aliasColObjMap,
@ -453,8 +458,7 @@ class BaseModelSqlv2 {
try {
data = await this.execAndParse(qb);
} catch (e) {
if (validateFormula || !haveFormulaColumn(await this.model.getColumns()))
throw e;
if (validateFormula || !haveFormulaColumn(columns)) throw e;
logger.log(e);
return this.list(args, {
ignoreViewFilterAndSort,
@ -474,13 +478,13 @@ class BaseModelSqlv2 {
ignoreViewFilterAndSort = false,
throwErrorIfInvalidParams = false,
): Promise<any> {
await this.model.getColumns();
const columns = await this.model.getColumns();
const { where } = this._getListArgs(args);
const qb = this.dbDriver(this.tnPath);
// qb.xwhere(where, await this.model.getAliasColMapping());
const aliasColObjMap = await this.model.getAliasColObjMap();
const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(
where,
aliasColObjMap,
@ -562,6 +566,8 @@ class BaseModelSqlv2 {
widgetFilterArr?: Filter[];
},
) {
const columns = await this.model.getColumns();
const { where, ...rest } = this._getListArgs(args as any);
const qb = this.dbDriver(this.tnPath);
@ -579,7 +585,7 @@ class BaseModelSqlv2 {
await this.shuffle({ qb });
}
const aliasColObjMap = await this.model.getAliasColObjMap();
const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap);
await conditionV2(
@ -621,7 +627,7 @@ class BaseModelSqlv2 {
args.column_name = args.column_name || '';
const cols = await this.model.getColumns();
const columns = await this.model.getColumns();
const groupByColumns: Record<string, Column> = {};
const selectors = [];
@ -630,7 +636,9 @@ class BaseModelSqlv2 {
await Promise.all(
args.column_name.split(',').map(async (col) => {
let column = cols.find((c) => c.column_name === col || c.title === col);
let column = columns.find(
(c) => c.column_name === col || c.title === col,
);
if (!column) {
throw NcError.fieldNotFound(col);
}
@ -712,7 +720,7 @@ class BaseModelSqlv2 {
break;
default:
{
const columnName = await getColumnName(column, cols);
const columnName = await getColumnName(column, columns);
selectors.push(
this.dbDriver.raw('?? as ??', [columnName, column.id]),
);
@ -735,7 +743,7 @@ class BaseModelSqlv2 {
await this.shuffle({ qb });
}
const aliasColObjMap = await this.model.getAliasColObjMap();
const aliasColObjMap = await this.model.getAliasColObjMap(columns);
let sorts = extractSortsObject(rest?.sort, aliasColObjMap);
@ -788,10 +796,7 @@ class BaseModelSqlv2 {
column.uidt as UITypes,
)
) {
const columnName = await getColumnName(
column,
await this.model.getColumns(),
);
const columnName = await getColumnName(column, columns);
const baseUsers = await BaseUser.getUsersList({
base_id: column.base_id,
@ -843,11 +848,12 @@ class BaseModelSqlv2 {
const groupBySelectors = [];
const getAlias = getAliasGenerator('__nc_gb');
const columns = await this.model.getColumns();
// todo: refactor and avoid duplicate code
await this.model.getColumns().then((cols) =>
Promise.all(
await Promise.all(
args.column_name.split(',').map(async (col) => {
let column = cols.find(
let column = columns.find(
(c) => c.column_name === col || c.title === col,
);
if (!column) {
@ -880,8 +886,7 @@ class BaseModelSqlv2 {
knex: this.dbDriver,
// column,
// alias,
columnOptions:
(await column.getColOptions()) as RollupColumn,
columnOptions: (await column.getColOptions()) as RollupColumn,
})
).builder.as(sanitize(column.id)),
);
@ -932,7 +937,7 @@ class BaseModelSqlv2 {
break;
default:
{
const columnName = await getColumnName(column, cols);
const columnName = await getColumnName(column, columns);
selectors.push(
this.dbDriver.raw('?? as ??', [columnName, column.id]),
);
@ -941,14 +946,13 @@ class BaseModelSqlv2 {
break;
}
}),
),
);
const qb = this.dbDriver(this.tnPath);
qb.count(`${this.model.primaryKey?.column_name || '*'} as count`);
qb.select(...selectors);
const aliasColObjMap = await this.model.getAliasColObjMap();
const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap);
await conditionV2(
@ -2452,7 +2456,7 @@ class BaseModelSqlv2 {
// const columns = _columns ?? (await this.model.getColumns());
// for (const column of columns) {
viewOrTableColumns =
_columns || viewColumns || (await this.model.getColumns());
viewColumns || _columns || (await this.model.getColumns());
}
for (const viewOrTableColumn of viewOrTableColumns) {
const column =
@ -2484,7 +2488,7 @@ class BaseModelSqlv2 {
{
const columnName = await getColumnName(
column,
await this.model.getColumns(),
_columns || (await this.model.getColumns()),
);
if (this.isMySQL) {
// MySQL stores timestamp in UTC but display in timezone
@ -2651,7 +2655,7 @@ class BaseModelSqlv2 {
case UITypes.LastModifiedBy: {
const columnName = await getColumnName(
column,
await this.model.getColumns(),
_columns || (await this.model.getColumns()),
);
res[sanitize(column.id || columnName)] = sanitize(
@ -2705,9 +2709,10 @@ class BaseModelSqlv2 {
data,
this.clientMeta,
this.dbDriver,
columns,
);
await this.validate(insertObj);
await this.validate(insertObj, columns);
if ('beforeInsert' in this) {
await this.beforeInsert(insertObj, trx, cookie);
@ -2952,13 +2957,16 @@ class BaseModelSqlv2 {
async updateByPk(id, data, trx?, cookie?, _disableOptimization = false) {
try {
const columns = await this.model.getColumns();
const updateObj = await this.model.mapAliasToColumn(
data,
this.clientMeta,
this.dbDriver,
columns,
);
await this.validate(data);
await this.validate(data, columns);
await this.beforeUpdate(data, trx, cookie);
@ -2973,7 +2981,7 @@ class BaseModelSqlv2 {
const query = this.dbDriver(this.tnPath)
.update(updateObj)
.where(await this._wherePk(id));
.where(await this._wherePk(id, true));
await this.execAndParse(query, null, { raw: true });
@ -2995,11 +3003,15 @@ class BaseModelSqlv2 {
}
}
async _wherePk(id) {
await this.model.getColumns();
async _wherePk(id, skipGetColumns = false) {
if (!skipGetColumns) await this.model.getColumns();
return _wherePk(this.model.primaryKeys, id);
}
comparePks(pk1, pk2) {
return equal(pk1, pk2);
}
public getTnPath(tb: { table_name: string } | string, alias?: string) {
const tn = typeof tb === 'string' ? tb : tb.table_name;
const schema = (this.dbDriver as any).searchPath?.();
@ -3088,23 +3100,25 @@ class BaseModelSqlv2 {
try {
const source = await Source.get(this.model.source_id);
await populatePk(this.model, data);
const columns = await this.model.getColumns();
const insertObj = await this.model.mapAliasToColumn(
data,
this.clientMeta,
this.dbDriver,
columns,
);
let rowId = null;
const nestedCols = (await this.model.getColumns()).filter((c) =>
isLinksOrLTAR(c),
);
const nestedCols = columns.filter((c) => isLinksOrLTAR(c));
const { postInsertOps, preInsertOps } = await this.prepareNestedLinkQb({
nestedCols,
data,
insertObj,
});
await this.validate(insertObj);
await this.validate(insertObj, columns);
await this.beforeInsert(insertObj, this.dbDriver, cookie);
@ -3402,11 +3416,9 @@ class BaseModelSqlv2 {
let agPkCol: Column;
if (!raw) {
const nestedCols = (await this.model.getColumns()).filter((c) =>
isLinksOrLTAR(c),
);
const columns = await this.model.getColumns();
await this.model.getColumns();
const nestedCols = columns.filter((c) => isLinksOrLTAR(c));
for (const d of datas) {
const insertObj = {};
@ -3687,12 +3699,12 @@ class BaseModelSqlv2 {
) {
let transaction;
try {
if (raw) await this.model.getColumns();
const columns = await this.model.getColumns();
// validate update data
if (!raw) {
for (const d of datas) {
await this.validate(d);
await this.validate(d, columns);
}
}
@ -3700,7 +3712,12 @@ class BaseModelSqlv2 {
? datas
: await Promise.all(
datas.map((d) =>
this.model.mapAliasToColumn(d, this.clientMeta, this.dbDriver),
this.model.mapAliasToColumn(
d,
this.clientMeta,
this.dbDriver,
columns,
),
),
);
@ -3708,8 +3725,10 @@ class BaseModelSqlv2 {
const newData = [];
const updatePkValues = [];
const toBeUpdated = [];
for (const d of updateDatas) {
const pkValues = await this._extractPksValues(d);
const pkAndData: { pk: any; data: any }[] = [];
const readChunkSize = 100;
for (const [i, d] of updateDatas.entries()) {
const pkValues = this._extractPksValues(d);
if (!pkValues) {
// throw or skip if no pk provided
if (throwExceptionIfNotExist) {
@ -3720,20 +3739,59 @@ class BaseModelSqlv2 {
if (!raw) {
await this.prepareNocoData(d, false, cookie);
const oldRecord = await this.readByPk(pkValues);
pkAndData.push({
pk: pkValues,
data: d,
});
if (
pkAndData.length >= readChunkSize ||
i === updateDatas.length - 1
) {
const tempToRead = pkAndData.splice(0, pkAndData.length);
const oldRecords = await this.list(
{
pks: tempToRead.map((v) => v.pk).join(','),
},
{
limitOverride: tempToRead.length,
},
);
if (oldRecords.length === tempToRead.length) {
prevData.push(...oldRecords);
} else {
for (const recordPk of tempToRead) {
const oldRecord = oldRecords.find((r) =>
this.comparePks(this._extractPksValues(r), recordPk),
);
if (!oldRecord) {
// throw or skip if no record found
if (throwExceptionIfNotExist) {
NcError.recordNotFound(JSON.stringify(pkValues));
NcError.recordNotFound(JSON.stringify(recordPk));
}
continue;
}
prevData.push(oldRecord);
}
const wherePk = await this._wherePk(pkValues);
}
for (const { pk, data } of tempToRead) {
const wherePk = await this._wherePk(pk, true);
toBeUpdated.push({ d: data, wherePk });
updatePkValues.push(pk);
}
}
} else {
const wherePk = await this._wherePk(pkValues, true);
toBeUpdated.push({ d, wherePk });
updatePkValues.push(pkValues);
}
}
transaction = await this.dbDriver.transaction();
@ -3744,9 +3802,17 @@ class BaseModelSqlv2 {
await transaction.commit();
if (!raw) {
for (const pkValues of updatePkValues) {
const updatedRecord = await this.readByPk(pkValues);
newData.push(updatedRecord);
while (updatePkValues.length) {
const updatedRecords = await this.list(
{
pks: updatePkValues.splice(0, readChunkSize).join(','),
},
{
limitOverride: readChunkSize,
},
);
newData.push(...updatedRecords);
}
}
@ -3783,23 +3849,27 @@ class BaseModelSqlv2 {
) {
try {
let count = 0;
const columns = await this.model.getColumns();
const updateData = await this.model.mapAliasToColumn(
data,
this.clientMeta,
this.dbDriver,
columns,
);
if (!args.skipValidationAndHooks) await this.validate(updateData);
if (!args.skipValidationAndHooks)
await this.validate(updateData, columns);
await this.prepareNocoData(updateData, false, cookie);
const pkValues = await this._extractPksValues(updateData);
const pkValues = this._extractPksValues(updateData);
if (pkValues) {
// pk is specified - by pass
} else {
await this.model.getColumns();
const { where } = this._getListArgs(args);
const qb = this.dbDriver(this.tnPath);
const aliasColObjMap = await this.model.getAliasColObjMap();
const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap, true);
const conditionObj = [
@ -3864,18 +3934,27 @@ class BaseModelSqlv2 {
isSingleRecordDeletion?: boolean;
} = {},
) {
const columns = await this.model.getColumns();
let transaction;
try {
const deleteIds = await Promise.all(
ids.map((d) =>
this.model.mapAliasToColumn(d, this.clientMeta, this.dbDriver),
this.model.mapAliasToColumn(
d,
this.clientMeta,
this.dbDriver,
columns,
),
),
);
const deleted = [];
const res = [];
for (const d of deleteIds) {
const pkValues = await this._extractPksValues(d);
const pkAndData: { pk: any; data: any }[] = [];
const readChunkSize = 100;
for (const [i, d] of deleteIds.entries()) {
const pkValues = this._extractPksValues(d);
if (!pkValues) {
// throw or skip if no pk provided
if (throwExceptionIfNotExist) {
@ -3884,17 +3963,41 @@ class BaseModelSqlv2 {
continue;
}
const deletedRecord = await this.readByPk(pkValues);
if (!deletedRecord) {
pkAndData.push({ pk: pkValues, data: d });
if (pkAndData.length >= readChunkSize || i === deleteIds.length - 1) {
const tempToRead = pkAndData.splice(0, pkAndData.length);
const oldRecords = await this.list(
{
pks: tempToRead.map((v) => v.pk).join(','),
},
{
limitOverride: tempToRead.length,
},
);
if (oldRecords.length === tempToRead.length) {
deleted.push(...oldRecords);
res.push(...tempToRead.map((v) => v.data));
} else {
for (const { pk, data } of tempToRead) {
const oldRecord = oldRecords.find((r) =>
this.comparePks(this._extractPksValues(r), pk),
);
if (!oldRecord) {
// throw or skip if no record found
if (throwExceptionIfNotExist) {
NcError.recordNotFound(JSON.stringify(pkValues));
NcError.recordNotFound(JSON.stringify(pk));
}
continue;
}
deleted.push(deletedRecord);
res.push(d);
deleted.push(oldRecord);
res.push(data);
}
}
}
}
const execQueries: ((
@ -3989,10 +4092,10 @@ class BaseModelSqlv2 {
) {
let trx: Knex.Transaction;
try {
await this.model.getColumns();
const columns = await this.model.getColumns();
const { where } = this._getListArgs(args);
const qb = this.dbDriver(this.tnPath);
const aliasColObjMap = await this.model.getAliasColObjMap();
const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap, true);
await conditionV2(
@ -4336,10 +4439,13 @@ class BaseModelSqlv2 {
protected async errorDelete(_e, _id, _trx, _cookie) {}
async validate(data: Record<string, any>): Promise<boolean> {
await this.model.getColumns();
async validate(
data: Record<string, any>,
columns?: Column[],
): Promise<boolean> {
const cols = columns || (await this.model.getColumns());
// let cols = Object.keys(this.columns);
for (let i = 0; i < this.model.columns.length; ++i) {
for (let i = 0; i < cols.length; ++i) {
const column = this.model.columns[i];
if (
@ -4836,9 +4942,8 @@ class BaseModelSqlv2 {
> {
try {
const { where, ...rest } = this._getListArgs(args as any);
const column = await this.model
.getColumns()
.then((cols) => cols?.find((col) => col.id === args.groupColumnId));
const columns = await this.model.getColumns();
const column = columns?.find((col) => col.id === args.groupColumnId);
if (!column) NcError.fieldNotFound(args.groupColumnId);
if (isVirtualCol(column))
@ -4876,7 +4981,7 @@ class BaseModelSqlv2 {
await this.selectObject({ qb, extractPkAndPv: true });
// todo: refactor and move to a method (applyFilterAndSort)
const aliasColObjMap = await this.model.getAliasColObjMap();
const aliasColObjMap = await this.model.getAliasColObjMap(columns);
let sorts = extractSortsObject(args?.sort, aliasColObjMap);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap);
// todo: replace with view id
@ -4998,9 +5103,8 @@ class BaseModelSqlv2 {
ignoreViewFilterAndSort?: boolean;
} & XcFilter,
) {
const column = await this.model
.getColumns()
.then((cols) => cols?.find((col) => col.id === args.groupColumnId));
const columns = await this.model.getColumns();
const column = columns?.find((col) => col.id === args.groupColumnId);
if (!column) NcError.fieldNotFound(args.groupColumnId);
if (isVirtualCol(column))
@ -5011,7 +5115,7 @@ class BaseModelSqlv2 {
.groupBy(column.column_name);
// todo: refactor and move to a common method (applyFilterAndSort)
const aliasColObjMap = await this.model.getAliasColObjMap();
const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(args.where, aliasColObjMap);
// todo: replace with view id
@ -6192,12 +6296,12 @@ class BaseModelSqlv2 {
args: { limit?; offset?; fieldSet?: Set<string> } = {},
) {
try {
const columns = await this.model.getColumns();
const { where, sort } = this._getListArgs(args as any);
// todo: get only required fields
const relColumn = (await this.model.getColumns()).find(
(c) => c.id === colId,
);
const relColumn = columns.find((c) => c.id === colId);
const row = await this.execAndParse(
this.dbDriver(this.tnPath).where(await this._wherePk(id)),

11
packages/nocodb/src/models/Model.ts
Unescape View File

@ -538,9 +538,10 @@ export default class Model implements TableType {
isPg: false,
},
knex,
columns?: Column[],
) {
const insertObj = {};
for (const col of await this.getColumns()) {
for (const col of columns || (await this.getColumns())) {
if (isVirtualCol(col)) continue;
let val =
data?.[col.column_name] !== undefined
@ -618,9 +619,9 @@ export default class Model implements TableType {
return insertObj;
}
async mapColumnToAlias(data) {
async mapColumnToAlias(data, columns?: Column[]) {
const res = {};
for (const col of await this.getColumns()) {
for (const col of columns || (await this.getColumns())) {
if (isVirtualCol(col)) continue;
let val =
data?.[col.title] !== undefined
@ -969,8 +970,8 @@ export default class Model implements TableType {
));
}
async getAliasColObjMap() {
return (await this.getColumns()).reduce(
async getAliasColObjMap(columns?: Column[]) {
return (columns || (await this.getColumns())).reduce(
(sortAgg, c) => ({ ...sortAgg, [c.title]: c }),
{},
);

Write Preview
Loading…
Cancel
Save