github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Merge pull request #8315 from nocodb/nc-feat/improve-bulk-update 

Nc feat/improve bulk update
pull/8319/head
Pranav C 7 months ago committed by GitHub
parent
e05ca5ea25 f4bbf64109
commit
caef9b0350
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 291 additions and 186 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 10
      packages/nc-gui/composables/useExtensionHelper.ts
  2. 456
      packages/nocodb/src/db/BaseModelSqlv2.ts
  3. 11
      packages/nocodb/src/models/Model.ts

10
packages/nc-gui/composables/useExtensionHelper.ts
Unescape View File

@ -47,7 +47,7 @@ const [useProvideExtensionHelper, useExtensionHelper] = useInjectionState((exten
}) => { }) => {
const { tableId, viewId, eachPage, done } = params const { tableId, viewId, eachPage, done } = params
let page = 0 let page = 1
const nextPage = async () => { const nextPage = async () => {
const { list: records, pageInfo } = await $api.dbViewRow.list( const { list: records, pageInfo } = await $api.dbViewRow.list(
@ -56,8 +56,8 @@ const [useProvideExtensionHelper, useExtensionHelper] = useInjectionState((exten
tableId, tableId,
viewId as string, viewId as string,
{ {
offset: (page - 1) * 25, offset: (page - 1) * 100,
limit: 25, limit: 100,
} as any, } as any,
) )
@ -175,14 +175,14 @@ const [useProvideExtensionHelper, useExtensionHelper] = useInjectionState((exten
if (insert.length) { if (insert.length) {
insertCounter += insert.length insertCounter += insert.length
for (let i = 0; i < insert.length; i += chunkSize) { while (insert.length) {
await $api.dbDataTableRow.create(tableId, insert.splice(0, chunkSize)) await $api.dbDataTableRow.create(tableId, insert.splice(0, chunkSize))
} }
} }
if (update.length) { if (update.length) {
updateCounter += update.length updateCounter += update.length
for (let i = 0; i < update.length; i += chunkSize) { while (update.length) {
await $api.dbDataTableRow.update(tableId, update.splice(0, chunkSize)) await $api.dbDataTableRow.update(tableId, update.splice(0, chunkSize))
} }
} }

456
packages/nocodb/src/db/BaseModelSqlv2.ts
Unescape View File

@ -4,6 +4,7 @@ import DataLoader from 'dataloader';
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc.js'; import utc from 'dayjs/plugin/utc.js';
import timezone from 'dayjs/plugin/timezone'; import timezone from 'dayjs/plugin/timezone';
import equal from 'fast-deep-equal';
import { nocoExecute } from 'nc-help'; import { nocoExecute } from 'nc-help';
import { import {
AuditOperationSubTypes, AuditOperationSubTypes,
@ -260,11 +261,12 @@ class BaseModelSqlv2 {
} = {}, } = {},
validateFormula = false, validateFormula = false,
): Promise<any> { ): Promise<any> {
const columns = await this.model.getColumns();
const { where, ...rest } = this._getListArgs(args as any); const { where, ...rest } = this._getListArgs(args as any);
const qb = this.dbDriver(this.tnPath); const qb = this.dbDriver(this.tnPath);
await this.selectObject({ ...args, qb, validateFormula }); await this.selectObject({ ...args, qb, validateFormula, columns });
const aliasColObjMap = await this.model.getAliasColObjMap(); const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const sorts = extractSortsObject(rest?.sort, aliasColObjMap); const sorts = extractSortsObject(rest?.sort, aliasColObjMap);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap); const filterObj = extractFilterFromXwhere(where, aliasColObjMap);
@ -296,8 +298,7 @@ class BaseModelSqlv2 {
try { try {
data = await this.execAndParse(qb, null, { first: true }); data = await this.execAndParse(qb, null, { first: true });
} catch (e) { } catch (e) {
if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) if (validateFormula || !haveFormulaColumn(columns)) throw e;
throw e;
logger.log(e); logger.log(e);
return this.findOne(args, true); return this.findOne(args, true);
} }
@ -319,6 +320,7 @@ class BaseModelSqlv2 {
sort?: string | string[]; sort?: string | string[];
fieldsSet?: Set<string>; fieldsSet?: Set<string>;
limitOverride?: number; limitOverride?: number;
pks?: string;
} = {}, } = {},
options: { options: {
ignoreViewFilterAndSort?: boolean; ignoreViewFilterAndSort?: boolean;
@ -336,6 +338,8 @@ class BaseModelSqlv2 {
limitOverride, limitOverride,
} = options; } = options;
const columns = await this.model.getColumns();
const { where, fields, ...rest } = this._getListArgs(args as any); const { where, fields, ...rest } = this._getListArgs(args as any);
const qb = this.dbDriver(this.tnPath); const qb = this.dbDriver(this.tnPath);
@ -345,12 +349,13 @@ class BaseModelSqlv2 {
fieldsSet: args.fieldsSet, fieldsSet: args.fieldsSet,
viewId: this.viewId, viewId: this.viewId,
validateFormula, validateFormula,
columns,
}); });
if (+rest?.shuffle) { if (+rest?.shuffle) {
await this.shuffle({ qb }); await this.shuffle({ qb });
} }
const aliasColObjMap = await this.model.getAliasColObjMap(); const aliasColObjMap = await this.model.getAliasColObjMap(columns);
let sorts = extractSortsObject( let sorts = extractSortsObject(
rest?.sort, rest?.sort,
aliasColObjMap, aliasColObjMap,
@ -453,8 +458,7 @@ class BaseModelSqlv2 {
try { try {
data = await this.execAndParse(qb); data = await this.execAndParse(qb);
} catch (e) { } catch (e) {
if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) if (validateFormula || !haveFormulaColumn(columns)) throw e;
throw e;
logger.log(e); logger.log(e);
return this.list(args, { return this.list(args, {
ignoreViewFilterAndSort, ignoreViewFilterAndSort,
@ -474,13 +478,13 @@ class BaseModelSqlv2 {
ignoreViewFilterAndSort = false, ignoreViewFilterAndSort = false,
throwErrorIfInvalidParams = false, throwErrorIfInvalidParams = false,
): Promise<any> { ): Promise<any> {
await this.model.getColumns(); const columns = await this.model.getColumns();
const { where } = this._getListArgs(args); const { where } = this._getListArgs(args);
const qb = this.dbDriver(this.tnPath); const qb = this.dbDriver(this.tnPath);
// qb.xwhere(where, await this.model.getAliasColMapping()); // qb.xwhere(where, await this.model.getAliasColMapping());
const aliasColObjMap = await this.model.getAliasColObjMap(); const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere( const filterObj = extractFilterFromXwhere(
where, where,
aliasColObjMap, aliasColObjMap,
@ -562,6 +566,8 @@ class BaseModelSqlv2 {
widgetFilterArr?: Filter[]; widgetFilterArr?: Filter[];
}, },
) { ) {
const columns = await this.model.getColumns();
const { where, ...rest } = this._getListArgs(args as any); const { where, ...rest } = this._getListArgs(args as any);
const qb = this.dbDriver(this.tnPath); const qb = this.dbDriver(this.tnPath);
@ -579,7 +585,7 @@ class BaseModelSqlv2 {
await this.shuffle({ qb }); await this.shuffle({ qb });
} }
const aliasColObjMap = await this.model.getAliasColObjMap(); const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap); const filterObj = extractFilterFromXwhere(where, aliasColObjMap);
await conditionV2( await conditionV2(
@ -621,7 +627,7 @@ class BaseModelSqlv2 {
args.column_name = args.column_name || ''; args.column_name = args.column_name || '';
const cols = await this.model.getColumns(); const columns = await this.model.getColumns();
const groupByColumns: Record<string, Column> = {}; const groupByColumns: Record<string, Column> = {};
const selectors = []; const selectors = [];
@ -630,7 +636,9 @@ class BaseModelSqlv2 {
await Promise.all( await Promise.all(
args.column_name.split(',').map(async (col) => { args.column_name.split(',').map(async (col) => {
let column = cols.find((c) => c.column_name === col || c.title === col); let column = columns.find(
(c) => c.column_name === col || c.title === col,
);
if (!column) { if (!column) {
throw NcError.fieldNotFound(col); throw NcError.fieldNotFound(col);
} }
@ -712,7 +720,7 @@ class BaseModelSqlv2 {
break; break;
default: default:
{ {
const columnName = await getColumnName(column, cols); const columnName = await getColumnName(column, columns);
selectors.push( selectors.push(
this.dbDriver.raw('?? as ??', [columnName, column.id]), this.dbDriver.raw('?? as ??', [columnName, column.id]),
); );
@ -735,7 +743,7 @@ class BaseModelSqlv2 {
await this.shuffle({ qb }); await this.shuffle({ qb });
} }
const aliasColObjMap = await this.model.getAliasColObjMap(); const aliasColObjMap = await this.model.getAliasColObjMap(columns);
let sorts = extractSortsObject(rest?.sort, aliasColObjMap); let sorts = extractSortsObject(rest?.sort, aliasColObjMap);
@ -788,10 +796,7 @@ class BaseModelSqlv2 {
column.uidt as UITypes, column.uidt as UITypes,
) )
) { ) {
const columnName = await getColumnName( const columnName = await getColumnName(column, columns);
column,
await this.model.getColumns(),
);
const baseUsers = await BaseUser.getUsersList({ const baseUsers = await BaseUser.getUsersList({
base_id: column.base_id, base_id: column.base_id,
@ -843,112 +848,111 @@ class BaseModelSqlv2 {
const groupBySelectors = []; const groupBySelectors = [];
const getAlias = getAliasGenerator('__nc_gb'); const getAlias = getAliasGenerator('__nc_gb');
const columns = await this.model.getColumns();
// todo: refactor and avoid duplicate code // todo: refactor and avoid duplicate code
await this.model.getColumns().then((cols) => await Promise.all(
Promise.all( args.column_name.split(',').map(async (col) => {
args.column_name.split(',').map(async (col) => { let column = columns.find(
let column = cols.find( (c) => c.column_name === col || c.title === col,
(c) => c.column_name === col || c.title === col, );
); if (!column) {
if (!column) { throw NcError.fieldNotFound(col);
throw NcError.fieldNotFound(col); }
}
// if qrCode or Barcode replace it with value column nd keep the alias // if qrCode or Barcode replace it with value column nd keep the alias
if ([UITypes.QrCode, UITypes.Barcode].includes(column.uidt)) if ([UITypes.QrCode, UITypes.Barcode].includes(column.uidt))
column = new Column({ column = new Column({
...(await column ...(await column
.getColOptions<BarcodeColumn | QrCodeColumn>() .getColOptions<BarcodeColumn | QrCodeColumn>()
.then((col) => col.getValueColumn())), .then((col) => col.getValueColumn())),
title: column.title, title: column.title,
id: column.id, id: column.id,
}); });
switch (column.uidt) { switch (column.uidt) {
case UITypes.Attachment: case UITypes.Attachment:
NcError.badRequest( NcError.badRequest(
'Group by using attachment column is not supported', 'Group by using attachment column is not supported',
); );
break; break;
case UITypes.Rollup: case UITypes.Rollup:
case UITypes.Links: case UITypes.Links:
selectors.push( selectors.push(
( (
await genRollupSelectv2({ await genRollupSelectv2({
baseModelSqlv2: this, baseModelSqlv2: this,
// tn: this.title, // tn: this.title,
knex: this.dbDriver, knex: this.dbDriver,
// column, // column,
// alias, // alias,
columnOptions: columnOptions: (await column.getColOptions()) as RollupColumn,
(await column.getColOptions()) as RollupColumn, })
}) ).builder.as(sanitize(column.id)),
).builder.as(sanitize(column.id)), );
groupBySelectors.push(sanitize(column.id));
break;
case UITypes.Formula: {
let selectQb;
try {
const _selectQb = await this.getSelectQueryBuilderForFormula(
column,
); );
groupBySelectors.push(sanitize(column.id));
break;
case UITypes.Formula: {
let selectQb;
try {
const _selectQb = await this.getSelectQueryBuilderForFormula(
column,
);
selectQb = this.dbDriver.raw(`?? as ??`, [
_selectQb.builder,
sanitize(column.id),
]);
} catch (e) {
logger.log(e);
// return dummy select
selectQb = this.dbDriver.raw(`'ERR' as ??`, [
sanitize(column.id),
]);
}
selectors.push(selectQb); selectQb = this.dbDriver.raw(`?? as ??`, [
groupBySelectors.push(column.id); _selectQb.builder,
break; sanitize(column.id),
]);
} catch (e) {
logger.log(e);
// return dummy select
selectQb = this.dbDriver.raw(`'ERR' as ??`, [
sanitize(column.id),
]);
} }
case UITypes.Lookup:
case UITypes.LinkToAnotherRecord:
{
const _selectQb = await generateLookupSelectQuery({
baseModelSqlv2: this,
column,
alias: null,
model: this.model,
getAlias,
});
const selectQb = this.dbDriver.raw(`?? as ??`, [
this.dbDriver.raw(_selectQb.builder).wrap('(', ')'),
sanitize(column.id),
]);
selectors.push(selectQb); selectors.push(selectQb);
groupBySelectors.push(sanitize(column.id)); groupBySelectors.push(column.id);
} break;
break;
default:
{
const columnName = await getColumnName(column, cols);
selectors.push(
this.dbDriver.raw('?? as ??', [columnName, column.id]),
);
groupBySelectors.push(sanitize(column.id));
}
break;
} }
}), case UITypes.Lookup:
), case UITypes.LinkToAnotherRecord:
{
const _selectQb = await generateLookupSelectQuery({
baseModelSqlv2: this,
column,
alias: null,
model: this.model,
getAlias,
});
const selectQb = this.dbDriver.raw(`?? as ??`, [
this.dbDriver.raw(_selectQb.builder).wrap('(', ')'),
sanitize(column.id),
]);
selectors.push(selectQb);
groupBySelectors.push(sanitize(column.id));
}
break;
default:
{
const columnName = await getColumnName(column, columns);
selectors.push(
this.dbDriver.raw('?? as ??', [columnName, column.id]),
);
groupBySelectors.push(sanitize(column.id));
}
break;
}
}),
); );
const qb = this.dbDriver(this.tnPath); const qb = this.dbDriver(this.tnPath);
qb.count(`${this.model.primaryKey?.column_name || '*'} as count`); qb.count(`${this.model.primaryKey?.column_name || '*'} as count`);
qb.select(...selectors); qb.select(...selectors);
const aliasColObjMap = await this.model.getAliasColObjMap(); const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap); const filterObj = extractFilterFromXwhere(where, aliasColObjMap);
await conditionV2( await conditionV2(
@ -2452,7 +2456,7 @@ class BaseModelSqlv2 {
// const columns = _columns ?? (await this.model.getColumns()); // const columns = _columns ?? (await this.model.getColumns());
// for (const column of columns) { // for (const column of columns) {
viewOrTableColumns = viewOrTableColumns =
_columns || viewColumns || (await this.model.getColumns()); viewColumns || _columns || (await this.model.getColumns());
} }
for (const viewOrTableColumn of viewOrTableColumns) { for (const viewOrTableColumn of viewOrTableColumns) {
const column = const column =
@ -2484,7 +2488,7 @@ class BaseModelSqlv2 {
{ {
const columnName = await getColumnName( const columnName = await getColumnName(
column, column,
await this.model.getColumns(), _columns || (await this.model.getColumns()),
); );
if (this.isMySQL) { if (this.isMySQL) {
// MySQL stores timestamp in UTC but display in timezone // MySQL stores timestamp in UTC but display in timezone
@ -2651,7 +2655,7 @@ class BaseModelSqlv2 {
case UITypes.LastModifiedBy: { case UITypes.LastModifiedBy: {
const columnName = await getColumnName( const columnName = await getColumnName(
column, column,
await this.model.getColumns(), _columns || (await this.model.getColumns()),
); );
res[sanitize(column.id || columnName)] = sanitize( res[sanitize(column.id || columnName)] = sanitize(
@ -2705,9 +2709,10 @@ class BaseModelSqlv2 {
data, data,
this.clientMeta, this.clientMeta,
this.dbDriver, this.dbDriver,
columns,
); );
await this.validate(insertObj); await this.validate(insertObj, columns);
if ('beforeInsert' in this) { if ('beforeInsert' in this) {
await this.beforeInsert(insertObj, trx, cookie); await this.beforeInsert(insertObj, trx, cookie);
@ -2952,13 +2957,16 @@ class BaseModelSqlv2 {
async updateByPk(id, data, trx?, cookie?, _disableOptimization = false) { async updateByPk(id, data, trx?, cookie?, _disableOptimization = false) {
try { try {
const columns = await this.model.getColumns();
const updateObj = await this.model.mapAliasToColumn( const updateObj = await this.model.mapAliasToColumn(
data, data,
this.clientMeta, this.clientMeta,
this.dbDriver, this.dbDriver,
columns,
); );
await this.validate(data); await this.validate(data, columns);
await this.beforeUpdate(data, trx, cookie); await this.beforeUpdate(data, trx, cookie);
@ -2973,7 +2981,7 @@ class BaseModelSqlv2 {
const query = this.dbDriver(this.tnPath) const query = this.dbDriver(this.tnPath)
.update(updateObj) .update(updateObj)
.where(await this._wherePk(id)); .where(await this._wherePk(id, true));
await this.execAndParse(query, null, { raw: true }); await this.execAndParse(query, null, { raw: true });
@ -2995,11 +3003,15 @@ class BaseModelSqlv2 {
} }
} }
async _wherePk(id) { async _wherePk(id, skipGetColumns = false) {
await this.model.getColumns(); if (!skipGetColumns) await this.model.getColumns();
return _wherePk(this.model.primaryKeys, id); return _wherePk(this.model.primaryKeys, id);
} }
comparePks(pk1, pk2) {
return equal(pk1, pk2);
}
public getTnPath(tb: { table_name: string } | string, alias?: string) { public getTnPath(tb: { table_name: string } | string, alias?: string) {
const tn = typeof tb === 'string' ? tb : tb.table_name; const tn = typeof tb === 'string' ? tb : tb.table_name;
const schema = (this.dbDriver as any).searchPath?.(); const schema = (this.dbDriver as any).searchPath?.();
@ -3088,23 +3100,25 @@ class BaseModelSqlv2 {
try { try {
const source = await Source.get(this.model.source_id); const source = await Source.get(this.model.source_id);
await populatePk(this.model, data); await populatePk(this.model, data);
const columns = await this.model.getColumns();
const insertObj = await this.model.mapAliasToColumn( const insertObj = await this.model.mapAliasToColumn(
data, data,
this.clientMeta, this.clientMeta,
this.dbDriver, this.dbDriver,
columns,
); );
let rowId = null; let rowId = null;
const nestedCols = (await this.model.getColumns()).filter((c) => const nestedCols = columns.filter((c) => isLinksOrLTAR(c));
isLinksOrLTAR(c),
);
const { postInsertOps, preInsertOps } = await this.prepareNestedLinkQb({ const { postInsertOps, preInsertOps } = await this.prepareNestedLinkQb({
nestedCols, nestedCols,
data, data,
insertObj, insertObj,
}); });
await this.validate(insertObj); await this.validate(insertObj, columns);
await this.beforeInsert(insertObj, this.dbDriver, cookie); await this.beforeInsert(insertObj, this.dbDriver, cookie);
@ -3402,11 +3416,9 @@ class BaseModelSqlv2 {
let agPkCol: Column; let agPkCol: Column;
if (!raw) { if (!raw) {
const nestedCols = (await this.model.getColumns()).filter((c) => const columns = await this.model.getColumns();
isLinksOrLTAR(c),
);
await this.model.getColumns(); const nestedCols = columns.filter((c) => isLinksOrLTAR(c));
for (const d of datas) { for (const d of datas) {
const insertObj = {}; const insertObj = {};
@ -3687,12 +3699,12 @@ class BaseModelSqlv2 {
) { ) {
let transaction; let transaction;
try { try {
if (raw) await this.model.getColumns(); const columns = await this.model.getColumns();
// validate update data // validate update data
if (!raw) { if (!raw) {
for (const d of datas) { for (const d of datas) {
await this.validate(d); await this.validate(d, columns);
} }
} }
@ -3700,7 +3712,12 @@ class BaseModelSqlv2 {
? datas ? datas
: await Promise.all( : await Promise.all(
datas.map((d) => datas.map((d) =>
this.model.mapAliasToColumn(d, this.clientMeta, this.dbDriver), this.model.mapAliasToColumn(
d,
this.clientMeta,
this.dbDriver,
columns,
),
), ),
); );
@ -3708,8 +3725,10 @@ class BaseModelSqlv2 {
const newData = []; const newData = [];
const updatePkValues = []; const updatePkValues = [];
const toBeUpdated = []; const toBeUpdated = [];
for (const d of updateDatas) { const pkAndData: { pk: any; data: any }[] = [];
const pkValues = await this._extractPksValues(d); const readChunkSize = 100;
for (const [i, d] of updateDatas.entries()) {
const pkValues = this._extractPksValues(d);
if (!pkValues) { if (!pkValues) {
// throw or skip if no pk provided // throw or skip if no pk provided
if (throwExceptionIfNotExist) { if (throwExceptionIfNotExist) {
@ -3720,19 +3739,58 @@ class BaseModelSqlv2 {
if (!raw) { if (!raw) {
await this.prepareNocoData(d, false, cookie); await this.prepareNocoData(d, false, cookie);
const oldRecord = await this.readByPk(pkValues); pkAndData.push({
if (!oldRecord) { pk: pkValues,
// throw or skip if no record found data: d,
if (throwExceptionIfNotExist) { });
NcError.recordNotFound(JSON.stringify(pkValues));
if (
pkAndData.length >= readChunkSize ||
i === updateDatas.length - 1
) {
const tempToRead = pkAndData.splice(0, pkAndData.length);
const oldRecords = await this.list(
{
pks: tempToRead.map((v) => v.pk).join(','),
},
{
limitOverride: tempToRead.length,
},
);
if (oldRecords.length === tempToRead.length) {
prevData.push(...oldRecords);
} else {
for (const recordPk of tempToRead) {
const oldRecord = oldRecords.find((r) =>
this.comparePks(this._extractPksValues(r), recordPk),
);
if (!oldRecord) {
// throw or skip if no record found
if (throwExceptionIfNotExist) {
NcError.recordNotFound(JSON.stringify(recordPk));
}
continue;
}
prevData.push(oldRecord);
}
}
for (const { pk, data } of tempToRead) {
const wherePk = await this._wherePk(pk, true);
toBeUpdated.push({ d: data, wherePk });
updatePkValues.push(pk);
} }
continue;
} }
prevData.push(oldRecord); } else {
const wherePk = await this._wherePk(pkValues, true);
toBeUpdated.push({ d, wherePk });
updatePkValues.push(pkValues);
} }
const wherePk = await this._wherePk(pkValues);
toBeUpdated.push({ d, wherePk });
updatePkValues.push(pkValues);
} }
transaction = await this.dbDriver.transaction(); transaction = await this.dbDriver.transaction();
@ -3744,9 +3802,17 @@ class BaseModelSqlv2 {
await transaction.commit(); await transaction.commit();
if (!raw) { if (!raw) {
for (const pkValues of updatePkValues) { while (updatePkValues.length) {
const updatedRecord = await this.readByPk(pkValues); const updatedRecords = await this.list(
newData.push(updatedRecord); {
pks: updatePkValues.splice(0, readChunkSize).join(','),
},
{
limitOverride: readChunkSize,
},
);
newData.push(...updatedRecords);
} }
} }
@ -3783,23 +3849,27 @@ class BaseModelSqlv2 {
) { ) {
try { try {
let count = 0; let count = 0;
const columns = await this.model.getColumns();
const updateData = await this.model.mapAliasToColumn( const updateData = await this.model.mapAliasToColumn(
data, data,
this.clientMeta, this.clientMeta,
this.dbDriver, this.dbDriver,
columns,
); );
if (!args.skipValidationAndHooks) await this.validate(updateData); if (!args.skipValidationAndHooks)
await this.validate(updateData, columns);
await this.prepareNocoData(updateData, false, cookie); await this.prepareNocoData(updateData, false, cookie);
const pkValues = await this._extractPksValues(updateData); const pkValues = this._extractPksValues(updateData);
if (pkValues) { if (pkValues) {
// pk is specified - by pass // pk is specified - by pass
} else { } else {
await this.model.getColumns();
const { where } = this._getListArgs(args); const { where } = this._getListArgs(args);
const qb = this.dbDriver(this.tnPath); const qb = this.dbDriver(this.tnPath);
const aliasColObjMap = await this.model.getAliasColObjMap(); const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap, true); const filterObj = extractFilterFromXwhere(where, aliasColObjMap, true);
const conditionObj = [ const conditionObj = [
@ -3864,18 +3934,27 @@ class BaseModelSqlv2 {
isSingleRecordDeletion?: boolean; isSingleRecordDeletion?: boolean;
} = {}, } = {},
) { ) {
const columns = await this.model.getColumns();
let transaction; let transaction;
try { try {
const deleteIds = await Promise.all( const deleteIds = await Promise.all(
ids.map((d) => ids.map((d) =>
this.model.mapAliasToColumn(d, this.clientMeta, this.dbDriver), this.model.mapAliasToColumn(
d,
this.clientMeta,
this.dbDriver,
columns,
),
), ),
); );
const deleted = []; const deleted = [];
const res = []; const res = [];
for (const d of deleteIds) { const pkAndData: { pk: any; data: any }[] = [];
const pkValues = await this._extractPksValues(d); const readChunkSize = 100;
for (const [i, d] of deleteIds.entries()) {
const pkValues = this._extractPksValues(d);
if (!pkValues) { if (!pkValues) {
// throw or skip if no pk provided // throw or skip if no pk provided
if (throwExceptionIfNotExist) { if (throwExceptionIfNotExist) {
@ -3884,17 +3963,41 @@ class BaseModelSqlv2 {
continue; continue;
} }
const deletedRecord = await this.readByPk(pkValues); pkAndData.push({ pk: pkValues, data: d });
if (!deletedRecord) {
// throw or skip if no record found if (pkAndData.length >= readChunkSize || i === deleteIds.length - 1) {
if (throwExceptionIfNotExist) { const tempToRead = pkAndData.splice(0, pkAndData.length);
NcError.recordNotFound(JSON.stringify(pkValues)); const oldRecords = await this.list(
{
pks: tempToRead.map((v) => v.pk).join(','),
},
{
limitOverride: tempToRead.length,
},
);
if (oldRecords.length === tempToRead.length) {
deleted.push(...oldRecords);
res.push(...tempToRead.map((v) => v.data));
} else {
for (const { pk, data } of tempToRead) {
const oldRecord = oldRecords.find((r) =>
this.comparePks(this._extractPksValues(r), pk),
);
if (!oldRecord) {
// throw or skip if no record found
if (throwExceptionIfNotExist) {
NcError.recordNotFound(JSON.stringify(pk));
}
continue;
}
deleted.push(oldRecord);
res.push(data);
}
} }
continue;
} }
deleted.push(deletedRecord);
res.push(d);
} }
const execQueries: (( const execQueries: ((
@ -3989,10 +4092,10 @@ class BaseModelSqlv2 {
) { ) {
let trx: Knex.Transaction; let trx: Knex.Transaction;
try { try {
await this.model.getColumns(); const columns = await this.model.getColumns();
const { where } = this._getListArgs(args); const { where } = this._getListArgs(args);
const qb = this.dbDriver(this.tnPath); const qb = this.dbDriver(this.tnPath);
const aliasColObjMap = await this.model.getAliasColObjMap(); const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap, true); const filterObj = extractFilterFromXwhere(where, aliasColObjMap, true);
await conditionV2( await conditionV2(
@ -4336,10 +4439,13 @@ class BaseModelSqlv2 {
protected async errorDelete(_e, _id, _trx, _cookie) {} protected async errorDelete(_e, _id, _trx, _cookie) {}
async validate(data: Record<string, any>): Promise<boolean> { async validate(
await this.model.getColumns(); data: Record<string, any>,
columns?: Column[],
): Promise<boolean> {
const cols = columns || (await this.model.getColumns());
// let cols = Object.keys(this.columns); // let cols = Object.keys(this.columns);
for (let i = 0; i < this.model.columns.length; ++i) { for (let i = 0; i < cols.length; ++i) {
const column = this.model.columns[i]; const column = this.model.columns[i];
if ( if (
@ -4836,9 +4942,8 @@ class BaseModelSqlv2 {
> { > {
try { try {
const { where, ...rest } = this._getListArgs(args as any); const { where, ...rest } = this._getListArgs(args as any);
const column = await this.model const columns = await this.model.getColumns();
.getColumns() const column = columns?.find((col) => col.id === args.groupColumnId);
.then((cols) => cols?.find((col) => col.id === args.groupColumnId));
if (!column) NcError.fieldNotFound(args.groupColumnId); if (!column) NcError.fieldNotFound(args.groupColumnId);
if (isVirtualCol(column)) if (isVirtualCol(column))
@ -4876,7 +4981,7 @@ class BaseModelSqlv2 {
await this.selectObject({ qb, extractPkAndPv: true }); await this.selectObject({ qb, extractPkAndPv: true });
// todo: refactor and move to a method (applyFilterAndSort) // todo: refactor and move to a method (applyFilterAndSort)
const aliasColObjMap = await this.model.getAliasColObjMap(); const aliasColObjMap = await this.model.getAliasColObjMap(columns);
let sorts = extractSortsObject(args?.sort, aliasColObjMap); let sorts = extractSortsObject(args?.sort, aliasColObjMap);
const filterObj = extractFilterFromXwhere(where, aliasColObjMap); const filterObj = extractFilterFromXwhere(where, aliasColObjMap);
// todo: replace with view id // todo: replace with view id
@ -4998,9 +5103,8 @@ class BaseModelSqlv2 {
ignoreViewFilterAndSort?: boolean; ignoreViewFilterAndSort?: boolean;
} & XcFilter, } & XcFilter,
) { ) {
const column = await this.model const columns = await this.model.getColumns();
.getColumns() const column = columns?.find((col) => col.id === args.groupColumnId);
.then((cols) => cols?.find((col) => col.id === args.groupColumnId));
if (!column) NcError.fieldNotFound(args.groupColumnId); if (!column) NcError.fieldNotFound(args.groupColumnId);
if (isVirtualCol(column)) if (isVirtualCol(column))
@ -5011,7 +5115,7 @@ class BaseModelSqlv2 {
.groupBy(column.column_name); .groupBy(column.column_name);
// todo: refactor and move to a common method (applyFilterAndSort) // todo: refactor and move to a common method (applyFilterAndSort)
const aliasColObjMap = await this.model.getAliasColObjMap(); const aliasColObjMap = await this.model.getAliasColObjMap(columns);
const filterObj = extractFilterFromXwhere(args.where, aliasColObjMap); const filterObj = extractFilterFromXwhere(args.where, aliasColObjMap);
// todo: replace with view id // todo: replace with view id
@ -6192,12 +6296,12 @@ class BaseModelSqlv2 {
args: { limit?; offset?; fieldSet?: Set<string> } = {}, args: { limit?; offset?; fieldSet?: Set<string> } = {},
) { ) {
try { try {
const columns = await this.model.getColumns();
const { where, sort } = this._getListArgs(args as any); const { where, sort } = this._getListArgs(args as any);
// todo: get only required fields // todo: get only required fields
const relColumn = (await this.model.getColumns()).find( const relColumn = columns.find((c) => c.id === colId);
(c) => c.id === colId,
);
const row = await this.execAndParse( const row = await this.execAndParse(
this.dbDriver(this.tnPath).where(await this._wherePk(id)), this.dbDriver(this.tnPath).where(await this._wherePk(id)),

11
packages/nocodb/src/models/Model.ts
Unescape View File

@ -538,9 +538,10 @@ export default class Model implements TableType {
isPg: false, isPg: false,
}, },
knex, knex,
columns?: Column[],
) { ) {
const insertObj = {}; const insertObj = {};
for (const col of await this.getColumns()) { for (const col of columns || (await this.getColumns())) {
if (isVirtualCol(col)) continue; if (isVirtualCol(col)) continue;
let val = let val =
data?.[col.column_name] !== undefined data?.[col.column_name] !== undefined
@ -618,9 +619,9 @@ export default class Model implements TableType {
return insertObj; return insertObj;
} }
async mapColumnToAlias(data) { async mapColumnToAlias(data, columns?: Column[]) {
const res = {}; const res = {};
for (const col of await this.getColumns()) { for (const col of columns || (await this.getColumns())) {
if (isVirtualCol(col)) continue; if (isVirtualCol(col)) continue;
let val = let val =
data?.[col.title] !== undefined data?.[col.title] !== undefined
@ -969,8 +970,8 @@ export default class Model implements TableType {
)); ));
} }
async getAliasColObjMap() { async getAliasColObjMap(columns?: Column[]) {
return (await this.getColumns()).reduce( return (columns || (await this.getColumns())).reduce(
(sortAgg, c) => ({ ...sortAgg, [c.title]: c }), (sortAgg, c) => ({ ...sortAgg, [c.title]: c }),
{}, {},
); );

Write Preview
Loading…
Cancel
Save