From a29f86152f8405ae9c4a3b9da56236f60c1a7a42 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 14 Jul 2022 20:04:10 +0530 Subject: [PATCH 01/22] wip: exclude unnecessary fields from select query Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 115 ++++++++++++------ 1 file changed, 77 insertions(+), 38 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index 6c20ec4089..0573639a4b 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -1,47 +1,49 @@ import autoBind from 'auto-bind'; -import _ from 'lodash'; - -import Model from '../../../../models/Model'; -import { XKnex } from '../../index'; -import LinkToAnotherRecordColumn from '../../../../models/LinkToAnotherRecordColumn'; -import RollupColumn from '../../../../models/RollupColumn'; -import LookupColumn from '../../../../models/LookupColumn'; import DataLoader from 'dataloader'; -import Column from '../../../../models/Column'; -import { XcFilter, XcFilterWithAlias } from '../BaseModel'; -import conditionV2 from './conditionV2'; -import Filter from '../../../../models/Filter'; -import sortV2 from './sortV2'; -import Sort from '../../../../models/Sort'; -import FormulaColumn from '../../../../models/FormulaColumn'; -import genRollupSelectv2 from './genRollupSelectv2'; -import formulaQueryBuilderv2 from './formulav2/formulaQueryBuilderv2'; +import ejs from 'ejs'; +import DOMPurify from 'isomorphic-dompurify'; import { QueryBuilder } from 'knex'; -import View from '../../../../models/View'; +import _ from 'lodash'; +import { customAlphabet } from 'nanoid'; import { AuditOperationSubTypes, AuditOperationTypes, + isSystemColumn, RelationTypes, SortType, UITypes, ViewTypes, } from 'nocodb-sdk'; -import formSubmissionEmailTemplate from '../../../../utils/common/formSubmissionEmailTemplate'; -import ejs from 'ejs'; -import Audit from '../../../../models/Audit'; -import FormView from '../../../../models/FormView'; -import Hook from '../../../../models/Hook'; +import Validator from 'validator'; +import { NcError } from '../../../../meta/helpers/catchError'; import NcPluginMgrv2 from '../../../../meta/helpers/NcPluginMgrv2'; import { _transformSubmittedFormDataForEmail, invokeWebhook, } from '../../../../meta/helpers/webhookHelpers'; -import Validator from 'validator'; +import Audit from '../../../../models/Audit'; +import Column from '../../../../models/Column'; +import Filter from '../../../../models/Filter'; +import FormulaColumn from '../../../../models/FormulaColumn'; +import FormView from '../../../../models/FormView'; +import GridViewColumn from '../../../../models/GridViewColumn'; +import Hook from '../../../../models/Hook'; +import LinkToAnotherRecordColumn from '../../../../models/LinkToAnotherRecordColumn'; +import LookupColumn from '../../../../models/LookupColumn'; + +import Model from '../../../../models/Model'; +import RollupColumn from '../../../../models/RollupColumn'; +import Sort from '../../../../models/Sort'; +import View from '../../../../models/View'; +import formSubmissionEmailTemplate from '../../../../utils/common/formSubmissionEmailTemplate'; +import { XKnex } from '../../index'; +import { XcFilter, XcFilterWithAlias } from '../BaseModel'; +import conditionV2 from './conditionV2'; import { customValidators } from './customValidators'; -import { NcError } from '../../../../meta/helpers/catchError'; -import { customAlphabet } from 'nanoid'; -import DOMPurify from 'isomorphic-dompurify'; +import formulaQueryBuilderv2 from './formulav2/formulaQueryBuilderv2'; +import genRollupSelectv2 from './genRollupSelectv2'; import { sanitize, unsanitize } from './helpers/sanitize'; +import sortV2 from './sortV2'; const GROUP_COL = '__nc_group_id'; @@ -57,6 +59,15 @@ async function populatePk(model: Model, insertObj: any) { } } +function checkColumnRequired(column: Column, fields: string[]) { + // if primary key or foreign key included in fields, it's required + if (column.pk || column.uidt === UITypes.ForeignKey) return true; + + // check fields defined and if not, then select all + // if defined check if it is in the fields + return !fields || fields.includes(column.title); +} + /** * Base class for models * @@ -175,10 +186,10 @@ class BaseModelSqlv2 { } = {}, ignoreFilterSort = false ): Promise { - const { where, ...rest } = this._getListArgs(args as any); + const { where, fields, ...rest } = this._getListArgs(args as any); const qb = this.dbDriver(this.tnPath); - await this.selectObject({ qb }); + await this.selectObject({ qb, fields }); if (+rest?.shuffle) { await this.shuffle({ qb }); } @@ -251,7 +262,9 @@ class BaseModelSqlv2 { if (!ignoreFilterSort) applyPaginate(qb, rest); const proto = await this.getProto(); - let data = await this.extractRawQueryAndExec(qb); + const data = await this.extractRawQueryAndExec(qb); + + console.log(qb.toQuery()); return data?.map((d) => { d.__proto__ = proto; @@ -362,7 +375,7 @@ class BaseModelSqlv2 { qb.groupBy(args.column_name); if (sorts) await sortV2(sorts, qb, this.dbDriver); applyPaginate(qb, rest); - let data = await qb; + const data = await qb; return data; } @@ -571,7 +584,10 @@ class BaseModelSqlv2 { } } - public async multipleMmList({ colId, parentIds }, args: { limit?; offset? } = {}) { + public async multipleMmList( + { colId, parentIds }, + args: { limit?; offset? } = {} + ) { const { where, ...rest } = this._getListArgs(args as any); const relColumn = (await this.model.getColumns()).find( (c) => c.id === colId @@ -879,7 +895,7 @@ class BaseModelSqlv2 { applyPaginate(qb, rest); const proto = await childModel.getProto(); - let data = await qb; + const data = await qb; return data.map((c) => { c.__proto__ = proto; @@ -979,7 +995,7 @@ class BaseModelSqlv2 { applyPaginate(qb, rest); const proto = await childModel.getProto(); - let data = await this.extractRawQueryAndExec(qb); + const data = await this.extractRawQueryAndExec(qb); return data.map((c) => { c.__proto__ = proto; @@ -1079,7 +1095,7 @@ class BaseModelSqlv2 { applyPaginate(qb, rest); const proto = await parentModel.getProto(); - let data = await this.extractRawQueryAndExec(qb); + const data = await this.extractRawQueryAndExec(qb); return data.map((c) => { c.__proto__ = proto; @@ -1272,7 +1288,7 @@ class BaseModelSqlv2 { this.config.limitMin ); obj.offset = Math.max(+(args.offset || args.o) || 0, 0); - obj.fields = args.fields || args.f || '*'; + obj.fields = args.fields || args.f; obj.sort = args.sort || args.s || this.model.primaryKey?.[0]?.tn; return obj; } @@ -1287,10 +1303,33 @@ class BaseModelSqlv2 { } } - public async selectObject({ qb }: { qb: QueryBuilder }): Promise { + public async selectObject({ + qb, + fields: _fields, + }: { + qb: QueryBuilder; + fields?: string[] | string; + }): Promise { + const view = await View.get(this.viewId); + const viewColumns = this.viewId && (await View.getColumns(this.viewId)); + const fields = Array.isArray(_fields) ? _fields : _fields?.split(','); const res = {}; - const columns = await this.model.getColumns(); - for (const column of columns) { + const columns = viewColumns || (await this.model.getColumns()); + for (const _column of columns) { + const column = + _column instanceof Column + ? _column + : await Column.get({ + colId: (_column as GridViewColumn).fk_column_id, + }); + if ( + !(column instanceof Column) && + !(column as GridViewColumn)?.show && + (!view?.show_system_fields || isSystemColumn(column)) + ) + continue; + if (!checkColumnRequired(column, fields)) continue; + switch (column.uidt) { case 'LinkToAnotherRecord': case 'Lookup': From 91b86b04e42c4685fb173683342a326338274eb8 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 14 Jul 2022 23:23:17 +0530 Subject: [PATCH 02/22] feat: exclude based on view and fields property - exclude columns in root list query based on filed param and view - include foreign key and primary keys by default Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index 0573639a4b..f32866528a 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -1314,18 +1314,22 @@ class BaseModelSqlv2 { const viewColumns = this.viewId && (await View.getColumns(this.viewId)); const fields = Array.isArray(_fields) ? _fields : _fields?.split(','); const res = {}; - const columns = viewColumns || (await this.model.getColumns()); - for (const _column of columns) { + const viewOrTableColumns = viewColumns || (await this.model.getColumns()); + for (const viewOrTableColumn of viewOrTableColumns) { const column = - _column instanceof Column - ? _column + viewOrTableColumn instanceof Column + ? viewOrTableColumn : await Column.get({ - colId: (_column as GridViewColumn).fk_column_id, + colId: (viewOrTableColumn as GridViewColumn).fk_column_id, }); + // hide if column marked as hidden in view + // of if column is system field and system field is hidden if ( - !(column instanceof Column) && - !(column as GridViewColumn)?.show && - (!view?.show_system_fields || isSystemColumn(column)) + !(viewOrTableColumn instanceof Column) && + (!(viewOrTableColumn as GridViewColumn)?.show || + (!view?.show_system_fields && + column.uidt !== UITypes.ForeignKey && + isSystemColumn(column))) ) continue; if (!checkColumnRequired(column, fields)) continue; From 4f4a02e7863d021a075a7b36c93c718e7e0acf08 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 14 Jul 2022 23:30:30 +0530 Subject: [PATCH 03/22] wip: extract dependency columns Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 34 ++++- .../sql-data-mapper/lib/sql/helpers/getAst.ts | 133 ++++++++++++++++-- .../lib/meta/api/dataApis/dataAliasApis.ts | 21 ++- packages/nocodb/src/lib/models/Column.ts | 2 +- 4 files changed, 165 insertions(+), 25 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index f32866528a..548603195b 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -59,10 +59,16 @@ async function populatePk(model: Model, insertObj: any) { } } -function checkColumnRequired(column: Column, fields: string[]) { +function checkColumnRequired( + column: Column, + fields: string[], + extractPkAndPv?: boolean +) { // if primary key or foreign key included in fields, it's required if (column.pk || column.uidt === UITypes.ForeignKey) return true; + if (extractPkAndPv && column.pv) return true; + // check fields defined and if not, then select all // if defined check if it is in the fields return !fields || fields.includes(column.title); @@ -189,7 +195,7 @@ class BaseModelSqlv2 { const { where, fields, ...rest } = this._getListArgs(args as any); const qb = this.dbDriver(this.tnPath); - await this.selectObject({ qb, fields }); + await this.selectObject({ qb, fields, viewId: this.viewId }); if (+rest?.shuffle) { await this.shuffle({ qb }); } @@ -264,7 +270,7 @@ class BaseModelSqlv2 { const proto = await this.getProto(); const data = await this.extractRawQueryAndExec(qb); - console.log(qb.toQuery()); + // console.log(qb.toQuery()); return data?.map((d) => { d.__proto__ = proto; @@ -404,7 +410,7 @@ class BaseModelSqlv2 { await parentTable.getColumns(); const qb = this.dbDriver(childTable.table_name); - await childModel.selectObject({ qb }); + await childModel.selectObject({ qb, extractPkAndPv: true }); const childQb = this.dbDriver.queryBuilder().from( this.dbDriver @@ -431,6 +437,9 @@ class BaseModelSqlv2 { .as('list') ); + + // console.log(childQb.toQuery()) + const children = await this.extractRawQueryAndExec(childQb); const proto = await ( await Model.getBaseModelSQL({ @@ -439,6 +448,7 @@ class BaseModelSqlv2 { }) ).getProto(); + return _.groupBy( children.map((c) => { c.__proto__ = proto; @@ -1303,15 +1313,22 @@ class BaseModelSqlv2 { } } + // todo: + // pass view id as argument + // add option to get only pk and pv public async selectObject({ qb, fields: _fields, + extractPkAndPv, + viewId, }: { qb: QueryBuilder; fields?: string[] | string; + extractPkAndPv?: boolean; + viewId?: string; }): Promise { - const view = await View.get(this.viewId); - const viewColumns = this.viewId && (await View.getColumns(this.viewId)); + const view = await View.get(viewId); + const viewColumns = viewId && (await View.getColumns(viewId)); const fields = Array.isArray(_fields) ? _fields : _fields?.split(','); const res = {}; const viewOrTableColumns = viewColumns || (await this.model.getColumns()); @@ -1325,14 +1342,17 @@ class BaseModelSqlv2 { // hide if column marked as hidden in view // of if column is system field and system field is hidden if ( + !extractPkAndPv && !(viewOrTableColumn instanceof Column) && (!(viewOrTableColumn as GridViewColumn)?.show || (!view?.show_system_fields && column.uidt !== UITypes.ForeignKey && + !column.pk && isSystemColumn(column))) ) continue; - if (!checkColumnRequired(column, fields)) continue; + + if (!checkColumnRequired(column, fields, extractPkAndPv)) continue; switch (column.uidt) { case 'LinkToAnotherRecord': diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts index 9e6af05f6e..6875349300 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts @@ -1,7 +1,9 @@ -import View from '../../../../../models/View'; -import { isSystemColumn, UITypes } from 'nocodb-sdk'; -import Model from '../../../../../models/Model'; +import { isSystemColumn, RelationTypes, UITypes } from 'nocodb-sdk'; +import Column from '../../../../../models/Column'; import LinkToAnotherRecordColumn from '../../../../../models/LinkToAnotherRecordColumn'; +import LookupColumn from '../../../../../models/LookupColumn'; +import Model from '../../../../../models/Model'; +import View from '../../../../../models/View'; const getAst = async ({ query, @@ -9,23 +11,34 @@ const getAst = async ({ includePkByDefault = true, model, view, + dependencyFields = { + nested: {}, + fields: new Set(), + }, }: { query?: RequestQuery; extractOnlyPrimaries?: boolean; includePkByDefault?: boolean; model: Model; view?: View; + dependencyFields?: DependantFields; }) => { if (!model.columns?.length) await model.getColumns(); // extract only pk and pv if (extractOnlyPrimaries) { - return { + const ast = { ...(model.primaryKeys ? model.primaryKeys.reduce((o, pk) => ({ ...o, [pk.title]: 1 }), {}) : {}), ...(model.primaryValue ? { [model.primaryValue.title]: 1 } : {}), }; + await Promise.all( + model.primaryKeys.map((c) => extractDependencies(c, dependencyFields)) + ); + await extractDependencies(model.primaryValue, dependencyFields); + + return { ast, dependencyFields }; } let fields = query?.fields || query?.f; @@ -45,7 +58,7 @@ const getAst = async ({ {} ); - return model.columns.reduce(async (obj, col) => { + const ast = await model.columns.reduce(async (obj, col) => { let value: number | boolean | { [key: string]: any } = 1; const nestedFields = query?.nested?.[col.title]?.fields || query?.nested?.[col.title]?.f; @@ -55,10 +68,19 @@ const getAst = async ({ .getColOptions() .then((colOpt) => colOpt.getRelatedTable()); - value = await getAst({ + const { ast } = await getAst({ model, query: query?.nested?.[col.title], + dependencyFields: (dependencyFields.nested[col.title] = + dependencyFields.nested[col.title] || { + nested: {}, + fields: new Set(), + }), }); + + value = ast; + + // todo: include field relative to the relation => pk / fk } else { value = (Array.isArray(fields) ? fields : fields.split(',')).reduce( (o, f) => ({ ...o, [f]: 1 }), @@ -74,22 +96,98 @@ const getAst = async ({ model, query: query?.nested?.[col.title], extractOnlyPrimaries: nestedFields !== '*', + dependencyFields:(dependencyFields.nested[col.title] = + dependencyFields.nested[col.title] || { + nested: {}, + fields: new Set(), + }) }); } + const isRequested = + allowedCols && (!includePkByDefault || !col.pk) + ? allowedCols[col.id] && + (!isSystemColumn(col) || view.show_system_fields) && + (!fields?.length || fields.includes(col.title)) && + value + : fields?.length + ? fields.includes(col.title) && value + : value; + if (isRequested) await extractDependencies(col, dependencyFields); + return { ...(await obj), - [col.title]: - allowedCols && (!includePkByDefault || !col.pk) - ? allowedCols[col.id] && - (!isSystemColumn(col) || view.show_system_fields) && - (!fields?.length || fields.includes(col.title)) && - value - : fields?.length - ? fields.includes(col.title) && value - : value, + [col.title]: isRequested, }; }, Promise.resolve({})); + + return { ast, dependencyFields }; +}; + +const extractDependencies = async ( + column: Column, + dependencyFields: DependantFields = { + nested: {}, + fields: new Set(), + } +) => { + switch (column.uidt) { + case UITypes.Lookup: + await extractLookupDependencies(column, dependencyFields); + break; + case UITypes.LinkToAnotherRecord: + await extractRelationDependencies(column, dependencyFields); + break; + default: + dependencyFields.fields.add(column.title); + break; + } +}; + +const extractLookupDependencies = async ( + lookUpColumn: Column, + dependencyFields: DependantFields = { + nested: {}, + fields: new Set(), + } +) => { + const lookupColumnOpts = await lookUpColumn.getColOptions(); + const relationColumn = await lookupColumnOpts.getRelationColumn(); + await extractRelationDependencies(relationColumn, dependencyFields); + await extractDependencies( + await lookupColumnOpts.getLookupColumn(), + (dependencyFields.nested[relationColumn.title] = dependencyFields.nested[ + relationColumn.title + ] || { + nested: {}, + fields: new Set(), + }) + ); +}; + +const extractRelationDependencies = async ( + relationColumn: Column, + dependencyFields: DependantFields = { + nested: {}, + fields: new Set(), + } +) => { + const relationColumnOpts = await relationColumn.getColOptions(); + + switch (relationColumnOpts.type) { + case RelationTypes.HAS_MANY: + dependencyFields.fields.add( + await relationColumnOpts.getParentColumn().then((col) => col.title) + ); + break; + case RelationTypes.BELONGS_TO: + case RelationTypes.MANY_TO_MANY: + dependencyFields.fields.add( + await relationColumnOpts.getChildColumn().then((col) => col.title) + ); + + break; + } }; type RequestQuery = { @@ -100,4 +198,9 @@ type RequestQuery = { }; }; +interface DependantFields { + fields?: Set; + nested?: DependantFields; +} + export default getAst; diff --git a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts b/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts index b982bbc4de..40ce3e782b 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts +++ b/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts @@ -97,7 +97,11 @@ async function getDataList(model, view: View, req) { dbDriver: NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ model, query: req.query, view }); + const { ast, dependencyFields } = await getAst({ + model, + query: req.query, + view, + }); const listArgs: any = { ...req.query }; try { @@ -107,8 +111,21 @@ async function getDataList(model, view: View, req) { listArgs.sortArr = JSON.parse(listArgs.sortArrJson); } catch (e) {} + console.log( + JSON.stringify( + dependencyFields, + (_v, o) => { + if (o instanceof Set) { + return [...o]; + } + return o; + }, + 2 + ) + ); + console.log(JSON.stringify(ast, null, 2)); const data = await nocoExecute( - requestObj, + ast, await baseModel.list(listArgs), {}, listArgs diff --git a/packages/nocodb/src/lib/models/Column.ts b/packages/nocodb/src/lib/models/Column.ts index 5589888ef3..96140f6c15 100644 --- a/packages/nocodb/src/lib/models/Column.ts +++ b/packages/nocodb/src/lib/models/Column.ts @@ -308,7 +308,7 @@ export default class Column implements ColumnType { } } - public async getColOptions(ncMeta = Noco.ncMeta): Promise { + public async getColOptions(ncMeta = Noco.ncMeta): Promise { let res: any; switch (this.uidt) { From 8d86348fe4f1281381f849d5b0e7d1ecb42a0718 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 28 Mar 2023 15:55:22 +0530 Subject: [PATCH 04/22] fix: corrections Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 8 +++---- .../sql-data-mapper/lib/sql/helpers/getAst.ts | 21 ++++++++++++------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index b11540acd4..f4000319fa 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -3,16 +3,17 @@ import groupBy from 'lodash/groupBy'; import DataLoader from 'dataloader'; import { AuditOperationSubTypes, - AuditOperationTypes, + AuditOperationTypes, isSystemColumn, isVirtualCol, RelationTypes, UITypes, ViewTypes, -} from 'nocodb-sdk'; +} from 'nocodb-sdk' import ejs from 'ejs'; import Validator from 'validator'; import { customAlphabet } from 'nanoid'; import DOMPurify from 'isomorphic-dompurify'; +import { GridViewColumn } from '../../../../models' import Model from '../../../../models/Model'; import Column from '../../../../models/Column'; import Filter, { @@ -1451,7 +1452,6 @@ class BaseModelSqlv2 { // add option to get only pk and pv public async selectObject({ qb, - columns: _columns,, columns: _columns, fields: _fields, extractPkAndPv, @@ -1469,7 +1469,7 @@ class BaseModelSqlv2 { const res = {}; // const columns = _columns ?? (await this.model.getColumns()); // for (const column of columns) { - const viewOrTableColumns = viewColumns || (await this.model.getColumns()); + const viewOrTableColumns = _columns || viewColumns || (await this.model.getColumns()); for (const viewOrTableColumn of viewOrTableColumns) { const column = viewOrTableColumn instanceof Column diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts index 813d296b9f..ec32885b5e 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts @@ -1,7 +1,11 @@ -import { isSystemColumn, UITypes } from 'nocodb-sdk'; -import View from '../../../../../models/View'; -import type Model from '../../../../../models/Model'; -import type LinkToAnotherRecordColumn from '../../../../../models/LinkToAnotherRecordColumn'; +import { isSystemColumn, RelationTypes, UITypes } from 'nocodb-sdk'; +import { View } from '../../../../../models'; +import type { + Column, + LinkToAnotherRecordColumn, + LookupColumn, + Model, +} from '../../../../../models'; const getAst = async ({ query, @@ -34,7 +38,8 @@ const getAst = async ({ await Promise.all( model.primaryKeys.map((c) => extractDependencies(c, dependencyFields)) ); - await extractDependencies(model.primaryValue, dependencyFields); + + await extractDependencies(model.displayValue, dependencyFields); return { ast, dependencyFields }; } @@ -94,11 +99,11 @@ const getAst = async ({ model, query: query?.nested?.[col.title], extractOnlyPrimaries: nestedFields !== '*', - dependencyFields:(dependencyFields.nested[col.title] = - dependencyFields.nested[col.title] || { + dependencyFields: (dependencyFields.nested[col.title] = dependencyFields + .nested[col.title] || { nested: {}, fields: new Set(), - }) + }), }); } From 9f1718304cace24fc65279699e6206533c0a570d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 28 Mar 2023 17:05:47 +0530 Subject: [PATCH 05/22] fix: corrections in getAst method usage Signed-off-by: Pranav C --- .../src/lib/controllers/dbData/oldData.ctl.ts | 16 +++++++------ .../publicControllers/publicDataExport.ctl.ts | 4 ++-- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 11 +++++++-- .../lib/meta/api/dataApis/dataAliasApis.ts | 8 +++++-- .../nocodb/src/lib/services/dbData/helpers.ts | 13 ++++++----- .../nocodb/src/lib/services/dbData/index.ts | 23 ++++++++++++------- .../src/lib/services/public/publicData.svc.ts | 23 +++++++++++-------- .../services/public/publicDataExport.svc.ts | 4 ++-- 8 files changed, 64 insertions(+), 38 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts b/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts index d5a573b013..4c91aa2ecc 100644 --- a/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts +++ b/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts @@ -21,7 +21,7 @@ export async function dataList(req: Request, res: Response) { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ + const { ast } = await getAst({ query: req.query, model, view, @@ -36,7 +36,7 @@ export async function dataList(req: Request, res: Response) { } catch (e) {} const data = await nocoExecute( - requestObj, + ast, await baseModel.list(listArgs), {}, listArgs @@ -132,13 +132,15 @@ async function dataRead(req: Request, res: Response) { dbDriver: await NcConnectionMgrv2.get(base), }); + const {ast} = await getAst({ + query: req.query, + model, + view, + }) + res.json( await nocoExecute( - await getAst({ - query: req.query, - model, - view, - }), + ast , await baseModel.readByPk(req.params.rowId), {}, {} diff --git a/packages/nocodb/src/lib/controllers/publicControllers/publicDataExport.ctl.ts b/packages/nocodb/src/lib/controllers/publicControllers/publicDataExport.ctl.ts index 9ab36afdc7..0cc375a595 100644 --- a/packages/nocodb/src/lib/controllers/publicControllers/publicDataExport.ctl.ts +++ b/packages/nocodb/src/lib/controllers/publicControllers/publicDataExport.ctl.ts @@ -136,7 +136,7 @@ async function getDbRows(model, view: View, req: Request) { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ + const { ast } = await getAst({ query: req.query, model, view, @@ -159,7 +159,7 @@ async function getDbRows(model, view: View, req: Request) { elapsed = temp[0] * 1000 + temp[1] / 1000000 ) { const rows = await nocoExecute( - requestObj, + ast, await baseModel.list({ ...listArgs, offset, limit }), {}, listArgs diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index f4000319fa..5015a48bd9 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -195,13 +195,14 @@ class BaseModelSqlv2 { filterArr?: Filter[]; sortArr?: Sort[]; sort?: string | string[]; + fieldsSet?:Set } = {}, ignoreViewFilterAndSort = false ): Promise { const { where, fields, ...rest } = this._getListArgs(args as any); const qb = this.dbDriver(this.tnPath); - await this.selectObject({ qb, fields, viewId: this.viewId }); + await this.selectObject({ qb, fieldsSet: args.fieldsSet, viewId: this.viewId }); if (+rest?.shuffle) { await this.shuffle({ qb }); } @@ -272,6 +273,9 @@ class BaseModelSqlv2 { if (!ignoreViewFilterAndSort) applyPaginate(qb, rest); const proto = await this.getProto(); + + console.log('list query', qb.toQuery()) + const data = await this.execAndParse(qb); // console.log(qb.toQuery()); @@ -1456,7 +1460,9 @@ class BaseModelSqlv2 { fields: _fields, extractPkAndPv, viewId, + fieldsSet }: { + fieldsSet?: Set; qb: Knex.QueryBuilder; columns?: Column[]; fields?: string[] | string; @@ -1480,6 +1486,7 @@ class BaseModelSqlv2 { // hide if column marked as hidden in view // of if column is system field and system field is hidden if ( + (!fieldsSet || !fieldsSet.has(column.title)) && !extractPkAndPv && !(viewOrTableColumn instanceof Column) && (!(viewOrTableColumn as GridViewColumn)?.show || @@ -2716,7 +2723,7 @@ class BaseModelSqlv2 { qb.limit(+rest?.limit || 25); qb.offset(+rest?.offset || 0); - await this.selectObject({ qb }); + await this.selectObject({ qb, extractPkAndPv:true }); // todo: refactor and move to a method (applyFilterAndSort) const aliasColObjMap = await this.model.getAliasColObjMap(); diff --git a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts b/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts index 40ce3e782b..4a5f6d4c62 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts +++ b/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts @@ -157,9 +157,11 @@ async function getFindOne(model, view: View, req) { } catch (e) {} const data = await baseModel.findOne(args); + + const { ast } = await getAst({ model, query: args, view }) return data ? await nocoExecute( - await getAst({ model, query: args, view }), + ast, data, {}, {} @@ -197,9 +199,11 @@ async function dataRead(req: Request, res: Response) { dbDriver: NcConnectionMgrv2.get(base), }); + const { ast } = await getAst({ model, query: req.query, view }) + res.json( await nocoExecute( - await getAst({ model, query: req.query, view }), + ast, await baseModel.readByPk(req.params.rowId), {}, {} diff --git a/packages/nocodb/src/lib/services/dbData/helpers.ts b/packages/nocodb/src/lib/services/dbData/helpers.ts index 6a61c14d9e..3e709022ad 100644 --- a/packages/nocodb/src/lib/services/dbData/helpers.ts +++ b/packages/nocodb/src/lib/services/dbData/helpers.ts @@ -242,13 +242,14 @@ export async function getDbRows(param: { temp = process.hrtime(startTime), elapsed = temp[0] * 1000 + temp[1] / 1000000 ) { + const {ast} = await getAst({ + query: query, + includePkByDefault: false, + model: view.model, + view, + }); const rows = await nocoExecute( - await getAst({ - query: query, - includePkByDefault: false, - model: view.model, - view, - }), + ast, await baseModel.list({ ...listArgs, offset, limit }), {}, query diff --git a/packages/nocodb/src/lib/services/dbData/index.ts b/packages/nocodb/src/lib/services/dbData/index.ts index bf6fa3efc0..80f6d57207 100644 --- a/packages/nocodb/src/lib/services/dbData/index.ts +++ b/packages/nocodb/src/lib/services/dbData/index.ts @@ -114,9 +114,10 @@ export async function getDataList(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ model, query, view }); + const { ast, dependencyFields } = await getAst({ model, query, view }); - const listArgs: any = { ...query }; + + const listArgs: any = { ...query, fieldsSet: dependencyFields?.fields }; try { listArgs.filterArr = JSON.parse(listArgs.filterArrJson); } catch (e) {} @@ -128,7 +129,7 @@ export async function getDataList(param: { let count = 0; try { data = await nocoExecute( - requestObj, + ast, await baseModel.list(listArgs), {}, listArgs @@ -170,10 +171,12 @@ export async function getFindOne(param: { args.sortArr = JSON.parse(args.sortArrJson); } catch (e) {} + const {ast} = await getAst({ model, query: args, view }) + const data = await baseModel.findOne(args); return data ? await nocoExecute( - await getAst({ model, query: args, view }), + ast, data, {}, {} @@ -225,8 +228,10 @@ export async function dataRead( NcError.notFound('Row not found'); } + const { ast } = await getAst({ model, query: param.query, view }) + return await nocoExecute( - await getAst({ model, query: param.query, view }), + ast, row, {}, param.query @@ -279,7 +284,7 @@ export async function getGroupedDataList(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ model, query, view }); + const { ast } = await getAst({ model, query, view }); const listArgs: any = { ...query }; try { @@ -299,7 +304,7 @@ export async function getGroupedDataList(param: { groupColumnId: param.columnId, }); data = await nocoExecute( - { key: 1, value: requestObj }, + { key: 1, value: ast }, groupedData, {}, listArgs @@ -650,8 +655,10 @@ export async function dataReadByViewId(param: { dbDriver: await NcConnectionMgrv2.get(base), }); + const { ast } = await getAst({ model, query: param.query }) + return await nocoExecute( - await getAst({ model, query: param.query }), + ast, await baseModel.readByPk(param.rowId), {}, {} diff --git a/packages/nocodb/src/lib/services/public/publicData.svc.ts b/packages/nocodb/src/lib/services/public/publicData.svc.ts index 707b6b5982..4a0a469117 100644 --- a/packages/nocodb/src/lib/services/public/publicData.svc.ts +++ b/packages/nocodb/src/lib/services/public/publicData.svc.ts @@ -59,20 +59,25 @@ export async function dataList(param: { let count = 0; try { + const { ast } = await getAst({ + query: param.query, + model, + view, + }) + data = await nocoExecute( - await getAst({ - query: param.query, - model, - view, - }), + ast, await baseModel.list(listArgs), {}, listArgs ); count = await baseModel.count(listArgs); } catch (e) { + console.log(e) // show empty result instead of throwing error here // e.g. search some text in a numeric field + + NcError.internalServerError('Please try after some time') } return new PagedResponseImpl(data, { ...param.query, count }); @@ -128,7 +133,7 @@ async function getGroupedDataList(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ model, query: param.query, view }); + const { ast } = await getAst({ model, query: param.query, view }); const listArgs: any = { ...query }; try { @@ -149,7 +154,7 @@ async function getGroupedDataList(param: { groupColumnId, }); data = await nocoExecute( - { key: 1, value: requestObj }, + { key: 1, value: ast }, groupedData, {}, listArgs @@ -304,7 +309,7 @@ export async function relDataList(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ + const { ast } = await getAst({ query: param.query, model, extractOnlyPrimaries: true, @@ -314,7 +319,7 @@ export async function relDataList(param: { let count = 0; try { data = data = await nocoExecute( - requestObj, + ast, await baseModel.list(param.query), {}, param.query diff --git a/packages/nocodb/src/lib/services/public/publicDataExport.svc.ts b/packages/nocodb/src/lib/services/public/publicDataExport.svc.ts index b673f950fe..9b58d362dd 100644 --- a/packages/nocodb/src/lib/services/public/publicDataExport.svc.ts +++ b/packages/nocodb/src/lib/services/public/publicDataExport.svc.ts @@ -46,7 +46,7 @@ export async function getDbRows(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ + const { ast } = await getAst({ query: param.query, model: param.model, view: param.view, @@ -69,7 +69,7 @@ export async function getDbRows(param: { elapsed = temp[0] * 1000 + temp[1] / 1000000 ) { const rows = await nocoExecute( - requestObj, + ast, await baseModel.list({ ...listArgs, offset, limit }), {}, listArgs From ed6e7f3d215527d9610c1afa32002be0e865187e Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 28 Mar 2023 17:29:48 +0530 Subject: [PATCH 06/22] fix: avoid fields from nested ast object Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/helpers/getAst.ts | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts index ec32885b5e..d53e8d57c6 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts @@ -95,16 +95,18 @@ const getAst = async ({ .getColOptions() .then((colOpt) => colOpt.getRelatedTable()); - value = await getAst({ - model, - query: query?.nested?.[col.title], - extractOnlyPrimaries: nestedFields !== '*', - dependencyFields: (dependencyFields.nested[col.title] = dependencyFields - .nested[col.title] || { - nested: {}, - fields: new Set(), - }), - }); + value = ( + await getAst({ + model, + query: query?.nested?.[col.title], + extractOnlyPrimaries: nestedFields !== '*', + dependencyFields: (dependencyFields.nested[col.title] = + dependencyFields.nested[col.title] || { + nested: {}, + fields: new Set(), + }), + }) + ).ast; } const isRequested = From 8476ea5cc41a4235067dd5ea74683450eb75a7b5 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 28 Mar 2023 18:35:33 +0530 Subject: [PATCH 07/22] feat: extract only required nested fields Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 57 ++++++++++++------- .../sql-data-mapper/lib/sql/helpers/getAst.ts | 24 ++++---- .../lib/meta/api/dataApis/dataAliasApis.ts | 29 +++------- .../nocodb/src/lib/services/dbData/index.ts | 2 +- 4 files changed, 57 insertions(+), 55 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index 5015a48bd9..c2f3f53c91 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -3,17 +3,17 @@ import groupBy from 'lodash/groupBy'; import DataLoader from 'dataloader'; import { AuditOperationSubTypes, - AuditOperationTypes, isSystemColumn, + AuditOperationTypes, + isSystemColumn, isVirtualCol, RelationTypes, UITypes, ViewTypes, -} from 'nocodb-sdk' +} from 'nocodb-sdk'; import ejs from 'ejs'; import Validator from 'validator'; import { customAlphabet } from 'nanoid'; import DOMPurify from 'isomorphic-dompurify'; -import { GridViewColumn } from '../../../../models' import Model from '../../../../models/Model'; import Column from '../../../../models/Column'; import Filter, { @@ -39,6 +39,7 @@ import genRollupSelectv2 from './genRollupSelectv2'; import sortV2 from './sortV2'; import conditionV2 from './conditionV2'; import { sanitize, unsanitize } from './helpers/sanitize'; +import type { GridViewColumn } from '../../../../models'; import type { SortType } from 'nocodb-sdk'; import type { Knex } from 'knex'; import type FormulaColumn from '../../../../models/FormulaColumn'; @@ -195,14 +196,18 @@ class BaseModelSqlv2 { filterArr?: Filter[]; sortArr?: Sort[]; sort?: string | string[]; - fieldsSet?:Set + fieldsSet?: Set; } = {}, ignoreViewFilterAndSort = false ): Promise { const { where, fields, ...rest } = this._getListArgs(args as any); const qb = this.dbDriver(this.tnPath); - await this.selectObject({ qb, fieldsSet: args.fieldsSet, viewId: this.viewId }); + await this.selectObject({ + qb, + fieldsSet: args.fieldsSet, + viewId: this.viewId, + }); if (+rest?.shuffle) { await this.shuffle({ qb }); } @@ -274,7 +279,7 @@ class BaseModelSqlv2 { if (!ignoreViewFilterAndSort) applyPaginate(qb, rest); const proto = await this.getProto(); - console.log('list query', qb.toQuery()) + console.log('list query', qb.toQuery()); const data = await this.execAndParse(qb); @@ -401,7 +406,7 @@ class BaseModelSqlv2 { return await qb; } - async multipleHmList({ colId, ids }, args: { limit?; offset? } = {}) { + async multipleHmList({ colId, ids }, args: { limit?; offset?; fieldsSet?:Set } = {}) { try { const { where, sort, ...rest } = this._getListArgs(args as any); // todo: get only required fields @@ -429,7 +434,7 @@ class BaseModelSqlv2 { const parentTn = this.getTnPath(parentTable); const qb = this.dbDriver(childTn); - await childModel.selectObject({ qb, extractPkAndPv: true }); + await childModel.selectObject({ qb, extractPkAndPv: true, fieldsSet: args.fieldsSet }); await this.applySortAndFilter({ table: childTable, where, qb, sort }); const childQb = this.dbDriver.queryBuilder().from( @@ -457,7 +462,6 @@ class BaseModelSqlv2 { .as('list') ); - // console.log(childQb.toQuery()) const children = await this.execAndParse(childQb, childTable); @@ -468,7 +472,6 @@ class BaseModelSqlv2 { }) ).getProto(); - return groupBy( children.map((c) => { c.__proto__ = proto; @@ -546,7 +549,10 @@ class BaseModelSqlv2 { } } - async hmList({ colId, id }, args: { limit?; offset? } = {}) { + async hmList( + { colId, id }, + args: { limit?; offset?; fieldSet?: Set } = {} + ) { try { const { where, sort, ...rest } = this._getListArgs(args as any); // todo: get only required fields @@ -586,7 +592,7 @@ class BaseModelSqlv2 { qb.limit(+rest?.limit || 25); qb.offset(+rest?.offset || 0); - await childModel.selectObject({ qb }); + await childModel.selectObject({ qb, fieldsSet: args.fieldSet }); const children = await this.execAndParse(qb, childTable); @@ -645,7 +651,7 @@ class BaseModelSqlv2 { public async multipleMmList( { colId, parentIds }, - args: { limit?; offset? } = {} + args: { limit?; offset?; fieldsSet?: Set } = {} ) { const { where, sort, ...rest } = this._getListArgs(args as any); const relColumn = (await this.model.getColumns()).find( @@ -678,7 +684,7 @@ class BaseModelSqlv2 { const qb = this.dbDriver(rtn).join(vtn, `${vtn}.${vrcn}`, `${rtn}.${rcn}`); - await childModel.selectObject({ qb }); + await childModel.selectObject({ qb, fieldsSet: args.fieldsSet }); await this.applySortAndFilter({ table: childTable, where, qb, sort }); @@ -724,7 +730,10 @@ class BaseModelSqlv2 { return parentIds.map((id) => gs[id] || []); } - public async mmList({ colId, parentId }, args: { limit?; offset? } = {}) { + public async mmList( + { colId, parentId }, + args: { limit?; offset?; fieldsSet?: Set } = {} + ) { const { where, sort, ...rest } = this._getListArgs(args as any); const relColumn = (await this.model.getColumns()).find( (c) => c.id === colId @@ -764,7 +773,7 @@ class BaseModelSqlv2 { .where(_wherePk(parentTable.primaryKeys, parentId)) ); - await childModel.selectObject({ qb }); + await childModel.selectObject({ qb, fieldsSet: args.fieldsSet }); await this.applySortAndFilter({ table: childTable, where, qb, sort }); @@ -1390,6 +1399,7 @@ class BaseModelSqlv2 { { // limit: ids.length, where: `(${pCol.column_name},in,${ids.join(',')})`, + fieldsSet: (readLoader as any).args?.fieldsSet }, true ); @@ -1402,13 +1412,15 @@ class BaseModelSqlv2 { }); // defining HasMany count method within GQL Type class - proto[column.title] = async function () { + proto[column.title] = async function (args?:any) { if ( this?.[cCol?.title] === null || this?.[cCol?.title] === undefined ) return null; + (readLoader as any).args = args; + return await readLoader.load(this?.[cCol?.title]); }; // todo : handle mm @@ -1460,7 +1472,7 @@ class BaseModelSqlv2 { fields: _fields, extractPkAndPv, viewId, - fieldsSet + fieldsSet, }: { fieldsSet?: Set; qb: Knex.QueryBuilder; @@ -1475,7 +1487,8 @@ class BaseModelSqlv2 { const res = {}; // const columns = _columns ?? (await this.model.getColumns()); // for (const column of columns) { - const viewOrTableColumns = _columns || viewColumns || (await this.model.getColumns()); + const viewOrTableColumns = + _columns || viewColumns || (await this.model.getColumns()); for (const viewOrTableColumn of viewOrTableColumns) { const column = viewOrTableColumn instanceof Column @@ -1486,14 +1499,14 @@ class BaseModelSqlv2 { // hide if column marked as hidden in view // of if column is system field and system field is hidden if ( - (!fieldsSet || !fieldsSet.has(column.title)) && + fieldsSet ? !fieldsSet.has(column.title) : ( !extractPkAndPv && !(viewOrTableColumn instanceof Column) && (!(viewOrTableColumn as GridViewColumn)?.show || (!view?.show_system_fields && column.uidt !== UITypes.ForeignKey && !column.pk && - isSystemColumn(column))) + isSystemColumn(column))) ) ) continue; @@ -2723,7 +2736,7 @@ class BaseModelSqlv2 { qb.limit(+rest?.limit || 25); qb.offset(+rest?.offset || 0); - await this.selectObject({ qb, extractPkAndPv:true }); + await this.selectObject({ qb, extractPkAndPv: true }); // todo: refactor and move to a method (applyFilterAndSort) const aliasColObjMap = await this.model.getAliasColObjMap(); diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts index d53e8d57c6..7a48172e65 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts @@ -14,8 +14,8 @@ const getAst = async ({ model, view, dependencyFields = { - nested: {}, - fields: new Set(), + nested: { ...(query?.nested || {}) }, + fieldsSet: new Set(), }, }: { query?: RequestQuery; @@ -77,7 +77,7 @@ const getAst = async ({ dependencyFields: (dependencyFields.nested[col.title] = dependencyFields.nested[col.title] || { nested: {}, - fields: new Set(), + fieldsSet: new Set(), }), }); @@ -103,7 +103,7 @@ const getAst = async ({ dependencyFields: (dependencyFields.nested[col.title] = dependencyFields.nested[col.title] || { nested: {}, - fields: new Set(), + fieldsSet: new Set(), }), }) ).ast; @@ -133,7 +133,7 @@ const extractDependencies = async ( column: Column, dependencyFields: DependantFields = { nested: {}, - fields: new Set(), + fieldsSet: new Set(), } ) => { switch (column.uidt) { @@ -144,7 +144,7 @@ const extractDependencies = async ( await extractRelationDependencies(column, dependencyFields); break; default: - dependencyFields.fields.add(column.title); + dependencyFields.fieldsSet.add(column.title); break; } }; @@ -153,7 +153,7 @@ const extractLookupDependencies = async ( lookUpColumn: Column, dependencyFields: DependantFields = { nested: {}, - fields: new Set(), + fieldsSet: new Set(), } ) => { const lookupColumnOpts = await lookUpColumn.getColOptions(); @@ -165,7 +165,7 @@ const extractLookupDependencies = async ( relationColumn.title ] || { nested: {}, - fields: new Set(), + fieldsSet: new Set(), }) ); }; @@ -174,20 +174,20 @@ const extractRelationDependencies = async ( relationColumn: Column, dependencyFields: DependantFields = { nested: {}, - fields: new Set(), + fieldsSet: new Set(), } ) => { const relationColumnOpts = await relationColumn.getColOptions(); switch (relationColumnOpts.type) { case RelationTypes.HAS_MANY: - dependencyFields.fields.add( + dependencyFields.fieldsSet.add( await relationColumnOpts.getParentColumn().then((col) => col.title) ); break; case RelationTypes.BELONGS_TO: case RelationTypes.MANY_TO_MANY: - dependencyFields.fields.add( + dependencyFields.fieldsSet.add( await relationColumnOpts.getChildColumn().then((col) => col.title) ); @@ -204,7 +204,7 @@ type RequestQuery = { }; interface DependantFields { - fields?: Set; + fieldsSet?: Set; nested?: DependantFields; } diff --git a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts b/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts index 4a5f6d4c62..e695411899 100644 --- a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts +++ b/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts @@ -1,14 +1,15 @@ -import { Request, Response, Router } from 'express'; -import Model from '../../../models/Model'; +import { Router } from 'express'; import { nocoExecute } from 'nc-help'; +import Model from '../../../models/Model'; import Base from '../../../models/Base'; import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; import { PagedResponseImpl } from '../../helpers/PagedResponse'; -import View from '../../../models/View'; import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import { getViewAndModelFromRequestByAliasOrId } from './helpers'; import apiMetrics from '../../helpers/apiMetrics'; import getAst from '../../../db/sql-data-mapper/lib/sql/helpers/getAst'; +import { getViewAndModelFromRequestByAliasOrId } from './helpers'; +import type View from '../../../models/View'; +import type { Request, Response } from 'express'; async function dataList(req: Request, res: Response) { const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); @@ -158,15 +159,8 @@ async function getFindOne(model, view: View, req) { const data = await baseModel.findOne(args); - const { ast } = await getAst({ model, query: args, view }) - return data - ? await nocoExecute( - ast, - data, - {}, - {} - ) - : {}; + const { ast } = await getAst({ model, query: args, view }); + return data ? await nocoExecute(ast, data, {}, {}) : {}; } async function getDataGroupBy(model, view: View, req) { @@ -199,15 +193,10 @@ async function dataRead(req: Request, res: Response) { dbDriver: NcConnectionMgrv2.get(base), }); - const { ast } = await getAst({ model, query: req.query, view }) + const { ast } = await getAst({ model, query: req.query, view }); res.json( - await nocoExecute( - ast, - await baseModel.readByPk(req.params.rowId), - {}, - {} - ) + await nocoExecute(ast, await baseModel.readByPk(req.params.rowId), {}, {}) ); } diff --git a/packages/nocodb/src/lib/services/dbData/index.ts b/packages/nocodb/src/lib/services/dbData/index.ts index 80f6d57207..741a2d8377 100644 --- a/packages/nocodb/src/lib/services/dbData/index.ts +++ b/packages/nocodb/src/lib/services/dbData/index.ts @@ -117,7 +117,7 @@ export async function getDataList(param: { const { ast, dependencyFields } = await getAst({ model, query, view }); - const listArgs: any = { ...query, fieldsSet: dependencyFields?.fields }; + const listArgs: any = dependencyFields; try { listArgs.filterArr = JSON.parse(listArgs.filterArrJson); } catch (e) {} From 64c3cd8184db34f1970884ab8fc60f3eaf404f78 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 29 Mar 2023 00:55:28 +0530 Subject: [PATCH 08/22] refactor: remove old files Signed-off-by: Pranav C --- .../lib/meta/api/dataApis/dataAliasApis.ts | 334 ------------------ 1 file changed, 334 deletions(-) delete mode 100644 packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts diff --git a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts b/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts deleted file mode 100644 index e695411899..0000000000 --- a/packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts +++ /dev/null @@ -1,334 +0,0 @@ -import { Router } from 'express'; -import { nocoExecute } from 'nc-help'; -import Model from '../../../models/Model'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; -import { PagedResponseImpl } from '../../helpers/PagedResponse'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import apiMetrics from '../../helpers/apiMetrics'; -import getAst from '../../../db/sql-data-mapper/lib/sql/helpers/getAst'; -import { getViewAndModelFromRequestByAliasOrId } from './helpers'; -import type View from '../../../models/View'; -import type { Request, Response } from 'express'; - -async function dataList(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - res.json(await getDataList(model, view, req)); -} - -async function dataFindOne(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - res.json(await getFindOne(model, view, req)); -} - -async function dataGroupBy(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - res.json(await getDataGroupBy(model, view, req)); -} - -async function dataCount(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const countArgs: any = { ...req.query }; - try { - countArgs.filterArr = JSON.parse(countArgs.filterArrJson); - } catch (e) {} - - const count = await baseModel.count(countArgs); - - res.json({ count }); -} - -async function dataInsert(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - res.json(await baseModel.insert(req.body, null, req)); -} - -async function dataUpdate(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - res.json(await baseModel.updateByPk(req.params.rowId, req.body, null, req)); -} - -async function dataDelete(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - const base = await Base.get(model.base_id); - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - const message = await baseModel.hasLTARData(req.params.rowId, model); - if (message.length) { - res.json({ message }); - return; - } - res.json(await baseModel.delByPk(req.params.rowId, null, req)); -} -async function getDataList(model, view: View, req) { - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const { ast, dependencyFields } = await getAst({ - model, - query: req.query, - view, - }); - - const listArgs: any = { ...req.query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - - console.log( - JSON.stringify( - dependencyFields, - (_v, o) => { - if (o instanceof Set) { - return [...o]; - } - return o; - }, - 2 - ) - ); - console.log(JSON.stringify(ast, null, 2)); - const data = await nocoExecute( - ast, - await baseModel.list(listArgs), - {}, - listArgs - ); - - const count = await baseModel.count(listArgs); - - return new PagedResponseImpl(data, { - ...req.query, - count, - }); -} - -async function getFindOne(model, view: View, req) { - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const args: any = { ...req.query }; - try { - args.filterArr = JSON.parse(args.filterArrJson); - } catch (e) {} - try { - args.sortArr = JSON.parse(args.sortArrJson); - } catch (e) {} - - const data = await baseModel.findOne(args); - - const { ast } = await getAst({ model, query: args, view }); - return data ? await nocoExecute(ast, data, {}, {}) : {}; -} - -async function getDataGroupBy(model, view: View, req) { - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const listArgs: any = { ...req.query }; - const data = await baseModel.groupBy({ ...req.query }); - const count = await baseModel.count(listArgs); - - return new PagedResponseImpl(data, { - ...req.query, - count, - }); -} - -async function dataRead(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - const { ast } = await getAst({ model, query: req.query, view }); - - res.json( - await nocoExecute(ast, await baseModel.readByPk(req.params.rowId), {}, {}) - ); -} - -async function dataExist(req: Request, res: Response) { - const { model, view } = await getViewAndModelFromRequestByAliasOrId(req); - - const base = await Base.get(model.base_id); - - const baseModel = await Model.getBaseModelSQL({ - id: model.id, - viewId: view?.id, - dbDriver: NcConnectionMgrv2.get(base), - }); - - res.json(await baseModel.exist(req.params.rowId)); -} -const router = Router({ mergeParams: true }); - -// table data crud apis -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName', - apiMetrics, - ncMetaAclMw(dataList, 'dataList') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/find-one', - apiMetrics, - ncMetaAclMw(dataFindOne, 'dataFindOne') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/groupby', - apiMetrics, - ncMetaAclMw(dataGroupBy, 'dataGroupBy') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/exist', - apiMetrics, - ncMetaAclMw(dataExist, 'dataExist') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/count', - apiMetrics, - ncMetaAclMw(dataCount, 'dataCount') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/count', - apiMetrics, - ncMetaAclMw(dataCount, 'dataCount') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId', - apiMetrics, - ncMetaAclMw(dataRead, 'dataRead') -); - -router.patch( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId', - apiMetrics, - ncMetaAclMw(dataUpdate, 'dataUpdate') -); - -router.delete( - '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId', - apiMetrics, - ncMetaAclMw(dataDelete, 'dataDelete') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName', - apiMetrics, - ncMetaAclMw(dataList, 'dataList') -); - -// table view data crud apis -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName', - apiMetrics, - ncMetaAclMw(dataList, 'dataList') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/find-one', - apiMetrics, - ncMetaAclMw(dataFindOne, 'dataFindOne') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/groupby', - apiMetrics, - ncMetaAclMw(dataGroupBy, 'dataGroupBy') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId/exist', - apiMetrics, - ncMetaAclMw(dataExist, 'dataExist') -); - -router.post( - '/api/v1/db/data/:orgs/:projectName/:tableName', - apiMetrics, - ncMetaAclMw(dataInsert, 'dataInsert') -); - -router.post( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName', - apiMetrics, - ncMetaAclMw(dataInsert, 'dataInsert') -); - -router.patch( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', - apiMetrics, - ncMetaAclMw(dataUpdate, 'dataUpdate') -); - -router.get( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', - apiMetrics, - ncMetaAclMw(dataRead, 'dataRead') -); - -router.delete( - '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId', - apiMetrics, - ncMetaAclMw(dataDelete, 'dataDelete') -); - -export default router; From edfb8f327918440a9ad31eb5f4864810f4be0f5d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 29 Mar 2023 10:13:32 +0530 Subject: [PATCH 09/22] refactor: generate formula and rollup on top of filtered, sorted and limited data Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 55 ++++++++++------ .../sql/formulav2/formulaQueryBuilderv2.ts | 66 +++++++++++++------ 2 files changed, 82 insertions(+), 39 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index c2f3f53c91..5288a84043 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -57,6 +57,9 @@ const GROUP_COL = '__nc_group_id'; const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14); const { v4: uuidv4 } = require('uuid'); +const INNER_QUERY_ALIAS = '__nc_inner'; +// const WRAPPER_QUERY_ALIAS = '__nc_wrapper'; + async function populatePk(model: Model, insertObj: any) { await model.getColumns(); for (const pkCol of model.primaryKeys) { @@ -202,14 +205,18 @@ class BaseModelSqlv2 { ): Promise { const { where, fields, ...rest } = this._getListArgs(args as any); - const qb = this.dbDriver(this.tnPath); + const innerQb = this.dbDriver(this.tnPath); + innerQb.select('*') + + const wrapperQb = this.dbDriver.from(innerQb.as(INNER_QUERY_ALIAS)) await this.selectObject({ - qb, + qb: wrapperQb, fieldsSet: args.fieldsSet, viewId: this.viewId, + alias: INNER_QUERY_ALIAS }); if (+rest?.shuffle) { - await this.shuffle({ qb }); + await this.shuffle({ qb: innerQb }); } const aliasColObjMap = await this.model.getAliasColObjMap(); @@ -235,7 +242,7 @@ class BaseModelSqlv2 { logical_op: 'and', }), ], - qb, + innerQb, this.dbDriver ); @@ -244,7 +251,7 @@ class BaseModelSqlv2 { ? args.sortArr : await Sort.list({ viewId: this.viewId }); - await sortV2(sorts, qb, this.dbDriver); + await sortV2(sorts, innerQb, this.dbDriver); } else { await conditionV2( [ @@ -259,31 +266,31 @@ class BaseModelSqlv2 { logical_op: 'and', }), ], - qb, + innerQb, this.dbDriver ); if (!sorts) sorts = args.sortArr; - await sortV2(sorts, qb, this.dbDriver); + await sortV2(sorts, innerQb, this.dbDriver); } // sort by primary key if not autogenerated string // if autogenerated string sort by created_at column if present if (this.model.primaryKey && this.model.primaryKey.ai) { - qb.orderBy(this.model.primaryKey.column_name); + innerQb.orderBy(this.model.primaryKey.column_name); } else if (this.model.columns.find((c) => c.column_name === 'created_at')) { - qb.orderBy('created_at'); + innerQb.orderBy('created_at'); } - if (!ignoreViewFilterAndSort) applyPaginate(qb, rest); + if (!ignoreViewFilterAndSort) applyPaginate(innerQb, rest); const proto = await this.getProto(); - console.log('list query', qb.toQuery()); + const data = await this.execAndParse(wrapperQb); - const data = await this.execAndParse(qb); + // todo: remove + console.log(wrapperQb.toQuery()); - // console.log(qb.toQuery()); return data?.map((d) => { d.__proto__ = proto; @@ -1247,7 +1254,7 @@ class BaseModelSqlv2 { }); } - private async getSelectQueryBuilderForFormula(column: Column) { + private async getSelectQueryBuilderForFormula(column: Column, tableAlias?:string ) { const formula = await column.getColOptions(); if (formula.error) throw new Error(`Formula error: ${formula.error}`); const qb = await formulaQueryBuilderv2( @@ -1255,7 +1262,9 @@ class BaseModelSqlv2 { null, this.dbDriver, this.model, - column + column, + { }, + tableAlias ); return qb; } @@ -1473,6 +1482,7 @@ class BaseModelSqlv2 { extractPkAndPv, viewId, fieldsSet, + alias }: { fieldsSet?: Set; qb: Knex.QueryBuilder; @@ -1480,6 +1490,7 @@ class BaseModelSqlv2 { fields?: string[] | string; extractPkAndPv?: boolean; viewId?: string; + alias?: string; }): Promise { const view = await View.get(viewId); const viewColumns = viewId && (await View.getColumns(viewId)); @@ -1531,7 +1542,8 @@ class BaseModelSqlv2 { case UITypes.Formula: try { const selectQb = await this.getSelectQueryBuilderForFormula( - qrValueColumn + qrValueColumn, + alias ); qb.select({ [column.column_name]: selectQb.builder, @@ -1563,7 +1575,7 @@ class BaseModelSqlv2 { case UITypes.Formula: try { const selectQb = await this.getSelectQueryBuilderForFormula( - barcodeValueColumn + barcodeValueColumn, ); qb.select({ [column.column_name]: selectQb.builder, @@ -1586,7 +1598,8 @@ class BaseModelSqlv2 { { try { const selectQb = await this.getSelectQueryBuilderForFormula( - column + column, + alias ); qb.select( this.dbDriver.raw(`?? as ??`, [ @@ -1594,7 +1607,8 @@ class BaseModelSqlv2 { sanitize(column.title), ]) ); - } catch { + } catch(e) { + console.log(e) // return dummy select qb.select( this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.title)]) @@ -1609,6 +1623,7 @@ class BaseModelSqlv2 { // tn: this.title, knex: this.dbDriver, // column, + alias, columnOptions: (await column.getColOptions()) as RollupColumn, }) ).builder.as(sanitize(column.title)) @@ -1616,7 +1631,7 @@ class BaseModelSqlv2 { break; default: res[sanitize(column.title || column.column_name)] = sanitize( - `${this.model.table_name}.${column.column_name}` + `${alias || this.model.table_name}.${column.column_name}` ); break; } diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index 74400802e2..b859b8ac66 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -53,7 +53,8 @@ async function _formulaQueryBuilder( alias, knex: XKnex, model: Model, - aliasToColumn = {} + aliasToColumn = {}, + tableAlias?: string ) { // formula may include double curly brackets in previous version // convert to single curly bracket here for compatibility @@ -74,7 +75,8 @@ async function _formulaQueryBuilder( alias, knex, model, - { ...aliasToColumn, [col.id]: null } + { ...aliasToColumn, [col.id]: null }, + tableAlias ); builder.sql = '(' + builder.sql + ')'; aliasToColumn[col.id] = builder; @@ -104,7 +106,9 @@ async function _formulaQueryBuilder( selectQb = knex(`${parentModel.table_name} as ${alias}`).where( `${alias}.${parentColumn.column_name}`, knex.raw(`??`, [ - `${childModel.table_name}.${childColumn.column_name}`, + `${tableAlias ?? childModel.table_name}.${ + childColumn.column_name + }`, ]) ); break; @@ -113,7 +117,9 @@ async function _formulaQueryBuilder( selectQb = knex(`${childModel.table_name} as ${alias}`).where( `${alias}.${childColumn.column_name}`, knex.raw(`??`, [ - `${parentModel.table_name}.${parentColumn.column_name}`, + `${tableAlias ?? parentModel.table_name}.${ + parentColumn.column_name + }`, ]) ); break; @@ -134,7 +140,9 @@ async function _formulaQueryBuilder( .where( `${assocAlias}.${mmChildColumn.column_name}`, knex.raw(`??`, [ - `${childModel.table_name}.${childColumn.column_name}`, + `${tableAlias ?? childModel.table_name}.${ + childColumn.column_name + }`, ]) ); } @@ -402,6 +410,7 @@ async function _formulaQueryBuilder( const qb = await genRollupSelectv2({ knex, columnOptions: (await col.getColOptions()) as RollupColumn, + alias: tableAlias, }); aliasToColumn[col.id] = knex.raw(qb.builder).wrap('(', ')'); } @@ -428,7 +437,9 @@ async function _formulaQueryBuilder( .where( `${parentModel.table_name}.${parentColumn.column_name}`, knex.raw(`??`, [ - `${childModel.table_name}.${childColumn.column_name}`, + `${tableAlias ?? childModel.table_name}.${ + childColumn.column_name + }`, ]) ); } else if (relation.type == 'hm') { @@ -437,7 +448,9 @@ async function _formulaQueryBuilder( .where( `${childModel.table_name}.${childColumn.column_name}`, knex.raw(`??`, [ - `${parentModel.table_name}.${parentColumn.column_name}`, + `${tableAlias ?? parentModel.table_name}.${ + parentColumn.column_name + }`, ]) ); @@ -490,7 +503,9 @@ async function _formulaQueryBuilder( .where( `${mmModel.table_name}.${mmChildColumn.column_name}`, knex.raw(`??`, [ - `${childModel.table_name}.${childColumn.column_name}`, + `${tableAlias ?? childModel.table_name}.${ + childColumn.column_name + }`, ]) ); selectQb = (fn) => @@ -775,18 +790,27 @@ async function _formulaQueryBuilder( return { builder: fn(tree, alias) }; } -function getTnPath(tb: Model, knex) { +function getTnPath(tb: Model, knex, tableAlias?: string) { const schema = knex.searchPath?.(); if (knex.clientType() === 'mssql' && schema) { - return knex.raw('??.??', [schema, tb.table_name]); - } else if (knex.clientType() === 'snowflake') { - return [ - knex.client.config.connection.database, - knex.client.config.connection.schema, + return knex.raw(`??.??${tableAlias ? ' as ??' : ''}`, [ + schema, tb.table_name, - ].join('.'); + ...(tableAlias ? [tableAlias] : []), + ]); + } else if (knex.clientType() === 'snowflake') { + return ( + [ + knex.client.config.connection.database, + knex.client.config.connection.schema, + tb.table_name, + ].join('.') + (tableAlias ? ` as ${tableAlias}` : '') + ); } else { - return tb.table_name; + return knex.raw(`??${tableAlias ? ' as ??' : ''}`, [ + tb.table_name, + ...(tableAlias ? [tableAlias] : []), + ]); } } @@ -796,7 +820,8 @@ export default async function formulaQueryBuilderv2( knex: XKnex, model: Model, column?: Column, - aliasToColumn = {} + aliasToColumn = {}, + tableAlias?: string ) { // register jsep curly hook once only jsep.plugins.register(jsepCurlyHook); @@ -806,13 +831,16 @@ export default async function formulaQueryBuilderv2( alias, knex, model, - aliasToColumn + aliasToColumn, + tableAlias ); try { // dry run qb.builder to see if it will break the grid view or not // if so, set formula error and show empty selectQb instead - await knex(getTnPath(model, knex)).select(qb.builder).as('dry-run-only'); + await knex(getTnPath(model, knex, tableAlias)) + .select(qb.builder) + .as('dry-run-only'); // if column is provided, i.e. formula has been created if (column) { From 6bef279c385deb657ac5b423a7f5a2487425f2f7 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 29 Mar 2023 10:58:06 +0530 Subject: [PATCH 10/22] refactor: cleanup Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 59 +++++++++++-------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index 5288a84043..c37bf83e1e 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -206,14 +206,14 @@ class BaseModelSqlv2 { const { where, fields, ...rest } = this._getListArgs(args as any); const innerQb = this.dbDriver(this.tnPath); - innerQb.select('*') + innerQb.select('*'); - const wrapperQb = this.dbDriver.from(innerQb.as(INNER_QUERY_ALIAS)) + const wrapperQb = this.dbDriver.from(innerQb.as(INNER_QUERY_ALIAS)); await this.selectObject({ qb: wrapperQb, fieldsSet: args.fieldsSet, viewId: this.viewId, - alias: INNER_QUERY_ALIAS + alias: INNER_QUERY_ALIAS, }); if (+rest?.shuffle) { await this.shuffle({ qb: innerQb }); @@ -288,10 +288,6 @@ class BaseModelSqlv2 { const data = await this.execAndParse(wrapperQb); - // todo: remove - console.log(wrapperQb.toQuery()); - - return data?.map((d) => { d.__proto__ = proto; return d; @@ -413,7 +409,10 @@ class BaseModelSqlv2 { return await qb; } - async multipleHmList({ colId, ids }, args: { limit?; offset?; fieldsSet?:Set } = {}) { + async multipleHmList( + { colId, ids }, + args: { limit?; offset?; fieldsSet?: Set } = {} + ) { try { const { where, sort, ...rest } = this._getListArgs(args as any); // todo: get only required fields @@ -441,7 +440,11 @@ class BaseModelSqlv2 { const parentTn = this.getTnPath(parentTable); const qb = this.dbDriver(childTn); - await childModel.selectObject({ qb, extractPkAndPv: true, fieldsSet: args.fieldsSet }); + await childModel.selectObject({ + qb, + extractPkAndPv: true, + fieldsSet: args.fieldsSet, + }); await this.applySortAndFilter({ table: childTable, where, qb, sort }); const childQb = this.dbDriver.queryBuilder().from( @@ -1254,7 +1257,10 @@ class BaseModelSqlv2 { }); } - private async getSelectQueryBuilderForFormula(column: Column, tableAlias?:string ) { + private async getSelectQueryBuilderForFormula( + column: Column, + tableAlias?: string + ) { const formula = await column.getColOptions(); if (formula.error) throw new Error(`Formula error: ${formula.error}`); const qb = await formulaQueryBuilderv2( @@ -1263,7 +1269,7 @@ class BaseModelSqlv2 { this.dbDriver, this.model, column, - { }, + {}, tableAlias ); return qb; @@ -1408,7 +1414,7 @@ class BaseModelSqlv2 { { // limit: ids.length, where: `(${pCol.column_name},in,${ids.join(',')})`, - fieldsSet: (readLoader as any).args?.fieldsSet + fieldsSet: (readLoader as any).args?.fieldsSet, }, true ); @@ -1421,7 +1427,7 @@ class BaseModelSqlv2 { }); // defining HasMany count method within GQL Type class - proto[column.title] = async function (args?:any) { + proto[column.title] = async function (args?: any) { if ( this?.[cCol?.title] === null || this?.[cCol?.title] === undefined @@ -1482,7 +1488,7 @@ class BaseModelSqlv2 { extractPkAndPv, viewId, fieldsSet, - alias + alias, }: { fieldsSet?: Set; qb: Knex.QueryBuilder; @@ -1510,14 +1516,15 @@ class BaseModelSqlv2 { // hide if column marked as hidden in view // of if column is system field and system field is hidden if ( - fieldsSet ? !fieldsSet.has(column.title) : ( - !extractPkAndPv && - !(viewOrTableColumn instanceof Column) && - (!(viewOrTableColumn as GridViewColumn)?.show || - (!view?.show_system_fields && - column.uidt !== UITypes.ForeignKey && - !column.pk && - isSystemColumn(column))) ) + fieldsSet + ? !fieldsSet.has(column.title) + : !extractPkAndPv && + !(viewOrTableColumn instanceof Column) && + (!(viewOrTableColumn as GridViewColumn)?.show || + (!view?.show_system_fields && + column.uidt !== UITypes.ForeignKey && + !column.pk && + isSystemColumn(column))) ) continue; @@ -1543,7 +1550,7 @@ class BaseModelSqlv2 { try { const selectQb = await this.getSelectQueryBuilderForFormula( qrValueColumn, - alias + alias ); qb.select({ [column.column_name]: selectQb.builder, @@ -1575,7 +1582,7 @@ class BaseModelSqlv2 { case UITypes.Formula: try { const selectQb = await this.getSelectQueryBuilderForFormula( - barcodeValueColumn, + barcodeValueColumn ); qb.select({ [column.column_name]: selectQb.builder, @@ -1607,8 +1614,8 @@ class BaseModelSqlv2 { sanitize(column.title), ]) ); - } catch(e) { - console.log(e) + } catch (e) { + console.log(e); // return dummy select qb.select( this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.title)]) From e803ff6a54b1d863d2f3c236d6b8a75793138194 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 29 Mar 2023 12:02:01 +0530 Subject: [PATCH 11/22] fix: pagination bug fix Signed-off-by: Pranav C --- .../src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 5 ++++- .../lib/sql/formulav2/formulaQueryBuilderv2.ts | 2 +- .../src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index c37bf83e1e..89f9805255 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -201,7 +201,8 @@ class BaseModelSqlv2 { sort?: string | string[]; fieldsSet?: Set; } = {}, - ignoreViewFilterAndSort = false + ignoreViewFilterAndSort = false, + ): Promise { const { where, fields, ...rest } = this._getListArgs(args as any); @@ -286,6 +287,8 @@ class BaseModelSqlv2 { if (!ignoreViewFilterAndSort) applyPaginate(innerQb, rest); const proto = await this.getProto(); + console.log(wrapperQb.toQuery()) + const data = await this.execAndParse(wrapperQb); return data?.map((d) => { diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index b859b8ac66..edf1da8221 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -821,7 +821,7 @@ export default async function formulaQueryBuilderv2( model: Model, column?: Column, aliasToColumn = {}, - tableAlias?: string + tableAlias?: string, ) { // register jsep curly hook once only jsep.plugins.register(jsepCurlyHook); diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts index 7a48172e65..6aac66c2f3 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts @@ -14,6 +14,7 @@ const getAst = async ({ model, view, dependencyFields = { + ...(query || {}), nested: { ...(query?.nested || {}) }, fieldsSet: new Set(), }, From fc7e69d1e16340be19a7a64e63997c214ff67fb7 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 29 Mar 2023 12:39:32 +0530 Subject: [PATCH 12/22] refactor: validate formula only if error occurred (WIP) Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 34 ++++++++++++++----- .../sql/formulav2/formulaQueryBuilderv2.ts | 3 ++ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index 89f9805255..4563567aea 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -58,6 +58,7 @@ const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14); const { v4: uuidv4 } = require('uuid'); const INNER_QUERY_ALIAS = '__nc_inner'; + // const WRAPPER_QUERY_ALIAS = '__nc_wrapper'; async function populatePk(model: Model, insertObj: any) { @@ -202,7 +203,7 @@ class BaseModelSqlv2 { fieldsSet?: Set; } = {}, ignoreViewFilterAndSort = false, - + validateFormula = false ): Promise { const { where, fields, ...rest } = this._getListArgs(args as any); @@ -215,6 +216,7 @@ class BaseModelSqlv2 { fieldsSet: args.fieldsSet, viewId: this.viewId, alias: INNER_QUERY_ALIAS, + validateFormula }); if (+rest?.shuffle) { await this.shuffle({ qb: innerQb }); @@ -287,10 +289,16 @@ class BaseModelSqlv2 { if (!ignoreViewFilterAndSort) applyPaginate(innerQb, rest); const proto = await this.getProto(); - console.log(wrapperQb.toQuery()) - - const data = await this.execAndParse(wrapperQb); + let data; + try { + data = await this.execAndParse(wrapperQb); + } catch (e) { + if (!validateFormula) { + return this.list(args, ignoreViewFilterAndSort, true); + } + throw e; + } return data?.map((d) => { d.__proto__ = proto; return d; @@ -1262,7 +1270,8 @@ class BaseModelSqlv2 { private async getSelectQueryBuilderForFormula( column: Column, - tableAlias?: string + tableAlias?: string, + validateFormula = false ) { const formula = await column.getColOptions(); if (formula.error) throw new Error(`Formula error: ${formula.error}`); @@ -1273,7 +1282,8 @@ class BaseModelSqlv2 { this.model, column, {}, - tableAlias + tableAlias, + validateFormula ); return qb; } @@ -1492,6 +1502,7 @@ class BaseModelSqlv2 { viewId, fieldsSet, alias, + validateFormula }: { fieldsSet?: Set; qb: Knex.QueryBuilder; @@ -1500,6 +1511,7 @@ class BaseModelSqlv2 { extractPkAndPv?: boolean; viewId?: string; alias?: string; + validateFormula?:boolean }): Promise { const view = await View.get(viewId); const viewColumns = viewId && (await View.getColumns(viewId)); @@ -1553,7 +1565,8 @@ class BaseModelSqlv2 { try { const selectQb = await this.getSelectQueryBuilderForFormula( qrValueColumn, - alias + alias, + validateFormula ); qb.select({ [column.column_name]: selectQb.builder, @@ -1585,7 +1598,9 @@ class BaseModelSqlv2 { case UITypes.Formula: try { const selectQb = await this.getSelectQueryBuilderForFormula( - barcodeValueColumn + barcodeValueColumn, + alias, + validateFormula ); qb.select({ [column.column_name]: selectQb.builder, @@ -1609,7 +1624,8 @@ class BaseModelSqlv2 { try { const selectQb = await this.getSelectQueryBuilderForFormula( column, - alias + alias, + validateFormula ); qb.select( this.dbDriver.raw(`?? as ??`, [ diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index edf1da8221..0b59219c43 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -822,6 +822,7 @@ export default async function formulaQueryBuilderv2( column?: Column, aliasToColumn = {}, tableAlias?: string, + validateFormula = false ) { // register jsep curly hook once only jsep.plugins.register(jsepCurlyHook); @@ -835,6 +836,8 @@ export default async function formulaQueryBuilderv2( tableAlias ); + if(!validateFormula) return qb; + try { // dry run qb.builder to see if it will break the grid view or not // if so, set formula error and show empty selectQb instead From ad2024a6bdfa7ee09c0b158c3c4f52cf79b0ebf8 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 29 Mar 2023 15:09:14 +0530 Subject: [PATCH 13/22] fix: define nested and fieldSet default value if missing Signed-off-by: Pranav C --- .../src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts index 6aac66c2f3..da3336a0f3 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts @@ -26,6 +26,10 @@ const getAst = async ({ view?: View; dependencyFields?: DependantFields; }) => { + // set default values of dependencyFields and nested + dependencyFields.nested = dependencyFields.nested || {}; + dependencyFields.fieldsSet = dependencyFields.fieldsSet || new Set(); + if (!model.columns?.length) await model.getColumns(); // extract only pk and pv From 0f790e9e07e39a63cd98ced8d97603cac10096e6 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 29 Mar 2023 15:44:52 +0530 Subject: [PATCH 14/22] refactor: if fieldsSet is defined use it for select query Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index 4563567aea..647c6ef601 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -216,7 +216,7 @@ class BaseModelSqlv2 { fieldsSet: args.fieldsSet, viewId: this.viewId, alias: INNER_QUERY_ALIAS, - validateFormula + validateFormula, }); if (+rest?.shuffle) { await this.shuffle({ qb: innerQb }); @@ -1502,7 +1502,7 @@ class BaseModelSqlv2 { viewId, fieldsSet, alias, - validateFormula + validateFormula, }: { fieldsSet?: Set; qb: Knex.QueryBuilder; @@ -1511,16 +1511,26 @@ class BaseModelSqlv2 { extractPkAndPv?: boolean; viewId?: string; alias?: string; - validateFormula?:boolean + validateFormula?: boolean; }): Promise { - const view = await View.get(viewId); - const viewColumns = viewId && (await View.getColumns(viewId)); - const fields = Array.isArray(_fields) ? _fields : _fields?.split(','); + let viewOrTableColumns: Column[] | { fk_column_id?: string }[]; + const res = {}; - // const columns = _columns ?? (await this.model.getColumns()); - // for (const column of columns) { - const viewOrTableColumns = - _columns || viewColumns || (await this.model.getColumns()); + let view: View; + let fields: string[] = []; + + if (fieldsSet?.size) { + viewOrTableColumns = await this.model.getColumns(); + } else { + view = await View.get(viewId); + const viewColumns = viewId && (await View.getColumns(viewId)); + fields = Array.isArray(_fields) ? _fields : _fields?.split(','); + + // const columns = _columns ?? (await this.model.getColumns()); + // for (const column of columns) { + viewOrTableColumns = + _columns || viewColumns || (await this.model.getColumns()); + } for (const viewOrTableColumn of viewOrTableColumns) { const column = viewOrTableColumn instanceof Column From 516178479354bb95c5ab840cbdb788b3396e74e6 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 29 Mar 2023 16:26:38 +0530 Subject: [PATCH 15/22] fix: fields corrections Signed-off-by: Pranav C --- .../src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index 647c6ef601..cb0d72ceab 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -1517,10 +1517,10 @@ class BaseModelSqlv2 { const res = {}; let view: View; - let fields: string[] = []; + let fields: string[]; if (fieldsSet?.size) { - viewOrTableColumns = await this.model.getColumns(); + viewOrTableColumns = _columns || (await this.model.getColumns()); } else { view = await View.get(viewId); const viewColumns = viewId && (await View.getColumns(viewId)); From 8bdd47a2456600c848e52a83f03a458ffa65b8dc Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 29 Mar 2023 18:53:33 +0530 Subject: [PATCH 16/22] fix: include primary key by default in fields Signed-off-by: Pranav C --- .../nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts index da3336a0f3..a0b4d1c92e 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts @@ -123,7 +123,7 @@ const getAst = async ({ : fields?.length ? fields.includes(col.title) && value : value; - if (isRequested) await extractDependencies(col, dependencyFields); + if (isRequested || col.pk) await extractDependencies(col, dependencyFields); return { ...(await obj), From da0a4534d8f31c1b5912ec85333fa368749f1279 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 30 Mar 2023 11:13:23 +0530 Subject: [PATCH 17/22] fix: include fieldsSet wherever necessary Signed-off-by: Pranav C --- packages/nocodb/src/lib/services/dbData/helpers.ts | 4 ++-- packages/nocodb/src/lib/services/dbData/index.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/nocodb/src/lib/services/dbData/helpers.ts b/packages/nocodb/src/lib/services/dbData/helpers.ts index 3e709022ad..7f1353062e 100644 --- a/packages/nocodb/src/lib/services/dbData/helpers.ts +++ b/packages/nocodb/src/lib/services/dbData/helpers.ts @@ -242,7 +242,7 @@ export async function getDbRows(param: { temp = process.hrtime(startTime), elapsed = temp[0] * 1000 + temp[1] / 1000000 ) { - const {ast} = await getAst({ + const {ast, dependencyFields} = await getAst({ query: query, includePkByDefault: false, model: view.model, @@ -250,7 +250,7 @@ export async function getDbRows(param: { }); const rows = await nocoExecute( ast, - await baseModel.list({ ...listArgs, offset, limit }), + await baseModel.list({ ...listArgs, offset, limit, ...dependencyFields }), {}, query ); diff --git a/packages/nocodb/src/lib/services/dbData/index.ts b/packages/nocodb/src/lib/services/dbData/index.ts index 741a2d8377..5a62023a14 100644 --- a/packages/nocodb/src/lib/services/dbData/index.ts +++ b/packages/nocodb/src/lib/services/dbData/index.ts @@ -171,9 +171,9 @@ export async function getFindOne(param: { args.sortArr = JSON.parse(args.sortArrJson); } catch (e) {} - const {ast} = await getAst({ model, query: args, view }) + const {ast, dependencyFields} = await getAst({ model, query: args, view }) - const data = await baseModel.findOne(args); + const data = await baseModel.findOne({ ...args, dependencyFields }); return data ? await nocoExecute( ast, @@ -228,7 +228,7 @@ export async function dataRead( NcError.notFound('Row not found'); } - const { ast } = await getAst({ model, query: param.query, view }) + const { ast, dependencyFields } = await getAst({ model, query: param.query, view }) return await nocoExecute( ast, From 12c5f6f0a7fef4ef14d7f7ef20ccfe74a4b82509 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 30 Mar 2023 11:15:09 +0530 Subject: [PATCH 18/22] chore: lint Signed-off-by: Pranav C --- .../src/lib/controllers/dbData/oldData.ctl.ts | 11 ++---- .../sql/formulav2/formulaQueryBuilderv2.ts | 2 +- .../nocodb/src/lib/services/dbData/helpers.ts | 2 +- .../nocodb/src/lib/services/dbData/index.ts | 37 ++++--------------- .../src/lib/services/public/publicData.svc.ts | 20 +++------- 5 files changed, 17 insertions(+), 55 deletions(-) diff --git a/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts b/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts index 4c91aa2ecc..15608b4b91 100644 --- a/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts +++ b/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts @@ -132,19 +132,14 @@ async function dataRead(req: Request, res: Response) { dbDriver: await NcConnectionMgrv2.get(base), }); - const {ast} = await getAst({ + const { ast } = await getAst({ query: req.query, model, view, - }) + }); res.json( - await nocoExecute( - ast , - await baseModel.readByPk(req.params.rowId), - {}, - {} - ) + await nocoExecute(ast, await baseModel.readByPk(req.params.rowId), {}, {}) ); } diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index 0b59219c43..3eb91300d9 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -836,7 +836,7 @@ export default async function formulaQueryBuilderv2( tableAlias ); - if(!validateFormula) return qb; + if (!validateFormula) return qb; try { // dry run qb.builder to see if it will break the grid view or not diff --git a/packages/nocodb/src/lib/services/dbData/helpers.ts b/packages/nocodb/src/lib/services/dbData/helpers.ts index 7f1353062e..92395e2f0b 100644 --- a/packages/nocodb/src/lib/services/dbData/helpers.ts +++ b/packages/nocodb/src/lib/services/dbData/helpers.ts @@ -242,7 +242,7 @@ export async function getDbRows(param: { temp = process.hrtime(startTime), elapsed = temp[0] * 1000 + temp[1] / 1000000 ) { - const {ast, dependencyFields} = await getAst({ + const { ast, dependencyFields } = await getAst({ query: query, includePkByDefault: false, model: view.model, diff --git a/packages/nocodb/src/lib/services/dbData/index.ts b/packages/nocodb/src/lib/services/dbData/index.ts index 5a62023a14..6bacb1f1b0 100644 --- a/packages/nocodb/src/lib/services/dbData/index.ts +++ b/packages/nocodb/src/lib/services/dbData/index.ts @@ -116,7 +116,6 @@ export async function getDataList(param: { const { ast, dependencyFields } = await getAst({ model, query, view }); - const listArgs: any = dependencyFields; try { listArgs.filterArr = JSON.parse(listArgs.filterArrJson); @@ -128,12 +127,7 @@ export async function getDataList(param: { let data = []; let count = 0; try { - data = await nocoExecute( - ast, - await baseModel.list(listArgs), - {}, - listArgs - ); + data = await nocoExecute(ast, await baseModel.list(listArgs), {}, listArgs); count = await baseModel.count(listArgs); } catch (e) { console.log(e); @@ -171,17 +165,10 @@ export async function getFindOne(param: { args.sortArr = JSON.parse(args.sortArrJson); } catch (e) {} - const {ast, dependencyFields} = await getAst({ model, query: args, view }) + const { ast, dependencyFields } = await getAst({ model, query: args, view }); const data = await baseModel.findOne({ ...args, dependencyFields }); - return data - ? await nocoExecute( - ast, - data, - {}, - {} - ) - : {}; + return data ? await nocoExecute(ast, data, {}, {}) : {}; } export async function getDataGroupBy(param: { @@ -228,14 +215,9 @@ export async function dataRead( NcError.notFound('Row not found'); } - const { ast, dependencyFields } = await getAst({ model, query: param.query, view }) + const { ast } = await getAst({ model, query: param.query, view }); - return await nocoExecute( - ast, - row, - {}, - param.query - ); + return await nocoExecute(ast, row, {}, param.query); } export async function dataExist( @@ -303,12 +285,7 @@ export async function getGroupedDataList(param: { ...listArgs, groupColumnId: param.columnId, }); - data = await nocoExecute( - { key: 1, value: ast }, - groupedData, - {}, - listArgs - ); + data = await nocoExecute({ key: 1, value: ast }, groupedData, {}, listArgs); const countArr = await baseModel.groupedListCount({ ...listArgs, groupColumnId: param.columnId, @@ -655,7 +632,7 @@ export async function dataReadByViewId(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const { ast } = await getAst({ model, query: param.query }) + const { ast } = await getAst({ model, query: param.query }); return await nocoExecute( ast, diff --git a/packages/nocodb/src/lib/services/public/publicData.svc.ts b/packages/nocodb/src/lib/services/public/publicData.svc.ts index 4a0a469117..0a29b7201b 100644 --- a/packages/nocodb/src/lib/services/public/publicData.svc.ts +++ b/packages/nocodb/src/lib/services/public/publicData.svc.ts @@ -63,21 +63,16 @@ export async function dataList(param: { query: param.query, model, view, - }) + }); - data = await nocoExecute( - ast, - await baseModel.list(listArgs), - {}, - listArgs - ); + data = await nocoExecute(ast, await baseModel.list(listArgs), {}, listArgs); count = await baseModel.count(listArgs); } catch (e) { - console.log(e) + console.log(e); // show empty result instead of throwing error here // e.g. search some text in a numeric field - NcError.internalServerError('Please try after some time') + NcError.internalServerError('Please try after some time'); } return new PagedResponseImpl(data, { ...param.query, count }); @@ -153,12 +148,7 @@ async function getGroupedDataList(param: { ...listArgs, groupColumnId, }); - data = await nocoExecute( - { key: 1, value: ast }, - groupedData, - {}, - listArgs - ); + data = await nocoExecute({ key: 1, value: ast }, groupedData, {}, listArgs); const countArr = await baseModel.groupedListCount({ ...listArgs, groupColumnId, From a4a6e924c969d65a78e2226218b79958d10854a3 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 30 Mar 2023 11:24:28 +0530 Subject: [PATCH 19/22] refactor: revert subquery implementation since it won't improve the performance Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index cb0d72ceab..b7f0551108 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -57,10 +57,6 @@ const GROUP_COL = '__nc_group_id'; const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14); const { v4: uuidv4 } = require('uuid'); -const INNER_QUERY_ALIAS = '__nc_inner'; - -// const WRAPPER_QUERY_ALIAS = '__nc_wrapper'; - async function populatePk(model: Model, insertObj: any) { await model.getColumns(); for (const pkCol of model.primaryKeys) { @@ -207,19 +203,16 @@ class BaseModelSqlv2 { ): Promise { const { where, fields, ...rest } = this._getListArgs(args as any); - const innerQb = this.dbDriver(this.tnPath); - innerQb.select('*'); + const qb = this.dbDriver(this.tnPath); - const wrapperQb = this.dbDriver.from(innerQb.as(INNER_QUERY_ALIAS)); await this.selectObject({ - qb: wrapperQb, + qb, fieldsSet: args.fieldsSet, viewId: this.viewId, - alias: INNER_QUERY_ALIAS, validateFormula, }); if (+rest?.shuffle) { - await this.shuffle({ qb: innerQb }); + await this.shuffle({ qb }); } const aliasColObjMap = await this.model.getAliasColObjMap(); @@ -245,7 +238,7 @@ class BaseModelSqlv2 { logical_op: 'and', }), ], - innerQb, + qb, this.dbDriver ); @@ -254,7 +247,7 @@ class BaseModelSqlv2 { ? args.sortArr : await Sort.list({ viewId: this.viewId }); - await sortV2(sorts, innerQb, this.dbDriver); + await sortV2(sorts, qb, this.dbDriver); } else { await conditionV2( [ @@ -269,30 +262,30 @@ class BaseModelSqlv2 { logical_op: 'and', }), ], - innerQb, + qb, this.dbDriver ); if (!sorts) sorts = args.sortArr; - await sortV2(sorts, innerQb, this.dbDriver); + await sortV2(sorts, qb, this.dbDriver); } // sort by primary key if not autogenerated string // if autogenerated string sort by created_at column if present if (this.model.primaryKey && this.model.primaryKey.ai) { - innerQb.orderBy(this.model.primaryKey.column_name); + qb.orderBy(this.model.primaryKey.column_name); } else if (this.model.columns.find((c) => c.column_name === 'created_at')) { - innerQb.orderBy('created_at'); + qb.orderBy('created_at'); } - if (!ignoreViewFilterAndSort) applyPaginate(innerQb, rest); + if (!ignoreViewFilterAndSort) applyPaginate(qb, rest); const proto = await this.getProto(); let data; try { - data = await this.execAndParse(wrapperQb); + data = await this.execAndParse(qb); } catch (e) { if (!validateFormula) { return this.list(args, ignoreViewFilterAndSort, true); From 9e406d55ca1c4eadc7776ecf38e780f8ae290610 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 30 Mar 2023 12:34:16 +0530 Subject: [PATCH 20/22] fix: avoid overriding query params(limit,offset) Signed-off-by: Pranav C --- packages/nocodb/src/lib/services/dbData/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/lib/services/dbData/helpers.ts b/packages/nocodb/src/lib/services/dbData/helpers.ts index 92395e2f0b..4fcee58b6b 100644 --- a/packages/nocodb/src/lib/services/dbData/helpers.ts +++ b/packages/nocodb/src/lib/services/dbData/helpers.ts @@ -250,7 +250,7 @@ export async function getDbRows(param: { }); const rows = await nocoExecute( ast, - await baseModel.list({ ...listArgs, offset, limit, ...dependencyFields }), + await baseModel.list({ ...listArgs, ...dependencyFields, offset, limit }), {}, query ); From 05ff4a300957f71a332dd2311b68f34465bc3f53 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 30 Mar 2023 13:04:41 +0530 Subject: [PATCH 21/22] feat: add formula validation for all read/list methods and invoke it only when there is atleast one formula column Signed-off-by: Pranav C --- .../sql-data-mapper/lib/sql/BaseModelSqlv2.ts | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index b7f0551108..c4b1f6a1ee 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -114,14 +114,23 @@ class BaseModelSqlv2 { autoBind(this); } - public async readByPk(id?: any): Promise { + public async readByPk(id?: any, validateFormula = false): Promise { const qb = this.dbDriver(this.tnPath); - await this.selectObject({ qb }); + await this.selectObject({ qb, validateFormula }); qb.where(_wherePk(this.model.primaryKeys, id)); - const data = (await this.execAndParse(qb))?.[0]; + let data; + + try { + data = (await this.execAndParse(qb))?.[0]; + } catch (e) { + if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) + throw e; + console.log(e); + return this.readByPk(id, true); + } if (data) { const proto = await this.getProto(); @@ -146,11 +155,12 @@ class BaseModelSqlv2 { where?: string; filterArr?: Filter[]; sort?: string | string[]; - } = {} + } = {}, + validateFormula = false ): Promise { const { where, ...rest } = this._getListArgs(args as any); const qb = this.dbDriver(this.tnPath); - await this.selectObject({ qb }); + await this.selectObject({ qb, validateFormula }); const aliasColObjMap = await this.model.getAliasColObjMap(); const sorts = extractSortsObject(rest?.sort, aliasColObjMap); @@ -179,7 +189,16 @@ class BaseModelSqlv2 { qb.orderBy(this.model.primaryKey.column_name); } - const data = await qb.first(); + let data; + + try { + data = await qb.first(); + } catch (e) { + if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) + throw e; + console.log(e); + return this.findOne(args, true); + } if (data) { const proto = await this.getProto(); @@ -287,10 +306,10 @@ class BaseModelSqlv2 { try { data = await this.execAndParse(qb); } catch (e) { - if (!validateFormula) { - return this.list(args, ignoreViewFilterAndSort, true); - } - throw e; + if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) + throw e; + console.log(e); + return this.list(args, ignoreViewFilterAndSort, true); } return data?.map((d) => { d.__proto__ = proto; @@ -3198,4 +3217,8 @@ function getCompositePk(primaryKeys: Column[], row) { return primaryKeys.map((c) => row[c.title]).join('___'); } +function haveFormulaColumn(columns: Column[]) { + return columns.some((c) => c.uidt === UITypes.Formula); +} + export { BaseModelSqlv2 }; From 0fa078f3aa406c35e276ba63543d90ffcb852604 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 30 Mar 2023 13:17:32 +0530 Subject: [PATCH 22/22] fix: validate formula on column create/update Signed-off-by: Pranav C --- .../nocodb/src/lib/services/column.svc.ts | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/lib/services/column.svc.ts b/packages/nocodb/src/lib/services/column.svc.ts index b3f35160c6..4ff0a6ea10 100644 --- a/packages/nocodb/src/lib/services/column.svc.ts +++ b/packages/nocodb/src/lib/services/column.svc.ts @@ -129,7 +129,16 @@ export async function columnUpdate(param: { try { // test the query to see if it is valid in db level const dbDriver = await NcConnectionMgrv2.get(base); - await formulaQueryBuilderv2(colBody.formula, null, dbDriver, table); + await formulaQueryBuilderv2( + colBody.formula, + null, + dbDriver, + table, + null, + {}, + null, + true + ); } catch (e) { console.error(e); NcError.badRequest('Invalid Formula'); @@ -934,7 +943,16 @@ export async function columnAdd(param: { try { // test the query to see if it is valid in db level const dbDriver = await NcConnectionMgrv2.get(base); - await formulaQueryBuilderv2(colBody.formula, null, dbDriver, table); + await formulaQueryBuilderv2( + colBody.formula, + null, + dbDriver, + table, + null, + {}, + null, + true + ); } catch (e) { console.error(e); NcError.badRequest('Invalid Formula');