diff --git a/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts b/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts index d5a573b013..15608b4b91 100644 --- a/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts +++ b/packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts @@ -21,7 +21,7 @@ export async function dataList(req: Request, res: Response) { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ + const { ast } = await getAst({ query: req.query, model, view, @@ -36,7 +36,7 @@ export async function dataList(req: Request, res: Response) { } catch (e) {} const data = await nocoExecute( - requestObj, + ast, await baseModel.list(listArgs), {}, listArgs @@ -132,17 +132,14 @@ async function dataRead(req: Request, res: Response) { dbDriver: await NcConnectionMgrv2.get(base), }); + const { ast } = await getAst({ + query: req.query, + model, + view, + }); + res.json( - await nocoExecute( - await getAst({ - query: req.query, - model, - view, - }), - await baseModel.readByPk(req.params.rowId), - {}, - {} - ) + await nocoExecute(ast, await baseModel.readByPk(req.params.rowId), {}, {}) ); } diff --git a/packages/nocodb/src/lib/controllers/publicControllers/publicDataExport.ctl.ts b/packages/nocodb/src/lib/controllers/publicControllers/publicDataExport.ctl.ts index 9ab36afdc7..0cc375a595 100644 --- a/packages/nocodb/src/lib/controllers/publicControllers/publicDataExport.ctl.ts +++ b/packages/nocodb/src/lib/controllers/publicControllers/publicDataExport.ctl.ts @@ -136,7 +136,7 @@ async function getDbRows(model, view: View, req: Request) { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ + const { ast } = await getAst({ query: req.query, model, view, @@ -159,7 +159,7 @@ async function getDbRows(model, view: View, req: Request) { elapsed = temp[0] * 1000 + temp[1] / 1000000 ) { const rows = await nocoExecute( - requestObj, + ast, await baseModel.list({ ...listArgs, offset, limit }), {}, listArgs diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts index 5d44e363c3..c4b1f6a1ee 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts @@ -4,6 +4,7 @@ import DataLoader from 'dataloader'; import { AuditOperationSubTypes, AuditOperationTypes, + isSystemColumn, isVirtualCol, RelationTypes, UITypes, @@ -38,6 +39,7 @@ import genRollupSelectv2 from './genRollupSelectv2'; import sortV2 from './sortV2'; import conditionV2 from './conditionV2'; import { sanitize, unsanitize } from './helpers/sanitize'; +import type { GridViewColumn } from '../../../../models'; import type { SortType } from 'nocodb-sdk'; import type { Knex } from 'knex'; import type FormulaColumn from '../../../../models/FormulaColumn'; @@ -64,6 +66,21 @@ async function populatePk(model: Model, insertObj: any) { } } +function checkColumnRequired( + column: Column, + fields: string[], + extractPkAndPv?: boolean +) { + // if primary key or foreign key included in fields, it's required + if (column.pk || column.uidt === UITypes.ForeignKey) return true; + + if (extractPkAndPv && column.pv) return true; + + // check fields defined and if not, then select all + // if defined check if it is in the fields + return !fields || fields.includes(column.title); +} + /** * Base class for models * @@ -97,14 +114,23 @@ class BaseModelSqlv2 { autoBind(this); } - public async readByPk(id?: any): Promise { + public async readByPk(id?: any, validateFormula = false): Promise { const qb = this.dbDriver(this.tnPath); - await this.selectObject({ qb }); + await this.selectObject({ qb, validateFormula }); qb.where(_wherePk(this.model.primaryKeys, id)); - const data = (await this.execAndParse(qb))?.[0]; + let data; + + try { + data = (await this.execAndParse(qb))?.[0]; + } catch (e) { + if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) + throw e; + console.log(e); + return this.readByPk(id, true); + } if (data) { const proto = await this.getProto(); @@ -129,11 +155,12 @@ class BaseModelSqlv2 { where?: string; filterArr?: Filter[]; sort?: string | string[]; - } = {} + } = {}, + validateFormula = false ): Promise { const { where, ...rest } = this._getListArgs(args as any); const qb = this.dbDriver(this.tnPath); - await this.selectObject({ qb }); + await this.selectObject({ qb, validateFormula }); const aliasColObjMap = await this.model.getAliasColObjMap(); const sorts = extractSortsObject(rest?.sort, aliasColObjMap); @@ -162,7 +189,16 @@ class BaseModelSqlv2 { qb.orderBy(this.model.primaryKey.column_name); } - const data = await qb.first(); + let data; + + try { + data = await qb.first(); + } catch (e) { + if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) + throw e; + console.log(e); + return this.findOne(args, true); + } if (data) { const proto = await this.getProto(); @@ -179,13 +215,21 @@ class BaseModelSqlv2 { filterArr?: Filter[]; sortArr?: Sort[]; sort?: string | string[]; + fieldsSet?: Set; } = {}, - ignoreViewFilterAndSort = false + ignoreViewFilterAndSort = false, + validateFormula = false ): Promise { - const { where, ...rest } = this._getListArgs(args as any); + const { where, fields, ...rest } = this._getListArgs(args as any); const qb = this.dbDriver(this.tnPath); - await this.selectObject({ qb }); + + await this.selectObject({ + qb, + fieldsSet: args.fieldsSet, + viewId: this.viewId, + validateFormula, + }); if (+rest?.shuffle) { await this.shuffle({ qb }); } @@ -256,8 +300,17 @@ class BaseModelSqlv2 { if (!ignoreViewFilterAndSort) applyPaginate(qb, rest); const proto = await this.getProto(); - const data = await this.execAndParse(qb); + let data; + + try { + data = await this.execAndParse(qb); + } catch (e) { + if (validateFormula || !haveFormulaColumn(await this.model.getColumns())) + throw e; + console.log(e); + return this.list(args, ignoreViewFilterAndSort, true); + } return data?.map((d) => { d.__proto__ = proto; return d; @@ -379,7 +432,10 @@ class BaseModelSqlv2 { return await qb; } - async multipleHmList({ colId, ids }, args: { limit?; offset? } = {}) { + async multipleHmList( + { colId, ids }, + args: { limit?; offset?; fieldsSet?: Set } = {} + ) { try { const { where, sort, ...rest } = this._getListArgs(args as any); // todo: get only required fields @@ -407,7 +463,11 @@ class BaseModelSqlv2 { const parentTn = this.getTnPath(parentTable); const qb = this.dbDriver(childTn); - await childModel.selectObject({ qb }); + await childModel.selectObject({ + qb, + extractPkAndPv: true, + fieldsSet: args.fieldsSet, + }); await this.applySortAndFilter({ table: childTable, where, qb, sort }); const childQb = this.dbDriver.queryBuilder().from( @@ -435,6 +495,8 @@ class BaseModelSqlv2 { .as('list') ); + // console.log(childQb.toQuery()) + const children = await this.execAndParse(childQb, childTable); const proto = await ( await Model.getBaseModelSQL({ @@ -520,7 +582,10 @@ class BaseModelSqlv2 { } } - async hmList({ colId, id }, args: { limit?; offset? } = {}) { + async hmList( + { colId, id }, + args: { limit?; offset?; fieldSet?: Set } = {} + ) { try { const { where, sort, ...rest } = this._getListArgs(args as any); // todo: get only required fields @@ -560,7 +625,7 @@ class BaseModelSqlv2 { qb.limit(+rest?.limit || 25); qb.offset(+rest?.offset || 0); - await childModel.selectObject({ qb }); + await childModel.selectObject({ qb, fieldsSet: args.fieldSet }); const children = await this.execAndParse(qb, childTable); @@ -619,7 +684,7 @@ class BaseModelSqlv2 { public async multipleMmList( { colId, parentIds }, - args: { limit?; offset? } = {} + args: { limit?; offset?; fieldsSet?: Set } = {} ) { const { where, sort, ...rest } = this._getListArgs(args as any); const relColumn = (await this.model.getColumns()).find( @@ -652,7 +717,7 @@ class BaseModelSqlv2 { const qb = this.dbDriver(rtn).join(vtn, `${vtn}.${vrcn}`, `${rtn}.${rcn}`); - await childModel.selectObject({ qb }); + await childModel.selectObject({ qb, fieldsSet: args.fieldsSet }); await this.applySortAndFilter({ table: childTable, where, qb, sort }); @@ -698,7 +763,10 @@ class BaseModelSqlv2 { return parentIds.map((id) => gs[id] || []); } - public async mmList({ colId, parentId }, args: { limit?; offset? } = {}) { + public async mmList( + { colId, parentId }, + args: { limit?; offset?; fieldsSet?: Set } = {} + ) { const { where, sort, ...rest } = this._getListArgs(args as any); const relColumn = (await this.model.getColumns()).find( (c) => c.id === colId @@ -738,7 +806,7 @@ class BaseModelSqlv2 { .where(_wherePk(parentTable.primaryKeys, parentId)) ); - await childModel.selectObject({ qb }); + await childModel.selectObject({ qb, fieldsSet: args.fieldsSet }); await this.applySortAndFilter({ table: childTable, where, qb, sort }); @@ -1212,7 +1280,11 @@ class BaseModelSqlv2 { }); } - private async getSelectQueryBuilderForFormula(column: Column) { + private async getSelectQueryBuilderForFormula( + column: Column, + tableAlias?: string, + validateFormula = false + ) { const formula = await column.getColOptions(); if (formula.error) throw new Error(`Formula error: ${formula.error}`); const qb = await formulaQueryBuilderv2( @@ -1220,7 +1292,10 @@ class BaseModelSqlv2 { null, this.dbDriver, this.model, - column + column, + {}, + tableAlias, + validateFormula ); return qb; } @@ -1364,6 +1439,7 @@ class BaseModelSqlv2 { { // limit: ids.length, where: `(${pCol.column_name},in,${ids.join(',')})`, + fieldsSet: (readLoader as any).args?.fieldsSet, }, true ); @@ -1376,13 +1452,15 @@ class BaseModelSqlv2 { }); // defining HasMany count method within GQL Type class - proto[column.title] = async function () { + proto[column.title] = async function (args?: any) { if ( this?.[cCol?.title] === null || this?.[cCol?.title] === undefined ) return null; + (readLoader as any).args = args; + return await readLoader.load(this?.[cCol?.title]); }; // todo : handle mm @@ -1410,7 +1488,7 @@ class BaseModelSqlv2 { this.config.limitMin ); obj.offset = Math.max(+(args.offset || args.o) || 0, 0); - obj.fields = args.fields || args.f || '*'; + obj.fields = args.fields || args.f; obj.sort = args.sort || args.s; return obj; } @@ -1425,16 +1503,70 @@ class BaseModelSqlv2 { } } + // todo: + // pass view id as argument + // add option to get only pk and pv public async selectObject({ qb, columns: _columns, + fields: _fields, + extractPkAndPv, + viewId, + fieldsSet, + alias, + validateFormula, }: { + fieldsSet?: Set; qb: Knex.QueryBuilder; columns?: Column[]; + fields?: string[] | string; + extractPkAndPv?: boolean; + viewId?: string; + alias?: string; + validateFormula?: boolean; }): Promise { + let viewOrTableColumns: Column[] | { fk_column_id?: string }[]; + const res = {}; - const columns = _columns ?? (await this.model.getColumns()); - for (const column of columns) { + let view: View; + let fields: string[]; + + if (fieldsSet?.size) { + viewOrTableColumns = _columns || (await this.model.getColumns()); + } else { + view = await View.get(viewId); + const viewColumns = viewId && (await View.getColumns(viewId)); + fields = Array.isArray(_fields) ? _fields : _fields?.split(','); + + // const columns = _columns ?? (await this.model.getColumns()); + // for (const column of columns) { + viewOrTableColumns = + _columns || viewColumns || (await this.model.getColumns()); + } + for (const viewOrTableColumn of viewOrTableColumns) { + const column = + viewOrTableColumn instanceof Column + ? viewOrTableColumn + : await Column.get({ + colId: (viewOrTableColumn as GridViewColumn).fk_column_id, + }); + // hide if column marked as hidden in view + // of if column is system field and system field is hidden + if ( + fieldsSet + ? !fieldsSet.has(column.title) + : !extractPkAndPv && + !(viewOrTableColumn instanceof Column) && + (!(viewOrTableColumn as GridViewColumn)?.show || + (!view?.show_system_fields && + column.uidt !== UITypes.ForeignKey && + !column.pk && + isSystemColumn(column))) + ) + continue; + + if (!checkColumnRequired(column, fields, extractPkAndPv)) continue; + switch (column.uidt) { case 'LinkToAnotherRecord': case 'Lookup': @@ -1454,7 +1586,9 @@ class BaseModelSqlv2 { case UITypes.Formula: try { const selectQb = await this.getSelectQueryBuilderForFormula( - qrValueColumn + qrValueColumn, + alias, + validateFormula ); qb.select({ [column.column_name]: selectQb.builder, @@ -1486,7 +1620,9 @@ class BaseModelSqlv2 { case UITypes.Formula: try { const selectQb = await this.getSelectQueryBuilderForFormula( - barcodeValueColumn + barcodeValueColumn, + alias, + validateFormula ); qb.select({ [column.column_name]: selectQb.builder, @@ -1509,7 +1645,9 @@ class BaseModelSqlv2 { { try { const selectQb = await this.getSelectQueryBuilderForFormula( - column + column, + alias, + validateFormula ); qb.select( this.dbDriver.raw(`?? as ??`, [ @@ -1517,7 +1655,8 @@ class BaseModelSqlv2 { sanitize(column.title), ]) ); - } catch { + } catch (e) { + console.log(e); // return dummy select qb.select( this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.title)]) @@ -1532,6 +1671,7 @@ class BaseModelSqlv2 { // tn: this.title, knex: this.dbDriver, // column, + alias, columnOptions: (await column.getColOptions()) as RollupColumn, }) ).builder.as(sanitize(column.title)) @@ -1539,7 +1679,7 @@ class BaseModelSqlv2 { break; default: res[sanitize(column.title || column.column_name)] = sanitize( - `${this.model.table_name}.${column.column_name}` + `${alias || this.model.table_name}.${column.column_name}` ); break; } @@ -2659,7 +2799,7 @@ class BaseModelSqlv2 { qb.limit(+rest?.limit || 25); qb.offset(+rest?.offset || 0); - await this.selectObject({ qb }); + await this.selectObject({ qb, extractPkAndPv: true }); // todo: refactor and move to a method (applyFilterAndSort) const aliasColObjMap = await this.model.getAliasColObjMap(); @@ -3077,4 +3217,8 @@ function getCompositePk(primaryKeys: Column[], row) { return primaryKeys.map((c) => row[c.title]).join('___'); } +function haveFormulaColumn(columns: Column[]) { + return columns.some((c) => c.uidt === UITypes.Formula); +} + export { BaseModelSqlv2 }; diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts index 74400802e2..3eb91300d9 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts @@ -53,7 +53,8 @@ async function _formulaQueryBuilder( alias, knex: XKnex, model: Model, - aliasToColumn = {} + aliasToColumn = {}, + tableAlias?: string ) { // formula may include double curly brackets in previous version // convert to single curly bracket here for compatibility @@ -74,7 +75,8 @@ async function _formulaQueryBuilder( alias, knex, model, - { ...aliasToColumn, [col.id]: null } + { ...aliasToColumn, [col.id]: null }, + tableAlias ); builder.sql = '(' + builder.sql + ')'; aliasToColumn[col.id] = builder; @@ -104,7 +106,9 @@ async function _formulaQueryBuilder( selectQb = knex(`${parentModel.table_name} as ${alias}`).where( `${alias}.${parentColumn.column_name}`, knex.raw(`??`, [ - `${childModel.table_name}.${childColumn.column_name}`, + `${tableAlias ?? childModel.table_name}.${ + childColumn.column_name + }`, ]) ); break; @@ -113,7 +117,9 @@ async function _formulaQueryBuilder( selectQb = knex(`${childModel.table_name} as ${alias}`).where( `${alias}.${childColumn.column_name}`, knex.raw(`??`, [ - `${parentModel.table_name}.${parentColumn.column_name}`, + `${tableAlias ?? parentModel.table_name}.${ + parentColumn.column_name + }`, ]) ); break; @@ -134,7 +140,9 @@ async function _formulaQueryBuilder( .where( `${assocAlias}.${mmChildColumn.column_name}`, knex.raw(`??`, [ - `${childModel.table_name}.${childColumn.column_name}`, + `${tableAlias ?? childModel.table_name}.${ + childColumn.column_name + }`, ]) ); } @@ -402,6 +410,7 @@ async function _formulaQueryBuilder( const qb = await genRollupSelectv2({ knex, columnOptions: (await col.getColOptions()) as RollupColumn, + alias: tableAlias, }); aliasToColumn[col.id] = knex.raw(qb.builder).wrap('(', ')'); } @@ -428,7 +437,9 @@ async function _formulaQueryBuilder( .where( `${parentModel.table_name}.${parentColumn.column_name}`, knex.raw(`??`, [ - `${childModel.table_name}.${childColumn.column_name}`, + `${tableAlias ?? childModel.table_name}.${ + childColumn.column_name + }`, ]) ); } else if (relation.type == 'hm') { @@ -437,7 +448,9 @@ async function _formulaQueryBuilder( .where( `${childModel.table_name}.${childColumn.column_name}`, knex.raw(`??`, [ - `${parentModel.table_name}.${parentColumn.column_name}`, + `${tableAlias ?? parentModel.table_name}.${ + parentColumn.column_name + }`, ]) ); @@ -490,7 +503,9 @@ async function _formulaQueryBuilder( .where( `${mmModel.table_name}.${mmChildColumn.column_name}`, knex.raw(`??`, [ - `${childModel.table_name}.${childColumn.column_name}`, + `${tableAlias ?? childModel.table_name}.${ + childColumn.column_name + }`, ]) ); selectQb = (fn) => @@ -775,18 +790,27 @@ async function _formulaQueryBuilder( return { builder: fn(tree, alias) }; } -function getTnPath(tb: Model, knex) { +function getTnPath(tb: Model, knex, tableAlias?: string) { const schema = knex.searchPath?.(); if (knex.clientType() === 'mssql' && schema) { - return knex.raw('??.??', [schema, tb.table_name]); - } else if (knex.clientType() === 'snowflake') { - return [ - knex.client.config.connection.database, - knex.client.config.connection.schema, + return knex.raw(`??.??${tableAlias ? ' as ??' : ''}`, [ + schema, tb.table_name, - ].join('.'); + ...(tableAlias ? [tableAlias] : []), + ]); + } else if (knex.clientType() === 'snowflake') { + return ( + [ + knex.client.config.connection.database, + knex.client.config.connection.schema, + tb.table_name, + ].join('.') + (tableAlias ? ` as ${tableAlias}` : '') + ); } else { - return tb.table_name; + return knex.raw(`??${tableAlias ? ' as ??' : ''}`, [ + tb.table_name, + ...(tableAlias ? [tableAlias] : []), + ]); } } @@ -796,7 +820,9 @@ export default async function formulaQueryBuilderv2( knex: XKnex, model: Model, column?: Column, - aliasToColumn = {} + aliasToColumn = {}, + tableAlias?: string, + validateFormula = false ) { // register jsep curly hook once only jsep.plugins.register(jsepCurlyHook); @@ -806,13 +832,18 @@ export default async function formulaQueryBuilderv2( alias, knex, model, - aliasToColumn + aliasToColumn, + tableAlias ); + if (!validateFormula) return qb; + try { // dry run qb.builder to see if it will break the grid view or not // if so, set formula error and show empty selectQb instead - await knex(getTnPath(model, knex)).select(qb.builder).as('dry-run-only'); + await knex(getTnPath(model, knex, tableAlias)) + .select(qb.builder) + .as('dry-run-only'); // if column is provided, i.e. formula has been created if (column) { diff --git a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts index cb00298318..a0b4d1c92e 100644 --- a/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts +++ b/packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts @@ -1,7 +1,11 @@ -import { isSystemColumn, UITypes } from 'nocodb-sdk'; -import View from '../../../../../models/View'; -import type Model from '../../../../../models/Model'; -import type LinkToAnotherRecordColumn from '../../../../../models/LinkToAnotherRecordColumn'; +import { isSystemColumn, RelationTypes, UITypes } from 'nocodb-sdk'; +import { View } from '../../../../../models'; +import type { + Column, + LinkToAnotherRecordColumn, + LookupColumn, + Model, +} from '../../../../../models'; const getAst = async ({ query, @@ -9,23 +13,40 @@ const getAst = async ({ includePkByDefault = true, model, view, + dependencyFields = { + ...(query || {}), + nested: { ...(query?.nested || {}) }, + fieldsSet: new Set(), + }, }: { query?: RequestQuery; extractOnlyPrimaries?: boolean; includePkByDefault?: boolean; model: Model; view?: View; + dependencyFields?: DependantFields; }) => { + // set default values of dependencyFields and nested + dependencyFields.nested = dependencyFields.nested || {}; + dependencyFields.fieldsSet = dependencyFields.fieldsSet || new Set(); + if (!model.columns?.length) await model.getColumns(); // extract only pk and pv if (extractOnlyPrimaries) { - return { + const ast = { ...(model.primaryKeys ? model.primaryKeys.reduce((o, pk) => ({ ...o, [pk.title]: 1 }), {}) : {}), ...(model.displayValue ? { [model.displayValue.title]: 1 } : {}), }; + await Promise.all( + model.primaryKeys.map((c) => extractDependencies(c, dependencyFields)) + ); + + await extractDependencies(model.displayValue, dependencyFields); + + return { ast, dependencyFields }; } let fields = query?.fields || query?.f; @@ -45,7 +66,7 @@ const getAst = async ({ {} ); - return model.columns.reduce(async (obj, col) => { + const ast = await model.columns.reduce(async (obj, col) => { let value: number | boolean | { [key: string]: any } = 1; const nestedFields = query?.nested?.[col.title]?.fields || query?.nested?.[col.title]?.f; @@ -55,10 +76,19 @@ const getAst = async ({ .getColOptions() .then((colOpt) => colOpt.getRelatedTable()); - value = await getAst({ + const { ast } = await getAst({ model, query: query?.nested?.[col.title], + dependencyFields: (dependencyFields.nested[col.title] = + dependencyFields.nested[col.title] || { + nested: {}, + fieldsSet: new Set(), + }), }); + + value = ast; + + // todo: include field relative to the relation => pk / fk } else { value = (Array.isArray(fields) ? fields : fields.split(',')).reduce( (o, f) => ({ ...o, [f]: 1 }), @@ -70,26 +100,104 @@ const getAst = async ({ .getColOptions() .then((colOpt) => colOpt.getRelatedTable()); - value = await getAst({ - model, - query: query?.nested?.[col.title], - extractOnlyPrimaries: nestedFields !== '*', - }); + value = ( + await getAst({ + model, + query: query?.nested?.[col.title], + extractOnlyPrimaries: nestedFields !== '*', + dependencyFields: (dependencyFields.nested[col.title] = + dependencyFields.nested[col.title] || { + nested: {}, + fieldsSet: new Set(), + }), + }) + ).ast; } + const isRequested = + allowedCols && (!includePkByDefault || !col.pk) + ? allowedCols[col.id] && + (!isSystemColumn(col) || view.show_system_fields) && + (!fields?.length || fields.includes(col.title)) && + value + : fields?.length + ? fields.includes(col.title) && value + : value; + if (isRequested || col.pk) await extractDependencies(col, dependencyFields); + return { ...(await obj), - [col.title]: - allowedCols && (!includePkByDefault || !col.pk) - ? allowedCols[col.id] && - (!isSystemColumn(col) || view.show_system_fields) && - (!fields?.length || fields.includes(col.title)) && - value - : fields?.length - ? fields.includes(col.title) && value - : value, + [col.title]: isRequested, }; }, Promise.resolve({})); + + return { ast, dependencyFields }; +}; + +const extractDependencies = async ( + column: Column, + dependencyFields: DependantFields = { + nested: {}, + fieldsSet: new Set(), + } +) => { + switch (column.uidt) { + case UITypes.Lookup: + await extractLookupDependencies(column, dependencyFields); + break; + case UITypes.LinkToAnotherRecord: + await extractRelationDependencies(column, dependencyFields); + break; + default: + dependencyFields.fieldsSet.add(column.title); + break; + } +}; + +const extractLookupDependencies = async ( + lookUpColumn: Column, + dependencyFields: DependantFields = { + nested: {}, + fieldsSet: new Set(), + } +) => { + const lookupColumnOpts = await lookUpColumn.getColOptions(); + const relationColumn = await lookupColumnOpts.getRelationColumn(); + await extractRelationDependencies(relationColumn, dependencyFields); + await extractDependencies( + await lookupColumnOpts.getLookupColumn(), + (dependencyFields.nested[relationColumn.title] = dependencyFields.nested[ + relationColumn.title + ] || { + nested: {}, + fieldsSet: new Set(), + }) + ); +}; + +const extractRelationDependencies = async ( + relationColumn: Column, + dependencyFields: DependantFields = { + nested: {}, + fieldsSet: new Set(), + } +) => { + const relationColumnOpts = await relationColumn.getColOptions(); + + switch (relationColumnOpts.type) { + case RelationTypes.HAS_MANY: + dependencyFields.fieldsSet.add( + await relationColumnOpts.getParentColumn().then((col) => col.title) + ); + break; + case RelationTypes.BELONGS_TO: + case RelationTypes.MANY_TO_MANY: + dependencyFields.fieldsSet.add( + await relationColumnOpts.getChildColumn().then((col) => col.title) + ); + + break; + } }; type RequestQuery = { @@ -100,4 +208,9 @@ type RequestQuery = { }; }; +interface DependantFields { + fieldsSet?: Set; + nested?: DependantFields; +} + export default getAst; diff --git a/packages/nocodb/src/lib/models/Column.ts b/packages/nocodb/src/lib/models/Column.ts index 30bfdc0662..5550142f2c 100644 --- a/packages/nocodb/src/lib/models/Column.ts +++ b/packages/nocodb/src/lib/models/Column.ts @@ -406,7 +406,7 @@ export default class Column implements ColumnType { } } - public async getColOptions(ncMeta = Noco.ncMeta): Promise { + public async getColOptions(ncMeta = Noco.ncMeta): Promise { let res: any; switch (this.uidt) { diff --git a/packages/nocodb/src/lib/services/column.svc.ts b/packages/nocodb/src/lib/services/column.svc.ts index b3f35160c6..4ff0a6ea10 100644 --- a/packages/nocodb/src/lib/services/column.svc.ts +++ b/packages/nocodb/src/lib/services/column.svc.ts @@ -129,7 +129,16 @@ export async function columnUpdate(param: { try { // test the query to see if it is valid in db level const dbDriver = await NcConnectionMgrv2.get(base); - await formulaQueryBuilderv2(colBody.formula, null, dbDriver, table); + await formulaQueryBuilderv2( + colBody.formula, + null, + dbDriver, + table, + null, + {}, + null, + true + ); } catch (e) { console.error(e); NcError.badRequest('Invalid Formula'); @@ -934,7 +943,16 @@ export async function columnAdd(param: { try { // test the query to see if it is valid in db level const dbDriver = await NcConnectionMgrv2.get(base); - await formulaQueryBuilderv2(colBody.formula, null, dbDriver, table); + await formulaQueryBuilderv2( + colBody.formula, + null, + dbDriver, + table, + null, + {}, + null, + true + ); } catch (e) { console.error(e); NcError.badRequest('Invalid Formula'); diff --git a/packages/nocodb/src/lib/services/dbData/helpers.ts b/packages/nocodb/src/lib/services/dbData/helpers.ts index 6a61c14d9e..4fcee58b6b 100644 --- a/packages/nocodb/src/lib/services/dbData/helpers.ts +++ b/packages/nocodb/src/lib/services/dbData/helpers.ts @@ -242,14 +242,15 @@ export async function getDbRows(param: { temp = process.hrtime(startTime), elapsed = temp[0] * 1000 + temp[1] / 1000000 ) { + const { ast, dependencyFields } = await getAst({ + query: query, + includePkByDefault: false, + model: view.model, + view, + }); const rows = await nocoExecute( - await getAst({ - query: query, - includePkByDefault: false, - model: view.model, - view, - }), - await baseModel.list({ ...listArgs, offset, limit }), + ast, + await baseModel.list({ ...listArgs, ...dependencyFields, offset, limit }), {}, query ); diff --git a/packages/nocodb/src/lib/services/dbData/index.ts b/packages/nocodb/src/lib/services/dbData/index.ts index bf6fa3efc0..6bacb1f1b0 100644 --- a/packages/nocodb/src/lib/services/dbData/index.ts +++ b/packages/nocodb/src/lib/services/dbData/index.ts @@ -114,9 +114,9 @@ export async function getDataList(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ model, query, view }); + const { ast, dependencyFields } = await getAst({ model, query, view }); - const listArgs: any = { ...query }; + const listArgs: any = dependencyFields; try { listArgs.filterArr = JSON.parse(listArgs.filterArrJson); } catch (e) {} @@ -127,12 +127,7 @@ export async function getDataList(param: { let data = []; let count = 0; try { - data = await nocoExecute( - requestObj, - await baseModel.list(listArgs), - {}, - listArgs - ); + data = await nocoExecute(ast, await baseModel.list(listArgs), {}, listArgs); count = await baseModel.count(listArgs); } catch (e) { console.log(e); @@ -170,15 +165,10 @@ export async function getFindOne(param: { args.sortArr = JSON.parse(args.sortArrJson); } catch (e) {} - const data = await baseModel.findOne(args); - return data - ? await nocoExecute( - await getAst({ model, query: args, view }), - data, - {}, - {} - ) - : {}; + const { ast, dependencyFields } = await getAst({ model, query: args, view }); + + const data = await baseModel.findOne({ ...args, dependencyFields }); + return data ? await nocoExecute(ast, data, {}, {}) : {}; } export async function getDataGroupBy(param: { @@ -225,12 +215,9 @@ export async function dataRead( NcError.notFound('Row not found'); } - return await nocoExecute( - await getAst({ model, query: param.query, view }), - row, - {}, - param.query - ); + const { ast } = await getAst({ model, query: param.query, view }); + + return await nocoExecute(ast, row, {}, param.query); } export async function dataExist( @@ -279,7 +266,7 @@ export async function getGroupedDataList(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ model, query, view }); + const { ast } = await getAst({ model, query, view }); const listArgs: any = { ...query }; try { @@ -298,12 +285,7 @@ export async function getGroupedDataList(param: { ...listArgs, groupColumnId: param.columnId, }); - data = await nocoExecute( - { key: 1, value: requestObj }, - groupedData, - {}, - listArgs - ); + data = await nocoExecute({ key: 1, value: ast }, groupedData, {}, listArgs); const countArr = await baseModel.groupedListCount({ ...listArgs, groupColumnId: param.columnId, @@ -650,8 +632,10 @@ export async function dataReadByViewId(param: { dbDriver: await NcConnectionMgrv2.get(base), }); + const { ast } = await getAst({ model, query: param.query }); + return await nocoExecute( - await getAst({ model, query: param.query }), + ast, await baseModel.readByPk(param.rowId), {}, {} diff --git a/packages/nocodb/src/lib/services/public/publicData.svc.ts b/packages/nocodb/src/lib/services/public/publicData.svc.ts index 707b6b5982..0a29b7201b 100644 --- a/packages/nocodb/src/lib/services/public/publicData.svc.ts +++ b/packages/nocodb/src/lib/services/public/publicData.svc.ts @@ -59,20 +59,20 @@ export async function dataList(param: { let count = 0; try { - data = await nocoExecute( - await getAst({ - query: param.query, - model, - view, - }), - await baseModel.list(listArgs), - {}, - listArgs - ); + const { ast } = await getAst({ + query: param.query, + model, + view, + }); + + data = await nocoExecute(ast, await baseModel.list(listArgs), {}, listArgs); count = await baseModel.count(listArgs); } catch (e) { + console.log(e); // show empty result instead of throwing error here // e.g. search some text in a numeric field + + NcError.internalServerError('Please try after some time'); } return new PagedResponseImpl(data, { ...param.query, count }); @@ -128,7 +128,7 @@ async function getGroupedDataList(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ model, query: param.query, view }); + const { ast } = await getAst({ model, query: param.query, view }); const listArgs: any = { ...query }; try { @@ -148,12 +148,7 @@ async function getGroupedDataList(param: { ...listArgs, groupColumnId, }); - data = await nocoExecute( - { key: 1, value: requestObj }, - groupedData, - {}, - listArgs - ); + data = await nocoExecute({ key: 1, value: ast }, groupedData, {}, listArgs); const countArr = await baseModel.groupedListCount({ ...listArgs, groupColumnId, @@ -304,7 +299,7 @@ export async function relDataList(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ + const { ast } = await getAst({ query: param.query, model, extractOnlyPrimaries: true, @@ -314,7 +309,7 @@ export async function relDataList(param: { let count = 0; try { data = data = await nocoExecute( - requestObj, + ast, await baseModel.list(param.query), {}, param.query diff --git a/packages/nocodb/src/lib/services/public/publicDataExport.svc.ts b/packages/nocodb/src/lib/services/public/publicDataExport.svc.ts index b673f950fe..9b58d362dd 100644 --- a/packages/nocodb/src/lib/services/public/publicDataExport.svc.ts +++ b/packages/nocodb/src/lib/services/public/publicDataExport.svc.ts @@ -46,7 +46,7 @@ export async function getDbRows(param: { dbDriver: await NcConnectionMgrv2.get(base), }); - const requestObj = await getAst({ + const { ast } = await getAst({ query: param.query, model: param.model, view: param.view, @@ -69,7 +69,7 @@ export async function getDbRows(param: { elapsed = temp[0] * 1000 + temp[1] / 1000000 ) { const rows = await nocoExecute( - requestObj, + ast, await baseModel.list({ ...listArgs, offset, limit }), {}, listArgs