Merge pull request #5379 from nocodb/feat/query-opt-imp

Select query improvements

packages/nocodb/src/lib/controllers/dbData/oldData.ctl.ts

@@ -21,7 +21,7 @@ export async function dataList(req: Request, res: Response) {
     dbDriver: await NcConnectionMgrv2.get(base),
   });
 
-  const requestObj = await getAst({
+  const { ast } = await getAst({
     query: req.query,
     model,
     view,
@@ -36,7 +36,7 @@ export async function dataList(req: Request, res: Response) {
   } catch (e) {}
 
   const data = await nocoExecute(
-    requestObj,
+    ast,
     await baseModel.list(listArgs),
     {},
     listArgs
@@ -132,17 +132,14 @@ async function dataRead(req: Request, res: Response) {
     dbDriver: await NcConnectionMgrv2.get(base),
   });
 
-  res.json(
-    await nocoExecute(
-      await getAst({
+  const { ast } = await getAst({
     query: req.query,
     model,
     view,
-      }),
-      await baseModel.readByPk(req.params.rowId),
-      {},
-      {}
-    )
+  });
+
+  res.json(
+    await nocoExecute(ast, await baseModel.readByPk(req.params.rowId), {}, {})
   );
 }
 

packages/nocodb/src/lib/controllers/publicControllers/publicDataExport.ctl.ts

@@ -136,7 +136,7 @@ async function getDbRows(model, view: View, req: Request) {
     dbDriver: await NcConnectionMgrv2.get(base),
   });
 
-  const requestObj = await getAst({
+  const { ast } = await getAst({
     query: req.query,
     model,
     view,
@@ -159,7 +159,7 @@ async function getDbRows(model, view: View, req: Request) {
       elapsed = temp[0] * 1000 + temp[1] / 1000000
   ) {
     const rows = await nocoExecute(
-      requestObj,
+      ast,
       await baseModel.list({ ...listArgs, offset, limit }),
       {},
       listArgs

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts

@@ -4,6 +4,7 @@ import DataLoader from 'dataloader';
 import {
   AuditOperationSubTypes,
   AuditOperationTypes,
+  isSystemColumn,
   isVirtualCol,
   RelationTypes,
   UITypes,
@@ -38,6 +39,7 @@ import genRollupSelectv2 from './genRollupSelectv2';
 import sortV2 from './sortV2';
 import conditionV2 from './conditionV2';
 import { sanitize, unsanitize } from './helpers/sanitize';
+import type { GridViewColumn } from '../../../../models';
 import type { SortType } from 'nocodb-sdk';
 import type { Knex } from 'knex';
 import type FormulaColumn from '../../../../models/FormulaColumn';
@@ -64,6 +66,21 @@ async function populatePk(model: Model, insertObj: any) {
   }
 }
 
+function checkColumnRequired(
+  column: Column<any>,
+  fields: string[],
+  extractPkAndPv?: boolean
+) {
+  // if primary key or foreign key included in fields, it's required
+  if (column.pk || column.uidt === UITypes.ForeignKey) return true;
+
+  if (extractPkAndPv && column.pv) return true;
+
+  // check fields defined and if not, then select all
+  // if defined check if it is in the fields
+  return !fields || fields.includes(column.title);
+}
+
 /**
  * Base class for models
  *
@@ -97,14 +114,23 @@ class BaseModelSqlv2 {
     autoBind(this);
   }
 
-  public async readByPk(id?: any): Promise<any> {
+  public async readByPk(id?: any, validateFormula = false): Promise<any> {
     const qb = this.dbDriver(this.tnPath);
 
-    await this.selectObject({ qb });
+    await this.selectObject({ qb, validateFormula });
 
     qb.where(_wherePk(this.model.primaryKeys, id));
 
-    const data = (await this.execAndParse(qb))?.[0];
+    let data;
+
+    try {
+      data = (await this.execAndParse(qb))?.[0];
+    } catch (e) {
+      if (validateFormula || !haveFormulaColumn(await this.model.getColumns()))
+        throw e;
+      console.log(e);
+      return this.readByPk(id, true);
+    }
 
     if (data) {
       const proto = await this.getProto();
@@ -129,11 +155,12 @@ class BaseModelSqlv2 {
       where?: string;
       filterArr?: Filter[];
       sort?: string | string[];
-    } = {}
+    } = {},
+    validateFormula = false
   ): Promise<any> {
     const { where, ...rest } = this._getListArgs(args as any);
     const qb = this.dbDriver(this.tnPath);
-    await this.selectObject({ qb });
+    await this.selectObject({ qb, validateFormula });
 
     const aliasColObjMap = await this.model.getAliasColObjMap();
     const sorts = extractSortsObject(rest?.sort, aliasColObjMap);
@@ -162,7 +189,16 @@ class BaseModelSqlv2 {
       qb.orderBy(this.model.primaryKey.column_name);
     }
 
-    const data = await qb.first();
+    let data;
+
+    try {
+      data = await qb.first();
+    } catch (e) {
+      if (validateFormula || !haveFormulaColumn(await this.model.getColumns()))
+        throw e;
+      console.log(e);
+      return this.findOne(args, true);
+    }
 
     if (data) {
       const proto = await this.getProto();
@@ -179,13 +215,21 @@ class BaseModelSqlv2 {
       filterArr?: Filter[];
       sortArr?: Sort[];
       sort?: string | string[];
+      fieldsSet?: Set<string>;
     } = {},
-    ignoreViewFilterAndSort = false
+    ignoreViewFilterAndSort = false,
+    validateFormula = false
   ): Promise<any> {
-    const { where, ...rest } = this._getListArgs(args as any);
+    const { where, fields, ...rest } = this._getListArgs(args as any);
 
     const qb = this.dbDriver(this.tnPath);
-    await this.selectObject({ qb });
+
+    await this.selectObject({
+      qb,
+      fieldsSet: args.fieldsSet,
+      viewId: this.viewId,
+      validateFormula,
+    });
     if (+rest?.shuffle) {
       await this.shuffle({ qb });
     }
@@ -256,8 +300,17 @@ class BaseModelSqlv2 {
 
     if (!ignoreViewFilterAndSort) applyPaginate(qb, rest);
     const proto = await this.getProto();
-    const data = await this.execAndParse(qb);
 
+    let data;
+
+    try {
+      data = await this.execAndParse(qb);
+    } catch (e) {
+      if (validateFormula || !haveFormulaColumn(await this.model.getColumns()))
+        throw e;
+      console.log(e);
+      return this.list(args, ignoreViewFilterAndSort, true);
+    }
     return data?.map((d) => {
       d.__proto__ = proto;
       return d;
@@ -379,7 +432,10 @@ class BaseModelSqlv2 {
     return await qb;
   }
 
-  async multipleHmList({ colId, ids }, args: { limit?; offset? } = {}) {
+  async multipleHmList(
+    { colId, ids },
+    args: { limit?; offset?; fieldsSet?: Set<string> } = {}
+  ) {
     try {
       const { where, sort, ...rest } = this._getListArgs(args as any);
       // todo: get only required fields
@@ -407,7 +463,11 @@ class BaseModelSqlv2 {
       const parentTn = this.getTnPath(parentTable);
 
       const qb = this.dbDriver(childTn);
-      await childModel.selectObject({ qb });
+      await childModel.selectObject({
+        qb,
+        extractPkAndPv: true,
+        fieldsSet: args.fieldsSet,
+      });
       await this.applySortAndFilter({ table: childTable, where, qb, sort });
 
       const childQb = this.dbDriver.queryBuilder().from(
@@ -435,6 +495,8 @@ class BaseModelSqlv2 {
           .as('list')
       );
 
+      // console.log(childQb.toQuery())
+
       const children = await this.execAndParse(childQb, childTable);
       const proto = await (
         await Model.getBaseModelSQL({
@@ -520,7 +582,10 @@ class BaseModelSqlv2 {
     }
   }
 
-  async hmList({ colId, id }, args: { limit?; offset? } = {}) {
+  async hmList(
+    { colId, id },
+    args: { limit?; offset?; fieldSet?: Set<string> } = {}
+  ) {
     try {
       const { where, sort, ...rest } = this._getListArgs(args as any);
       // todo: get only required fields
@@ -560,7 +625,7 @@ class BaseModelSqlv2 {
       qb.limit(+rest?.limit || 25);
       qb.offset(+rest?.offset || 0);
 
-      await childModel.selectObject({ qb });
+      await childModel.selectObject({ qb, fieldsSet: args.fieldSet });
 
       const children = await this.execAndParse(qb, childTable);
 
@@ -619,7 +684,7 @@ class BaseModelSqlv2 {
 
   public async multipleMmList(
     { colId, parentIds },
-    args: { limit?; offset? } = {}
+    args: { limit?; offset?; fieldsSet?: Set<string> } = {}
   ) {
     const { where, sort, ...rest } = this._getListArgs(args as any);
     const relColumn = (await this.model.getColumns()).find(
@@ -652,7 +717,7 @@ class BaseModelSqlv2 {
 
     const qb = this.dbDriver(rtn).join(vtn, `${vtn}.${vrcn}`, `${rtn}.${rcn}`);
 
-    await childModel.selectObject({ qb });
+    await childModel.selectObject({ qb, fieldsSet: args.fieldsSet });
 
     await this.applySortAndFilter({ table: childTable, where, qb, sort });
 
@@ -698,7 +763,10 @@ class BaseModelSqlv2 {
     return parentIds.map((id) => gs[id] || []);
   }
 
-  public async mmList({ colId, parentId }, args: { limit?; offset? } = {}) {
+  public async mmList(
+    { colId, parentId },
+    args: { limit?; offset?; fieldsSet?: Set<string> } = {}
+  ) {
     const { where, sort, ...rest } = this._getListArgs(args as any);
     const relColumn = (await this.model.getColumns()).find(
       (c) => c.id === colId
@@ -738,7 +806,7 @@ class BaseModelSqlv2 {
           .where(_wherePk(parentTable.primaryKeys, parentId))
       );
 
-    await childModel.selectObject({ qb });
+    await childModel.selectObject({ qb, fieldsSet: args.fieldsSet });
 
     await this.applySortAndFilter({ table: childTable, where, qb, sort });
 
@@ -1212,7 +1280,11 @@ class BaseModelSqlv2 {
     });
   }
 
-  private async getSelectQueryBuilderForFormula(column: Column<any>) {
+  private async getSelectQueryBuilderForFormula(
+    column: Column<any>,
+    tableAlias?: string,
+    validateFormula = false
+  ) {
     const formula = await column.getColOptions<FormulaColumn>();
     if (formula.error) throw new Error(`Formula error: ${formula.error}`);
     const qb = await formulaQueryBuilderv2(
@@ -1220,7 +1292,10 @@ class BaseModelSqlv2 {
       null,
       this.dbDriver,
       this.model,
-      column
+      column,
+      {},
+      tableAlias,
+      validateFormula
     );
     return qb;
   }
@@ -1364,6 +1439,7 @@ class BaseModelSqlv2 {
                     {
                       // limit: ids.length,
                       where: `(${pCol.column_name},in,${ids.join(',')})`,
+                      fieldsSet: (readLoader as any).args?.fieldsSet,
                     },
                     true
                   );
@@ -1376,13 +1452,15 @@ class BaseModelSqlv2 {
               });
 
               // defining HasMany count method within GQL Type class
-              proto[column.title] = async function () {
+              proto[column.title] = async function (args?: any) {
                 if (
                   this?.[cCol?.title] === null ||
                   this?.[cCol?.title] === undefined
                 )
                   return null;
 
+                (readLoader as any).args = args;
+
                 return await readLoader.load(this?.[cCol?.title]);
               };
               // todo : handle mm
@@ -1410,7 +1488,7 @@ class BaseModelSqlv2 {
       this.config.limitMin
     );
     obj.offset = Math.max(+(args.offset || args.o) || 0, 0);
-    obj.fields = args.fields || args.f || '*';
+    obj.fields = args.fields || args.f;
     obj.sort = args.sort || args.s;
     return obj;
   }
@@ -1425,16 +1503,70 @@ class BaseModelSqlv2 {
     }
   }
 
+  // todo:
+  //  pass view id as argument
+  //  add option to get only pk and pv
   public async selectObject({
     qb,
     columns: _columns,
+    fields: _fields,
+    extractPkAndPv,
+    viewId,
+    fieldsSet,
+    alias,
+    validateFormula,
   }: {
+    fieldsSet?: Set<string>;
     qb: Knex.QueryBuilder;
     columns?: Column[];
+    fields?: string[] | string;
+    extractPkAndPv?: boolean;
+    viewId?: string;
+    alias?: string;
+    validateFormula?: boolean;
   }): Promise<void> {
+    let viewOrTableColumns: Column[] | { fk_column_id?: string }[];
+
     const res = {};
-    const columns = _columns ?? (await this.model.getColumns());
-    for (const column of columns) {
+    let view: View;
+    let fields: string[];
+
+    if (fieldsSet?.size) {
+      viewOrTableColumns = _columns || (await this.model.getColumns());
+    } else {
+      view = await View.get(viewId);
+      const viewColumns = viewId && (await View.getColumns(viewId));
+      fields = Array.isArray(_fields) ? _fields : _fields?.split(',');
+
+      // const columns = _columns ?? (await this.model.getColumns());
+      // for (const column of columns) {
+      viewOrTableColumns =
+        _columns || viewColumns || (await this.model.getColumns());
+    }
+    for (const viewOrTableColumn of viewOrTableColumns) {
+      const column =
+        viewOrTableColumn instanceof Column
+          ? viewOrTableColumn
+          : await Column.get({
+              colId: (viewOrTableColumn as GridViewColumn).fk_column_id,
+            });
+      // hide if column marked as hidden in view
+      // of if column is system field and system field is hidden
+      if (
+        fieldsSet
+          ? !fieldsSet.has(column.title)
+          : !extractPkAndPv &&
+            !(viewOrTableColumn instanceof Column) &&
+            (!(viewOrTableColumn as GridViewColumn)?.show ||
+              (!view?.show_system_fields &&
+                column.uidt !== UITypes.ForeignKey &&
+                !column.pk &&
+                isSystemColumn(column)))
+      )
+        continue;
+
+      if (!checkColumnRequired(column, fields, extractPkAndPv)) continue;
+
       switch (column.uidt) {
         case 'LinkToAnotherRecord':
         case 'Lookup':
@@ -1454,7 +1586,9 @@ class BaseModelSqlv2 {
             case UITypes.Formula:
               try {
                 const selectQb = await this.getSelectQueryBuilderForFormula(
-                  qrValueColumn
+                  qrValueColumn,
+                  alias,
+                  validateFormula
                 );
                 qb.select({
                   [column.column_name]: selectQb.builder,
@@ -1486,7 +1620,9 @@ class BaseModelSqlv2 {
             case UITypes.Formula:
               try {
                 const selectQb = await this.getSelectQueryBuilderForFormula(
-                  barcodeValueColumn
+                  barcodeValueColumn,
+                  alias,
+                  validateFormula
                 );
                 qb.select({
                   [column.column_name]: selectQb.builder,
@@ -1509,7 +1645,9 @@ class BaseModelSqlv2 {
           {
             try {
               const selectQb = await this.getSelectQueryBuilderForFormula(
-                column
+                column,
+                alias,
+                validateFormula
               );
               qb.select(
                 this.dbDriver.raw(`?? as ??`, [
@@ -1517,7 +1655,8 @@ class BaseModelSqlv2 {
                   sanitize(column.title),
                 ])
               );
-            } catch {
+            } catch (e) {
+              console.log(e);
               // return dummy select
               qb.select(
                 this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.title)])
@@ -1532,6 +1671,7 @@ class BaseModelSqlv2 {
                 // tn: this.title,
                 knex: this.dbDriver,
                 // column,
+                alias,
                 columnOptions: (await column.getColOptions()) as RollupColumn,
               })
             ).builder.as(sanitize(column.title))
@@ -1539,7 +1679,7 @@ class BaseModelSqlv2 {
           break;
         default:
           res[sanitize(column.title || column.column_name)] = sanitize(
-            `${this.model.table_name}.${column.column_name}`
+            `${alias || this.model.table_name}.${column.column_name}`
           );
           break;
       }
@@ -2659,7 +2799,7 @@ class BaseModelSqlv2 {
       qb.limit(+rest?.limit || 25);
       qb.offset(+rest?.offset || 0);
 
-      await this.selectObject({ qb });
+      await this.selectObject({ qb, extractPkAndPv: true });
 
       // todo: refactor and move to a method (applyFilterAndSort)
       const aliasColObjMap = await this.model.getAliasColObjMap();
@@ -3077,4 +3217,8 @@ function getCompositePk(primaryKeys: Column[], row) {
   return primaryKeys.map((c) => row[c.title]).join('___');
 }
 
+function haveFormulaColumn(columns: Column[]) {
+  return columns.some((c) => c.uidt === UITypes.Formula);
+}
+
 export { BaseModelSqlv2 };

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts

@@ -53,7 +53,8 @@ async function _formulaQueryBuilder(
   alias,
   knex: XKnex,
   model: Model,
-  aliasToColumn = {}
+  aliasToColumn = {},
+  tableAlias?: string
 ) {
   // formula may include double curly brackets in previous version
   // convert to single curly bracket here for compatibility
@@ -74,7 +75,8 @@ async function _formulaQueryBuilder(
             alias,
             knex,
             model,
-            { ...aliasToColumn, [col.id]: null }
+            { ...aliasToColumn, [col.id]: null },
+            tableAlias
           );
           builder.sql = '(' + builder.sql + ')';
           aliasToColumn[col.id] = builder;
@@ -104,7 +106,9 @@ async function _formulaQueryBuilder(
                 selectQb = knex(`${parentModel.table_name} as ${alias}`).where(
                   `${alias}.${parentColumn.column_name}`,
                   knex.raw(`??`, [
-                    `${childModel.table_name}.${childColumn.column_name}`,
+                    `${tableAlias ?? childModel.table_name}.${
+                      childColumn.column_name
+                    }`,
                   ])
                 );
                 break;
@@ -113,7 +117,9 @@ async function _formulaQueryBuilder(
                 selectQb = knex(`${childModel.table_name} as ${alias}`).where(
                   `${alias}.${childColumn.column_name}`,
                   knex.raw(`??`, [
-                    `${parentModel.table_name}.${parentColumn.column_name}`,
+                    `${tableAlias ?? parentModel.table_name}.${
+                      parentColumn.column_name
+                    }`,
                   ])
                 );
                 break;
@@ -134,7 +140,9 @@ async function _formulaQueryBuilder(
                     .where(
                       `${assocAlias}.${mmChildColumn.column_name}`,
                       knex.raw(`??`, [
-                        `${childModel.table_name}.${childColumn.column_name}`,
+                        `${tableAlias ?? childModel.table_name}.${
+                          childColumn.column_name
+                        }`,
                       ])
                     );
                 }
@@ -402,6 +410,7 @@ async function _formulaQueryBuilder(
           const qb = await genRollupSelectv2({
             knex,
             columnOptions: (await col.getColOptions()) as RollupColumn,
+            alias: tableAlias,
           });
           aliasToColumn[col.id] = knex.raw(qb.builder).wrap('(', ')');
         }
@@ -428,7 +437,9 @@ async function _formulaQueryBuilder(
               .where(
                 `${parentModel.table_name}.${parentColumn.column_name}`,
                 knex.raw(`??`, [
-                  `${childModel.table_name}.${childColumn.column_name}`,
+                  `${tableAlias ?? childModel.table_name}.${
+                    childColumn.column_name
+                  }`,
                 ])
               );
           } else if (relation.type == 'hm') {
@@ -437,7 +448,9 @@ async function _formulaQueryBuilder(
               .where(
                 `${childModel.table_name}.${childColumn.column_name}`,
                 knex.raw(`??`, [
-                  `${parentModel.table_name}.${parentColumn.column_name}`,
+                  `${tableAlias ?? parentModel.table_name}.${
+                    parentColumn.column_name
+                  }`,
                 ])
               );
 
@@ -490,7 +503,9 @@ async function _formulaQueryBuilder(
               .where(
                 `${mmModel.table_name}.${mmChildColumn.column_name}`,
                 knex.raw(`??`, [
-                  `${childModel.table_name}.${childColumn.column_name}`,
+                  `${tableAlias ?? childModel.table_name}.${
+                    childColumn.column_name
+                  }`,
                 ])
               );
             selectQb = (fn) =>
@@ -775,18 +790,27 @@ async function _formulaQueryBuilder(
   return { builder: fn(tree, alias) };
 }
 
-function getTnPath(tb: Model, knex) {
+function getTnPath(tb: Model, knex, tableAlias?: string) {
   const schema = knex.searchPath?.();
   if (knex.clientType() === 'mssql' && schema) {
-    return knex.raw('??.??', [schema, tb.table_name]);
+    return knex.raw(`??.??${tableAlias ? ' as ??' : ''}`, [
+      schema,
+      tb.table_name,
+      ...(tableAlias ? [tableAlias] : []),
+    ]);
   } else if (knex.clientType() === 'snowflake') {
-    return [
+    return (
+      [
         knex.client.config.connection.database,
         knex.client.config.connection.schema,
         tb.table_name,
-    ].join('.');
+      ].join('.') + (tableAlias ? ` as ${tableAlias}` : '')
+    );
   } else {
-    return tb.table_name;
+    return knex.raw(`??${tableAlias ? ' as ??' : ''}`, [
+      tb.table_name,
+      ...(tableAlias ? [tableAlias] : []),
+    ]);
   }
 }
 
@@ -796,7 +820,9 @@ export default async function formulaQueryBuilderv2(
   knex: XKnex,
   model: Model,
   column?: Column,
-  aliasToColumn = {}
+  aliasToColumn = {},
+  tableAlias?: string,
+  validateFormula = false
 ) {
   // register jsep curly hook once only
   jsep.plugins.register(jsepCurlyHook);
@@ -806,13 +832,18 @@ export default async function formulaQueryBuilderv2(
     alias,
     knex,
     model,
-    aliasToColumn
+    aliasToColumn,
+    tableAlias
   );
 
+  if (!validateFormula) return qb;
+
   try {
     // dry run qb.builder to see if it will break the grid view or not
     // if so, set formula error and show empty selectQb instead
-    await knex(getTnPath(model, knex)).select(qb.builder).as('dry-run-only');
+    await knex(getTnPath(model, knex, tableAlias))
+      .select(qb.builder)
+      .as('dry-run-only');
 
     // if column is provided, i.e. formula has been created
     if (column) {

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts

@@ -1,7 +1,11 @@
-import { isSystemColumn, UITypes } from 'nocodb-sdk';
-import View from '../../../../../models/View';
-import type Model from '../../../../../models/Model';
-import type LinkToAnotherRecordColumn from '../../../../../models/LinkToAnotherRecordColumn';
+import { isSystemColumn, RelationTypes, UITypes } from 'nocodb-sdk';
+import { View } from '../../../../../models';
+import type {
+  Column,
+  LinkToAnotherRecordColumn,
+  LookupColumn,
+  Model,
+} from '../../../../../models';
 
 const getAst = async ({
   query,
@@ -9,23 +13,40 @@ const getAst = async ({
   includePkByDefault = true,
   model,
   view,
+  dependencyFields = {
+    ...(query || {}),
+    nested: { ...(query?.nested || {}) },
+    fieldsSet: new Set(),
+  },
 }: {
   query?: RequestQuery;
   extractOnlyPrimaries?: boolean;
   includePkByDefault?: boolean;
   model: Model;
   view?: View;
+  dependencyFields?: DependantFields;
 }) => {
+  // set default values of dependencyFields and nested
+  dependencyFields.nested = dependencyFields.nested || {};
+  dependencyFields.fieldsSet = dependencyFields.fieldsSet || new Set();
+
   if (!model.columns?.length) await model.getColumns();
 
   // extract only pk and pv
   if (extractOnlyPrimaries) {
-    return {
+    const ast = {
       ...(model.primaryKeys
         ? model.primaryKeys.reduce((o, pk) => ({ ...o, [pk.title]: 1 }), {})
         : {}),
       ...(model.displayValue ? { [model.displayValue.title]: 1 } : {}),
     };
+    await Promise.all(
+      model.primaryKeys.map((c) => extractDependencies(c, dependencyFields))
+    );
+
+    await extractDependencies(model.displayValue, dependencyFields);
+
+    return { ast, dependencyFields };
   }
 
   let fields = query?.fields || query?.f;
@@ -45,7 +66,7 @@ const getAst = async ({
       {}
     );
 
-  return model.columns.reduce(async (obj, col) => {
+  const ast = await model.columns.reduce(async (obj, col) => {
     let value: number | boolean | { [key: string]: any } = 1;
     const nestedFields =
       query?.nested?.[col.title]?.fields || query?.nested?.[col.title]?.f;
@@ -55,10 +76,19 @@ const getAst = async ({
           .getColOptions<LinkToAnotherRecordColumn>()
           .then((colOpt) => colOpt.getRelatedTable());
 
-        value = await getAst({
+        const { ast } = await getAst({
           model,
           query: query?.nested?.[col.title],
+          dependencyFields: (dependencyFields.nested[col.title] =
+            dependencyFields.nested[col.title] || {
+              nested: {},
+              fieldsSet: new Set(),
+            }),
         });
+
+        value = ast;
+
+        // todo: include field relative to the relation => pk / fk
       } else {
         value = (Array.isArray(fields) ? fields : fields.split(',')).reduce(
           (o, f) => ({ ...o, [f]: 1 }),
@@ -70,16 +100,21 @@ const getAst = async ({
         .getColOptions<LinkToAnotherRecordColumn>()
         .then((colOpt) => colOpt.getRelatedTable());
 
-      value = await getAst({
+      value = (
+        await getAst({
           model,
           query: query?.nested?.[col.title],
           extractOnlyPrimaries: nestedFields !== '*',
-      });
+          dependencyFields: (dependencyFields.nested[col.title] =
+            dependencyFields.nested[col.title] || {
+              nested: {},
+              fieldsSet: new Set(),
+            }),
+        })
+      ).ast;
     }
 
-    return {
-      ...(await obj),
-      [col.title]:
+    const isRequested =
       allowedCols && (!includePkByDefault || !col.pk)
         ? allowedCols[col.id] &&
           (!isSystemColumn(col) || view.show_system_fields) &&
@@ -87,9 +122,82 @@ const getAst = async ({
           value
         : fields?.length
         ? fields.includes(col.title) && value
-          : value,
+        : value;
+    if (isRequested || col.pk) await extractDependencies(col, dependencyFields);
+
+    return {
+      ...(await obj),
+      [col.title]: isRequested,
     };
   }, Promise.resolve({}));
+
+  return { ast, dependencyFields };
+};
+
+const extractDependencies = async (
+  column: Column,
+  dependencyFields: DependantFields = {
+    nested: {},
+    fieldsSet: new Set(),
+  }
+) => {
+  switch (column.uidt) {
+    case UITypes.Lookup:
+      await extractLookupDependencies(column, dependencyFields);
+      break;
+    case UITypes.LinkToAnotherRecord:
+      await extractRelationDependencies(column, dependencyFields);
+      break;
+    default:
+      dependencyFields.fieldsSet.add(column.title);
+      break;
+  }
+};
+
+const extractLookupDependencies = async (
+  lookUpColumn: Column<LookupColumn>,
+  dependencyFields: DependantFields = {
+    nested: {},
+    fieldsSet: new Set(),
+  }
+) => {
+  const lookupColumnOpts = await lookUpColumn.getColOptions();
+  const relationColumn = await lookupColumnOpts.getRelationColumn();
+  await extractRelationDependencies(relationColumn, dependencyFields);
+  await extractDependencies(
+    await lookupColumnOpts.getLookupColumn(),
+    (dependencyFields.nested[relationColumn.title] = dependencyFields.nested[
+      relationColumn.title
+    ] || {
+      nested: {},
+      fieldsSet: new Set(),
+    })
+  );
+};
+
+const extractRelationDependencies = async (
+  relationColumn: Column<LinkToAnotherRecordColumn>,
+  dependencyFields: DependantFields = {
+    nested: {},
+    fieldsSet: new Set(),
+  }
+) => {
+  const relationColumnOpts = await relationColumn.getColOptions();
+
+  switch (relationColumnOpts.type) {
+    case RelationTypes.HAS_MANY:
+      dependencyFields.fieldsSet.add(
+        await relationColumnOpts.getParentColumn().then((col) => col.title)
+      );
+      break;
+    case RelationTypes.BELONGS_TO:
+    case RelationTypes.MANY_TO_MANY:
+      dependencyFields.fieldsSet.add(
+        await relationColumnOpts.getChildColumn().then((col) => col.title)
+      );
+
+      break;
+  }
 };
 
 type RequestQuery = {
@@ -100,4 +208,9 @@ type RequestQuery = {
   };
 };
 
+interface DependantFields {
+  fieldsSet?: Set<string>;
+  nested?: DependantFields;
+}
+
 export default getAst;

packages/nocodb/src/lib/models/Column.ts

@@ -406,7 +406,7 @@ export default class Column<T = any> implements ColumnType {
     }
   }
 
-  public async getColOptions<T>(ncMeta = Noco.ncMeta): Promise<T> {
+  public async getColOptions<U = T>(ncMeta = Noco.ncMeta): Promise<U> {
     let res: any;
 
     switch (this.uidt) {

packages/nocodb/src/lib/services/column.svc.ts

@@ -129,7 +129,16 @@ export async function columnUpdate(param: {
         try {
           // test the query to see if it is valid in db level
           const dbDriver = await NcConnectionMgrv2.get(base);
-          await formulaQueryBuilderv2(colBody.formula, null, dbDriver, table);
+          await formulaQueryBuilderv2(
+            colBody.formula,
+            null,
+            dbDriver,
+            table,
+            null,
+            {},
+            null,
+            true
+          );
         } catch (e) {
           console.error(e);
           NcError.badRequest('Invalid Formula');
@@ -934,7 +943,16 @@ export async function columnAdd(param: {
       try {
         // test the query to see if it is valid in db level
         const dbDriver = await NcConnectionMgrv2.get(base);
-        await formulaQueryBuilderv2(colBody.formula, null, dbDriver, table);
+        await formulaQueryBuilderv2(
+          colBody.formula,
+          null,
+          dbDriver,
+          table,
+          null,
+          {},
+          null,
+          true
+        );
       } catch (e) {
         console.error(e);
         NcError.badRequest('Invalid Formula');

packages/nocodb/src/lib/services/dbData/helpers.ts

@@ -242,14 +242,15 @@ export async function getDbRows(param: {
       temp = process.hrtime(startTime),
       elapsed = temp[0] * 1000 + temp[1] / 1000000
   ) {
-    const rows = await nocoExecute(
-      await getAst({
+    const { ast, dependencyFields } = await getAst({
       query: query,
       includePkByDefault: false,
       model: view.model,
       view,
-      }),
-      await baseModel.list({ ...listArgs, offset, limit }),
+    });
+    const rows = await nocoExecute(
+      ast,
+      await baseModel.list({ ...listArgs, ...dependencyFields, offset, limit }),
       {},
       query
     );

packages/nocodb/src/lib/services/dbData/index.ts

@@ -114,9 +114,9 @@ export async function getDataList(param: {
     dbDriver: await NcConnectionMgrv2.get(base),
   });
 
-  const requestObj = await getAst({ model, query, view });
+  const { ast, dependencyFields } = await getAst({ model, query, view });
 
-  const listArgs: any = { ...query };
+  const listArgs: any = dependencyFields;
   try {
     listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
   } catch (e) {}
@@ -127,12 +127,7 @@ export async function getDataList(param: {
   let data = [];
   let count = 0;
   try {
-    data = await nocoExecute(
-      requestObj,
-      await baseModel.list(listArgs),
-      {},
-      listArgs
-    );
+    data = await nocoExecute(ast, await baseModel.list(listArgs), {}, listArgs);
     count = await baseModel.count(listArgs);
   } catch (e) {
     console.log(e);
@@ -170,15 +165,10 @@ export async function getFindOne(param: {
     args.sortArr = JSON.parse(args.sortArrJson);
   } catch (e) {}
 
-  const data = await baseModel.findOne(args);
-  return data
-    ? await nocoExecute(
-        await getAst({ model, query: args, view }),
-        data,
-        {},
-        {}
-      )
-    : {};
+  const { ast, dependencyFields } = await getAst({ model, query: args, view });
+
+  const data = await baseModel.findOne({ ...args, dependencyFields });
+  return data ? await nocoExecute(ast, data, {}, {}) : {};
 }
 
 export async function getDataGroupBy(param: {
@@ -225,12 +215,9 @@ export async function dataRead(
     NcError.notFound('Row not found');
   }
 
-  return await nocoExecute(
-    await getAst({ model, query: param.query, view }),
-    row,
-    {},
-    param.query
-  );
+  const { ast } = await getAst({ model, query: param.query, view });
+
+  return await nocoExecute(ast, row, {}, param.query);
 }
 
 export async function dataExist(
@@ -279,7 +266,7 @@ export async function getGroupedDataList(param: {
     dbDriver: await NcConnectionMgrv2.get(base),
   });
 
-  const requestObj = await getAst({ model, query, view });
+  const { ast } = await getAst({ model, query, view });
 
   const listArgs: any = { ...query };
   try {
@@ -298,12 +285,7 @@ export async function getGroupedDataList(param: {
     ...listArgs,
     groupColumnId: param.columnId,
   });
-  data = await nocoExecute(
-    { key: 1, value: requestObj },
-    groupedData,
-    {},
-    listArgs
-  );
+  data = await nocoExecute({ key: 1, value: ast }, groupedData, {}, listArgs);
   const countArr = await baseModel.groupedListCount({
     ...listArgs,
     groupColumnId: param.columnId,
@@ -650,8 +632,10 @@ export async function dataReadByViewId(param: {
       dbDriver: await NcConnectionMgrv2.get(base),
     });
 
+    const { ast } = await getAst({ model, query: param.query });
+
     return await nocoExecute(
-      await getAst({ model, query: param.query }),
+      ast,
       await baseModel.readByPk(param.rowId),
       {},
       {}

packages/nocodb/src/lib/services/public/publicData.svc.ts

@@ -59,20 +59,20 @@ export async function dataList(param: {
   let count = 0;
 
   try {
-    data = await nocoExecute(
-      await getAst({
+    const { ast } = await getAst({
       query: param.query,
       model,
       view,
-      }),
-      await baseModel.list(listArgs),
-      {},
-      listArgs
-    );
+    });
+
+    data = await nocoExecute(ast, await baseModel.list(listArgs), {}, listArgs);
     count = await baseModel.count(listArgs);
   } catch (e) {
+    console.log(e);
     // show empty result instead of throwing error here
     // e.g. search some text in a numeric field
+
+    NcError.internalServerError('Please try after some time');
   }
 
   return new PagedResponseImpl(data, { ...param.query, count });
@@ -128,7 +128,7 @@ async function getGroupedDataList(param: {
     dbDriver: await NcConnectionMgrv2.get(base),
   });
 
-  const requestObj = await getAst({ model, query: param.query, view });
+  const { ast } = await getAst({ model, query: param.query, view });
 
   const listArgs: any = { ...query };
   try {
@@ -148,12 +148,7 @@ async function getGroupedDataList(param: {
       ...listArgs,
       groupColumnId,
     });
-    data = await nocoExecute(
-      { key: 1, value: requestObj },
-      groupedData,
-      {},
-      listArgs
-    );
+    data = await nocoExecute({ key: 1, value: ast }, groupedData, {}, listArgs);
     const countArr = await baseModel.groupedListCount({
       ...listArgs,
       groupColumnId,
@@ -304,7 +299,7 @@ export async function relDataList(param: {
     dbDriver: await NcConnectionMgrv2.get(base),
   });
 
-  const requestObj = await getAst({
+  const { ast } = await getAst({
     query: param.query,
     model,
     extractOnlyPrimaries: true,
@@ -314,7 +309,7 @@ export async function relDataList(param: {
   let count = 0;
   try {
     data = data = await nocoExecute(
-      requestObj,
+      ast,
       await baseModel.list(param.query),
       {},
       param.query

packages/nocodb/src/lib/services/public/publicDataExport.svc.ts

@@ -46,7 +46,7 @@ export async function getDbRows(param: {
     dbDriver: await NcConnectionMgrv2.get(base),
   });
 
-  const requestObj = await getAst({
+  const { ast } = await getAst({
     query: param.query,
     model: param.model,
     view: param.view,
@@ -69,7 +69,7 @@ export async function getDbRows(param: {
       elapsed = temp[0] * 1000 + temp[1] / 1000000
   ) {
     const rows = await nocoExecute(
-      requestObj,
+      ast,
       await baseModel.list({ ...listArgs, offset, limit }),
       {},
       listArgs