Merge branch 'feat/query-optimization' into feat/query-opt-imp

# Conflicts:
#	packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts
#	packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts
#	packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts

@@ -64,6 +64,21 @@ async function populatePk(model: Model, insertObj: any) {
   }
 }
 
+function checkColumnRequired(
+  column: Column<any>,
+  fields: string[],
+  extractPkAndPv?: boolean
+) {
+  // if primary key or foreign key included in fields, it's required
+  if (column.pk || column.uidt === UITypes.ForeignKey) return true;
+
+  if (extractPkAndPv && column.pv) return true;
+
+  // check fields defined and if not, then select all
+  // if defined check if it is in the fields
+  return !fields || fields.includes(column.title);
+}
+
 /**
  * Base class for models
  *
@@ -182,10 +197,10 @@ class BaseModelSqlv2 {
     } = {},
     ignoreViewFilterAndSort = false
   ): Promise<any> {
-    const { where, ...rest } = this._getListArgs(args as any);
+    const { where, fields, ...rest } = this._getListArgs(args as any);
 
     const qb = this.dbDriver(this.tnPath);
-    await this.selectObject({ qb });
+    await this.selectObject({ qb, fields, viewId: this.viewId });
     if (+rest?.shuffle) {
       await this.shuffle({ qb });
     }
@@ -258,6 +273,8 @@ class BaseModelSqlv2 {
     const proto = await this.getProto();
     const data = await this.execAndParse(qb);
 
+    // console.log(qb.toQuery());
+
     return data?.map((d) => {
       d.__proto__ = proto;
       return d;
@@ -407,7 +424,7 @@ class BaseModelSqlv2 {
       const parentTn = this.getTnPath(parentTable);
 
       const qb = this.dbDriver(childTn);
-      await childModel.selectObject({ qb });
+      await childModel.selectObject({ qb, extractPkAndPv: true });
       await this.applySortAndFilter({ table: childTable, where, qb, sort });
 
       const childQb = this.dbDriver.queryBuilder().from(
@@ -435,6 +452,9 @@ class BaseModelSqlv2 {
           .as('list')
       );
 
+
+      // console.log(childQb.toQuery())
+
       const children = await this.execAndParse(childQb, childTable);
       const proto = await (
         await Model.getBaseModelSQL({
@@ -443,6 +463,7 @@ class BaseModelSqlv2 {
         })
       ).getProto();
 
+
       return groupBy(
         children.map((c) => {
           c.__proto__ = proto;
@@ -1410,7 +1431,7 @@ class BaseModelSqlv2 {
       this.config.limitMin
     );
     obj.offset = Math.max(+(args.offset || args.o) || 0, 0);
-    obj.fields = args.fields || args.f || '*';
+    obj.fields = args.fields || args.f;
     obj.sort = args.sort || args.s;
     return obj;
   }
@@ -1425,16 +1446,52 @@ class BaseModelSqlv2 {
     }
   }
 
+  // todo:
+  //  pass view id as argument
+  //  add option to get only pk and pv
   public async selectObject({
     qb,
+    columns: _columns,,
     columns: _columns,
+    fields: _fields,
+    extractPkAndPv,
+    viewId,
   }: {
     qb: Knex.QueryBuilder;
     columns?: Column[];
+    fields?: string[] | string;
+    extractPkAndPv?: boolean;
+    viewId?: string;
   }): Promise<void> {
+    const view = await View.get(viewId);
+    const viewColumns = viewId && (await View.getColumns(viewId));
+    const fields = Array.isArray(_fields) ? _fields : _fields?.split(',');
     const res = {};
-    const columns = _columns ?? (await this.model.getColumns());
+    // const columns = _columns ?? (await this.model.getColumns());
-    for (const column of columns) {
+    // for (const column of columns) {
+    const viewOrTableColumns = viewColumns || (await this.model.getColumns());
+    for (const viewOrTableColumn of viewOrTableColumns) {
+      const column =
+        viewOrTableColumn instanceof Column
+          ? viewOrTableColumn
+          : await Column.get({
+              colId: (viewOrTableColumn as GridViewColumn).fk_column_id,
+            });
+      // hide if column marked as hidden in view
+      // of if column is system field and system field is hidden
+      if (
+        !extractPkAndPv &&
+        !(viewOrTableColumn instanceof Column) &&
+        (!(viewOrTableColumn as GridViewColumn)?.show ||
+          (!view?.show_system_fields &&
+            column.uidt !== UITypes.ForeignKey &&
+            !column.pk &&
+            isSystemColumn(column)))
+      )
+        continue;
+
+      if (!checkColumnRequired(column, fields, extractPkAndPv)) continue;
+
       switch (column.uidt) {
         case 'LinkToAnotherRecord':
         case 'Lookup':

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/helpers/getAst.ts

@@ -9,23 +9,34 @@ const getAst = async ({
   includePkByDefault = true,
   model,
   view,
+  dependencyFields = {
+    nested: {},
+    fields: new Set(),
+  },
 }: {
   query?: RequestQuery;
   extractOnlyPrimaries?: boolean;
   includePkByDefault?: boolean;
   model: Model;
   view?: View;
+  dependencyFields?: DependantFields;
 }) => {
   if (!model.columns?.length) await model.getColumns();
 
   // extract only pk and pv
   if (extractOnlyPrimaries) {
-    return {
+    const ast = {
       ...(model.primaryKeys
         ? model.primaryKeys.reduce((o, pk) => ({ ...o, [pk.title]: 1 }), {})
         : {}),
       ...(model.displayValue ? { [model.displayValue.title]: 1 } : {}),
     };
+    await Promise.all(
+      model.primaryKeys.map((c) => extractDependencies(c, dependencyFields))
+    );
+    await extractDependencies(model.primaryValue, dependencyFields);
+
+    return { ast, dependencyFields };
   }
 
   let fields = query?.fields || query?.f;
@@ -45,7 +56,7 @@ const getAst = async ({
       {}
     );
 
-  return model.columns.reduce(async (obj, col) => {
+  const ast = await model.columns.reduce(async (obj, col) => {
     let value: number | boolean | { [key: string]: any } = 1;
     const nestedFields =
       query?.nested?.[col.title]?.fields || query?.nested?.[col.title]?.f;
@@ -55,10 +66,19 @@ const getAst = async ({
           .getColOptions<LinkToAnotherRecordColumn>()
           .then((colOpt) => colOpt.getRelatedTable());
 
-        value = await getAst({
+        const { ast } = await getAst({
           model,
           query: query?.nested?.[col.title],
+          dependencyFields: (dependencyFields.nested[col.title] =
+            dependencyFields.nested[col.title] || {
+              nested: {},
+              fields: new Set(),
+            }),
         });
+
+        value = ast;
+
+        // todo: include field relative to the relation => pk / fk
       } else {
         value = (Array.isArray(fields) ? fields : fields.split(',')).reduce(
           (o, f) => ({ ...o, [f]: 1 }),
@@ -74,22 +94,98 @@ const getAst = async ({
         model,
         query: query?.nested?.[col.title],
         extractOnlyPrimaries: nestedFields !== '*',
+        dependencyFields:(dependencyFields.nested[col.title] =
+        dependencyFields.nested[col.title] || {
+          nested: {},
+          fields: new Set(),
+        })
       });
     }
 
+    const isRequested =
+      allowedCols && (!includePkByDefault || !col.pk)
+        ? allowedCols[col.id] &&
+          (!isSystemColumn(col) || view.show_system_fields) &&
+          (!fields?.length || fields.includes(col.title)) &&
+          value
+        : fields?.length
+        ? fields.includes(col.title) && value
+        : value;
+    if (isRequested) await extractDependencies(col, dependencyFields);
+
     return {
       ...(await obj),
-      [col.title]:
+      [col.title]: isRequested,
-        allowedCols && (!includePkByDefault || !col.pk)
-          ? allowedCols[col.id] &&
-            (!isSystemColumn(col) || view.show_system_fields) &&
-            (!fields?.length || fields.includes(col.title)) &&
-            value
-          : fields?.length
-          ? fields.includes(col.title) && value
-          : value,
     };
   }, Promise.resolve({}));
+
+  return { ast, dependencyFields };
+};
+
+const extractDependencies = async (
+  column: Column,
+  dependencyFields: DependantFields = {
+    nested: {},
+    fields: new Set(),
+  }
+) => {
+  switch (column.uidt) {
+    case UITypes.Lookup:
+      await extractLookupDependencies(column, dependencyFields);
+      break;
+    case UITypes.LinkToAnotherRecord:
+      await extractRelationDependencies(column, dependencyFields);
+      break;
+    default:
+      dependencyFields.fields.add(column.title);
+      break;
+  }
+};
+
+const extractLookupDependencies = async (
+  lookUpColumn: Column<LookupColumn>,
+  dependencyFields: DependantFields = {
+    nested: {},
+    fields: new Set(),
+  }
+) => {
+  const lookupColumnOpts = await lookUpColumn.getColOptions();
+  const relationColumn = await lookupColumnOpts.getRelationColumn();
+  await extractRelationDependencies(relationColumn, dependencyFields);
+  await extractDependencies(
+    await lookupColumnOpts.getLookupColumn(),
+    (dependencyFields.nested[relationColumn.title] = dependencyFields.nested[
+      relationColumn.title
+    ] || {
+      nested: {},
+      fields: new Set(),
+    })
+  );
+};
+
+const extractRelationDependencies = async (
+  relationColumn: Column<LinkToAnotherRecordColumn>,
+  dependencyFields: DependantFields = {
+    nested: {},
+    fields: new Set(),
+  }
+) => {
+  const relationColumnOpts = await relationColumn.getColOptions();
+
+  switch (relationColumnOpts.type) {
+    case RelationTypes.HAS_MANY:
+      dependencyFields.fields.add(
+        await relationColumnOpts.getParentColumn().then((col) => col.title)
+      );
+      break;
+    case RelationTypes.BELONGS_TO:
+    case RelationTypes.MANY_TO_MANY:
+      dependencyFields.fields.add(
+        await relationColumnOpts.getChildColumn().then((col) => col.title)
+      );
+
+      break;
+  }
 };
 
 type RequestQuery = {
@@ -100,4 +196,9 @@ type RequestQuery = {
   };
 };
 
+interface DependantFields {
+  fields?: Set<string>;
+  nested?: DependantFields;
+}
+
 export default getAst;

packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts

@@ -0,0 +1,341 @@
+import { Request, Response, Router } from 'express';
+import Model from '../../../models/Model';
+import { nocoExecute } from 'nc-help';
+import Base from '../../../models/Base';
+import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
+import { PagedResponseImpl } from '../../helpers/PagedResponse';
+import View from '../../../models/View';
+import ncMetaAclMw from '../../helpers/ncMetaAclMw';
+import { getViewAndModelFromRequestByAliasOrId } from './helpers';
+import apiMetrics from '../../helpers/apiMetrics';
+import getAst from '../../../db/sql-data-mapper/lib/sql/helpers/getAst';
+
+async function dataList(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+  res.json(await getDataList(model, view, req));
+}
+
+async function dataFindOne(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+  res.json(await getFindOne(model, view, req));
+}
+
+async function dataGroupBy(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+  res.json(await getDataGroupBy(model, view, req));
+}
+
+async function dataCount(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  const countArgs: any = { ...req.query };
+  try {
+    countArgs.filterArr = JSON.parse(countArgs.filterArrJson);
+  } catch (e) {}
+
+  const count = await baseModel.count(countArgs);
+
+  res.json({ count });
+}
+
+async function dataInsert(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  res.json(await baseModel.insert(req.body, null, req));
+}
+
+async function dataUpdate(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  res.json(await baseModel.updateByPk(req.params.rowId, req.body, null, req));
+}
+
+async function dataDelete(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+  const base = await Base.get(model.base_id);
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+  const message = await baseModel.hasLTARData(req.params.rowId, model);
+  if (message.length) {
+    res.json({ message });
+    return;
+  }
+  res.json(await baseModel.delByPk(req.params.rowId, null, req));
+}
+async function getDataList(model, view: View, req) {
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  const { ast, dependencyFields } = await getAst({
+    model,
+    query: req.query,
+    view,
+  });
+
+  const listArgs: any = { ...req.query };
+  try {
+    listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
+  } catch (e) {}
+  try {
+    listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
+  } catch (e) {}
+
+  console.log(
+    JSON.stringify(
+      dependencyFields,
+      (_v, o) => {
+        if (o instanceof Set) {
+          return [...o];
+        }
+        return o;
+      },
+      2
+    )
+  );
+  console.log(JSON.stringify(ast, null, 2));
+  const data = await nocoExecute(
+    ast,
+    await baseModel.list(listArgs),
+    {},
+    listArgs
+  );
+
+  const count = await baseModel.count(listArgs);
+
+  return new PagedResponseImpl(data, {
+    ...req.query,
+    count,
+  });
+}
+
+async function getFindOne(model, view: View, req) {
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  const args: any = { ...req.query };
+  try {
+    args.filterArr = JSON.parse(args.filterArrJson);
+  } catch (e) {}
+  try {
+    args.sortArr = JSON.parse(args.sortArrJson);
+  } catch (e) {}
+
+  const data = await baseModel.findOne(args);
+  return data
+    ? await nocoExecute(
+        await getAst({ model, query: args, view }),
+        data,
+        {},
+        {}
+      )
+    : {};
+}
+
+async function getDataGroupBy(model, view: View, req) {
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  const listArgs: any = { ...req.query };
+  const data = await baseModel.groupBy({ ...req.query });
+  const count = await baseModel.count(listArgs);
+
+  return new PagedResponseImpl(data, {
+    ...req.query,
+    count,
+  });
+}
+
+async function dataRead(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  res.json(
+    await nocoExecute(
+      await getAst({ model, query: req.query, view }),
+      await baseModel.readByPk(req.params.rowId),
+      {},
+      {}
+    )
+  );
+}
+
+async function dataExist(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  res.json(await baseModel.exist(req.params.rowId));
+}
+const router = Router({ mergeParams: true });
+
+// table data crud apis
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName',
+  apiMetrics,
+  ncMetaAclMw(dataList, 'dataList')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/find-one',
+  apiMetrics,
+  ncMetaAclMw(dataFindOne, 'dataFindOne')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/groupby',
+  apiMetrics,
+  ncMetaAclMw(dataGroupBy, 'dataGroupBy')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/exist',
+  apiMetrics,
+  ncMetaAclMw(dataExist, 'dataExist')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/count',
+  apiMetrics,
+  ncMetaAclMw(dataCount, 'dataCount')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/count',
+  apiMetrics,
+  ncMetaAclMw(dataCount, 'dataCount')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId',
+  apiMetrics,
+  ncMetaAclMw(dataRead, 'dataRead')
+);
+
+router.patch(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId',
+  apiMetrics,
+  ncMetaAclMw(dataUpdate, 'dataUpdate')
+);
+
+router.delete(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId',
+  apiMetrics,
+  ncMetaAclMw(dataDelete, 'dataDelete')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName',
+  apiMetrics,
+  ncMetaAclMw(dataList, 'dataList')
+);
+
+// table view data crud apis
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName',
+  apiMetrics,
+  ncMetaAclMw(dataList, 'dataList')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/find-one',
+  apiMetrics,
+  ncMetaAclMw(dataFindOne, 'dataFindOne')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/groupby',
+  apiMetrics,
+  ncMetaAclMw(dataGroupBy, 'dataGroupBy')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId/exist',
+  apiMetrics,
+  ncMetaAclMw(dataExist, 'dataExist')
+);
+
+router.post(
+  '/api/v1/db/data/:orgs/:projectName/:tableName',
+  apiMetrics,
+  ncMetaAclMw(dataInsert, 'dataInsert')
+);
+
+router.post(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName',
+  apiMetrics,
+  ncMetaAclMw(dataInsert, 'dataInsert')
+);
+
+router.patch(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId',
+  apiMetrics,
+  ncMetaAclMw(dataUpdate, 'dataUpdate')
+);
+
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId',
+  apiMetrics,
+  ncMetaAclMw(dataRead, 'dataRead')
+);
+
+router.delete(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId',
+  apiMetrics,
+  ncMetaAclMw(dataDelete, 'dataDelete')
+);
+
+export default router;

packages/nocodb/src/lib/models/Column.ts

@@ -406,7 +406,7 @@ export default class Column<T = any> implements ColumnType {
     }
   }
 
-  public async getColOptions<T>(ncMeta = Noco.ncMeta): Promise<T> {
+  public async getColOptions<U = T>(ncMeta = Noco.ncMeta): Promise<U> {
     let res: any;
 
     switch (this.uidt) {