feat: nested field filtering (#1856)

* feat: allow user to include/exclude nested fields and by default only extract pk&pv

Signed-off-by: Pranav C <pranavxc@gmail.com>

* chore: api metrics

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: add missing await and code cleanup

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: corrections in ast population

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: map default primary value column on table create and include field param for belongsTo column

Signed-off-by: Pranav C <pranavxc@gmail.com>

* fix: old data list api response correction and count api

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/noco-docs/content/en/setup-and-usages/usage-information.md

@@ -42,7 +42,7 @@ Here is an example :
     "os_release" : "5.10.25-linuxkit",
     "node_version" : "14.18.2",
     "docker" : "true",
-    "xc_version" : "0.84.15",
+    "xc_version" : "a0885e8e6a38d9fbb5d39e7d04a44da7773d4f",
     "env" : "dev",
 }
 

packages/nocodb/package.json

@@ -150,7 +150,7 @@
     "mysql2": "^2.2.5",
     "nanoid": "^3.1.20",
     "nc-common": "0.0.6",
-    "nc-help": "^0.2.44",
+    "nc-help": "0.2.46",
     "nc-lib-gui": "0.90.7",
     "nc-plugin": "^0.1.1",
     "ncp": "^2.0.0",

packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSqlv2.ts

@@ -21,7 +21,6 @@ import View from '../../../noco-models/View';
 import {
   AuditOperationSubTypes,
   AuditOperationTypes,
-  isSystemColumn,
   RelationTypes,
   SortType,
   UITypes,
@@ -288,69 +287,8 @@ class BaseModelSqlv2 {
     return ((await qb) as any).count;
   }
 
-  public async defaultResolverReq(
-    query?: any,
-    extractOnlyPrimaries = false,
-    includePkByDefault = true
-  ) {
-    await this.model.getColumns();
-    if (extractOnlyPrimaries) {
-      return {
-        [this.model.primaryKey.title]: 1,
-        [this.model.primaryValue.title]: 1
-      };
-    }
-
-    let fields = query?.fields || query?.f;
-    if (fields && fields !== '*') {
-      fields = Array.isArray(fields) ? fields : fields.split(',');
-    } else {
-      fields = null;
-    }
-
-    let allowedCols = null;
-    if (this.viewId)
-      allowedCols = (await View.getColumns(this.viewId)).reduce(
-        (o, c) => ({
-          ...o,
-          [c.fk_column_id]: c.show
-        }),
-        {}
-      );
-
-    const view = await View.get(this.viewId);
-
-    return this.model.getColumns().then(columns =>
-      Promise.resolve(
-        columns.reduce(
-          (obj, col) => ({
-            ...obj,
-            [col.title]:
-              allowedCols && (!includePkByDefault || !col.pk)
-                ? allowedCols[col.id] &&
-                  (!isSystemColumn(col) || view.show_system_fields) &&
-                  (!fields?.length || fields.includes(col.title))
-                : fields?.length
-                ? fields.includes(col.title)
-                : 1
-          }),
-          {}
-        )
-      )
-    );
-  }
-
   async multipleHmList({ colId, ids }, args?: { limit?; offset? }) {
     try {
-      // const {
-      //   where,
-      //   limit,
-      //   offset,
-      //   conditionGraph,
-      //   sort
-      //   // ...restArgs
-      // } = this.dbModels[child]._getChildListArgs(args);
-      // let { fields } = restArgs;
       // todo: get only required fields
       let fields = '*';
 

packages/nocodb/src/lib/dataMapper/lib/sql/helpers/getAst.ts

@@ -0,0 +1,99 @@
+import View from '../../../../noco-models/View';
+import { isSystemColumn, UITypes } from 'nocodb-sdk';
+import Model from '../../../../noco-models/Model';
+import LinkToAnotherRecordColumn from '../../../../noco-models/LinkToAnotherRecordColumn';
+
+const getAst = async ({
+  query,
+  extractOnlyPrimaries = false,
+  includePkByDefault = true,
+  model,
+  view
+}: {
+  query?: RequestQuery;
+  extractOnlyPrimaries?: boolean;
+  includePkByDefault?: boolean;
+  model: Model;
+  view?: View;
+}) => {
+  if (!model.columns?.length) await model.getColumns();
+  if (extractOnlyPrimaries) {
+    return {
+      [model.primaryKey.title]: 1,
+      [model.primaryValue.title]: 1
+    };
+  }
+
+  let fields = query?.fields || query?.f;
+  if (fields && fields !== '*') {
+    fields = Array.isArray(fields) ? fields : fields.split(',');
+  } else {
+    fields = null;
+  }
+
+  let allowedCols = null;
+  if (view)
+    allowedCols = (await View.getColumns(view.id)).reduce(
+      (o, c) => ({
+        ...o,
+        [c.fk_column_id]: c.show
+      }),
+      {}
+    );
+
+  return model.columns.reduce(async (obj, col) => {
+    let value: number | boolean | { [key: string]: any } = 1;
+    const nestedFields =
+      query?.nested?.[col.title]?.fields || query?.nested?.[col.title]?.f;
+    if (nestedFields && nestedFields !== '*') {
+      if (col.uidt === UITypes.LinkToAnotherRecord) {
+        const model = await col
+          .getColOptions<LinkToAnotherRecordColumn>()
+          .then(colOpt => colOpt.getRelatedTable());
+
+        value = await getAst({
+          model,
+          query: query?.nested?.[col.title]
+        });
+      } else {
+        value = (Array.isArray(fields) ? fields : fields.split(',')).reduce(
+          (o, f) => ({ ...o, [f]: 1 }),
+          {}
+        );
+      }
+    } else if (col.uidt === UITypes.LinkToAnotherRecord) {
+      const model = await col
+        .getColOptions<LinkToAnotherRecordColumn>()
+        .then(colOpt => colOpt.getRelatedTable());
+
+      value = await getAst({
+        model,
+        query: query?.nested,
+        extractOnlyPrimaries: true
+      });
+    }
+
+    return {
+      ...(await obj),
+      [col.title]:
+        allowedCols && (!includePkByDefault || !col.pk)
+          ? allowedCols[col.id] &&
+            (!isSystemColumn(col) || view.show_system_fields) &&
+            (!fields?.length || fields.includes(col.title)) &&
+            value
+          : fields?.length
+          ? fields.includes(col.title)
+          : value
+    };
+  }, Promise.resolve({}));
+};
+
+type RequestQuery = {
+  [fields in 'f' | 'fields']?: string | string[];
+} & {
+  nested?: {
+    [field: string]: RequestQuery;
+  };
+};
+
+export default getAst;

packages/nocodb/src/lib/noco-models/Model.ts

@@ -51,14 +51,6 @@ export default class Model implements TableType {
   columnsById?: { [id: string]: Column };
   views?: View[];
 
-  // private static baseModels: {
-  //   [baseId: string]: {
-  //     [dbAlias: string]: {
-  //       [tableIdOrName: string]: BaseModelSqlv2;
-  //     };
-  //   };
-  // } = {};
-
   constructor(data: Partial<TableType | Model>) {
     Object.assign(this, data);
   }

packages/nocodb/src/lib/noco/Noco.ts

@@ -568,14 +568,28 @@ export default class Noco {
       }
       this.config.auth.jwt.secret = secret;
     }
+    let serverId = (
+      await Noco._ncMeta.metaGet('', '', 'nc_store', {
+        key: 'nc_server_id'
+      })
+    )?.value;
+    if (!serverId) {
+      await Noco._ncMeta.metaInsert('', '', 'nc_store', {
+        key: 'nc_server_id',
+        value: serverId = Tele.id
+      });
+    }
+    process.env.NC_SERVER_UUID = serverId;
   }
 
   public static get ncMeta(): NcMetaIO {
     return this._ncMeta;
   }
+
   public get ncMeta(): NcMetaIO {
     return Noco._ncMeta;
   }
+
   public static getConfig(): NcConfig {
     return Noco.config;
   }

packages/nocodb/src/lib/noco/meta/api/dataApis/bulkDataAliasApis.ts

@@ -4,6 +4,7 @@ import Base from '../../../../noco-models/Base';
 import NcConnectionMgrv2 from '../../../common/NcConnectionMgrv2';
 import ncMetaAclMw from '../../helpers/ncMetaAclMw';
 import { getViewAndModelFromRequestByAliasOrId } from './helpers';
+import apiMetrics from '../../helpers/apiMetrics';
 
 async function bulkDataInsert(req: Request, res: Response) {
   const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
@@ -72,22 +73,27 @@ const router = Router({ mergeParams: true });
 
 router.post(
   '/api/v1/db/data/bulk/:orgs/:projectName/:tableName',
+  apiMetrics,
   ncMetaAclMw(bulkDataInsert, 'bulkDataInsert')
 );
 router.patch(
   '/api/v1/db/data/bulk/:orgs/:projectName/:tableName',
+  apiMetrics,
   ncMetaAclMw(bulkDataUpdate, 'bulkDataUpdate')
 );
 router.patch(
   '/api/v1/db/data/bulk/:orgs/:projectName/:tableName/all',
+  apiMetrics,
   ncMetaAclMw(bulkDataUpdateAll, 'bulkDataUpdateAll')
 );
 router.delete(
   '/api/v1/db/data/bulk/:orgs/:projectName/:tableName',
+  apiMetrics,
   ncMetaAclMw(bulkDataDelete, 'bulkDataDelete')
 );
 router.delete(
   '/api/v1/db/data/bulk/:orgs/:projectName/:tableName/all',
+  apiMetrics,
   ncMetaAclMw(bulkDataDeleteAll, 'bulkDataDeleteAll')
 );
 

packages/nocodb/src/lib/noco/meta/api/dataApis/dataAliasApis.ts

@@ -7,6 +7,8 @@ import { PagedResponseImpl } from '../../helpers/PagedResponse';
 import View from '../../../../noco-models/View';
 import ncMetaAclMw from '../../helpers/ncMetaAclMw';
 import { getViewAndModelFromRequestByAliasOrId } from './helpers';
+import apiMetrics from '../../helpers/apiMetrics';
+import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
 
 async function dataList(req: Request, res: Response) {
   const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
@@ -86,7 +88,7 @@ async function getDataList(model, view: View, req) {
     dbDriver: NcConnectionMgrv2.get(base)
   });
 
-  const requestObj = await baseModel.defaultResolverReq(req.query);
+  const requestObj = await getAst({ model, query: req.query, view });
 
   const listArgs: any = { ...req.query };
   try {
@@ -129,7 +131,7 @@ async function getFindOne(model, view: View, req) {
   } catch (e) {}
 
   return await nocoExecute(
-    await baseModel.defaultResolverReq(),
+    await getAst({ model, query: args, view }),
     await baseModel.findOne(args),
     {},
     {}
@@ -149,7 +151,7 @@ async function dataRead(req: Request, res: Response) {
 
   res.json(
     await nocoExecute(
-      await baseModel.defaultResolverReq(),
+      await getAst({ model, query: req.query, view }),
       await baseModel.readByPk(req.params.rowId),
       {},
       {}
@@ -162,70 +164,85 @@ const router = Router({ mergeParams: true });
 // table data crud apis
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName',
+  apiMetrics,
   ncMetaAclMw(dataList, 'dataList')
 );
 
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/find-one',
+  apiMetrics,
   ncMetaAclMw(dataFindOne, 'dataFindOne')
 );
 
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/count',
+  apiMetrics,
   ncMetaAclMw(dataCount, 'dataCount')
 );
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/count',
+  apiMetrics,
   ncMetaAclMw(dataCount, 'dataCount')
 );
 
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId',
+  apiMetrics,
   ncMetaAclMw(dataRead, 'dataRead')
 );
 router.patch(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId',
+  apiMetrics,
   ncMetaAclMw(dataUpdate, 'dataUpdate')
 );
 router.delete(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId',
+  apiMetrics,
   ncMetaAclMw(dataDelete, 'dataDelete')
 );
 
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName',
+  apiMetrics,
   ncMetaAclMw(dataList, 'dataList')
 );
 
 // table view data crud apis
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName',
+  apiMetrics,
   ncMetaAclMw(dataList, 'dataList')
 );
 
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/find-one',
+  apiMetrics,
   ncMetaAclMw(dataFindOne, 'dataFindOne')
 );
 
 router.post(
   '/api/v1/db/data/:orgs/:projectName/:tableName',
+  apiMetrics,
   ncMetaAclMw(dataInsert, 'dataInsert')
 );
 router.post(
   '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName',
+  apiMetrics,
   ncMetaAclMw(dataInsert, 'dataInsert')
 );
 router.patch(
   '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId',
+  apiMetrics,
   ncMetaAclMw(dataUpdate, 'dataUpdate')
 );
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId',
+  apiMetrics,
   ncMetaAclMw(dataRead, 'dataRead')
 );
 router.delete(
   '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId',
+  apiMetrics,
   ncMetaAclMw(dataDelete, 'dataDelete')
 );
 

packages/nocodb/src/lib/noco/meta/api/dataApis/dataAliasExportApis.ts

@@ -4,6 +4,7 @@ import {
   extractCsvData,
   getViewAndModelFromRequestByAliasOrId
 } from './helpers';
+import apiMetrics from '../../helpers/apiMetrics';
 
 async function csvDataExport(req: Request, res: Response) {
   const { view } = await getViewAndModelFromRequestByAliasOrId(req);
@@ -23,10 +24,12 @@ const router = Router({ mergeParams: true });
 
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/export/csv',
+  apiMetrics,
   ncMetaAclMw(csvDataExport, 'exportCsv')
 );
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/export/csv',
+  apiMetrics,
   ncMetaAclMw(csvDataExport, 'exportCsv')
 );
 

packages/nocodb/src/lib/noco/meta/api/dataApis/dataAliasNestedApis.ts

@@ -6,6 +6,7 @@ import { PagedResponseImpl } from '../../helpers/PagedResponse';
 import ncMetaAclMw from '../../helpers/ncMetaAclMw';
 import { getViewAndModelFromRequestByAliasOrId } from './helpers';
 import { NcError } from '../../helpers/catchError';
+import apiMetrics from '../../helpers/apiMetrics';
 
 export async function mmList(req: Request, res: Response, next) {
   const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
@@ -256,32 +257,39 @@ const router = Router({ mergeParams: true });
 
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/mm/:columnName/exclude',
+  apiMetrics,
   ncMetaAclMw(mmExcludedList, 'mmExcludedList')
 );
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/hm/:columnName/exclude',
+  apiMetrics,
   ncMetaAclMw(hmExcludedList, 'hmExcludedList')
 );
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/bt/:columnName/exclude',
+  apiMetrics,
   ncMetaAclMw(btExcludedList, 'btExcludedList')
 );
 
 router.post(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/:relationType/:columnName/:refRowId',
+  apiMetrics,
   ncMetaAclMw(relationDataAdd, 'relationDataAdd')
 );
 router.delete(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/:relationType/:columnName/:refRowId',
+  apiMetrics,
   ncMetaAclMw(relationDataRemove, 'relationDataRemove')
 );
 
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/mm/:columnName',
+  apiMetrics,
   ncMetaAclMw(mmList, 'mmList')
 );
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/hm/:columnName',
+  apiMetrics,
   ncMetaAclMw(hmList, 'hmList')
 );
 

packages/nocodb/src/lib/noco/meta/api/dataApis/dataApis.ts

@@ -7,6 +7,8 @@ import { PagedResponseImpl } from '../../helpers/PagedResponse';
 import View from '../../../../noco-models/View';
 import ncMetaAclMw from '../../helpers/ncMetaAclMw';
 import { NcError } from '../../helpers/catchError';
+import apiMetrics from '../../helpers/apiMetrics';
+import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
 
 export async function dataList(req: Request, res: Response, next) {
   const view = await View.get(req.params.viewId);
@@ -315,24 +317,14 @@ async function dataRead(req: Request, res: Response, next) {
       id: model.id,
       dbDriver: NcConnectionMgrv2.get(base)
     });
-    const key = `${model.title}Read`;
 
     res.json(
-      (
       await nocoExecute(
-          {
-            [key]: await baseModel.defaultResolverReq()
-          },
-          {
-            [key]: async id => {
-              return await baseModel.readByPk(id);
-              // return row ? new ctx.types[model.title](row) : null;
-            }
-          },
+        await getAst({ model, query: req.query }),
+        await baseModel.readByPk(req.params.rowId),
         {},
-          { nested: { [key]: req.params.rowId } }
+        {}
       )
-      )?.[key]
     );
   } catch (e) {
     console.log(e);
@@ -450,10 +442,7 @@ async function getDataList(model, view: View, req) {
     dbDriver: NcConnectionMgrv2.get(base)
   });
 
-  const key = `${model._tn}List`;
-  const requestObj = {
-    [key]: await baseModel.defaultResolverReq(req.query)
-  };
+  const requestObj = await getAst({ query: req.query, model, view });
 
   const listArgs: any = { ...req.query };
   try {
@@ -463,18 +452,12 @@ async function getDataList(model, view: View, req) {
     listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
   } catch (e) {}
 
-  const data = (
-    await nocoExecute(
+  const data = await nocoExecute(
     requestObj,
-      {
-        [key]: async args => {
-          return await baseModel.list(args);
-        }
-      },
+    await baseModel.list(listArgs),
     {},
-      { nested: { [key]: listArgs } }
-    )
-  )?.[key];
+    listArgs
+  );
 
   const count = await baseModel.count(listArgs);
 
@@ -539,7 +522,7 @@ async function relationDataAdd(req, res) {
 
 const router = Router({ mergeParams: true });
 
-// router.get('/data/:orgs/:projectName/:tableName', ncMetaAclMw(dataListNew));
+// router.get('/data/:orgs/:projectName/:tableName',apiMetrics,ncMetaAclMw(dataListNew));
 // router.get(
 //   '/data/:orgs/:projectName/:tableName/views/:viewName',
 //   ncMetaAclMw(dataListNew)
@@ -558,14 +541,38 @@ const router = Router({ mergeParams: true });
 //   ncMetaAclMw(dataDeleteNew)
 // );
 
-router.get('/data/:viewId/', ncMetaAclMw(dataList, 'dataList'));
-router.post('/data/:viewId/', ncMetaAclMw(dataInsert, 'dataInsert'));
-router.get('/data/:viewId/:rowId', ncMetaAclMw(dataRead, 'dataRead'));
-router.patch('/data/:viewId/:rowId', ncMetaAclMw(dataUpdate, 'dataUpdate'));
-router.delete('/data/:viewId/:rowId', ncMetaAclMw(dataDelete, 'dataDelete'));
+router.get('/data/:viewId/', apiMetrics, ncMetaAclMw(dataList, 'dataList'));
+router.post(
+  '/data/:viewId/',
+  apiMetrics,
+  ncMetaAclMw(dataInsert, 'dataInsert')
+);
+router.get(
+  '/data/:viewId/:rowId',
+  apiMetrics,
+  ncMetaAclMw(dataRead, 'dataRead')
+);
+router.patch(
+  '/data/:viewId/:rowId',
+  apiMetrics,
+  ncMetaAclMw(dataUpdate, 'dataUpdate')
+);
+router.delete(
+  '/data/:viewId/:rowId',
+  apiMetrics,
+  ncMetaAclMw(dataDelete, 'dataDelete')
+);
 
-router.get('/data/:viewId/:rowId/mm/:colId', ncMetaAclMw(mmList, 'mmList'));
-router.get('/data/:viewId/:rowId/hm/:colId', ncMetaAclMw(hmList, 'hmList'));
+router.get(
+  '/data/:viewId/:rowId/mm/:colId',
+  apiMetrics,
+  ncMetaAclMw(mmList, 'mmList')
+);
+router.get(
+  '/data/:viewId/:rowId/hm/:colId',
+  apiMetrics,
+  ncMetaAclMw(hmList, 'hmList')
+);
 
 router.get(
   '/data/:viewId/:rowId/mm/:colId/exclude',

packages/nocodb/src/lib/noco/meta/api/dataApis/helpers.ts

@@ -13,6 +13,7 @@ import LookupColumn from '../../../../noco-models/LookupColumn';
 import LinkToAnotherRecordColumn from '../../../../noco-models/LinkToAnotherRecordColumn';
 
 import papaparse from 'papaparse';
+import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
 export async function getViewAndModelFromRequestByAliasOrId(
   req:
     | Request<{ projectName: string; tableName: string; viewName?: string }>
@@ -71,7 +72,12 @@ export async function extractCsvData(view: View, req: Request) {
       elapsed = temp[0] * 1000 + temp[1] / 1000000
   ) {
     const rows = await nocoExecute(
-      await baseModel.defaultResolverReq(req.query, false, false),
+      await getAst({
+        query: req.query,
+        includePkByDefault: false,
+        model: view.model,
+        view
+      }),
       await baseModel.list({ ...req.query, offset, limit }),
       {},
       req.query

packages/nocodb/src/lib/noco/meta/api/dataApis/oldDataApis.ts

@@ -3,15 +3,66 @@ import Model from '../../../../noco-models/Model';
 import { nocoExecute } from 'nc-help';
 import Base from '../../../../noco-models/Base';
 import NcConnectionMgrv2 from '../../../common/NcConnectionMgrv2';
-import { PagedResponseImpl } from '../../helpers/PagedResponse';
 import View from '../../../../noco-models/View';
 import ncMetaAclMw from '../../helpers/ncMetaAclMw';
 import Project from '../../../../noco-models/Project';
 import { NcError } from '../../helpers/catchError';
+import apiMetrics from '../../helpers/apiMetrics';
+import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
 
 export async function dataList(req: Request, res: Response) {
   const { model, view } = await getViewAndModelFromRequest(req);
-  res.json(await getDataList(model, view, req));
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base)
+  });
+
+  const requestObj = await getAst({
+    query: req.query,
+    model,
+    view
+  });
+
+  const listArgs: any = { ...req.query };
+  try {
+    listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
+  } catch (e) {}
+  try {
+    listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
+  } catch (e) {}
+
+  const data = await nocoExecute(
+    requestObj,
+    await baseModel.list(listArgs),
+    {},
+    listArgs
+  );
+
+  res.json(data);
+}
+export async function dataCount(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequest(req);
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base)
+  });
+
+  const listArgs: any = { ...req.query };
+  try {
+    listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
+  } catch (e) {}
+
+  const count = await baseModel.count(listArgs);
+
+  res.json({
+    count
+  });
 }
 
 async function dataInsert(req: Request, res: Response) {
@@ -52,39 +103,7 @@ async function dataDelete(req: Request, res: Response) {
 
   res.json(await baseModel.delByPk(req.params.rowId, null, req));
 }
-async function getDataList(model, view: View, req) {
-  const base = await Base.get(model.base_id);
 
-  const baseModel = await Model.getBaseModelSQL({
-    id: model.id,
-    viewId: view?.id,
-    dbDriver: NcConnectionMgrv2.get(base)
-  });
-
-  const requestObj = await baseModel.defaultResolverReq(req.query);
-
-  const listArgs: any = { ...req.query };
-  try {
-    listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
-  } catch (e) {}
-  try {
-    listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
-  } catch (e) {}
-
-  const data = await nocoExecute(
-    requestObj,
-    await baseModel.list(listArgs),
-    {},
-    listArgs
-  );
-
-  const count = await baseModel.count(listArgs);
-
-  return new PagedResponseImpl(data, {
-    ...req.query,
-    count
-  });
-}
 async function getViewAndModelFromRequest(req) {
   const project = await Project.getWithInfo(req.params.projectId);
   const model = await Model.getByAliasOrId({
@@ -115,7 +134,11 @@ async function dataRead(req: Request, res: Response) {
 
   res.json(
     await nocoExecute(
-      await baseModel.defaultResolverReq(),
+      await getAst({
+        query: req.query,
+        model,
+        view
+      }),
       await baseModel.readByPk(req.params.rowId),
       {},
       {}
@@ -130,20 +153,29 @@ router.get(
   ncMetaAclMw(dataList, 'dataList')
 );
 
+router.get(
+  '/nc/:projectId/api/v1/:tableName/count',
+  ncMetaAclMw(dataCount, 'dataCount')
+);
+
 router.post(
   '/nc/:projectId/api/v1/:tableName',
+  apiMetrics,
   ncMetaAclMw(dataInsert, 'dataInsert')
 );
 router.get(
   '/nc/:projectId/api/v1/:tableName/:rowId',
+  apiMetrics,
   ncMetaAclMw(dataRead, 'dataRead')
 );
 router.patch(
   '/nc/:projectId/api/v1/:tableName/:rowId',
+  apiMetrics,
   ncMetaAclMw(dataUpdate, 'dataUpdate')
 );
 router.delete(
   '/nc/:projectId/api/v1/:tableName/:rowId',
+  apiMetrics,
   ncMetaAclMw(dataDelete, 'dataDelete')
 );
 

packages/nocodb/src/lib/noco/meta/api/index.ts

@@ -95,7 +95,6 @@ export default function(router: Router, server) {
       credentials: true
     }
   });
-
   io.use(function(socket, next) {
     passport.authenticate(
       'jwt',
@@ -110,7 +109,10 @@ export default function(router: Router, server) {
       }
     )(socket.handshake, {}, next);
   }).on('connection', socket => {
-    const id = getHash(Tele.id + (socket?.handshake as any)?.user?.id);
+    const id = getHash(
+      (process.env.NC_SERVER_UUID || Tele.id) +
+        (socket?.handshake as any)?.user?.id
+    );
 
     socket.on('page', args => {
       Tele.page({ ...args, id });

packages/nocodb/src/lib/noco/meta/api/publicApis/publicDataApis.ts

@@ -16,6 +16,7 @@ import { nanoid } from 'nanoid';
 import { mimeIcons } from '../../../../utils/mimeTypes';
 import slash from 'slash';
 import { sanitizeUrlPath } from '../attachmentApis';
+import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
 
 export async function dataList(req: Request, res: Response) {
   try {
@@ -49,7 +50,11 @@ export async function dataList(req: Request, res: Response) {
     } catch (e) {}
 
     const data = await nocoExecute(
-      await baseModel.defaultResolverReq(req.query),
+      await getAst({
+        query: req.query,
+        model,
+        view
+      }),
       await baseModel.list(listArgs),
       {},
       listArgs
@@ -186,23 +191,18 @@ async function relDataList(req, res) {
     dbDriver: NcConnectionMgrv2.get(base)
   });
 
-  const key = `${model.title}List`;
-  const requestObj = {
-    [key]: await baseModel.defaultResolverReq(req.query, true)
-  };
+  const requestObj = await getAst({
+    query: req.query,
+    model,
+    extractOnlyPrimaries: true
+  });
 
-  const data = (
-    await nocoExecute(
+  const data = await nocoExecute(
     requestObj,
-      {
-        [key]: async args => {
-          return await baseModel.list(args);
-        }
-      },
+    await baseModel.list(req.query),
     {},
-      { nested: { [key]: req.query } }
-    )
-  )?.[key];
+    req.query
+  );
 
   const count = await baseModel.count(req.query);
 

packages/nocodb/src/lib/noco/meta/api/publicApis/publicDataExportApis.ts

@@ -10,6 +10,7 @@ import Column from '../../../../noco-models/Column';
 import LinkToAnotherRecordColumn from '../../../../noco-models/LinkToAnotherRecordColumn';
 import LookupColumn from '../../../../noco-models/LookupColumn';
 import catchError, { NcError } from '../../helpers/catchError';
+import getAst from '../../../../dataMapper/lib/sql/helpers/getAst';
 
 async function exportCsv(req: Request, res: Response) {
   const view = await View.getByUUID(req.params.publicDataUuid);
@@ -50,10 +51,12 @@ async function exportCsv(req: Request, res: Response) {
     dbDriver: NcConnectionMgrv2.get(base)
   });
 
-  const key = `${model.title}List`;
-  const requestObj = {
-    [key]: await baseModel.defaultResolverReq(req.query, false, false)
-  };
+  const requestObj = await getAst({
+    query: req.query,
+    model,
+    view,
+    includePkByDefault: false
+  });
 
   let offset = +req.query.offset || 0;
   const limit = 100;
@@ -70,22 +73,12 @@ async function exportCsv(req: Request, res: Response) {
       temp = process.hrtime(startTime),
       elapsed = temp[0] * 1000 + temp[1] / 1000000
   ) {
-    const rows = (
-      await nocoExecute(
+    const rows = await nocoExecute(
       requestObj,
-        {
-          [key]: async args => {
-            return await baseModel.list({ ...args, offset, limit });
-          }
-        },
+      await baseModel.list({ ...listArgs, offset, limit }),
       {},
-        {
-          nested: {
-            [key]: listArgs
-          }
-        }
-      )
-    )?.[key];
+      listArgs
+    );
 
     if (!rows?.length) {
       offset = -1;

packages/nocodb/src/lib/noco/meta/api/swagger/helpers/templates/params.ts

@@ -184,6 +184,8 @@ export const getNestedParams = async (
           nestedFieldParam(column.title),
           nestedSortParam(column.title)
         ];
+      } else {
+        return [...(await paramsArr), nestedFieldParam(column.title)];
       }
     }
 

packages/nocodb/src/lib/noco/meta/api/tableApis.ts

@@ -157,7 +157,7 @@ export async function tableCreate(req: Request<any, any, TableReqType>, res) {
     ip: (req as any).clientIp
   }).then(() => {});
 
-  mapDefaultPrimaryValue(req.body.columns);
+  mapDefaultPrimaryValue(columns);
 
   Tele.emit('evt', { evt_type: 'table:created' });
 

packages/nocodb/src/lib/noco/meta/helpers/apiMetrics.ts

@@ -0,0 +1,19 @@
+import { Request } from 'express';
+import { Tele } from 'nc-help';
+
+const countMap = {};
+
+const metrics = async (req: Request) => {
+  if (!req?.route?.path) return;
+  const event = `a:api:${req.route.path}:${req.method}`;
+  countMap[event] = (countMap[event] || 0) + 1;
+  if (countMap[event] >= 50) {
+    Tele.event({ event });
+    countMap[event] = 0;
+  }
+};
+
+export default async (req: Request, _res, next) => {
+  metrics(req).then(() => {});
+  next();
+};