From 599d307a49a167a53d4a3485fbd955f56c5b6060 Mon Sep 17 00:00:00 2001
From: Pranav C <pranavxc@gmail.com>
Date: Thu, 28 Oct 2021 20:44:27 +0530
Subject: [PATCH] feat: csv export public/private api

Signed-off-by: Pranav C <pranavxc@gmail.com>
---
 .../components/columnFilterMenu.vue           |   2 +-
 .../spreadsheet/components/csvExport.vue      | 130 ++++++++---
 .../project/spreadsheet/public/xcTable.vue    |   5 +-
 .../project/spreadsheet/rowsXcDataTable.vue   |  16 +-
 packages/nc-gui/store/sqlMgr.js               |  12 +-
 .../lib/dataMapper/lib/sql/BaseModelSql.ts    | 162 ++++++++------
 .../nocodb/src/lib/noco/meta/NcMetaMgr.ts     | 208 +++++++++++++++++-
 .../nocodb/src/lib/noco/meta/NcMetaMgrEE.ts   |  84 +------
 8 files changed, 420 insertions(+), 199 deletions(-)

diff --git a/packages/nc-gui/components/project/spreadsheet/components/columnFilterMenu.vue b/packages/nc-gui/components/project/spreadsheet/components/columnFilterMenu.vue
index 254bd98fa6..293b4f87f8 100644
--- a/packages/nc-gui/components/project/spreadsheet/components/columnFilterMenu.vue
+++ b/packages/nc-gui/components/project/spreadsheet/components/columnFilterMenu.vue
@@ -17,7 +17,7 @@
           v-on="on"
         >
           <v-icon small class="mr-1" color="grey  darken-3">
-            mdi-filter
+            mdi-filter-outline
           </v-icon>
           Filter
           <v-icon small color="#777">
diff --git a/packages/nc-gui/components/project/spreadsheet/components/csvExport.vue b/packages/nc-gui/components/project/spreadsheet/components/csvExport.vue
index 175833ced8..ae3eb2d26d 100644
--- a/packages/nc-gui/components/project/spreadsheet/components/csvExport.vue
+++ b/packages/nc-gui/components/project/spreadsheet/components/csvExport.vue
@@ -1,27 +1,71 @@
 <template>
-  <v-btn
-    outlined
-    class="caption"
-    small
-    text
-    @click="exportCsv"
+  <v-menu
+    open-on-hover
+    bottom
+    offset-y
   >
-    Export
-  </v-btn>
+    <template #activator="{on}">
+      <v-btn
+        outlined
+        class="caption"
+        small
+        text
+        v-on="on"
+      >
+        <v-icon small color="#777">
+          mdi-flash-outline
+        </v-icon>
+        Actions
+
+        <v-icon small color="#777">
+          mdi-menu-down
+        </v-icon>
+      </v-btn>
+    </template>
+
+    <v-list dense>
+      <v-list-item
+        dense
+        @click="exportCsv"
+      >
+        <v-list-item-title>
+          <v-icon small class="mr-1">
+            mdi-download-outline
+          </v-icon>
+          <span class="caption">
+            Download as CSV
+          </span>
+        </v-list-item-title>
+      </v-list-item>
+      <v-list-item
+        dense
+        @click="comingSoon"
+      >
+        <v-list-item-title>
+          <v-icon small class="mr-1" color="grey">
+            mdi-upload-outline
+          </v-icon>
+          <span class="caption grey--text">
+            Upload CSV
+          </span>
+        </v-list-item-title>
+      </v-list-item>
+    </v-list>
+  </v-menu>
 </template>
 
 <script>
 
-import Papaparse from 'papaparse'
+// import Papaparse from 'papaparse'
 import FileSaver from 'file-saver'
 
 export default {
   name: 'CsvExport',
   props: {
-    data: Array,
     meta: Object,
     nodes: Object,
-    availableColumns: Array
+    selectedView: Object,
+    publicViewId: String
   },
 
   methods: {
@@ -100,37 +144,55 @@ export default {
       }))
     },
     async exportCsv() {
+      // const fields = this.availableColumns.map(c => c._cn)
       // const blob = new Blob([Papaparse.unparse(await this.extractCsvData())], { type: 'text/plain;charset=utf-8' })
 
+      let offset = 0
+      let c = 1
+
       try {
-        const data = await this.$store.dispatch('sqlMgr/ActSqlOp', [
-          {
-            dbAlias: this.nodes.dbAlias,
-            env: '_noco'
-          },
-          'xcExportAsCsv',
-          {
-            query: {},
-            model_name: this.meta.tn
-          },
-          null,
-          {
-            responseType: 'blob'
+        while (!isNaN(offset) && offset > -1) {
+          const res = await this.$store.dispatch('sqlMgr/ActSqlOp', [
+            this.publicViewId
+              ? null
+              : {
+                  dbAlias: this.nodes.dbAlias,
+                  env: '_noco'
+                },
+            this.publicViewId ? 'sharedViewExportAsCsv' : 'xcExportAsCsv',
+            {
+              query: { offset },
+              ...(this.publicViewId
+                ? {
+                    view_id: this.publicViewId
+                  }
+                : {
+                    view_name: this.selectedView.title,
+                    model_name: this.meta.tn
+                  })
+            },
+            null,
+            {
+              responseType: 'blob'
+            },
+            null,
+            true
+          ])
+          const data = res.data
+          offset = +res.headers['nc-export-offset']
+          const blob = new Blob([data], { type: 'text/plain;charset=utf-8' })
+          FileSaver.saveAs(blob, `${this.meta._tn}_exported_${c++}.csv`)
+          if (offset > -1) {
+            this.$toast.info('Downloading more files').goAway(3000)
+          } else {
+            this.$toast.success('Successfully exported all table data').goAway(3000)
           }
-        ])
-        // const url = window.URL.createObjectURL(new Blob([data], { type: 'application/zip' }))
-        // const link = document.createElement('a')
-        // link.href = url
-        // link.setAttribute('download', 'meta.zip') // or any other extension
-        // document.body.appendChild(link)
-        // link.click()
-        const blob = new Blob([data], { type: 'text/plain;charset=utf-8' })
-        FileSaver.saveAs(blob, `${this.meta._tn}_exported.csv`)
-        this.$toast.success('Successfully exported metadata').goAway(3000)
+        }
       } catch (e) {
         this.$toast.error(e.message).goAway(3000)
       }
     }
+
   }
 
 }
diff --git a/packages/nc-gui/components/project/spreadsheet/public/xcTable.vue b/packages/nc-gui/components/project/spreadsheet/public/xcTable.vue
index 83da332216..98852585f5 100644
--- a/packages/nc-gui/components/project/spreadsheet/public/xcTable.vue
+++ b/packages/nc-gui/components/project/spreadsheet/public/xcTable.vue
@@ -91,6 +91,8 @@
 
         <column-filter-menu v-model="filters" :field-list="realFieldList" />
 
+        <csv-export :public-view-id="$route.params.id" :meta="meta" />
+
       <!--      <v-menu>
         <template #activator="{ on, attrs }">
           <v-icon
@@ -194,11 +196,12 @@ import SortListMenu from '../components/sortListMenu'
 import ColumnFilterMenu from '../components/columnFilterMenu'
 import XcGridView from '../views/xcGridView'
 import { SqlUI } from '@/helpers/sqlUi'
+import CsvExport from '~/components/project/spreadsheet/components/csvExport'
 // import ExpandedForm from "../expandedForm";
 
 export default {
   name: 'XcTable',
-  components: { XcGridView, ColumnFilterMenu, SortListMenu, FieldsMenu },
+  components: { CsvExport, XcGridView, ColumnFilterMenu, SortListMenu, FieldsMenu },
   mixins: [spreadsheet],
   props: {
     env: String,
diff --git a/packages/nc-gui/components/project/spreadsheet/rowsXcDataTable.vue b/packages/nc-gui/components/project/spreadsheet/rowsXcDataTable.vue
index cc1cc43e77..d760716938 100644
--- a/packages/nc-gui/components/project/spreadsheet/rowsXcDataTable.vue
+++ b/packages/nc-gui/components/project/spreadsheet/rowsXcDataTable.vue
@@ -61,14 +61,6 @@
 
       <v-spacer class="h-100" @dblclick="debug=true" />
 
-      <csv-export
-        :available-columns="availableColumns"
-        :data="data"
-        :meta="meta"
-        :nodes="nodes"
-        class="mr-1"
-      />
-
       <template v-if="!isForm">
         <debug-metas v-if="debug" class="mr-3" />
         <v-tooltip bottom>
@@ -146,6 +138,14 @@
           :field-list="[...realFieldList, ...formulaFieldList]"
           dense
         />
+
+        <csv-export
+          :meta="meta"
+          :nodes="nodes"
+          :selected-view="selectedView"
+          class="mr-1"
+        />
+
         <v-tooltip
           v-if="_isUIAllowed('table-delete')"
           bottom
diff --git a/packages/nc-gui/store/sqlMgr.js b/packages/nc-gui/store/sqlMgr.js
index 8406e4e843..e225a014a1 100644
--- a/packages/nc-gui/store/sqlMgr.js
+++ b/packages/nc-gui/store/sqlMgr.js
@@ -354,7 +354,7 @@ export const actions = {
     }
   },
 
-  async ActSqlOp({ commit, state, rootState, dispatch }, [args, op, opArgs, cusHeaders, cusAxiosOptions, queryParams]) {
+  async ActSqlOp({ commit, state, rootState, dispatch }, [args, op, opArgs, cusHeaders, cusAxiosOptions, queryParams, returnResponse]) {
     const params = {}
     if (this.$router.currentRoute && this.$router.currentRoute.params && this.$router.currentRoute.params.project_id) {
       params.project_id = this.$router.currentRoute.params.project_id
@@ -370,7 +370,7 @@ export const actions = {
       if (cusHeaders) {
         Object.assign(headers, cusHeaders)
       }
-      const data = (await this.$axios({
+      const res = (await this.$axios({
         url: '?q=sqlOp_' + op,
         baseURL: `${this.$axios.defaults.baseURL}/dashboard`,
         data: { api: op, ...args, ...params, args: opArgs },
@@ -379,7 +379,9 @@ export const actions = {
         params: (args && args.query) || {},
         ...(cusAxiosOptions || {})
 
-      })).data
+      }))
+
+      const data = res.data
 
       // clear meta cache on relation create/delete
       // todo: clear only necessary metas
@@ -401,6 +403,10 @@ export const actions = {
         }
       }
 
+      if (returnResponse) {
+        return res
+      }
+
       return data
     } catch (e) {
       const err = new Error(e.response.data.msg)
diff --git a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts
index 7e3c6cf7cf..ede0e187df 100644
--- a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts
+++ b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts
@@ -2414,84 +2414,119 @@ class BaseModelSql extends BaseModel {
     }, []);
   }
 
-  public async extractCsvData(args: any = {}) {
-    const rows = await this.nestedList(args);
-
+  // todo: optimize
+  public async extractCsvData(args: any = {}, fields = null) {
+    const defaultNestedQueryParams = { ...this.defaultNestedQueryParams };
+
+    // // get all nested props by default
+    // for (const key of Object.keys(defaultNestedQueryParams)) {
+    //   if (key.indexOf('fields') > -1) {
+    //     defaultNestedQueryParams[key] = '*';
+    //   }
+    // }
+
+    let offset = +args.offset || 0;
+    const limit = 100;
+    // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024;
+    const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 500;
     const csvRows = [];
+    const startTime = process.hrtime();
+    let elapsed, temp;
+
+    for (
+      elapsed = 0;
+      elapsed < timeout;
+      offset += limit,
+        temp = process.hrtime(startTime),
+        elapsed = temp[0] * 1000 + temp[1] / 1000000
+    ) {
+      const rows = await this.nestedList({
+        ...defaultNestedQueryParams,
+        ...args,
+        offset,
+        limit
+      });
+
+      if (!rows?.length) {
+        offset = -1;
+        break;
+      }
 
-    for (const row of rows) {
-      const csvRow = {};
+      for (const row of rows) {
+        const csvRow = {};
 
-      for (const column of this.columns) {
-        if (column._cn in row) {
-          csvRow[column._cn] = this.serializeCellValue({
-            value: row[column._cn],
-            column
-          });
+        for (const column of this.columns) {
+          if (column._cn in row) {
+            csvRow[column._cn] = this.serializeCellValue({
+              value: row[column._cn],
+              column
+            });
+          }
         }
-      }
 
-      for (const vColumn of this.virtualColumns) {
-        if (vColumn._cn in row && !vColumn.bt && !vColumn.hm && !vColumn.mm) {
-          if (vColumn.lk) {
-          } else {
-            csvRow[vColumn._cn] = row[vColumn._cn];
+        for (const vColumn of this.virtualColumns) {
+          if (vColumn._cn in row && !vColumn.bt && !vColumn.hm && !vColumn.mm) {
+            if (vColumn.lk) {
+            } else {
+              csvRow[vColumn._cn] = row[vColumn._cn];
+            }
           }
         }
-      }
 
-      for (const [prop, col] of Object.entries(this.nestedProps)) {
-        const refModel = this._nestedPropsModels[prop];
+        for (const [prop, col] of Object.entries(this.nestedProps)) {
+          const refModel = this._nestedPropsModels[prop];
 
-        const mapPropFn = (alias, colAlias) => {
-          if (Array.isArray(row[prop])) {
-            csvRow[alias] = row[prop].map(r =>
-              refModel.serializeCellValue({
-                value: r[colAlias],
+          const mapPropFn = (alias, colAlias) => {
+            if (Array.isArray(row[prop])) {
+              csvRow[alias] = row[prop].map(r =>
+                refModel.serializeCellValue({
+                  value: r[colAlias],
+                  columnName: colAlias
+                })
+              );
+            } else if (row[prop]) {
+              csvRow[alias] = refModel.serializeCellValue({
+                value: row?.[prop]?.[colAlias],
                 columnName: colAlias
-              })
-            );
-          } else if (row[prop]) {
-            csvRow[alias] = refModel.serializeCellValue({
-              value: row[colAlias],
-              columnName: colAlias
-            });
-          }
-        };
-
-        if (prop in row) {
-          // todo: optimize
-          for (const vColumn of this.virtualColumns) {
-            if (vColumn.lk) {
-              if (
-                col.hm &&
-                vColumn.lk.type === 'hm' &&
-                col.hm.tn === vColumn.lk.ltn
-              ) {
-                mapPropFn(col._cn, vColumn.lk._lcn);
-              } else if (
-                col.mm &&
-                vColumn.lk.type === 'mm' &&
-                col.mm.rtn === vColumn.lk.ltn
-              ) {
-                mapPropFn(col._cn, vColumn.lk._lcn);
-              }
-              if (
-                col.bt &&
-                vColumn.lk.type === 'bt' &&
-                col.bt.rtn === vColumn.lk.ltn
-              ) {
-                mapPropFn(col._cn, vColumn.lk._lcn);
+              });
+            }
+          };
+
+          if (prop in row) {
+            // todo: optimize
+            for (const vColumn of this.virtualColumns) {
+              if (vColumn.lk) {
+                if (
+                  col.hm &&
+                  vColumn.lk.type === 'hm' &&
+                  col.hm.tn === vColumn.lk.ltn
+                ) {
+                  mapPropFn(vColumn._cn, vColumn.lk._lcn);
+                } else if (
+                  col.mm &&
+                  vColumn.lk.type === 'mm' &&
+                  col.mm.rtn === vColumn.lk.ltn
+                ) {
+                  mapPropFn(vColumn._cn, vColumn.lk._lcn);
+                }
+                if (
+                  col.bt &&
+                  vColumn.lk.type === 'bt' &&
+                  col.bt.rtn === vColumn.lk.ltn
+                ) {
+                  mapPropFn(vColumn._cn, vColumn.lk._lcn);
+                }
               }
             }
           }
+          mapPropFn(col._cn, refModel.primaryColAlias);
         }
-        mapPropFn(col._cn, refModel.primaryColAlias);
+        csvRows.push(csvRow);
       }
-      csvRows.push(csvRow);
     }
 
-    return Papaparse.unparse(csvRows);
+    const data = Papaparse.unparse({ fields, data: csvRows });
+    return { data, offset, elapsed };
   }
 
   public serializeCellValue({
@@ -2506,7 +2541,7 @@ class BaseModelSql extends BaseModel {
       return value;
     }
     const column =
-      args.column || this.columns.find(c => c._cn === this.columnToAlias);
+      args.column || this.columns.find(c => c._cn === args.columnName);
 
     switch (column?.uidt) {
       case 'Attachment': {
@@ -2518,7 +2553,8 @@ class BaseModelSql extends BaseModel {
         } catch {}
 
         return (data || []).map(
-          attachment => `${attachment.title}(${encodeURI(attachment.url)})`
+          attachment =>
+            `${encodeURI(attachment.title)}(${encodeURI(attachment.url)})`
         );
       }
       default:
diff --git a/packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts b/packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts
index ec21d984f4..7c876d6e6f 100644
--- a/packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts
+++ b/packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts
@@ -5,7 +5,7 @@ import path from 'path';
 import archiver from 'archiver';
 import axios from 'axios';
 import bodyParser from 'body-parser';
-import { Handler, Router } from 'express';
+import express, { Handler, Router } from 'express';
 import extract from 'extract-zip';
 import isDocker from 'is-docker';
 import multer from 'multer';
@@ -1307,6 +1307,9 @@ export default class NcMetaMgr {
         case 'sharedViewGet':
           result = await this.sharedViewGet(req, args);
           break;
+        case 'sharedViewExportAsCsv':
+          result = await this.sharedViewExportAsCsv(req, args, res);
+          break;
 
         case 'sharedViewInsert':
           result = await this.sharedViewInsert(req, args);
@@ -1333,6 +1336,10 @@ export default class NcMetaMgr {
       return next(e);
     }
 
+    if (typeof result?.cb === 'function') {
+      return await result.cb();
+    }
+
     res.json(result);
   }
 
@@ -2771,7 +2778,12 @@ export default class NcMetaMgr {
   }
 
   protected getDbAlias(args): string {
-    return args?.dbAlias || args?.args?.dbAlias;
+    return (
+      args?.dbAlias ||
+      args?.args?.dbAlias ||
+      args?.db_alias ||
+      args?.args?.db_alias
+    );
   }
 
   protected isProjectRest() {
@@ -3822,6 +3834,49 @@ export default class NcMetaMgr {
     return { ...sharedViewMeta, ...viewMeta };
   }
 
+  protected async sharedViewExportAsCsv(_req, args: any, res): Promise<any> {
+    const sharedViewMeta = await this.xcMeta
+      .knex('nc_shared_views')
+      .where({
+        view_id: args.args.view_id
+      })
+      .first();
+
+    if (!sharedViewMeta) {
+      throw new Error('Meta not found');
+    }
+
+    const viewMeta = await this.xcMeta.metaGet(
+      sharedViewMeta.project_id,
+      sharedViewMeta.db_alias,
+      'nc_models',
+      {
+        title: sharedViewMeta.view_name
+      }
+    );
+
+    if (!viewMeta) {
+      throw new Error('Not found');
+    }
+
+    if (
+      sharedViewMeta &&
+      sharedViewMeta.password &&
+      sharedViewMeta.password !== args.args.password
+    ) {
+      throw new Error(this.INVALID_PASSWORD_ERROR);
+    }
+
+    return this.xcExportAsCsv(
+      {
+        ...sharedViewMeta,
+        args: { ...sharedViewMeta, offset: 0 }
+      },
+      _req,
+      res
+    );
+  }
+
   protected async xcAuthHookGet(args: any): Promise<any> {
     try {
       return await this.xcMeta.metaGet(args.project_id, 'db', 'nc_hooks', {
@@ -3944,24 +3999,78 @@ export default class NcMetaMgr {
     return { data: { list: procedures } };
   }
 
-  protected async xcExportAsCsv(args, _req, res) {
+  protected async xcExportAsCsv(args, _req, res: express.Response) {
+    const projectId = this.getProjectId(args);
+    const dbAlias = this.getDbAlias(args);
     const apiBuilder = this.app?.projectBuilders
-      ?.find(pb => pb.id === this.getProjectId(args))
-      ?.apiBuilders?.find(ab => ab.dbAlias === this.getDbAlias(args));
+      ?.find(pb => pb.id === projectId)
+      ?.apiBuilders?.find(ab => ab.dbAlias === dbAlias);
 
     const model = apiBuilder?.xcModels?.[args.args.model_name];
 
-    const csvData = await model.extractCsvData(args.args.query || {});
+    const meta = apiBuilder?.getMeta(args.args.model_name);
+
+    const selectedView = await this.xcMeta.metaGet(
+      projectId,
+      dbAlias,
+      'nc_models',
+      {
+        title: args.args.view_name
+      }
+    );
+
+    const queryParams = JSON.parse(selectedView.query_params);
+    const sort = this.serializeSortParam(queryParams);
+
+    const csvData = await model.extractCsvData(
+      {
+        ...(args.args.query || {}),
+        fields: meta.columns
+          .filter(c => queryParams?.showFields?.[c._cn])
+          .map(c => c._cn)
+          .join(','),
+        sort: sort,
+        where: this.serializeToXwhere(queryParams?.filters),
+        ...this.serializeNestedParams(meta, queryParams)
+      },
+      // filter only visible columns
+      Object.entries(queryParams?.showFields || {})
+        .filter(v => v[1])
+        .map(v => v[0])
+        .sort(
+          (a, b) =>
+            queryParams?.fieldsOrder?.indexOf(a) -
+            queryParams?.fieldsOrder?.indexOf(b)
+        )
+    );
 
     return {
       cb: async () => {
         res.set({
+          'Access-Control-Expose-Headers': 'nc-export-offset',
+          'nc-export-offset': csvData.offset,
+          'nc-export-elapsed-time': csvData.elapsed,
           'Content-Disposition': `attachment; filename="${args.args.model_name}-export.csv"`
         });
-        res.send(csvData);
+        res.send(csvData.data);
       }
     };
   }
+
+  private serializeSortParam(queryParams) {
+    const sort = [];
+    if (queryParams.sortList) {
+      sort.push(
+        ...(queryParams?.sortList
+          ?.map(sort => {
+            return sort.field ? `${sort.order}${sort.field}` : '';
+          })
+          .filter(Boolean) || [])
+      );
+    }
+    return sort.join(',');
+  }
+
   // @ts-ignore
   protected async xcVisibilityMetaGet(args) {
     try {
@@ -4787,6 +4896,91 @@ export default class NcMetaMgr {
 
     return result;
   }
+
+  protected serializeToXwhere(filters) {
+    // todo: move  this logic to a common library
+    // todo: replace with condition prop
+    const privateViewWhere = filters?.reduce?.((condition, filt, i) => {
+      if (!i && !filt.logicOp) {
+        return condition;
+      }
+      if (!(filt.field && filt.op)) {
+        return condition;
+      }
+
+      condition += i ? `~${filt.logicOp}` : '';
+      switch (filt.op) {
+        case 'is equal':
+          return condition + `(${filt.field},eq,${filt.value})`;
+        case 'is not equal':
+          return condition + `~not(${filt.field},eq,${filt.value})`;
+        case 'is like':
+          return condition + `(${filt.field},like,%${filt.value}%)`;
+        case 'is not like':
+          return condition + `~not(${filt.field},like,%${filt.value}%)`;
+        case 'is empty':
+          return condition + `(${filt.field},in,)`;
+        case 'is not empty':
+          return condition + `~not(${filt.field},in,)`;
+        case 'is null':
+          return condition + `(${filt.field},is,null)`;
+        case 'is not null':
+          return condition + `~not(${filt.field},is,null)`;
+        case '<':
+          return condition + `(${filt.field},lt,${filt.value})`;
+        case '<=':
+          return condition + `(${filt.field},le,${filt.value})`;
+        case '>':
+          return condition + `(${filt.field},gt,${filt.value})`;
+        case '>=':
+          return condition + `(${filt.field},ge,${filt.value})`;
+      }
+      return condition;
+    }, '');
+    return privateViewWhere;
+  }
+
+  protected serializeNestedParams(meta, queryParams) {
+    const nestedParams: any = {
+      hm: [],
+      mm: [],
+      bt: []
+    };
+
+    for (const v of meta.v) {
+      if (!queryParams?.showFields?.[v._cn]) continue;
+      if (v.bt || v.lk?.type === 'bt') {
+        const tn = v.bt?.rtn || v.lk?.rtn;
+        if (!nestedParams.bt.includes(tn)) nestedParams.bt.push(tn);
+        if (v.lk) {
+          const key = `bf${nestedParams.bt.indexOf(tn)}`;
+          nestedParams[key] =
+            (nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
+        }
+      } else if (v.hm || v.lk?.type === 'hm') {
+        const tn = v.hm?.tn || v.lk?.tn;
+        if (!nestedParams.hm.includes(tn)) nestedParams.hm.push(tn);
+        if (v.lk) {
+          const key = `hf${nestedParams.hm.indexOf(tn)}`;
+          nestedParams[key] =
+            (nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
+        }
+      } else if (v.mm || v.lk?.type === 'mm') {
+        const tn = v.mm?.rtn || v.lk?.rtn;
+        if (!nestedParams.mm.includes(tn)) nestedParams.mm.push(tn);
+        if (v.lk) {
+          const key = `mf${nestedParams.mm.indexOf(tn)}`;
+          nestedParams[key] =
+            (nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
+        }
+      }
+    }
+
+    nestedParams.mm = nestedParams.mm.join(',');
+    nestedParams.hm = nestedParams.hm.join(',');
+    nestedParams.bt = nestedParams.bt.join(',');
+    return nestedParams;
+  }
 }
 
 export class XCEeError extends Error {
diff --git a/packages/nocodb/src/lib/noco/meta/NcMetaMgrEE.ts b/packages/nocodb/src/lib/noco/meta/NcMetaMgrEE.ts
index 22d08b043e..4519031e94 100644
--- a/packages/nocodb/src/lib/noco/meta/NcMetaMgrEE.ts
+++ b/packages/nocodb/src/lib/noco/meta/NcMetaMgrEE.ts
@@ -169,49 +169,7 @@ export default class NcMetaMgrEE extends NcMetaMgr {
       if (req.query.where) {
         where += req.query.where;
       }
-
-      // todo: move  this logic to a common library
-      // todo: replace with condition prop
-      const privateViewWhere = queryParams?.filters?.reduce?.(
-        (condition, filt, i) => {
-          if (!i && !filt.logicOp) {
-            return condition;
-          }
-          if (!(filt.field && filt.op)) {
-            return condition;
-          }
-
-          condition += i ? `~${filt.logicOp}` : '';
-          switch (filt.op) {
-            case 'is equal':
-              return condition + `(${filt.field},eq,${filt.value})`;
-            case 'is not equal':
-              return condition + `~not(${filt.field},eq,${filt.value})`;
-            case 'is like':
-              return condition + `(${filt.field},like,%${filt.value}%)`;
-            case 'is not like':
-              return condition + `~not(${filt.field},like,%${filt.value}%)`;
-            case 'is empty':
-              return condition + `(${filt.field},in,)`;
-            case 'is not empty':
-              return condition + `~not(${filt.field},in,)`;
-            case 'is null':
-              return condition + `(${filt.field},is,null)`;
-            case 'is not null':
-              return condition + `~not(${filt.field},is,null)`;
-            case '<':
-              return condition + `(${filt.field},lt,${filt.value})`;
-            case '<=':
-              return condition + `(${filt.field},le,${filt.value})`;
-            case '>':
-              return condition + `(${filt.field},gt,${filt.value})`;
-            case '>=':
-              return condition + `(${filt.field},ge,${filt.value})`;
-          }
-          return condition;
-        },
-        ''
-      );
+      const privateViewWhere = this.serializeToXwhere(queryParams?.filters);
 
       if (privateViewWhere) {
         where += where ? `~and(${privateViewWhere})` : privateViewWhere;
@@ -231,45 +189,7 @@ export default class NcMetaMgrEE extends NcMetaMgr {
       }
 
       const fields = meta.columns.map(c => c._cn).join(',');
-
-      const nestedParams: any = {
-        hm: [],
-        mm: [],
-        bt: []
-      };
-
-      for (const v of meta.v) {
-        if (!queryParams?.showFields?.[v._cn]) continue;
-        if (v.bt || v.lk?.type === 'bt') {
-          const tn = v.bt?.rtn || v.lk?.rtn;
-          if (!nestedParams.bt.includes(tn)) nestedParams.bt.push(tn);
-          if (v.lk) {
-            const key = `bf${nestedParams.bt.indexOf(tn)}`;
-            nestedParams[key] =
-              (nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
-          }
-        } else if (v.hm || v.lk?.type === 'hm') {
-          const tn = v.hm?.tn || v.lk?.tn;
-          if (!nestedParams.hm.includes(tn)) nestedParams.hm.push(tn);
-          if (v.lk) {
-            const key = `hf${nestedParams.hm.indexOf(tn)}`;
-            nestedParams[key] =
-              (nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
-          }
-        } else if (v.mm || v.lk?.type === 'mm') {
-          const tn = v.mm?.rtn || v.lk?.rtn;
-          if (!nestedParams.mm.includes(tn)) nestedParams.mm.push(tn);
-          if (v.lk) {
-            const key = `mf${nestedParams.mm.indexOf(tn)}`;
-            nestedParams[key] =
-              (nestedParams[key] ? `${nestedParams[key]},` : '') + tn;
-          }
-        }
-      }
-
-      nestedParams.mm = nestedParams.mm.join(',');
-      nestedParams.hm = nestedParams.hm.join(',');
-      nestedParams.bt = nestedParams.bt.join(',');
+      const nestedParams = this.serializeNestedParams(meta, queryParams);
 
       return {
         model_name: sharedViewMeta.model_name,