refactor: generate formula and rollup on top of filtered, sorted and limited data

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts

@@ -57,6 +57,9 @@ const GROUP_COL = '__nc_group_id';
 const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14);
 const { v4: uuidv4 } = require('uuid');
 
+const INNER_QUERY_ALIAS = '__nc_inner';
+// const WRAPPER_QUERY_ALIAS = '__nc_wrapper';
+
 async function populatePk(model: Model, insertObj: any) {
   await model.getColumns();
   for (const pkCol of model.primaryKeys) {
@@ -202,14 +205,18 @@ class BaseModelSqlv2 {
   ): Promise<any> {
     const { where, fields, ...rest } = this._getListArgs(args as any);
 
-    const qb = this.dbDriver(this.tnPath);
+    const innerQb = this.dbDriver(this.tnPath);
+    innerQb.select('*')
+
+    const wrapperQb = this.dbDriver.from(innerQb.as(INNER_QUERY_ALIAS))
     await this.selectObject({
-      qb,
+      qb: wrapperQb,
       fieldsSet: args.fieldsSet,
       viewId: this.viewId,
+      alias: INNER_QUERY_ALIAS
     });
     if (+rest?.shuffle) {
-      await this.shuffle({ qb });
+      await this.shuffle({ qb: innerQb });
     }
 
     const aliasColObjMap = await this.model.getAliasColObjMap();
@@ -235,7 +242,7 @@ class BaseModelSqlv2 {
             logical_op: 'and',
           }),
         ],
-        qb,
+        innerQb,
         this.dbDriver
       );
 
@@ -244,7 +251,7 @@ class BaseModelSqlv2 {
           ? args.sortArr
           : await Sort.list({ viewId: this.viewId });
 
-      await sortV2(sorts, qb, this.dbDriver);
+      await sortV2(sorts, innerQb, this.dbDriver);
     } else {
       await conditionV2(
         [
@@ -259,31 +266,31 @@ class BaseModelSqlv2 {
             logical_op: 'and',
           }),
         ],
-        qb,
+        innerQb,
         this.dbDriver
       );
 
       if (!sorts) sorts = args.sortArr;
 
-      await sortV2(sorts, qb, this.dbDriver);
+      await sortV2(sorts, innerQb, this.dbDriver);
     }
 
     // sort by primary key if not autogenerated string
     // if autogenerated string sort by created_at column if present
     if (this.model.primaryKey && this.model.primaryKey.ai) {
-      qb.orderBy(this.model.primaryKey.column_name);
+      innerQb.orderBy(this.model.primaryKey.column_name);
     } else if (this.model.columns.find((c) => c.column_name === 'created_at')) {
-      qb.orderBy('created_at');
+      innerQb.orderBy('created_at');
     }
 
-    if (!ignoreViewFilterAndSort) applyPaginate(qb, rest);
+    if (!ignoreViewFilterAndSort) applyPaginate(innerQb, rest);
     const proto = await this.getProto();
 
-    console.log('list query', qb.toQuery());
+    const data = await this.execAndParse(wrapperQb);
 
-    const data = await this.execAndParse(qb);
+    // todo: remove
+    console.log(wrapperQb.toQuery());
 
-    // console.log(qb.toQuery());
 
     return data?.map((d) => {
       d.__proto__ = proto;
@@ -1247,7 +1254,7 @@ class BaseModelSqlv2 {
     });
   }
 
-  private async getSelectQueryBuilderForFormula(column: Column<any>) {
+  private async getSelectQueryBuilderForFormula(column: Column<any>, tableAlias?:string ) {
     const formula = await column.getColOptions<FormulaColumn>();
     if (formula.error) throw new Error(`Formula error: ${formula.error}`);
     const qb = await formulaQueryBuilderv2(
@@ -1255,7 +1262,9 @@ class BaseModelSqlv2 {
       null,
       this.dbDriver,
       this.model,
-      column
+      column,
+      {  },
+      tableAlias
     );
     return qb;
   }
@@ -1473,6 +1482,7 @@ class BaseModelSqlv2 {
     extractPkAndPv,
     viewId,
     fieldsSet,
+                              alias
   }: {
     fieldsSet?: Set<string>;
     qb: Knex.QueryBuilder;
@@ -1480,6 +1490,7 @@ class BaseModelSqlv2 {
     fields?: string[] | string;
     extractPkAndPv?: boolean;
     viewId?: string;
+    alias?: string;
   }): Promise<void> {
     const view = await View.get(viewId);
     const viewColumns = viewId && (await View.getColumns(viewId));
@@ -1531,7 +1542,8 @@ class BaseModelSqlv2 {
             case UITypes.Formula:
               try {
                 const selectQb = await this.getSelectQueryBuilderForFormula(
-                  qrValueColumn
+                  qrValueColumn,
+                   alias
                 );
                 qb.select({
                   [column.column_name]: selectQb.builder,
@@ -1563,7 +1575,7 @@ class BaseModelSqlv2 {
             case UITypes.Formula:
               try {
                 const selectQb = await this.getSelectQueryBuilderForFormula(
-                  barcodeValueColumn
+                  barcodeValueColumn,
                 );
                 qb.select({
                   [column.column_name]: selectQb.builder,
@@ -1586,7 +1598,8 @@ class BaseModelSqlv2 {
           {
             try {
               const selectQb = await this.getSelectQueryBuilderForFormula(
-                column
+                column,
+                alias
               );
               qb.select(
                 this.dbDriver.raw(`?? as ??`, [
@@ -1594,7 +1607,8 @@ class BaseModelSqlv2 {
                   sanitize(column.title),
                 ])
               );
-            } catch {
+            } catch(e) {
+              console.log(e)
               // return dummy select
               qb.select(
                 this.dbDriver.raw(`'ERR' as ??`, [sanitize(column.title)])
@@ -1609,6 +1623,7 @@ class BaseModelSqlv2 {
                 // tn: this.title,
                 knex: this.dbDriver,
                 // column,
+                alias,
                 columnOptions: (await column.getColOptions()) as RollupColumn,
               })
             ).builder.as(sanitize(column.title))
@@ -1616,7 +1631,7 @@ class BaseModelSqlv2 {
           break;
         default:
           res[sanitize(column.title || column.column_name)] = sanitize(
-            `${this.model.table_name}.${column.column_name}`
+            `${alias || this.model.table_name}.${column.column_name}`
           );
           break;
       }

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/formulav2/formulaQueryBuilderv2.ts

@@ -53,7 +53,8 @@ async function _formulaQueryBuilder(
   alias,
   knex: XKnex,
   model: Model,
-  aliasToColumn = {}
+  aliasToColumn = {},
+  tableAlias?: string
 ) {
   // formula may include double curly brackets in previous version
   // convert to single curly bracket here for compatibility
@@ -74,7 +75,8 @@ async function _formulaQueryBuilder(
             alias,
             knex,
             model,
-            { ...aliasToColumn, [col.id]: null }
+            { ...aliasToColumn, [col.id]: null },
+            tableAlias
           );
           builder.sql = '(' + builder.sql + ')';
           aliasToColumn[col.id] = builder;
@@ -104,7 +106,9 @@ async function _formulaQueryBuilder(
                 selectQb = knex(`${parentModel.table_name} as ${alias}`).where(
                   `${alias}.${parentColumn.column_name}`,
                   knex.raw(`??`, [
-                    `${childModel.table_name}.${childColumn.column_name}`,
+                    `${tableAlias ?? childModel.table_name}.${
+                      childColumn.column_name
+                    }`,
                   ])
                 );
                 break;
@@ -113,7 +117,9 @@ async function _formulaQueryBuilder(
                 selectQb = knex(`${childModel.table_name} as ${alias}`).where(
                   `${alias}.${childColumn.column_name}`,
                   knex.raw(`??`, [
-                    `${parentModel.table_name}.${parentColumn.column_name}`,
+                    `${tableAlias ?? parentModel.table_name}.${
+                      parentColumn.column_name
+                    }`,
                   ])
                 );
                 break;
@@ -134,7 +140,9 @@ async function _formulaQueryBuilder(
                     .where(
                       `${assocAlias}.${mmChildColumn.column_name}`,
                       knex.raw(`??`, [
-                        `${childModel.table_name}.${childColumn.column_name}`,
+                        `${tableAlias ?? childModel.table_name}.${
+                          childColumn.column_name
+                        }`,
                       ])
                     );
                 }
@@ -402,6 +410,7 @@ async function _formulaQueryBuilder(
           const qb = await genRollupSelectv2({
             knex,
             columnOptions: (await col.getColOptions()) as RollupColumn,
+            alias: tableAlias,
           });
           aliasToColumn[col.id] = knex.raw(qb.builder).wrap('(', ')');
         }
@@ -428,7 +437,9 @@ async function _formulaQueryBuilder(
               .where(
                 `${parentModel.table_name}.${parentColumn.column_name}`,
                 knex.raw(`??`, [
-                  `${childModel.table_name}.${childColumn.column_name}`,
+                  `${tableAlias ?? childModel.table_name}.${
+                    childColumn.column_name
+                  }`,
                 ])
               );
           } else if (relation.type == 'hm') {
@@ -437,7 +448,9 @@ async function _formulaQueryBuilder(
               .where(
                 `${childModel.table_name}.${childColumn.column_name}`,
                 knex.raw(`??`, [
-                  `${parentModel.table_name}.${parentColumn.column_name}`,
+                  `${tableAlias ?? parentModel.table_name}.${
+                    parentColumn.column_name
+                  }`,
                 ])
               );
 
@@ -490,7 +503,9 @@ async function _formulaQueryBuilder(
               .where(
                 `${mmModel.table_name}.${mmChildColumn.column_name}`,
                 knex.raw(`??`, [
-                  `${childModel.table_name}.${childColumn.column_name}`,
+                  `${tableAlias ?? childModel.table_name}.${
+                    childColumn.column_name
+                  }`,
                 ])
               );
             selectQb = (fn) =>
@@ -775,18 +790,27 @@ async function _formulaQueryBuilder(
   return { builder: fn(tree, alias) };
 }
 
-function getTnPath(tb: Model, knex) {
+function getTnPath(tb: Model, knex, tableAlias?: string) {
   const schema = knex.searchPath?.();
   if (knex.clientType() === 'mssql' && schema) {
-    return knex.raw('??.??', [schema, tb.table_name]);
-  } else if (knex.clientType() === 'snowflake') {
-    return [
-      knex.client.config.connection.database,
-      knex.client.config.connection.schema,
+    return knex.raw(`??.??${tableAlias ? ' as ??' : ''}`, [
+      schema,
       tb.table_name,
-    ].join('.');
+      ...(tableAlias ? [tableAlias] : []),
+    ]);
+  } else if (knex.clientType() === 'snowflake') {
+    return (
+      [
+        knex.client.config.connection.database,
+        knex.client.config.connection.schema,
+        tb.table_name,
+      ].join('.') + (tableAlias ? ` as ${tableAlias}` : '')
+    );
   } else {
-    return tb.table_name;
+    return knex.raw(`??${tableAlias ? ' as ??' : ''}`, [
+      tb.table_name,
+      ...(tableAlias ? [tableAlias] : []),
+    ]);
   }
 }
 
@@ -796,7 +820,8 @@ export default async function formulaQueryBuilderv2(
   knex: XKnex,
   model: Model,
   column?: Column,
-  aliasToColumn = {}
+  aliasToColumn = {},
+  tableAlias?: string
 ) {
   // register jsep curly hook once only
   jsep.plugins.register(jsepCurlyHook);
@@ -806,13 +831,16 @@ export default async function formulaQueryBuilderv2(
     alias,
     knex,
     model,
-    aliasToColumn
+    aliasToColumn,
+    tableAlias
   );
 
   try {
     // dry run qb.builder to see if it will break the grid view or not
     // if so, set formula error and show empty selectQb instead
-    await knex(getTnPath(model, knex)).select(qb.builder).as('dry-run-only');
+    await knex(getTnPath(model, knex, tableAlias))
+      .select(qb.builder)
+      .as('dry-run-only');
 
     // if column is provided, i.e. formula has been created
     if (column) {