Merge pull request #3814 from nocodb/feat/kanban-grouped-list-api

Feat/kanban grouped list api

packages/nocodb/src/lib/db/sql-data-mapper/lib/BaseModel.ts

@@ -1,5 +1,7 @@
 /* eslint-disable @typescript-eslint/ban-types,prefer-const */
 import Knex from 'knex';
+import Filter from '../../../models/Filter'
+import Sort from '../../../models/Sort'
 
 const autoBind = require('auto-bind');
 const _ = require('lodash');
@@ -1515,6 +1517,8 @@ export interface XcFilter {
   offset?: string | number;
   sort?: string;
   fields?: string;
+  filterArr?: Filter[]
+  sortArr?: Sort[]
 }
 
 export interface XcFilterWithAlias extends XcFilter {

packages/nocodb/src/lib/db/sql-data-mapper/lib/sql/BaseModelSqlv2.ts

@@ -2,6 +2,7 @@ import autoBind from 'auto-bind';
 import _ from 'lodash';
 
 import Model from '../../../../models/Model';
+import SelectOption from '../../../../models/SelectOption';
 import { XKnex } from '../../index';
 import LinkToAnotherRecordColumn from '../../../../models/LinkToAnotherRecordColumn';
 import RollupColumn from '../../../../models/RollupColumn';
@@ -21,6 +22,7 @@ import View from '../../../../models/View';
 import {
   AuditOperationSubTypes,
   AuditOperationTypes,
+  isVirtualCol,
   RelationTypes,
   SortType,
   UITypes,
@@ -1290,9 +1292,15 @@ class BaseModelSqlv2 {
     }
   }
 
-  public async selectObject({ qb }: { qb: QueryBuilder }): Promise<void> {
+  public async selectObject({
+    qb,
+    columns: _columns,
+  }: {
+    qb: QueryBuilder;
+    columns?: Column[];
+  }): Promise<void> {
     const res = {};
-    const columns = await this.model.getColumns();
+    const columns = _columns ?? (await this.model.getColumns());
     for (const column of columns) {
       switch (column.uidt) {
         case 'LinkToAnotherRecord':
@@ -2322,6 +2330,221 @@ class BaseModelSqlv2 {
     });
   }
 
+  public async groupedList(
+    args: {
+      groupColumnId: string;
+      ignoreFilterSort?: boolean;
+      sortArr?: any;
+      filterArr?: any;
+    } & Partial<XcFilter>
+  ): Promise<
+    {
+      key: string;
+      value: Record<string, unknown>[];
+    }[]
+  > {
+    const column = await this.model
+      .getColumns()
+      .then((cols) => cols?.find((col) => col.id === args.groupColumnId));
+
+    if (!column) NcError.notFound('Column not found');
+    if (isVirtualCol(column))
+      NcError.notImplemented('Grouping for virtual columns not implemented');
+
+    // extract distinct group column values
+    let groupingValues;
+    if (column.uidt === UITypes.SingleSelect) {
+      const colOptions = await column.getColOptions<SelectOption[]>();
+      groupingValues = colOptions?.map((opt) => opt.title);
+    } else {
+      groupingValues = (
+        await this.dbDriver(this.model.table_name)
+          .select(column.column_name)
+          .distinct()
+      ).map((row) => row[column.column_name]);
+    }
+
+    const qb = this.dbDriver(this.model.table_name);
+
+    qb.limit(+args?.limit || 20);
+    qb.offset(+args?.offset || 0);
+
+    await this.selectObject({ qb });
+
+    // todo: refactor and move to a method (applyFilterAndSort)
+    const aliasColObjMap = await this.model.getAliasColObjMap();
+    let sorts = extractSortsObject(args?.sort, aliasColObjMap);
+    const filterObj = extractFilterFromXwhere(args.where, aliasColObjMap);
+    // todo: replace with view id
+    if (!args.ignoreFilterSort && this.viewId) {
+      await conditionV2(
+        [
+          new Filter({
+            children:
+              (await Filter.rootFilterList({ viewId: this.viewId })) || [],
+            is_group: true,
+          }),
+          new Filter({
+            children: args.filterArr || [],
+            is_group: true,
+            logical_op: 'and',
+          }),
+          new Filter({
+            children: filterObj,
+            is_group: true,
+            logical_op: 'and',
+          }),
+        ],
+        qb,
+        this.dbDriver
+      );
+
+      if (!sorts)
+        sorts = args.sortArr?.length
+          ? args.sortArr
+          : await Sort.list({ viewId: this.viewId });
+
+      if (sorts?.['length']) await sortV2(sorts, qb, this.dbDriver);
+    } else {
+      await conditionV2(
+        [
+          new Filter({
+            children: args.filterArr || [],
+            is_group: true,
+            logical_op: 'and',
+          }),
+          new Filter({
+            children: filterObj,
+            is_group: true,
+            logical_op: 'and',
+          }),
+        ],
+        qb,
+        this.dbDriver
+      );
+
+      if (!sorts) sorts = args.sortArr;
+
+      if (sorts?.['length']) await sortV2(sorts, qb, this.dbDriver);
+    }
+
+    // sort by primary key if not autogenerated string
+    // if autogenerated string sort by created_at column if present
+    if (this.model.primaryKey && this.model.primaryKey.ai) {
+      qb.orderBy(this.model.primaryKey.column_name);
+    } else if (this.model.columns.find((c) => c.column_name === 'created_at')) {
+      qb.orderBy('created_at');
+    }
+
+    const nullListQb = qb.clone().whereNull(column.title);
+
+    const groupedQb = this.dbDriver.from(
+      this.dbDriver
+        .unionAll(
+          [
+            this.isSqlite
+              ? this.dbDriver.select().from(nullListQb)
+              : nullListQb,
+            ...groupingValues.map((r) => {
+              const query = qb.clone().where(column.title, r);
+
+              return this.isSqlite ? this.dbDriver.select().from(query) : query;
+            }),
+          ],
+          !this.isSqlite
+        )
+        .as('__nc_grouped_list')
+    );
+
+    const proto = await this.getProto();
+
+    const result = (await groupedQb)?.map((d) => {
+      d.__proto__ = proto;
+      return d;
+    });
+
+    // todo: handle null values
+    const groupedResult: Record<string, Record<string, unknown>[]> = _.groupBy(
+      result,
+      column.title
+    );
+
+    const r = Object.entries(groupedResult).map(([key, value]) => ({
+      key,
+      value,
+    }));
+
+    return r;
+  }
+
+  public async groupedListCount(args: { groupColumnId: string ; ignoreFilterSort?:boolean} & XcFilter) {
+    const column = await this.model
+      .getColumns()
+      .then((cols) => cols?.find((col) => col.id === args.groupColumnId));
+
+    if (!column) NcError.notFound('Column not found');
+    if (isVirtualCol(column))
+      NcError.notImplemented('Grouping for virtual columns not implemented');
+
+    const qb = this.dbDriver(this.model.table_name)
+      .count('*', { as: 'count' })
+      .groupBy(column.column_name);
+
+    // todo: refactor and move to a common method (applyFilterAndSort)
+    const aliasColObjMap = await this.model.getAliasColObjMap();
+    const filterObj = extractFilterFromXwhere(args.where, aliasColObjMap);
+    // todo: replace with view id
+
+    if (!args.ignoreFilterSort && this.viewId) {
+      await conditionV2(
+        [
+          new Filter({
+            children:
+              (await Filter.rootFilterList({ viewId: this.viewId })) || [],
+            is_group: true,
+          }),
+          new Filter({
+            children: args.filterArr || [],
+            is_group: true,
+            logical_op: 'and',
+          }),
+          new Filter({
+            children: filterObj,
+            is_group: true,
+            logical_op: 'and',
+          }),
+        ],
+        qb,
+        this.dbDriver
+      );
+    } else {
+      await conditionV2(
+        [
+          new Filter({
+            children: args.filterArr || [],
+            is_group: true,
+            logical_op: 'and',
+          }),
+          new Filter({
+            children: filterObj,
+            is_group: true,
+            logical_op: 'and',
+          }),
+        ],
+        qb,
+        this.dbDriver
+      );
+    }
+
+
+    await this.selectObject({
+      qb,
+      columns: [new Column({ ...column, title: 'key' })],
+    });
+
+    return await qb;
+  }
+
   private async extractRawQueryAndExec(qb: QueryBuilder) {
     let query = qb.toQuery();
     if (!this.isPg && !this.isMssql) {

packages/nocodb/src/lib/meta/api/dataApis/dataAliasApis.ts

@@ -222,6 +222,67 @@ async function dataExist(req: Request, res: Response) {
   res.json(await baseModel.exist(req.params.rowId));
 }
 
+// todo: Handle the error case where view doesnt belong to model
+async function groupedDataList(req: Request, res: Response) {
+  const { model, view } = await getViewAndModelFromRequestByAliasOrId(req);
+  res.json(await getGroupedDataList(model, view, req));
+}
+
+async function getGroupedDataList(model, view: View, req) {
+  const base = await Base.get(model.base_id);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: model.id,
+    viewId: view?.id,
+    dbDriver: NcConnectionMgrv2.get(base),
+  });
+
+  const requestObj = await getAst({ model, query: req.query, view });
+
+  const listArgs: any = { ...req.query };
+  try {
+    listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
+  } catch (e) {}
+  try {
+    listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
+  } catch (e) {}
+
+  let data = [];
+  // let count = 0
+  try {
+    const groupedData = await baseModel.groupedList({
+      ...listArgs,
+      groupColumnId: req.params.columnId,
+    });
+    data = await nocoExecute(
+      { key: 1, value: requestObj },
+      groupedData,
+      {},
+      listArgs
+    );
+    const countArr = await baseModel.groupedListCount({
+      ...listArgs,
+      groupColumnId: req.params.columnId,
+    });
+    data = data.map((item) => {
+      // todo: use map to avoid loop
+      const count = countArr.find(
+        (countItem) => countItem.key === item.key
+      )?.count;
+
+      item.value = new PagedResponseImpl(item.value, {
+        ...req.query,
+        count,
+      });
+      return item;
+    });
+  } catch (e) {
+    // show empty result instead of throwing error here
+    // e.g. search some text in a numeric field
+  }
+  return data;
+}
+
 const router = Router({ mergeParams: true });
 
 // table data crud apis
@@ -243,6 +304,12 @@ router.get(
   ncMetaAclMw(dataGroupBy, 'dataGroupBy')
 );
 
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/group/:columnId',
+  apiMetrics,
+  ncMetaAclMw(groupedDataList, 'groupedDataList')
+);
+
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/:rowId/exist',
   apiMetrics,
@@ -304,6 +371,12 @@ router.get(
   ncMetaAclMw(dataGroupBy, 'dataGroupBy')
 );
 
+router.get(
+  '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/group/:columnId',
+  apiMetrics,
+  ncMetaAclMw(groupedDataList, 'groupedDataList')
+);
+
 router.get(
   '/api/v1/db/data/:orgs/:projectName/:tableName/views/:viewName/:rowId/exist',
   apiMetrics,

packages/nocodb/tests/unit/TestDbMngr.ts

@@ -110,7 +110,7 @@ export default class TestDbMngr {
       await TestDbMngr.resetMetaSqlite();
       TestDbMngr.metaKnex = knex(TestDbMngr.getMetaDbConfig());
       return
-    } 
+    }
 
     TestDbMngr.metaKnex = knex(TestDbMngr.getDbConfigWithNoDb());
     await TestDbMngr.resetDatabase(TestDbMngr.metaKnex, TestDbMngr.dbName);
@@ -129,12 +129,12 @@ export default class TestDbMngr {
       await TestDbMngr.seedSakila();
       TestDbMngr.sakilaKnex = knex(TestDbMngr.getSakilaDbConfig());
       return
-    } 
-    
+    }
+
     TestDbMngr.sakilaKnex = knex(TestDbMngr.getDbConfigWithNoDb());
     await TestDbMngr.resetDatabase(TestDbMngr.sakilaKnex, TestDbMngr.sakilaDbName);
     await TestDbMngr.sakilaKnex.destroy();
-    
+
     TestDbMngr.sakilaKnex = knex(TestDbMngr.getSakilaDbConfig());
     await TestDbMngr.useDatabase(TestDbMngr.sakilaKnex, TestDbMngr.sakilaDbName);
   }
@@ -217,7 +217,7 @@ export default class TestDbMngr {
     return sakilaDbConfig;
   }
 
-  static async seedSakila() {     
+  static async seedSakila() {
     const testsDir = __dirname.replace('tests/unit', 'tests');
 
     if(TestDbMngr.isSqlite()){
@@ -263,4 +263,4 @@ export default class TestDbMngr {
       );
     }
   }
-}
+}