feat: normal data pis

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb-nest/src/app.module.ts

@@ -35,9 +35,10 @@ import { OrgTokensModule } from './modules/org-tokens/org-tokens.module';
 import { OrgUsersModule } from './modules/org-users/org-users.module';
 import { MetaDiffsModule } from './modules/meta-diffs/meta-diffs.module';
 import { AuditsModule } from './modules/audits/audits.module';
+import { DatasModule } from './modules/datas/datas.module';
 
 @Module({
-  imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule, AuditsModule],
+  imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule, AuditsModule, DatasModule],
   controllers: [],
   providers: [Connection, MetaService, JwtStrategy, ExtractProjectIdMiddleware],
   exports: [Connection, MetaService],

packages/nocodb-nest/src/modules/datas/datas.controller.spec.ts

@@ -0,0 +1,20 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { DatasController } from './datas.controller';
+import { DatasService } from './datas.service';
+
+describe('DatasController', () => {
+  let controller: DatasController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [DatasController],
+      providers: [DatasService],
+    }).compile();
+
+    controller = module.get<DatasController>(DatasController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/datas/datas.controller.ts

@@ -0,0 +1,206 @@
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  Param,
+  Patch,
+  Post,
+  Request,
+} from '@nestjs/common';
+import { Acl } from '../../middlewares/extract-project-id/extract-project-id.middleware';
+import { DatasService } from './datas.service';
+
+@Controller('datas')
+export class DatasController {
+  constructor(private readonly datasService: DatasService) {}
+
+  @Get('/data/:viewId/')
+  @Acl('dataList')
+  async dataList(@Request() req, @Param('viewId') viewId: string) {
+    return await this.datasService.dataListByViewId({
+      viewId: viewId,
+      query: req.query,
+    });
+  }
+
+  @Get('/data/:viewId/:rowId/mm/:colId')
+  @Acl('mmList')
+  async mmList(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Param('colId') colId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    return await this.datasService.mmList({
+      viewId: viewId,
+      colId: colId,
+      rowId: rowId,
+      query: req.query,
+    });
+  }
+
+  @Get('/data/:viewId/:rowId/mm/:colId/exclude')
+  @Acl('mmExcludedList')
+  async mmExcludedList(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Param('colId') colId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    return await this.datasService.mmExcludedList({
+      viewId: viewId,
+      colId: colId,
+      rowId: rowId,
+      query: req.query,
+    });
+  }
+
+  @Get('/data/:viewId/:rowId/hm/:colId/exclude')
+  @Acl('hmExcludedList')
+  async hmExcludedList(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Param('colId') colId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    await this.datasService.hmExcludedList({
+      viewId: viewId,
+      colId: colId,
+      rowId: rowId,
+      query: req.query,
+    });
+  }
+
+  @Get('/data/:viewId/:rowId/bt/:colId/exclude')
+  @Acl('btExcludedList')
+  async btExcludedList(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Param('colId') colId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    return await this.datasService.btExcludedList({
+      viewId: viewId,
+      colId: colId,
+      rowId: rowId,
+      query: req.query,
+    });
+  }
+
+  @Get('/data/:viewId/:rowId/hm/:colId')
+  @Acl('hmList')
+  async hmList(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Param('colId') colId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    return await this.datasService.hmList({
+      viewId: viewId,
+      colId: colId,
+      rowId: rowId,
+      query: req.query,
+    });
+  }
+
+  @Get('/data/:viewId/:rowId')
+  @Acl('dataRead')
+  async dataRead(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    return await this.datasService.dataReadByViewId({
+      viewId,
+      rowId,
+      query: req.query,
+    });
+  }
+
+  @Post('/data/:viewId/')
+  @Acl('dataInsert')
+  async dataInsert(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Body() body: any,
+  ) {
+    return await this.datasService.dataInsertByViewId({
+      viewId: viewId,
+      body: body,
+      cookie: req,
+    });
+  }
+
+  @Patch('/data/:viewId/:rowId')
+  @Acl('dataUpdate')
+  async dataUpdate(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Param('rowId') rowId: string,
+    @Body() body: any,
+  ) {
+    return await this.datasService.dataUpdateByViewId({
+      viewId: viewId,
+      rowId: rowId,
+      body: body,
+      cookie: req,
+    });
+  }
+
+  @Delete('/data/:viewId/:rowId')
+  @Acl('dataDelete')
+  async dataDelete(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    return await this.datasService.dataDeleteByViewId({
+      viewId: viewId,
+      rowId: rowId,
+      cookie: req,
+    });
+  }
+
+  @Delete('/data/:viewId/:rowId/:relationType/:colId/:childId')
+  @Acl('relationDataDelete')
+  async relationDataDelete(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Param('rowId') rowId: string,
+    @Param('relationType') relationType: string,
+    @Param('colId') colId: string,
+    @Param('childId') childId: string,
+  ) {
+    await this.datasService.relationDataDelete({
+      viewId: viewId,
+      colId: colId,
+      childId: childId,
+      rowId: rowId,
+      cookie: req,
+    });
+
+    return { msg: 'The relation data has been deleted successfully' };
+  }
+
+  @Post('/data/:viewId/:rowId/:relationType/:colId/:childId')
+  @Acl('relationDataAdd')
+  async relationDataAdd(
+    @Request() req,
+    @Param('viewId') viewId: string,
+    @Param('rowId') rowId: string,
+    @Param('relationType') relationType: string,
+    @Param('colId') colId: string,
+    @Param('childId') childId: string,
+  ) {
+    await this.datasService.relationDataAdd({
+      viewId: viewId,
+      colId: colId,
+      childId: childId,
+      rowId: rowId,
+      cookie: req,
+    });
+
+    return { msg: 'The relation data has been created successfully' };
+  }
+}

packages/nocodb-nest/src/modules/datas/datas.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { DatasService } from './datas.service';
+import { DatasController } from './datas.controller';
+
+@Module({
+  controllers: [DatasController],
+  providers: [DatasService]
+})
+export class DatasModule {}

packages/nocodb-nest/src/modules/datas/datas.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { DatasService } from './datas.service';
+
+describe('DatasService', () => {
+  let service: DatasService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [DatasService],
+    }).compile();
+
+    service = module.get<DatasService>(DatasService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/datas/datas.service.ts

@@ -0,0 +1,768 @@
+import { Injectable } from '@nestjs/common';
+import { NcError } from '../../helpers/catchError';
+import getAst from '../../helpers/getAst';
+import { PagedResponseImpl } from '../../helpers/PagedResponse';
+import { Base, Model, View } from '../../models';
+import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2';
+import { getViewAndModelByAliasOrId, PathParams } from './helpers';
+import { nocoExecute } from 'nc-help';
+
+@Injectable()
+export class DatasService {
+  async dataList(param: PathParams & { query: any }) {
+    const { model, view } = await getViewAndModelByAliasOrId(param);
+    const responseData = await this.getDataList({
+      model,
+      view,
+      query: param.query,
+    });
+    return responseData;
+  }
+
+  async dataFindOne(param: PathParams & { query: any }) {
+    const { model, view } = await getViewAndModelByAliasOrId(param);
+    return await this.getFindOne({ model, view, query: param.query });
+  }
+
+  async dataGroupBy(param: PathParams & { query: any }) {
+    const { model, view } = await getViewAndModelByAliasOrId(param);
+    return await this.getDataGroupBy({ model, view, query: param.query });
+  }
+
+  async dataCount(param: PathParams & { query: any }) {
+    const { model, view } = await getViewAndModelByAliasOrId(param);
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const countArgs: any = { ...param.query };
+    try {
+      countArgs.filterArr = JSON.parse(countArgs.filterArrJson);
+    } catch (e) {}
+
+    const count: number = await baseModel.count(countArgs);
+
+    return { count };
+  }
+
+  async dataInsert(param: PathParams & { body: unknown; cookie: any }) {
+    const { model, view } = await getViewAndModelByAliasOrId(param);
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    return await baseModel.insert(param.body, null, param.cookie);
+  }
+
+  async dataUpdate(
+    param: PathParams & { body: unknown; cookie: any; rowId: string },
+  ) {
+    const { model, view } = await getViewAndModelByAliasOrId(param);
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    return await baseModel.updateByPk(
+      param.rowId,
+      param.body,
+      null,
+      param.cookie,
+    );
+  }
+
+  async dataDelete(param: PathParams & { rowId: string; cookie: any }) {
+    const { model, view } = await getViewAndModelByAliasOrId(param);
+    const base = await Base.get(model.base_id);
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    // todo: Should have error http status code
+    const message = await baseModel.hasLTARData(param.rowId, model);
+    if (message.length) {
+      return { message };
+    }
+    return await baseModel.delByPk(param.rowId, null, param.cookie);
+  }
+
+  async getDataList(param: { model: Model; view: View; query: any }) {
+    const { model, view, query = {} } = param;
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const { ast, dependencyFields } = await getAst({ model, query, view });
+
+    const listArgs: any = dependencyFields;
+    try {
+      listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
+    } catch (e) {}
+    try {
+      listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
+    } catch (e) {}
+
+    let data = [];
+    let count = 0;
+    try {
+      data = await nocoExecute(
+        ast,
+        await baseModel.list(listArgs),
+        {},
+        listArgs,
+      );
+      count = await baseModel.count(listArgs);
+    } catch (e) {
+      console.log(e);
+      NcError.internalServerError(
+        'Internal Server Error, check server log for more details',
+      );
+    }
+
+    return new PagedResponseImpl(data, {
+      ...query,
+      count,
+    });
+  }
+
+  async getFindOne(param: { model: Model; view: View; query: any }) {
+    const { model, view, query = {} } = param;
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const args: any = { ...query };
+    try {
+      args.filterArr = JSON.parse(args.filterArrJson);
+    } catch (e) {}
+    try {
+      args.sortArr = JSON.parse(args.sortArrJson);
+    } catch (e) {}
+
+    const { ast, dependencyFields } = await getAst({
+      model,
+      query: args,
+      view,
+    });
+
+    const data = await baseModel.findOne({ ...args, dependencyFields });
+    return data ? await nocoExecute(ast, data, {}, {}) : {};
+  }
+
+  async getDataGroupBy(param: { model: Model; view: View; query?: any }) {
+    const { model, view, query = {} } = param;
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const listArgs: any = { ...query };
+    const data = await baseModel.groupBy({ ...query });
+    const count = await baseModel.count(listArgs);
+
+    return new PagedResponseImpl(data, {
+      ...query,
+      count,
+    });
+  }
+
+  async dataRead(param: PathParams & { query: any; rowId: string }) {
+    const { model, view } = await getViewAndModelByAliasOrId(param);
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const row = await baseModel.readByPk(param.rowId);
+
+    if (!row) {
+      NcError.notFound('Row not found');
+    }
+
+    const { ast } = await getAst({ model, query: param.query, view });
+
+    return await nocoExecute(ast, row, {}, param.query);
+  }
+
+  async dataExist(param: PathParams & { rowId: string; query: any }) {
+    const { model, view } = await getViewAndModelByAliasOrId(param);
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    return await baseModel.exist(param.rowId);
+  }
+
+  // todo: Handle the error case where view doesnt belong to model
+  async groupedDataList(param: PathParams & { query: any; columnId: string }) {
+    const { model, view } = await getViewAndModelByAliasOrId(param);
+    const groupedData = await this.getGroupedDataList({
+      model,
+      view,
+      query: param.query,
+      columnId: param.columnId,
+    });
+    return groupedData;
+  }
+
+  async getGroupedDataList(param: {
+    model;
+    view: View;
+    query: any;
+    columnId: string;
+  }) {
+    const { model, view, query = {} } = param;
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const { ast } = await getAst({ model, query, view });
+
+    const listArgs: any = { ...query };
+    try {
+      listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
+    } catch (e) {}
+    try {
+      listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
+    } catch (e) {}
+    try {
+      listArgs.options = JSON.parse(listArgs.optionsArrJson);
+    } catch (e) {}
+
+    let data = [];
+
+    const groupedData = await baseModel.groupedList({
+      ...listArgs,
+      groupColumnId: param.columnId,
+    });
+    data = await nocoExecute({ key: 1, value: ast }, groupedData, {}, listArgs);
+    const countArr = await baseModel.groupedListCount({
+      ...listArgs,
+      groupColumnId: param.columnId,
+    });
+    data = data.map((item) => {
+      // todo: use map to avoid loop
+      const count =
+        countArr.find((countItem: any) => countItem.key === item.key)?.count ??
+        0;
+
+      item.value = new PagedResponseImpl(item.value, {
+        ...query,
+        count: count,
+      });
+      return item;
+    });
+
+    return data;
+  }
+
+  async dataListByViewId(param: { viewId: string; query: any }) {
+    const view = await View.get(param.viewId);
+
+    const model = await Model.getByIdOrName({
+      id: view?.fk_model_id || param.viewId,
+    });
+
+    if (!model) NcError.notFound('Table not found');
+
+    return await this.getDataList({ model, view, query: param.query });
+  }
+
+  async mmList(param: {
+    viewId: string;
+    colId: string;
+    query: any;
+    rowId: string;
+  }) {
+    const view = await View.get(param.viewId);
+
+    const model = await Model.getByIdOrName({
+      id: view?.fk_model_id || param.viewId,
+    });
+
+    if (!model) NcError.notFound('Table not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const key = `${model.title}List`;
+    const requestObj: any = {
+      [key]: 1,
+    };
+
+    const data = (
+      await nocoExecute(
+        requestObj,
+        {
+          [key]: async (args) => {
+            return await baseModel.mmList(
+              {
+                colId: param.colId,
+                parentId: param.rowId,
+              },
+              args,
+            );
+          },
+        },
+        {},
+
+        { nested: { [key]: param.query } },
+      )
+    )?.[key];
+
+    const count: any = await baseModel.mmListCount({
+      colId: param.colId,
+      parentId: param.rowId,
+    });
+
+    return new PagedResponseImpl(data, {
+      count,
+      ...param.query,
+    });
+  }
+
+  async mmExcludedList(param: {
+    viewId: string;
+    colId: string;
+    query: any;
+    rowId: string;
+  }) {
+    const view = await View.get(param.viewId);
+
+    const model = await Model.getByIdOrName({
+      id: view?.fk_model_id || param.viewId,
+    });
+
+    if (!model) NcError.notFound('Table not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const key = 'List';
+    const requestObj: any = {
+      [key]: 1,
+    };
+
+    const data = (
+      await nocoExecute(
+        requestObj,
+        {
+          [key]: async (args) => {
+            return await baseModel.getMmChildrenExcludedList(
+              {
+                colId: param.colId,
+                pid: param.rowId,
+              },
+              args,
+            );
+          },
+        },
+        {},
+
+        { nested: { [key]: param.query } },
+      )
+    )?.[key];
+
+    const count = await baseModel.getMmChildrenExcludedListCount(
+      {
+        colId: param.colId,
+        pid: param.rowId,
+      },
+      param.query,
+    );
+
+    return new PagedResponseImpl(data, {
+      count,
+      ...param.query,
+    });
+  }
+
+  async hmExcludedList(param: {
+    viewId: string;
+    colId: string;
+    query: any;
+    rowId: string;
+  }) {
+    const view = await View.get(param.viewId);
+
+    const model = await Model.getByIdOrName({
+      id: view?.fk_model_id || param.viewId,
+    });
+
+    if (!model) NcError.notFound('Table not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const key = 'List';
+    const requestObj: any = {
+      [key]: 1,
+    };
+
+    const data = (
+      await nocoExecute(
+        requestObj,
+        {
+          [key]: async (args) => {
+            return await baseModel.getHmChildrenExcludedList(
+              {
+                colId: param.colId,
+                pid: param.rowId,
+              },
+              args,
+            );
+          },
+        },
+        {},
+
+        { nested: { [key]: param.query } },
+      )
+    )?.[key];
+
+    const count = await baseModel.getHmChildrenExcludedListCount(
+      {
+        colId: param.colId,
+        pid: param.rowId,
+      },
+      param.query,
+    );
+
+    return new PagedResponseImpl(data, {
+      count,
+      ...param.query,
+    });
+  }
+
+  async btExcludedList(param: {
+    viewId: string;
+    colId: string;
+    query: any;
+    rowId: string;
+  }) {
+    const view = await View.get(param.viewId);
+
+    const model = await Model.getByIdOrName({
+      id: view?.fk_model_id || param.viewId,
+    });
+
+    if (!model) return NcError.notFound('Table not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const key = 'List';
+    const requestObj: any = {
+      [key]: 1,
+    };
+
+    const data = (
+      await nocoExecute(
+        requestObj,
+        {
+          [key]: async (args) => {
+            return await baseModel.getBtChildrenExcludedList(
+              {
+                colId: param.colId,
+                cid: param.rowId,
+              },
+              args,
+            );
+          },
+        },
+        {},
+
+        { nested: { [key]: param.query } },
+      )
+    )?.[key];
+
+    const count = await baseModel.getBtChildrenExcludedListCount(
+      {
+        colId: param.colId,
+        cid: param.rowId,
+      },
+      param.query,
+    );
+
+    return new PagedResponseImpl(data, {
+      count,
+      ...param.query,
+    });
+  }
+
+  async hmList(param: {
+    viewId: string;
+    colId: string;
+    query: any;
+    rowId: string;
+  }) {
+    const view = await View.get(param.viewId);
+
+    const model = await Model.getByIdOrName({
+      id: view?.fk_model_id || param.viewId,
+    });
+
+    if (!model) NcError.notFound('Table not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const key = `${model.title}List`;
+    const requestObj: any = {
+      [key]: 1,
+    };
+
+    const data = (
+      await nocoExecute(
+        requestObj,
+        {
+          [key]: async (args) => {
+            return await baseModel.hmList(
+              {
+                colId: param.colId,
+                id: param.rowId,
+              },
+              args,
+            );
+          },
+        },
+        {},
+        { nested: { [key]: param.query } },
+      )
+    )?.[key];
+
+    const count = await baseModel.hmListCount({
+      colId: param.colId,
+      id: param.rowId,
+    });
+
+    return new PagedResponseImpl(data, {
+      totalRows: count,
+    } as any);
+  }
+
+  async dataReadByViewId(param: { viewId: string; rowId: string; query: any }) {
+    try {
+      const model = await Model.getByIdOrName({
+        id: param.viewId,
+      });
+      if (!model) NcError.notFound('Table not found');
+
+      const base = await Base.get(model.base_id);
+
+      const baseModel = await Model.getBaseModelSQL({
+        id: model.id,
+        dbDriver: await NcConnectionMgrv2.get(base),
+      });
+
+      const { ast } = await getAst({ model, query: param.query });
+
+      return await nocoExecute(
+        ast,
+        await baseModel.readByPk(param.rowId),
+        {},
+        {},
+      );
+    } catch (e) {
+      console.log(e);
+      NcError.internalServerError(
+        'Internal Server Error, check server log for more details',
+      );
+    }
+  }
+
+  async dataInsertByViewId(param: { viewId: string; body: any; cookie: any }) {
+    const model = await Model.getByIdOrName({
+      id: param.viewId,
+    });
+    if (!model) return NcError.notFound('Table not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    return await baseModel.insert(param.body, null, param.cookie);
+  }
+
+  async dataUpdateByViewId(param: {
+    viewId: string;
+    rowId: string;
+    body: any;
+    cookie: any;
+  }) {
+    const model = await Model.getByIdOrName({
+      id: param.viewId,
+    });
+    if (!model) NcError.notFound('Table not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    return await baseModel.updateByPk(
+      param.rowId,
+      param.body,
+      null,
+      param.cookie,
+    );
+  }
+
+  async dataDeleteByViewId(param: {
+    viewId: string;
+    rowId: string;
+    cookie: any;
+  }) {
+    const model = await Model.getByIdOrName({
+      id: param.viewId,
+    });
+    if (!model) NcError.notFound('Table not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    return await baseModel.delByPk(param.rowId, null, param.cookie);
+  }
+
+  async relationDataDelete(param: {
+    viewId: string;
+    colId: string;
+    childId: string;
+    rowId: string;
+    cookie: any;
+  }) {
+    const view = await View.get(param.viewId);
+
+    const model = await Model.getByIdOrName({
+      id: view?.fk_model_id || param.viewId,
+    });
+
+    if (!model) NcError.notFound('Table not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    await baseModel.removeChild({
+      colId: param.colId,
+      childId: param.childId,
+      rowId: param.rowId,
+      cookie: param.cookie,
+    });
+
+    return true;
+  }
+
+  async relationDataAdd(param: {
+    viewId: string;
+    colId: string;
+    childId: string;
+    rowId: string;
+    cookie: any;
+  }) {
+    const view = await View.get(param.viewId);
+
+    const model = await Model.getByIdOrName({
+      id: view?.fk_model_id || param.viewId,
+    });
+
+    if (!model) NcError.notFound('Table not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    await baseModel.addChild({
+      colId: param.colId,
+      childId: param.childId,
+      rowId: param.rowId,
+      cookie: param.cookie,
+    });
+
+    return true;
+  }
+}

packages/nocodb-nest/src/modules/datas/helpers.ts

@@ -0,0 +1,278 @@
+import { nocoExecute } from 'nc-help';
+import { isSystemColumn, UITypes } from 'nocodb-sdk';
+import * as XLSX from 'xlsx';
+import papaparse from 'papaparse';
+import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst';
+import { NcError } from '../../meta/helpers/catchError';
+import { Model, View } from '../../models';
+import Base from '../../models/Base';
+import Column from '../../models/Column';
+import Project from '../../models/Project';
+import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2';
+import type LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn';
+import type LookupColumn from '../../models/LookupColumn';
+import type { BaseModelSqlv2 } from '../../db/sql-data-mapper/lib/sql/BaseModelSqlv2';
+import type { Request } from 'express';
+
+export interface PathParams {
+  projectName: string;
+  tableName: string;
+  viewName?: string;
+}
+
+export async function getViewAndModelByAliasOrId(param: {
+  projectName: string;
+  tableName: string;
+  viewName?: string;
+}) {
+  const project = await Project.getWithInfoByTitleOrId(param.projectName);
+
+  const model = await Model.getByAliasOrId({
+    project_id: project.id,
+    aliasOrId: param.tableName,
+  });
+  const view =
+    param.viewName &&
+    (await View.getByTitleOrId({
+      titleOrId: param.viewName,
+      fk_model_id: model.id,
+    }));
+  if (!model) NcError.notFound('Table not found');
+  return { model, view };
+}
+
+export async function extractXlsxData(view: View, req: Request) {
+  const base = await Base.get(view.base_id);
+
+  await view.getModelWithInfo();
+  await view.getColumns();
+
+  view.model.columns = view.columns
+    .filter((c) => c.show)
+    .map(
+      (c) =>
+        new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any)
+    )
+    .filter((column) => !isSystemColumn(column) || view.show_system_fields);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: view.model.id,
+    viewId: view?.id,
+    dbDriver: await NcConnectionMgrv2.get(base),
+  });
+
+  const { offset, dbRows, elapsed } = await getDbRows({
+    baseModel,
+    view,
+    siteUrl: (req as any).ncSiteUrl,
+    query: req.query,
+  });
+
+  const fields = req.query.fields as string[];
+
+  const data = XLSX.utils.json_to_sheet(dbRows, { header: fields });
+
+  return { offset, dbRows, elapsed, data };
+}
+
+export async function extractCsvData(view: View, req: Request) {
+  const base = await Base.get(view.base_id);
+  const fields = req.query.fields;
+
+  await view.getModelWithInfo();
+  await view.getColumns();
+
+  view.model.columns = view.columns
+    .filter((c) => c.show)
+    .map(
+      (c) =>
+        new Column({ ...c, ...view.model.columnsById[c.fk_column_id] } as any)
+    )
+    .filter((column) => !isSystemColumn(column) || view.show_system_fields);
+
+  const baseModel = await Model.getBaseModelSQL({
+    id: view.model.id,
+    viewId: view?.id,
+    dbDriver: await NcConnectionMgrv2.get(base),
+  });
+
+  const { offset, dbRows, elapsed } = await getDbRows({
+    baseModel,
+    view,
+    query: req.query,
+    siteUrl: (req as any).ncSiteUrl,
+  });
+
+  const data = papaparse.unparse(
+    {
+      fields: view.model.columns
+        .sort((c1, c2) =>
+          Array.isArray(fields)
+            ? fields.indexOf(c1.title as any) - fields.indexOf(c2.title as any)
+            : 0
+        )
+        .filter(
+          (c) =>
+            !fields || !Array.isArray(fields) || fields.includes(c.title as any)
+        )
+        .map((c) => c.title),
+      data: dbRows,
+    },
+    {
+      escapeFormulae: true,
+    }
+  );
+
+  return { offset, dbRows, elapsed, data };
+}
+
+export async function serializeCellValue({
+  value,
+  column,
+  siteUrl,
+}: {
+  column?: Column;
+  value: any;
+  siteUrl: string;
+}) {
+  if (!column) {
+    return value;
+  }
+
+  if (!value) return value;
+
+  switch (column?.uidt) {
+    case UITypes.Attachment: {
+      let data = value;
+      try {
+        if (typeof value === 'string') {
+          data = JSON.parse(value);
+        }
+      } catch {}
+
+      return (data || []).map(
+        (attachment) =>
+          `${encodeURI(attachment.title)}(${encodeURI(
+            attachment.path ? `${siteUrl}/${attachment.path}` : attachment.url
+          )})`
+      );
+    }
+    case UITypes.Lookup:
+      {
+        const colOptions = await column.getColOptions<LookupColumn>();
+        const lookupColumn = await colOptions.getLookupColumn();
+        return (
+          await Promise.all(
+            [...(Array.isArray(value) ? value : [value])].map(async (v) =>
+              serializeCellValue({
+                value: v,
+                column: lookupColumn,
+                siteUrl,
+              })
+            )
+          )
+        ).join(', ');
+      }
+      break;
+    case UITypes.LinkToAnotherRecord:
+      {
+        const colOptions =
+          await column.getColOptions<LinkToAnotherRecordColumn>();
+        const relatedModel = await colOptions.getRelatedTable();
+        await relatedModel.getColumns();
+        return [...(Array.isArray(value) ? value : [value])]
+          .map((v) => {
+            return v[relatedModel.displayValue?.title];
+          })
+          .join(', ');
+      }
+      break;
+    default:
+      if (value && typeof value === 'object') {
+        return JSON.stringify(value);
+      }
+      return value;
+  }
+}
+
+export async function getColumnByIdOrName(
+  columnNameOrId: string,
+  model: Model
+) {
+  const column = (await model.getColumns()).find(
+    (c) =>
+      c.title === columnNameOrId ||
+      c.id === columnNameOrId ||
+      c.column_name === columnNameOrId
+  );
+
+  if (!column)
+    NcError.notFound(`Column with id/name '${columnNameOrId}' is not found`);
+
+  return column;
+}
+
+export async function getDbRows(param: {
+  baseModel: BaseModelSqlv2;
+  view: View;
+  query: any;
+  siteUrl: string;
+}) {
+  const { baseModel, view, query = {}, siteUrl } = param;
+  let offset = +query.offset || 0;
+  const limit = 100;
+  // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024;
+  const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000;
+  const dbRows = [];
+  const startTime = process.hrtime();
+  let elapsed, temp;
+
+  const listArgs: any = { ...query };
+  try {
+    listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
+  } catch (e) {}
+  try {
+    listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
+  } catch (e) {}
+
+  for (
+    elapsed = 0;
+    elapsed < timeout;
+    offset += limit,
+      temp = process.hrtime(startTime),
+      elapsed = temp[0] * 1000 + temp[1] / 1000000
+  ) {
+    const { ast, dependencyFields } = await getAst({
+      query: query,
+      includePkByDefault: false,
+      model: view.model,
+      view,
+    });
+    const rows = await nocoExecute(
+      ast,
+      await baseModel.list({ ...listArgs, ...dependencyFields, offset, limit }),
+      {},
+      query
+    );
+
+    if (!rows?.length) {
+      offset = -1;
+      break;
+    }
+
+    for (const row of rows) {
+      const dbRow = { ...row };
+
+      for (const column of view.model.columns) {
+        if (isSystemColumn(column) && !view.show_system_fields) continue;
+        dbRow[column.title] = await serializeCellValue({
+          value: row[column.title],
+          column,
+          siteUrl,
+        });
+      }
+      dbRows.push(dbRow);
+    }
+  }
+  return { offset, dbRows, elapsed };
+}