Merge pull request #5747 from nocodb/feat/data-apis

feat: New table data apis endpoints

packages/nocodb/src/app.module.ts

@@ -29,7 +29,6 @@ import type { MiddlewareConsumer } from '@nestjs/common';
     JobsModule,
     NestJsEventEmitter.forRoot(),
   ],
-  controllers: [],
   providers: [
     AuthService,
     {

packages/nocodb/src/controllers/data-table.controller.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { DataTableController } from './data-table.controller';
+
+describe('DataTableController', () => {
+  let controller: DataTableController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [DataTableController],
+    }).compile();
+
+    controller = module.get<DataTableController>(DataTableController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

packages/nocodb/src/controllers/data-table.controller.ts

@@ -0,0 +1,189 @@
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  HttpCode,
+  Param,
+  Patch,
+  Post,
+  Query,
+  Request,
+  Response,
+  UseGuards,
+} from '@nestjs/common';
+import { GlobalGuard } from '../guards/global/global.guard';
+import { parseHrtimeToSeconds } from '../helpers';
+import {
+  Acl,
+  ExtractProjectIdMiddleware,
+} from '../middlewares/extract-project-id/extract-project-id.middleware';
+import { DataTableService } from '../services/data-table.service';
+
+@Controller()
+@UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
+export class DataTableController {
+  constructor(private readonly dataTableService: DataTableService) {}
+
+  // todo: Handle the error case where view doesnt belong to model
+  @Get('/api/v1/tables/:modelId/rows')
+  @Acl('dataList')
+  async dataList(
+    @Request() req,
+    @Response() res,
+    @Param('modelId') modelId: string,
+    @Query('viewId') viewId: string,
+  ) {
+    const startTime = process.hrtime();
+    const responseData = await this.dataTableService.dataList({
+      query: req.query,
+      modelId: modelId,
+      viewId: viewId,
+    });
+    const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime));
+    res.setHeader('xc-db-response', elapsedSeconds);
+    res.json(responseData);
+  }
+
+  @Get(['/api/v1/tables/:modelId/rows/count'])
+  @Acl('dataCount')
+  async dataCount(
+    @Request() req,
+    @Response() res,
+    @Param('modelId') modelId: string,
+    @Query('viewId') viewId: string,
+  ) {
+    const countResult = await this.dataTableService.dataCount({
+      query: req.query,
+      modelId,
+      viewId,
+    });
+
+    res.json(countResult);
+  }
+
+  @Post(['/api/v1/tables/:modelId/rows'])
+  @HttpCode(200)
+  @Acl('dataInsert')
+  async dataInsert(
+    @Request() req,
+    @Param('modelId') modelId: string,
+    @Query('viewId') viewId: string,
+    @Body() body: any,
+  ) {
+    return await this.dataTableService.dataInsert({
+      modelId: modelId,
+      body: body,
+      viewId,
+      cookie: req,
+    });
+  }
+
+  @Patch(['/api/v1/tables/:modelId/rows'])
+  @Acl('dataUpdate')
+  async dataUpdate(
+    @Request() req,
+    @Param('modelId') modelId: string,
+    @Query('viewId') viewId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    return await this.dataTableService.dataUpdate({
+      modelId: modelId,
+      body: req.body,
+      cookie: req,
+      viewId,
+    });
+  }
+
+  @Delete(['/api/v1/tables/:modelId/rows'])
+  @Acl('dataDelete')
+  async dataDelete(
+    @Request() req,
+    @Param('modelId') modelId: string,
+    @Query('viewId') viewId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    return await this.dataTableService.dataDelete({
+      modelId: modelId,
+      cookie: req,
+      viewId,
+      body: req.body,
+    });
+  }
+
+  @Get(['/api/v1/tables/:modelId/rows/:rowId'])
+  @Acl('dataRead')
+  async dataRead(
+    @Request() req,
+    @Param('modelId') modelId: string,
+    @Query('viewId') viewId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    return await this.dataTableService.dataRead({
+      modelId,
+      rowId: rowId,
+      query: req.query,
+      viewId,
+    });
+  }
+
+  @Get(['/api/v1/tables/:modelId/links/:columnId/rows/:rowId'])
+  @Acl('nestedDataList')
+  async nestedDataList(
+    @Request() req,
+    @Param('modelId') modelId: string,
+    @Query('viewId') viewId: string,
+    @Param('columnId') columnId: string,
+    @Param('rowId') rowId: string,
+  ) {
+    return await this.dataTableService.nestedDataList({
+      modelId,
+      rowId: rowId,
+      query: req.query,
+      viewId,
+      columnId,
+    });
+  }
+
+  @Post(['/api/v1/tables/:modelId/links/:columnId/rows/:rowId'])
+  @Acl('nestedDataLink')
+  async nestedLink(
+    @Request() req,
+    @Param('modelId') modelId: string,
+    @Query('viewId') viewId: string,
+    @Param('columnId') columnId: string,
+    @Param('rowId') rowId: string,
+    @Body() refRowIds: string | string[] | number | number[],
+  ) {
+    return await this.dataTableService.nestedLink({
+      modelId,
+      rowId: rowId,
+      query: req.query,
+      viewId,
+      columnId,
+      refRowIds,
+      cookie: req,
+    });
+  }
+
+  @Delete(['/api/v1/tables/:modelId/links/:columnId/rows/:rowId'])
+  @Acl('nestedDataUnlink')
+  async nestedUnlink(
+    @Request() req,
+    @Param('modelId') modelId: string,
+    @Query('viewId') viewId: string,
+    @Param('columnId') columnId: string,
+    @Param('rowId') rowId: string,
+    @Body() refRowIds: string | string[] | number | number[],
+  ) {
+    return await this.dataTableService.nestedUnlink({
+      modelId,
+      rowId: rowId,
+      query: req.query,
+      viewId,
+      columnId,
+      refRowIds,
+      cookie: req,
+    });
+  }
+}

packages/nocodb/src/db/BaseModelSqlv2.ts

@@ -18,6 +18,7 @@ import { customAlphabet } from 'nanoid';
 import DOMPurify from 'isomorphic-dompurify';
 import { v4 as uuidv4 } from 'uuid';
 import { Knex } from 'knex';
+import { extractLimitAndOffset } from '../helpers';
 import { NcError } from '../helpers/catchError';
 import getAst from '../helpers/getAst';
 import {
@@ -1511,20 +1512,12 @@ class BaseModelSqlv2 {
   }
 
   _getListArgs(args: XcFilterWithAlias): XcFilter {
-    const obj: XcFilter = {};
+    const obj: XcFilter = extractLimitAndOffset(args);
-    obj.where = args.where || args.w || '';
+    obj.where = args.filter || args.where || args.w || '';
     obj.having = args.having || args.h || '';
     obj.shuffle = args.shuffle || args.r || '';
     obj.condition = args.condition || args.c || {};
     obj.conditionGraph = args.conditionGraph || {};
-    obj.limit = Math.max(
-      Math.min(
-        args.limit || args.l || this.config.limitDefault,
-        this.config.limitMax,
-      ),
-      this.config.limitMin,
-    );
-    obj.offset = Math.max(+(args.offset || args.o) || 0, 0);
     obj.fields = args.fields || args.f;
     obj.sort = args.sort || args.s;
     return obj;
@@ -2244,12 +2237,14 @@ class BaseModelSqlv2 {
       foreign_key_checks = true,
       skip_hooks = false,
       raw = false,
+      insertOneByOneAsFallback = false,
     }: {
       chunkSize?: number;
       cookie?: any;
       foreign_key_checks?: boolean;
       skip_hooks?: boolean;
       raw?: boolean;
+      insertOneByOneAsFallback?: boolean;
     } = {},
   ) {
     let trx;
@@ -2403,12 +2398,28 @@ class BaseModelSqlv2 {
         }
       }
 
-      const response =
+      let response;
-        this.isPg || this.isMssql
+
-          ? await trx
+      // insert one by one as fallback to get ids for sqlite and mysql
-              .batchInsert(this.tnPath, insertDatas, chunkSize)
+      if (insertOneByOneAsFallback && (this.isSqlite || this.isMySQL)) {
-              .returning(this.model.primaryKey?.column_name)
+        // sqlite and mysql doesnt support returning, so insert one by one and return ids
-          : await trx.batchInsert(this.tnPath, insertDatas, chunkSize);
+        response = [];
+
+        const aiPkCol = this.model.primaryKeys.find((pk) => pk.ai);
+
+        for (const insertData of insertDatas) {
+          const query = trx(this.tnPath).insert(insertData);
+          const id = (await query)[0];
+          response.push(aiPkCol ? { [aiPkCol.title]: id } : id);
+        }
+      } else {
+        response =
+          this.isPg || this.isMssql
+            ? await trx
+                .batchInsert(this.tnPath, insertDatas, chunkSize)
+                .returning(this.model.primaryKey?.column_name)
+            : await trx.batchInsert(this.tnPath, insertDatas, chunkSize);
+      }
 
       if (!foreign_key_checks) {
         if (this.isPg) {
@@ -2433,7 +2444,11 @@ class BaseModelSqlv2 {
 
   async bulkUpdate(
     datas: any[],
-    { cookie, raw = false }: { cookie?: any; raw?: boolean } = {},
+    {
+      cookie,
+      raw = false,
+      throwExceptionIfNotExist = false,
+    }: { cookie?: any; raw?: boolean; throwExceptionIfNotExist?: boolean } = {},
   ) {
     let transaction;
     try {
@@ -2476,9 +2491,12 @@ class BaseModelSqlv2 {
 
       if (!raw) {
         for (const pkValues of updatePkValues) {
-          newData.push(
+          const oldRecord = await this.readByPk(pkValues);
-            await this.readByPk(pkValues, false, {}, { ignoreView: true }),
+          if (!oldRecord && throwExceptionIfNotExist)
-          );
+            NcError.unprocessableEntity(
+              `Record with pk ${JSON.stringify(pkValues)} not found`,
+            );
+          newData.push(oldRecord);
         }
       }
 
@@ -2553,7 +2571,13 @@ class BaseModelSqlv2 {
     }
   }
 
-  async bulkDelete(ids: any[], { cookie }: { cookie?: any } = {}) {
+  async bulkDelete(
+    ids: any[],
+    {
+      cookie,
+      throwExceptionIfNotExist = false,
+    }: { cookie?: any; throwExceptionIfNotExist?: boolean } = {},
+  ) {
     let transaction;
     try {
       const deleteIds = await Promise.all(
@@ -2570,9 +2594,14 @@ class BaseModelSqlv2 {
           // pk not specified - bypass
           continue;
         }
-        deleted.push(
+
-          await this.readByPk(pkValues, false, {}, { ignoreView: true }),
+        const oldRecord = await this.readByPk(pkValues);
-        );
+        if (!oldRecord && throwExceptionIfNotExist)
+          NcError.unprocessableEntity(
+            `Record with pk ${JSON.stringify(pkValues)} not found`,
+          );
+        deleted.push(oldRecord);
+
         res.push(d);
       }
 
@@ -2635,12 +2664,6 @@ class BaseModelSqlv2 {
 
       transaction = await this.dbDriver.transaction();
 
-      if (base.is_meta && execQueries.length > 0) {
-        for (const execQuery of execQueries) {
-          await execQuery(transaction, idsVals);
-        }
-      }
-
       for (const d of res) {
         await transaction(this.tnPath).del().where(d);
       }
@@ -2685,8 +2708,8 @@ class BaseModelSqlv2 {
         qb,
         this.dbDriver,
       );
-
       const execQueries: ((trx: Transaction, qb: any) => Promise<any>)[] = [];
+      // qb.del();
 
       for (const column of this.model.columns) {
         if (column.uidt !== UITypes.LinkToAnotherRecord) continue;
@@ -2779,7 +2802,6 @@ class BaseModelSqlv2 {
 
       return count;
     } catch (e) {
-      if (trx) await trx.rollback();
       throw e;
     }
   }
@@ -3715,6 +3737,464 @@ class BaseModelSqlv2 {
     }
     return data;
   }
+
+  async addLinks({
+    cookie,
+    childIds,
+    colId,
+    rowId,
+  }: {
+    cookie: any;
+    childIds: (string | number)[];
+    colId: string;
+    rowId: string;
+  }) {
+    const columns = await this.model.getColumns();
+    const column = columns.find((c) => c.id === colId);
+
+    if (!column || column.uidt !== UITypes.LinkToAnotherRecord)
+      NcError.notFound(`Link column ${colId} not found`);
+
+    const row = await this.dbDriver(this.tnPath)
+      .where(await this._wherePk(rowId))
+      .first();
+
+    // validate rowId
+    if (!row) {
+      NcError.notFound(`Row with id '${rowId}' not found`);
+    }
+
+    if (!childIds.length) return;
+
+    const colOptions = await column.getColOptions<LinkToAnotherRecordColumn>();
+
+    const childColumn = await colOptions.getChildColumn();
+    const parentColumn = await colOptions.getParentColumn();
+    const parentTable = await parentColumn.getModel();
+    const childTable = await childColumn.getModel();
+    await childTable.getColumns();
+    await parentTable.getColumns();
+
+    const childTn = this.getTnPath(childTable);
+    const parentTn = this.getTnPath(parentTable);
+
+    switch (colOptions.type) {
+      case RelationTypes.MANY_TO_MANY:
+        {
+          const vChildCol = await colOptions.getMMChildColumn();
+          const vParentCol = await colOptions.getMMParentColumn();
+          const vTable = await colOptions.getMMModel();
+
+          const vTn = this.getTnPath(vTable);
+
+          let insertData: Record<string, any>[];
+
+          // validate Ids
+          {
+            const childRowsQb = this.dbDriver(parentTn)
+              .select(parentColumn.column_name)
+              .select(`${vTable.table_name}.${vChildCol.column_name}`)
+              .leftJoin(vTn, (qb) => {
+                qb.on(
+                  `${vTable.table_name}.${vParentCol.column_name}`,
+                  `${parentTable.table_name}.${parentColumn.column_name}`,
+                ).andOn(
+                  `${vTable.table_name}.${vChildCol.column_name}`,
+                  row[childColumn.column_name],
+                );
+              });
+            // .where(_wherePk(parentTable.primaryKeys, childId))
+
+            if (parentTable.primaryKeys.length > 1) {
+              childRowsQb.where((qb) => {
+                for (const childId of childIds) {
+                  qb.orWhere(_wherePk(parentTable.primaryKeys, childId));
+                }
+              });
+            } else {
+              childRowsQb.whereIn(parentTable.primaryKey.column_name, childIds);
+            }
+
+            if (parentTable.primaryKey.column_name !== parentColumn.column_name)
+              childRowsQb.select(parentTable.primaryKey.column_name);
+
+            const childRows = await childRowsQb;
+
+            if (childRows.length !== childIds.length) {
+              const missingIds = childIds.filter(
+                (id) =>
+                  !childRows.find((r) => r[parentColumn.column_name] === id),
+              );
+
+              NcError.unprocessableEntity(
+                `Child record with id [${missingIds.join(', ')}] not found`,
+              );
+            }
+
+            insertData = childRows
+              // skip existing links
+              .filter((childRow) => !childRow[vChildCol.column_name])
+              // generate insert data for new links
+              .map((childRow) => ({
+                [vParentCol.column_name]: childRow[parentColumn.column_name],
+                [vChildCol.column_name]: row[childColumn.column_name],
+              }));
+
+            // if no new links, return true
+            if (!insertData.length) return true;
+          }
+
+          // if (this.isSnowflake) {
+          //   const parentPK = this.dbDriver(parentTn)
+          //     .select(parentColumn.column_name)
+          //     // .where(_wherePk(parentTable.primaryKeys, childId))
+          //     .whereIn(parentTable.primaryKey.column_name, childIds)
+          //     .first();
+          //
+          //   const childPK = this.dbDriver(childTn)
+          //     .select(childColumn.column_name)
+          //     .where(_wherePk(childTable.primaryKeys, rowId))
+          //     .first();
+          //
+          //   await this.dbDriver.raw(
+          //     `INSERT INTO ?? (??, ??) SELECT (${parentPK.toQuery()}), (${childPK.toQuery()})`,
+          //     [vTn, vParentCol.column_name, vChildCol.column_name],
+          //   );
+          // } else {
+          //   await this.dbDriver(vTn).insert({
+          //     [vParentCol.column_name]: this.dbDriver(parentTn)
+          //       .select(parentColumn.column_name)
+          //       // .where(_wherePk(parentTable.primaryKeys, childId))
+          //       .where(parentTable.primaryKey.column_name, childIds)
+          //       .first(),
+          //     [vChildCol.column_name]: this.dbDriver(childTn)
+          //       .select(childColumn.column_name)
+          //       .where(_wherePk(childTable.primaryKeys, rowId))
+          //       .first(),
+          //   });
+
+          // todo: use bulk insert
+          await this.dbDriver(vTn).insert(insertData);
+          // }
+        }
+        break;
+      case RelationTypes.HAS_MANY:
+        {
+          // validate Ids
+          {
+            const childRowsQb = this.dbDriver(childTn).select(
+              childTable.primaryKey.column_name,
+            );
+
+            if (childTable.primaryKeys.length > 1) {
+              childRowsQb.where((qb) => {
+                for (const childId of childIds) {
+                  qb.orWhere(_wherePk(childTable.primaryKeys, childId));
+                }
+              });
+            } else {
+              childRowsQb.whereIn(parentTable.primaryKey.column_name, childIds);
+            }
+
+            const childRows = await childRowsQb;
+
+            if (childRows.length !== childIds.length) {
+              const missingIds = childIds.filter(
+                (id) =>
+                  !childRows.find((r) => r[parentColumn.column_name] === id),
+              );
+
+              NcError.unprocessableEntity(
+                `Child record with id [${missingIds.join(', ')}] not found`,
+              );
+            }
+          }
+
+          await this.dbDriver(childTn)
+            .update({
+              [childColumn.column_name]: this.dbDriver.from(
+                this.dbDriver(parentTn)
+                  .select(parentColumn.column_name)
+                  .where(_wherePk(parentTable.primaryKeys, rowId))
+                  .first()
+                  .as('___cn_alias'),
+              ),
+            })
+            // .where(_wherePk(childTable.primaryKeys, childId));
+            .whereIn(childTable.primaryKey.column_name, childIds);
+        }
+        break;
+      case RelationTypes.BELONGS_TO:
+        {
+          // validate Ids
+          {
+            const childRowsQb = this.dbDriver(parentTn)
+              .select(parentTable.primaryKey.column_name)
+              .where(_wherePk(parentTable.primaryKeys, childIds[0]))
+              .first();
+
+            const childRow = await childRowsQb;
+
+            if (!childRow) {
+              NcError.unprocessableEntity(
+                `Child record with id [${childIds[0]}] not found`,
+              );
+            }
+          }
+
+          await this.dbDriver(childTn)
+            .update({
+              [childColumn.column_name]: this.dbDriver.from(
+                this.dbDriver(parentTn)
+                  .select(parentColumn.column_name)
+                  .where(_wherePk(parentTable.primaryKeys, childIds[0]))
+                  // .whereIn(parentTable.primaryKey.column_name, childIds)
+                  .first()
+                  .as('___cn_alias'),
+              ),
+            })
+            .where(_wherePk(childTable.primaryKeys, rowId));
+        }
+        break;
+    }
+
+    // const response = await this.readByPk(rowId);
+    // await this.afterInsert(response, this.dbDriver, cookie);
+    // await this.afterAddChild(rowId, childId, cookie);
+  }
+
+  async removeLinks({
+    cookie,
+    childIds,
+    colId,
+    rowId,
+  }: {
+    cookie: any;
+    childIds: (string | number)[];
+    colId: string;
+    rowId: string;
+  }) {
+    const columns = await this.model.getColumns();
+    const column = columns.find((c) => c.id === colId);
+
+    if (!column || column.uidt !== UITypes.LinkToAnotherRecord)
+      NcError.notFound(`Link column ${colId} not found`);
+
+    const row = await this.dbDriver(this.tnPath)
+      .where(await this._wherePk(rowId))
+      .first();
+
+    // validate rowId
+    if (!row) {
+      NcError.notFound(`Row with id '${rowId}' not found`);
+    }
+
+    if (!childIds.length) return;
+
+    const colOptions = await column.getColOptions<LinkToAnotherRecordColumn>();
+
+    const childColumn = await colOptions.getChildColumn();
+    const parentColumn = await colOptions.getParentColumn();
+    const parentTable = await parentColumn.getModel();
+    const childTable = await childColumn.getModel();
+    await childTable.getColumns();
+    await parentTable.getColumns();
+
+    const childTn = this.getTnPath(childTable);
+    const parentTn = this.getTnPath(parentTable);
+
+    const prevData = await this.readByPk(rowId);
+
+    switch (colOptions.type) {
+      case RelationTypes.MANY_TO_MANY:
+        {
+          const vChildCol = await colOptions.getMMChildColumn();
+          const vParentCol = await colOptions.getMMParentColumn();
+          const vTable = await colOptions.getMMModel();
+
+          // validate Ids
+          {
+            const childRowsQb = this.dbDriver(parentTn)
+              .select(parentColumn.column_name)
+              // .where(_wherePk(parentTable.primaryKeys, childId))
+              .whereIn(parentTable.primaryKey.column_name, childIds);
+
+            if (parentTable.primaryKey.column_name !== parentColumn.column_name)
+              childRowsQb.select(parentTable.primaryKey.column_name);
+
+            const childRows = await childRowsQb;
+
+            if (childRows.length !== childIds.length) {
+              const missingIds = childIds.filter(
+                (id) =>
+                  !childRows.find((r) => r[parentColumn.column_name] === id),
+              );
+
+              NcError.unprocessableEntity(
+                `Child record with id [${missingIds.join(', ')}] not found`,
+              );
+            }
+          }
+
+          const vTn = this.getTnPath(vTable);
+
+          await this.dbDriver(vTn)
+            .where({
+              [vChildCol.column_name]: this.dbDriver(childTn)
+                .select(childColumn.column_name)
+                .where(_wherePk(childTable.primaryKeys, rowId))
+                .first(),
+            })
+            .whereIn(
+              [vParentCol.column_name],
+              this.dbDriver(parentTn)
+                .select(parentColumn.column_name)
+                .whereIn(parentTable.primaryKey.column_name, childIds),
+            )
+            .delete();
+        }
+        break;
+      case RelationTypes.HAS_MANY:
+        {
+          // validate Ids
+          {
+            const childRowsQb = this.dbDriver(childTn)
+              .select(childTable.primaryKey.column_name)
+              .whereIn(childTable.primaryKey.column_name, childIds);
+
+            const childRows = await childRowsQb;
+
+            if (childRows.length !== childIds.length) {
+              const missingIds = childIds.filter(
+                (id) =>
+                  !childRows.find((r) => r[parentColumn.column_name] === id),
+              );
+
+              NcError.unprocessableEntity(
+                `Child record with id [${missingIds.join(', ')}] not found`,
+              );
+            }
+          }
+
+          await this.dbDriver(childTn)
+            // .where({
+            //   [childColumn.cn]: this.dbDriver(parentTable.tn)
+            //     .select(parentColumn.cn)
+            //     .where(parentTable.primaryKey.cn, rowId)
+            //     .first()
+            // })
+            // .where(_wherePk(childTable.primaryKeys, childId))
+            .whereIn(childTable.primaryKey.column_name, childIds)
+            .update({ [childColumn.column_name]: null });
+        }
+        break;
+      case RelationTypes.BELONGS_TO:
+        {
+          // validate Ids
+          {
+            if (childIds.length > 1)
+              NcError.unprocessableEntity(
+                'Request must contain only one parent id',
+              );
+
+            const childRowsQb = this.dbDriver(parentTn)
+              .select(parentTable.primaryKey.column_name)
+              .where(_wherePk(parentTable.primaryKeys, childIds[0]))
+              .first();
+
+            const childRow = await childRowsQb;
+
+            if (!childRow) {
+              NcError.unprocessableEntity(
+                `Child record with id [${childIds[0]}] not found`,
+              );
+            }
+          }
+
+          await this.dbDriver(childTn)
+            // .where({
+            //   [childColumn.cn]: this.dbDriver(parentTable.tn)
+            //     .select(parentColumn.cn)
+            //     .where(parentTable.primaryKey.cn, childId)
+            //     .first()
+            // })
+            // .where(_wherePk(childTable.primaryKeys, rowId))
+            .where(childTable.primaryKey.column_name, rowId)
+            .update({ [childColumn.column_name]: null });
+        }
+        break;
+    }
+
+    // const newData = await this.readByPk(rowId);
+    // await this.afterUpdate(prevData, newData, this.dbDriver, cookie);
+    // await this.afterRemoveChild(rowId, childIds, cookie);
+  }
+
+  async btRead(
+    { colId, id }: { colId; id },
+    args: { limit?; offset?; fieldSet?: Set<string> } = {},
+  ) {
+    try {
+      const { where, sort } = this._getListArgs(args as any);
+      // todo: get only required fields
+
+      const relColumn = (await this.model.getColumns()).find(
+        (c) => c.id === colId,
+      );
+
+      const row = await this.dbDriver(this.tnPath)
+        .where(await this._wherePk(id))
+        .first();
+
+      // validate rowId
+      if (!row) {
+        NcError.notFound(`Row with id ${id} not found`);
+      }
+
+      const parentCol = await (
+        (await relColumn.getColOptions()) as LinkToAnotherRecordColumn
+      ).getParentColumn();
+      const parentTable = await parentCol.getModel();
+      const chilCol = await (
+        (await relColumn.getColOptions()) as LinkToAnotherRecordColumn
+      ).getChildColumn();
+      const childTable = await chilCol.getModel();
+
+      const parentModel = await Model.getBaseModelSQL({
+        model: parentTable,
+        dbDriver: this.dbDriver,
+      });
+      await childTable.getColumns();
+
+      const childTn = this.getTnPath(childTable);
+      const parentTn = this.getTnPath(parentTable);
+
+      const qb = this.dbDriver(parentTn);
+      await this.applySortAndFilter({ table: parentTable, where, qb, sort });
+
+      qb.where(
+        parentCol.column_name,
+        this.dbDriver(childTn)
+          .select(chilCol.column_name)
+          // .where(parentTable.primaryKey.cn, p)
+          .where(_wherePk(childTable.primaryKeys, id)),
+      );
+
+      await parentModel.selectObject({ qb, fieldsSet: args.fieldSet });
+
+      const parent = (await this.execAndParse(qb, childTable))?.[0];
+
+      const proto = await parentModel.getProto();
+
+      if (parent) {
+        parent.__proto__ = proto;
+      }
+      return parent;
+    } catch (e) {
+      console.log(e);
+      throw e;
+    }
+  }
 }
 
 function extractSortsObject(

packages/nocodb/src/db/sql-data-mapper/lib/BaseModel.ts

@@ -1503,6 +1503,7 @@ abstract class BaseModel {
 
 export interface XcFilter {
   where?: string;
+  filter?: string;
   having?: string;
   condition?: any;
   conditionGraph?: any;

packages/nocodb/src/filters/global-exception/global-exception.filter.ts

@@ -8,6 +8,7 @@ import {
   NotFound,
   NotImplemented,
   Unauthorized,
+  UnprocessableEntity,
 } from '../../helpers/catchError';
 import type { ArgumentsHost, ExceptionFilter } from '@nestjs/common';
 import type { Response } from 'express';
@@ -15,6 +16,7 @@ import type { Response } from 'express';
 @Catch()
 export class GlobalExceptionFilter implements ExceptionFilter {
   private logger = new Logger(GlobalExceptionFilter.name);
+
   catch(exception: any, host: ArgumentsHost) {
     const ctx = host.switchToHttp();
     const response = ctx.getResponse<Response>();
@@ -58,6 +60,8 @@ export class GlobalExceptionFilter implements ExceptionFilter {
       return response
         .status(400)
         .json({ msg: exception.message, errors: exception.errors });
+    } else if (exception instanceof UnprocessableEntity) {
+      return response.status(422).json({ msg: exception.message });
     }
 
     // handle different types of exceptions

packages/nocodb/src/helpers/PagedResponse.ts

@@ -1,11 +1,6 @@
+import { extractLimitAndOffset } from '.';
 import type { PaginatedType } from 'nocodb-sdk';
 
-const config: any = {
-  limitDefault: Math.max(+process.env.DB_QUERY_LIMIT_DEFAULT || 25, 1),
-  limitMin: Math.max(+process.env.DB_QUERY_LIMIT_MIN || 1, 1),
-  limitMax: Math.max(+process.env.DB_QUERY_LIMIT_MAX || 1000, 1),
-};
-
 export class PagedResponseImpl<T> {
   constructor(
     list: T[],
@@ -17,12 +12,7 @@ export class PagedResponseImpl<T> {
       o?: number;
     } = {},
   ) {
-    const limit = Math.max(
+    const { offset, limit } = extractLimitAndOffset(args);
-      Math.min(args.limit || args.l || config.limitDefault, config.limitMax),
-      config.limitMin,
-    );
-
-    const offset = Math.max(+(args.offset || args.o) || 0, 0);
 
     let count = args.count ?? null;
 
@@ -40,8 +30,17 @@ export class PagedResponseImpl<T> {
         this.pageInfo.page ===
         (Math.ceil(this.pageInfo.totalRows / this.pageInfo.pageSize) || 1);
     }
+
+    if (offset && offset >= count) {
+      this.errors = [
+        {
+          message: 'Offset is beyond the total number of rows',
+        },
+      ];
+    }
   }
 
   list: Array<T>;
   pageInfo: PaginatedType;
+  errors?: any[];
 }

packages/nocodb/src/helpers/catchError.ts

@@ -413,6 +413,8 @@ export default function (
         return res.status(501).json({ msg: e.message });
       } else if (e instanceof AjvError) {
         return res.status(400).json({ msg: e.message, errors: e.errors });
+      } else if (e instanceof UnprocessableEntity) {
+        return res.status(422).json({ msg: e.message });
       }
       next(e);
     }
@@ -431,6 +433,8 @@ export class InternalServerError extends Error {}
 
 export class NotImplemented extends Error {}
 
+export class UnprocessableEntity extends Error {}
+
 export class AjvError extends Error {
   constructor(param: { message: string; errors: ErrorObject[] }) {
     super(param.message);
@@ -468,4 +472,8 @@ export class NcError {
   static ajvValidationError(param: { message: string; errors: ErrorObject[] }) {
     throw new AjvError(param);
   }
+
+  static unprocessableEntity(message = 'Unprocessable entity') {
+    throw new UnprocessableEntity(message);
+  }
 }

packages/nocodb/src/helpers/extractLimitAndOffset.ts

@@ -0,0 +1,39 @@
+const config = {
+  limitDefault: Math.max(+process.env.DB_QUERY_LIMIT_DEFAULT || 25, 1),
+  limitMin: Math.max(+process.env.DB_QUERY_LIMIT_MIN || 1, 1),
+  limitMax: Math.max(+process.env.DB_QUERY_LIMIT_MAX || 1000, 1),
+};
+
+export function extractLimitAndOffset(
+  args: {
+    limit?: number | string;
+    offset?: number | string;
+    l?: number | string;
+    o?: number | string;
+  } = {},
+) {
+  const obj: {
+    limit?: number;
+    offset?: number;
+  } = {};
+
+  // use default value if invalid limit
+  // for example, if limit is not a number, it will be ignored
+  // if limit is less than 1, it will be ignored
+  const limit = +(args.limit || args.l);
+  obj.limit = Math.max(
+    Math.min(
+      limit && limit > 0 && Number.isInteger(limit)
+        ? limit
+        : config.limitDefault,
+      config.limitMax,
+    ),
+    config.limitMin,
+  );
+
+  // skip any invalid offset, ignore negative and non-integer values
+  const offset = +(args.offset || args.o) || 0;
+  obj.offset = Math.max(Number.isInteger(offset) ? offset : 0, 0);
+
+  return obj;
+}

packages/nocodb/src/helpers/index.ts

@@ -2,5 +2,6 @@ import { populateMeta } from './populateMeta';
 export * from './columnHelpers';
 export * from './apiHelpers';
 export * from './cacheHelpers';
+export * from './extractLimitAndOffset';
 
 export { populateMeta };

packages/nocodb/src/modules/datas/datas.module.ts

@@ -3,8 +3,10 @@ import { MulterModule } from '@nestjs/platform-express';
 import multer from 'multer';
 import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants';
 import { DataAliasController } from '../../controllers/data-alias.controller';
+import { DataTableController } from '../../controllers/data-table.controller';
 import { PublicDatasExportController } from '../../controllers/public-datas-export.controller';
 import { PublicDatasController } from '../../controllers/public-datas.controller';
+import { DataTableService } from '../../services/data-table.service';
 import { DatasService } from '../../services/datas.service';
 import { DatasController } from '../../controllers/datas.controller';
 import { BulkDataAliasController } from '../../controllers/bulk-data-alias.controller';
@@ -29,6 +31,7 @@ import { PublicDatasService } from '../../services/public-datas.service';
   controllers: [
     ...(process.env.NC_WORKER_CONTAINER !== 'true'
       ? [
+          DataTableController,
           DatasController,
           BulkDataAliasController,
           DataAliasController,
@@ -41,6 +44,7 @@ import { PublicDatasService } from '../../services/public-datas.service';
       : []),
   ],
   providers: [
+    DataTableService,
     DatasService,
     BulkDataAliasService,
     DataAliasNestedService,

packages/nocodb/src/services/data-table.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { DataTableService } from './data-table.service';
+
+describe('DataTableService', () => {
+  let service: DataTableService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [DataTableService],
+    }).compile();
+
+    service = module.get<DataTableService>(DataTableService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

packages/nocodb/src/services/data-table.service.ts

@@ -0,0 +1,434 @@
+import { Injectable } from '@nestjs/common';
+import { RelationTypes, UITypes } from 'nocodb-sdk';
+import { nocoExecute } from 'nc-help';
+import { NcError } from '../helpers/catchError';
+import getAst from '../helpers/getAst';
+import { PagedResponseImpl } from '../helpers/PagedResponse';
+import { Base, Column, Model, View } from '../models';
+import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2';
+import { DatasService } from './datas.service';
+import type { LinkToAnotherRecordColumn } from '../models';
+
+@Injectable()
+export class DataTableService {
+  constructor(private datasService: DatasService) {}
+
+  async dataList(param: {
+    projectId?: string;
+    modelId: string;
+    query: any;
+    viewId?: string;
+  }) {
+    const { model, view } = await this.getModelAndView(param);
+
+    return await this.datasService.getDataList({
+      model,
+      view,
+      query: param.query,
+    });
+  }
+
+  async dataRead(param: {
+    projectId?: string;
+    modelId: string;
+    rowId: string;
+    viewId?: string;
+    query: any;
+  }) {
+    const { model, view } = await this.getModelAndView(param);
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const row = await baseModel.readByPk(param.rowId, false, param.query);
+
+    if (!row) {
+      NcError.notFound('Row not found');
+    }
+
+    return row;
+  }
+
+  async dataInsert(param: {
+    projectId?: string;
+    viewId?: string;
+    modelId: string;
+    body: any;
+    cookie: any;
+  }) {
+    const { model, view } = await this.getModelAndView(param);
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    // if array then do bulk insert
+    const result = await baseModel.bulkInsert(
+      Array.isArray(param.body) ? param.body : [param.body],
+      { cookie: param.cookie, insertOneByOneAsFallback: true },
+    );
+
+    return Array.isArray(param.body) ? result : result[0];
+  }
+
+  async dataUpdate(param: {
+    projectId?: string;
+    modelId: string;
+    viewId?: string;
+    // rowId: string;
+    body: any;
+    cookie: any;
+  }) {
+    const { model, view } = await this.getModelAndView(param);
+
+    await this.checkForDuplicateRow({ rows: param.body, model });
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const res = await baseModel.bulkUpdate(
+      Array.isArray(param.body) ? param.body : [param.body],
+      { cookie: param.cookie, throwExceptionIfNotExist: true },
+    );
+
+    return this.extractIdObj({ body: param.body, model });
+  }
+
+  async dataDelete(param: {
+    projectId?: string;
+    modelId: string;
+    viewId?: string;
+    // rowId: string;
+    cookie: any;
+    body: any;
+  }) {
+    const { model, view } = await this.getModelAndView(param);
+
+    await this.checkForDuplicateRow({ rows: param.body, model });
+
+    const base = await Base.get(model.base_id);
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    await baseModel.bulkDelete(
+      Array.isArray(param.body) ? param.body : [param.body],
+      { cookie: param.cookie, throwExceptionIfNotExist: true },
+    );
+
+    return this.extractIdObj({ body: param.body, model });
+  }
+
+  async dataCount(param: {
+    projectId?: string;
+    viewId?: string;
+    modelId: string;
+    query: any;
+  }) {
+    const { model, view } = await this.getModelAndView(param);
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const countArgs: any = { ...param.query };
+    try {
+      countArgs.filterArr = JSON.parse(countArgs.filterArrJson);
+    } catch (e) {}
+
+    const count: number = await baseModel.count(countArgs);
+
+    return { count };
+  }
+
+  private async getModelAndView(param: {
+    projectId?: string;
+    viewId?: string;
+    modelId: string;
+  }) {
+    const model = await Model.get(param.modelId);
+
+    if (!model) {
+      NcError.notFound(`Table with id '${param.modelId}' not found`);
+    }
+
+    if (param.projectId && model.project_id !== param.projectId) {
+      throw new Error('Table not belong to project');
+    }
+
+    let view: View;
+
+    if (param.viewId) {
+      view = await View.get(param.viewId);
+      if (!view || (view.fk_model_id && view.fk_model_id !== param.modelId)) {
+        NcError.unprocessableEntity(`View with id '${param.viewId}' not found`);
+      }
+    }
+
+    return { model, view };
+  }
+
+  private async extractIdObj({
+    model,
+    body,
+  }: {
+    body: Record<string, any> | Record<string, any>[];
+    model: Model;
+  }) {
+    const pkColumns = await model
+      .getColumns()
+      .then((cols) => cols.filter((col) => col.pk));
+
+    const result = (Array.isArray(body) ? body : [body]).map((row) => {
+      return pkColumns.reduce((acc, col) => {
+        acc[col.title] = row[col.title];
+        return acc;
+      }, {});
+    });
+
+    return Array.isArray(body) ? result : result[0];
+  }
+
+  private async checkForDuplicateRow({
+    rows,
+    model,
+  }: {
+    rows: any[] | any;
+    model: Model;
+  }) {
+    if (!rows || !Array.isArray(rows) || rows.length === 1) {
+      return;
+    }
+
+    await model.getColumns();
+
+    const keys = new Set();
+
+    for (const row of rows) {
+      let pk;
+      // if only one primary key then extract the value
+      if (model.primaryKeys.length === 1) pk = row[model.primaryKey.title];
+      // if composite primary key then join the values with ___
+      else pk = model.primaryKeys.map((pk) => row[pk.title]).join('___');
+      // if duplicate then throw error
+      if (keys.has(pk)) {
+        NcError.unprocessableEntity('Duplicate row with id ' + pk);
+      }
+      keys.add(pk);
+    }
+  }
+
+  async nestedDataList(param: {
+    viewId: string;
+    modelId: string;
+    query: any;
+    rowId: string | string[] | number | number[];
+    columnId: string;
+  }) {
+    const { model, view } = await this.getModelAndView(param);
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    if (!(await baseModel.exist(param.rowId))) {
+      NcError.notFound(`Row with id '${param.rowId}' not found`);
+    }
+
+    const column = await this.getColumn(param);
+
+    const colOptions = await column.getColOptions<LinkToAnotherRecordColumn>();
+
+    const relatedModel = await colOptions.getRelatedTable();
+
+    const { ast, dependencyFields } = await getAst({
+      model: relatedModel,
+      query: param.query,
+      extractOnlyPrimaries: !(param.query?.f || param.query?.fields),
+    });
+
+    const listArgs: any = dependencyFields;
+    try {
+      listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
+    } catch (e) {}
+    try {
+      listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
+    } catch (e) {}
+
+    let data: any[];
+    let count: number;
+    if (colOptions.type === RelationTypes.MANY_TO_MANY) {
+      data = await baseModel.mmList(
+        {
+          colId: column.id,
+          parentId: param.rowId,
+        },
+        listArgs as any,
+      );
+      count = (await baseModel.mmListCount({
+        colId: column.id,
+        parentId: param.rowId,
+      })) as number;
+    } else if (colOptions.type === RelationTypes.HAS_MANY) {
+      data = await baseModel.hmList(
+        {
+          colId: column.id,
+          id: param.rowId,
+        },
+        listArgs as any,
+      );
+      count = (await baseModel.hmListCount({
+        colId: column.id,
+        id: param.rowId,
+      })) as number;
+    } else {
+      data = await baseModel.btRead(
+        {
+          colId: column.id,
+          id: param.rowId,
+        },
+        param.query as any,
+      );
+    }
+
+    data = await nocoExecute(ast, data, {}, listArgs);
+
+    if (colOptions.type === RelationTypes.BELONGS_TO) return data;
+
+    return new PagedResponseImpl(data, {
+      count,
+      ...param.query,
+    });
+  }
+
+  private async getColumn(param: { modelId: string; columnId: string }) {
+    const column = await Column.get({ colId: param.columnId });
+
+    if (!column)
+      NcError.notFound(`Column with id '${param.columnId}' not found`);
+
+    if (column.fk_model_id !== param.modelId)
+      NcError.badRequest('Column not belong to model');
+
+    if (column.uidt !== UITypes.LinkToAnotherRecord)
+      NcError.badRequest('Column is not LTAR');
+    return column;
+  }
+
+  async nestedLink(param: {
+    cookie: any;
+    viewId: string;
+    modelId: string;
+    columnId: string;
+    query: any;
+    refRowIds: string | string[] | number | number[];
+    rowId: string;
+  }) {
+    this.validateIds(param.refRowIds);
+
+    const { model, view } = await this.getModelAndView(param);
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const column = await this.getColumn(param);
+
+    await baseModel.addLinks({
+      colId: column.id,
+      childIds: Array.isArray(param.refRowIds)
+        ? param.refRowIds
+        : [param.refRowIds],
+      rowId: param.rowId,
+      cookie: param.cookie,
+    });
+
+    return true;
+  }
+
+  async nestedUnlink(param: {
+    cookie: any;
+    viewId: string;
+    modelId: string;
+    columnId: string;
+    query: any;
+    refRowIds: string | string[] | number | number[];
+    rowId: string;
+  }) {
+    this.validateIds(param.refRowIds);
+
+    const { model, view } = await this.getModelAndView(param);
+    if (!model)
+      NcError.notFound('Table with id ' + param.modelId + ' not found');
+
+    const base = await Base.get(model.base_id);
+
+    const baseModel = await Model.getBaseModelSQL({
+      id: model.id,
+      viewId: view?.id,
+      dbDriver: await NcConnectionMgrv2.get(base),
+    });
+
+    const column = await this.getColumn(param);
+
+    await baseModel.removeLinks({
+      colId: column.id,
+      childIds: Array.isArray(param.refRowIds)
+        ? param.refRowIds
+        : [param.refRowIds],
+      rowId: param.rowId,
+      cookie: param.cookie,
+    });
+
+    return true;
+  }
+
+  private validateIds(rowIds: any[] | any) {
+    if (Array.isArray(rowIds)) {
+      const map = new Map<string, boolean>();
+      const set = new Set<string>();
+      for (const rowId of rowIds) {
+        if (rowId === undefined || rowId === null)
+          NcError.unprocessableEntity('Invalid row id ' + rowId);
+        if (map.has(rowId)) {
+          set.add(rowId);
+        } else {
+          map.set(rowId, true);
+        }
+      }
+
+      if (set.size > 0)
+        NcError.unprocessableEntity(
+          'Child record with id [' + [...set].join(', ') + '] are duplicated',
+        );
+    } else if (rowIds === undefined || rowIds === null) {
+      NcError.unprocessableEntity('Invalid row id ' + rowIds);
+    }
+  }
+}

packages/nocodb/src/services/datas.service.ts

@@ -117,7 +117,7 @@ export class DatasService {
 
   async getDataList(param: {
     model: Model;
-    view: View;
+    view?: View;
     query: any;
     baseModel?: BaseModelSqlv2;
   }) {

packages/nocodb/tests/unit/factory/column.ts

@@ -1,13 +1,13 @@
 import { UITypes } from 'nocodb-sdk';
 import request from 'supertest';
-import Column from '../../../src/models/Column';
-import FormViewColumn from '../../../src/models/FormViewColumn';
-import GalleryViewColumn from '../../../src/models/GalleryViewColumn';
-import GridViewColumn from '../../../src/models/GridViewColumn';
 import Model from '../../../src/models/Model';
-import Project from '../../../src/models/Project';
+import { isPg, isSqlite } from '../init/db';
-import View from '../../../src/models/View';
+import type Column from '../../../src/models/Column';
-import { isSqlite, isPg } from '../init/db';
+import type FormViewColumn from '../../../src/models/FormViewColumn';
+import type GalleryViewColumn from '../../../src/models/GalleryViewColumn';
+import type GridViewColumn from '../../../src/models/GridViewColumn';
+import type Project from '../../../src/models/Project';
+import type View from '../../../src/models/View';
 
 const defaultColumns = function (context) {
   return [
@@ -46,6 +46,122 @@ const defaultColumns = function (context) {
   ];
 };
 
+const customColumns = function (type: string, options: any = {}) {
+  switch (type) {
+    case 'textBased':
+      return [
+        {
+          column_name: 'Id',
+          title: 'Id',
+          uidt: UITypes.ID,
+        },
+        {
+          column_name: 'SingleLineText',
+          title: 'SingleLineText',
+          uidt: UITypes.SingleLineText,
+        },
+        {
+          column_name: 'MultiLineText',
+          title: 'MultiLineText',
+          uidt: UITypes.LongText,
+        },
+        {
+          column_name: 'Email',
+          title: 'Email',
+          uidt: UITypes.Email,
+        },
+        {
+          column_name: 'Phone',
+          title: 'Phone',
+          uidt: UITypes.PhoneNumber,
+        },
+        {
+          column_name: 'Url',
+          title: 'Url',
+          uidt: UITypes.URL,
+        },
+      ];
+    case 'numberBased':
+      return [
+        {
+          column_name: 'Id',
+          title: 'Id',
+          uidt: UITypes.ID,
+        },
+        {
+          column_name: 'Number',
+          title: 'Number',
+          uidt: UITypes.Number,
+        },
+        {
+          column_name: 'Decimal',
+          title: 'Decimal',
+          uidt: UITypes.Decimal,
+        },
+        {
+          column_name: 'Currency',
+          title: 'Currency',
+          uidt: UITypes.Currency,
+        },
+        {
+          column_name: 'Percent',
+          title: 'Percent',
+          uidt: UITypes.Percent,
+        },
+        {
+          column_name: 'Duration',
+          title: 'Duration',
+          uidt: UITypes.Duration,
+        },
+        {
+          column_name: 'Rating',
+          title: 'Rating',
+          uidt: UITypes.Rating,
+        },
+      ];
+    case 'dateBased':
+      return [
+        {
+          column_name: 'Id',
+          title: 'Id',
+          uidt: UITypes.ID,
+        },
+        {
+          column_name: 'Date',
+          title: 'Date',
+          uidt: UITypes.Date,
+        },
+        {
+          column_name: 'DateTime',
+          title: 'DateTime',
+          uidt: UITypes.DateTime,
+        },
+      ];
+    case 'selectBased':
+      return [
+        {
+          column_name: 'Id',
+          title: 'Id',
+          uidt: UITypes.ID,
+        },
+        {
+          column_name: 'SingleSelect',
+          title: 'SingleSelect',
+          uidt: UITypes.SingleSelect,
+          dtxp: "'jan','feb','mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'",
+        },
+        {
+          column_name: 'MultiSelect',
+          title: 'MultiSelect',
+          uidt: UITypes.MultiSelect,
+          dtxp: "'jan','feb','mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'",
+        },
+      ];
+    case 'custom':
+      return [{ title: 'Id', column_name: 'Id', uidt: UITypes.ID }, ...options];
+  }
+};
+
 const createColumn = async (context, table, columnAttr) => {
   await request(context.app)
     .post(`/api/v1/db/meta/tables/${table.id}/columns`)
@@ -55,7 +171,7 @@ const createColumn = async (context, table, columnAttr) => {
     });
 
   const column: Column = (await table.getColumns()).find(
-    (column) => column.title === columnAttr.title
+    (column) => column.title === columnAttr.title,
   );
   return column;
 };
@@ -76,7 +192,7 @@ const createRollupColumn = async (
     table: Model;
     relatedTableName: string;
     relatedTableColumnTitle: string;
-  }
+  },
 ) => {
   const childBases = await project.getBases();
   const childTable = await Model.getByIdOrName({
@@ -86,13 +202,13 @@ const createRollupColumn = async (
   });
   const childTableColumns = await childTable.getColumns();
   const childTableColumn = await childTableColumns.find(
-    (column) => column.title === relatedTableColumnTitle
+    (column) => column.title === relatedTableColumnTitle,
   );
 
   const ltarColumn = (await table.getColumns()).find(
     (column) =>
       column.uidt === UITypes.LinkToAnotherRecord &&
-      column.colOptions?.fk_related_model_id === childTable.id
+      column.colOptions?.fk_related_model_id === childTable.id,
   );
 
   const rollupColumn = await createColumn(context, table, {
@@ -122,7 +238,7 @@ const createLookupColumn = async (
     table: Model;
     relatedTableName: string;
     relatedTableColumnTitle: string;
-  }
+  },
 ) => {
   const childBases = await project.getBases();
   const childTable = await Model.getByIdOrName({
@@ -132,19 +248,19 @@ const createLookupColumn = async (
   });
   const childTableColumns = await childTable.getColumns();
   const childTableColumn = await childTableColumns.find(
-    (column) => column.title === relatedTableColumnTitle
+    (column) => column.title === relatedTableColumnTitle,
   );
 
   if (!childTableColumn) {
     throw new Error(
-      `Could not find column ${relatedTableColumnTitle} in ${relatedTableName}`
+      `Could not find column ${relatedTableColumnTitle} in ${relatedTableName}`,
     );
   }
 
   const ltarColumn = (await table.getColumns()).find(
     (column) =>
       column.uidt === UITypes.LinkToAnotherRecord &&
-      column.colOptions?.fk_related_model_id === childTable.id
+      column.colOptions?.fk_related_model_id === childTable.id,
   );
   const lookupColumn = await createColumn(context, table, {
     title: title,
@@ -168,15 +284,15 @@ const createQrCodeColumn = async (
     title: string;
     table: Model;
     referencedQrValueTableColumnTitle: string;
-  }
+  },
 ) => {
   const referencedQrValueTableColumnId = await table
     .getColumns()
     .then(
       (cols) =>
         cols.find(
-          (column) => column.title == referencedQrValueTableColumnTitle
+          (column) => column.title == referencedQrValueTableColumnTitle,
-        )['id']
+        )['id'],
     );
 
   const qrCodeColumn = await createColumn(context, table, {
@@ -198,15 +314,15 @@ const createBarcodeColumn = async (
     title: string;
     table: Model;
     referencedBarcodeValueTableColumnTitle: string;
-  }
+  },
 ) => {
   const referencedBarcodeValueTableColumnId = await table
     .getColumns()
     .then(
       (cols) =>
         cols.find(
-          (column) => column.title == referencedBarcodeValueTableColumnTitle
+          (column) => column.title == referencedBarcodeValueTableColumnTitle,
-        )['id']
+        )['id'],
     );
 
   const barcodeColumn = await createColumn(context, table, {
@@ -230,7 +346,7 @@ const createLtarColumn = async (
     parentTable: Model;
     childTable: Model;
     type: string;
-  }
+  },
 ) => {
   const ltarColumn = await createColumn(context, parentTable, {
     title: title,
@@ -246,7 +362,7 @@ const createLtarColumn = async (
 
 const updateViewColumn = async (
   context,
-  { view, column, attr }: { column: Column; view: View; attr: any }
+  { view, column, attr }: { column: Column; view: View; attr: any },
 ) => {
   const res = await request(context.app)
     .patch(`/api/v1/db/meta/views/${view.id}/columns/${column.id}`)
@@ -263,6 +379,7 @@ const updateViewColumn = async (
 };
 
 export {
+  customColumns,
   defaultColumns,
   createColumn,
   createQrCodeColumn,

packages/nocodb/tests/unit/factory/row.ts

@@ -1,11 +1,12 @@
-import { ColumnType, UITypes } from 'nocodb-sdk';
+import { UITypes } from 'nocodb-sdk';
 import request from 'supertest';
-import Column from '../../../src/models/Column';
-import Filter from '../../../src/models/Filter';
 import Model from '../../../src/models/Model';
-import Project from '../../../src/models/Project';
-import Sort from '../../../src/models/Sort';
 import NcConnectionMgrv2 from '../../../src/utils/common/NcConnectionMgrv2';
+import type { ColumnType } from 'nocodb-sdk';
+import type Column from '../../../src/models/Column';
+import type Filter from '../../../src/models/Filter';
+import type Project from '../../../src/models/Project';
+import type Sort from '../../../src/models/Sort';
 
 const rowValue = (column: ColumnType, index: number) => {
   switch (column.uidt) {
@@ -175,9 +176,17 @@ const rowMixedValue = (column: ColumnType, index: number) => {
     case UITypes.Date:
       // set startDate as 400 days before today
       // eslint-disable-next-line no-case-declarations
-      const result = new Date();
+      const d1 = new Date();
-      result.setDate(result.getDate() - 400 + index);
+      d1.setDate(d1.getDate() - 400 + index);
-      return result.toISOString().slice(0, 10);
+      return d1.toISOString().slice(0, 10);
+    case UITypes.DateTime:
+      // set startDate as 400 days before today
+      // eslint-disable-next-line no-case-declarations
+      const d2 = new Date();
+      d2.setDate(d2.getDate() - 400 + index);
+      // set time to 12:00:00
+      d2.setHours(12, 0, 0, 0);
+      return d2.toISOString();
     case UITypes.URL:
       return urls[index % urls.length];
     case UITypes.SingleSelect:
@@ -228,7 +237,7 @@ const listRow = async ({
 
 const getOneRow = async (
   context,
-  { project, table }: { project: Project; table: Model }
+  { project, table }: { project: Project; table: Model },
 ) => {
   const response = await request(context.app)
     .get(`/api/v1/db/data/noco/${project.id}/${table.id}/find-one`)
@@ -266,7 +275,7 @@ const createRow = async (
     project: Project;
     table: Model;
     index?: number;
-  }
+  },
 ) => {
   const columns = await table.getColumns();
   const rowData = generateDefaultRowAttributes({ columns, index });
@@ -289,7 +298,7 @@ const createBulkRows = async (
     project: Project;
     table: Model;
     values: any[];
-  }
+  },
 ) => {
   await request(context.app)
     .post(`/api/v1/db/data/bulk/noco/${project.id}/${table.id}`)
@@ -317,7 +326,7 @@ const createChildRow = async (
     rowId?: string;
     childRowId?: string;
     type: string;
-  }
+  },
 ) => {
   if (!rowId) {
     const row = await createRow(context, { project, table });
@@ -331,7 +340,7 @@ const createChildRow = async (
 
   await request(context.app)
     .post(
-      `/api/v1/db/data/noco/${project.id}/${table.id}/${rowId}/${type}/${column.title}/${childRowId}`
+      `/api/v1/db/data/noco/${project.id}/${table.id}/${rowId}/${type}/${column.title}/${childRowId}`,
     )
     .set('xc-auth', context.token);
 

packages/nocodb/tests/unit/factory/view.ts

@@ -1,9 +1,20 @@
 import { ViewTypes } from 'nocodb-sdk';
 import request from 'supertest';
-import Model from '../../../src/models/Model';
 import View from '../../../src/models/View';
+import type Model from '../../../src/models/Model';
 
-const createView = async (context, {title, table, type}: {title: string, table: Model, type: ViewTypes}) => {
+const createView = async (
+  context,
+  {
+    title,
+    table,
+    type,
+  }: {
+    title: string;
+    table: Model;
+    type: ViewTypes;
+  },
+) => {
   const viewTypeStr = (type) => {
     switch (type) {
       case ViewTypes.GALLERY:
@@ -26,13 +37,70 @@ const createView = async (context, {title, table, type}: {title: string, table:
       title,
       type,
     });
-  if(response.status !== 200) {
+  if (response.status !== 200) {
-    throw new Error('createView',response.body.message);
+    throw new Error('createView', response.body.message);
   }
 
-  const view = await View.getByTitleOrId({fk_model_id: table.id, titleOrId:title}) as View;
+  const view = (await View.getByTitleOrId({
+    fk_model_id: table.id,
+    titleOrId: title,
+  })) as View;
+  return view;
+};
 
-  return view
+const updateView = async (
-}
+  context,
+  {
+    table,
+    view,
+    filter = [],
+    sort = [],
+    field = [],
+  }: {
+    table: Model;
+    view: View;
+    filter?: any[];
+    sort?: any[];
+    field?: any[];
+  },
+) => {
+  if (filter.length) {
+    for (let i = 0; i < filter.length; i++) {
+      await request(context.app)
+        .post(`/api/v1/db/meta/views/${view.id}/filters`)
+        .set('xc-auth', context.token)
+        .send(filter[i])
+        .expect(200);
+    }
+  }
+
+  if (sort.length) {
+    for (let i = 0; i < sort.length; i++) {
+      await request(context.app)
+        .post(`/api/v1/db/meta/views/${view.id}/sorts`)
+        .set('xc-auth', context.token)
+        .send(sort[i])
+        .expect(200);
+    }
+  }
+
+  if (field.length) {
+    for (let i = 0; i < field.length; i++) {
+      const columns = await table.getColumns();
+      const viewColumns = await view.getColumns();
+
+      const columnId = columns.find((c) => c.title === field[i]).id;
+      const viewColumnId = viewColumns.find(
+        (c) => c.fk_column_id === columnId,
+      ).id;
+      // configure view to hide selected fields
+      await request(context.app)
+        .patch(`/api/v1/db/meta/views/${view.id}/columns/${viewColumnId}`)
+        .set('xc-auth', context.token)
+        .send({ show: false })
+        .expect(200);
+    }
+  }
+};
 
-export {createView}
+export { createView, updateView };

packages/nocodb/tests/unit/init/index.ts

@@ -25,7 +25,7 @@ const serverInit = async () => {
 
 const isFirstTimeRun = () => !server;
 
-export default async function () {
+export default async function (isSakila = true) {
   const { default: TestDbMngr } = await import('../TestDbMngr');
 
   if (isFirstTimeRun()) {
@@ -33,7 +33,10 @@ export default async function () {
     server = await serverInit();
   }
 
-  await cleanUpSakila();
+  if (isSakila) {
+    await cleanUpSakila();
+  }
+
   await cleanupMeta();
 
   const { token } = await createUser({ app: server }, { roles: 'editor' });

packages/nocodb/tests/unit/rest/index.test.ts

@@ -8,6 +8,7 @@ import tableRowTests from './tests/tableRow.test';
 import viewRowTests from './tests/viewRow.test';
 import attachmentTests from './tests/attachment.test';
 import filterTest from './tests/filter.test';
+import newDataApisTest from './tests/newDataApis.test';
 
 function restTests() {
   authTests();
@@ -19,6 +20,7 @@ function restTests() {
   columnTypeSpecificTests();
   attachmentTests();
   filterTest();
+  newDataApisTest();
 }
 
 export default function () {