Browse Source

Merge branch 'develop' into import-at-users-option

pull/6127/head
briankinney 11 months ago
parent
commit
b0d79427e1
  1. 6
      packages/nc-gui/components/smartsheet/Pagination.vue
  2. 4
      packages/nc-gui/components/smartsheet/header/Icon.vue
  3. 2
      packages/nc-gui/components/tabs/Smartsheet.vue
  4. 1
      packages/nc-gui/components/virtual-cell/components/ItemChip.vue
  5. 1
      packages/nc-gui/components/virtual-cell/components/ListChildItems.vue
  6. 1
      packages/nc-gui/components/virtual-cell/components/ListItems.vue
  7. 4
      packages/nc-gui/composables/useMultiSelect/index.ts
  8. 2
      packages/nc-gui/package-lock.json
  9. 2
      packages/nc-lib-gui/package.json
  10. 4
      packages/nocodb-sdk/package-lock.json
  11. 2
      packages/nocodb-sdk/package.json
  12. 20
      packages/nocodb/package-lock.json
  13. 4
      packages/nocodb/package.json
  14. 1
      packages/nocodb/src/app.module.ts
  15. 18
      packages/nocodb/src/controllers/data-table.controller.spec.ts
  16. 189
      packages/nocodb/src/controllers/data-table.controller.ts
  17. 2
      packages/nocodb/src/controllers/test/TestResetService/index.ts
  18. 648
      packages/nocodb/src/db/BaseModelSqlv2.ts
  19. 41
      packages/nocodb/src/db/sql-client/lib/mssql/MssqlClient.ts
  20. 2
      packages/nocodb/src/db/sql-client/lib/oracle/OracleClient.ts
  21. 2
      packages/nocodb/src/db/sql-client/lib/pg/PgClient.ts
  22. 73
      packages/nocodb/src/db/sql-client/lib/snowflake/SnowflakeClient.ts
  23. 1
      packages/nocodb/src/db/sql-client/lib/sqlite/SqliteClient.ts
  24. 44
      packages/nocodb/src/db/sql-data-mapper/lib/BaseModel.ts
  25. 2
      packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts
  26. 1
      packages/nocodb/src/db/sql-migrator/lib/KnexMigratorv2.ts
  27. 4
      packages/nocodb/src/filters/global-exception/global-exception.filter.ts
  28. 23
      packages/nocodb/src/helpers/PagedResponse.ts
  29. 2
      packages/nocodb/src/helpers/apiMetrics.ts
  30. 8
      packages/nocodb/src/helpers/catchError.ts
  31. 39
      packages/nocodb/src/helpers/extractLimitAndOffset.ts
  32. 1
      packages/nocodb/src/helpers/index.ts
  33. 2
      packages/nocodb/src/helpers/webhookHelpers.ts
  34. 2
      packages/nocodb/src/meta/migrations/v1/nc_011_remove_old_ses_plugin.ts
  35. 1
      packages/nocodb/src/models/Model.ts
  36. 1
      packages/nocodb/src/models/View.ts
  37. 4
      packages/nocodb/src/modules/datas/datas.module.ts
  38. 2
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts
  39. 79
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
  40. 1
      packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts
  41. 38
      packages/nocodb/src/plugins/storage/Local.ts
  42. 4
      packages/nocodb/src/services/auth.service.ts
  43. 18
      packages/nocodb/src/services/data-table.service.spec.ts
  44. 434
      packages/nocodb/src/services/data-table.service.ts
  45. 2
      packages/nocodb/src/services/datas.service.ts
  46. 8
      packages/nocodb/src/services/tables.service.ts
  47. 1
      packages/nocodb/src/utils/common/XcAudit.ts
  48. 163
      packages/nocodb/tests/unit/factory/column.ts
  49. 35
      packages/nocodb/tests/unit/factory/row.ts
  50. 84
      packages/nocodb/tests/unit/factory/view.ts
  51. 7
      packages/nocodb/tests/unit/init/index.ts
  52. 2
      packages/nocodb/tests/unit/rest/index.test.ts
  53. 2867
      packages/nocodb/tests/unit/rest/tests/newDataApis.test.ts

6
packages/nc-gui/components/smartsheet/Pagination.vue

@ -2,9 +2,11 @@
import { ChangePageInj, PaginationDataInj, computed, iconMap, inject, isRtlLang, useI18n } from '#imports'
import type { Language } from '~/lib'
const props = defineProps<{
interface Props {
alignCountOnRight?: boolean
}>()
}
const { alignCountOnRight } = defineProps<Props>()
const { locale } = useI18n()

4
packages/nc-gui/components/smartsheet/header/Icon.vue

@ -1,8 +1,8 @@
<script lang="ts" setup>
import { ColumnType, isVirtualCol } from 'nocodb-sdk'
import type { ColumnType } from 'nocodb-sdk'
import { isVirtualCol } from 'nocodb-sdk'
const { column } = defineProps<{ column: ColumnType }>()
</script>
<template>

2
packages/nc-gui/components/tabs/Smartsheet.vue

@ -72,7 +72,7 @@ const onDrop = async (event: DragEvent) => {
event.preventDefault()
try {
// Access the dropped data
const data = JSON.parse(event.dataTransfer?.getData('text/json')!)
const data = JSON.parse(event.dataTransfer!.getData('text/json'))
// Do something with the received data
// if dragged item is not from the same base, return

1
packages/nc-gui/components/virtual-cell/components/ItemChip.vue

@ -9,7 +9,6 @@ import {
inject,
isAttachment,
ref,
renderValue,
useExpandedFormDetached,
useLTARStoreOrThrow,
} from '#imports'

1
packages/nc-gui/components/virtual-cell/components/ListChildItems.vue

@ -13,7 +13,6 @@ import {
iconMap,
inject,
ref,
renderValue,
useLTARStoreOrThrow,
useSmartsheetRowStoreOrThrow,
useVModel,

1
packages/nc-gui/components/virtual-cell/components/ListItems.vue

@ -12,7 +12,6 @@ import {
inject,
isDrawerExist,
ref,
renderValue,
useLTARStoreOrThrow,
useSelectedCellKeyupListener,
useSmartsheetRowStoreOrThrow,

4
packages/nc-gui/composables/useMultiSelect/index.ts

@ -26,7 +26,6 @@ import {
useI18n,
useMetas,
useProject,
useUIPermission,
} from '#imports'
const MAIN_MOUSE_PRESSED = 0
@ -80,9 +79,6 @@ export function useMultiSelect(
() => !(activeCell.row === null || activeCell.col === null || isNaN(activeCell.row) || isNaN(activeCell.col)),
)
const { isUIAllowed } = useUIPermission()
const hasEditPermission = $computed(() => isUIAllowed('xcDatatableEditable'))
function makeActive(row: number, col: number) {
if (activeCell.row === row && activeCell.col === col) {
return

2
packages/nc-gui/package-lock.json generated

@ -110,7 +110,7 @@
}
},
"../nocodb-sdk": {
"version": "0.109.5",
"version": "0.109.6",
"license": "AGPL-3.0-or-later",
"dependencies": {
"axios": "^0.21.1",

2
packages/nc-lib-gui/package.json

@ -1,6 +1,6 @@
{
"name": "nc-lib-gui",
"version": "0.109.5",
"version": "0.109.6",
"description": "NocoDB GUI",
"author": {
"name": "NocoDB",

4
packages/nocodb-sdk/package-lock.json generated

@ -1,12 +1,12 @@
{
"name": "nocodb-sdk",
"version": "0.109.5",
"version": "0.109.6",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "nocodb-sdk",
"version": "0.109.5",
"version": "0.109.6",
"license": "AGPL-3.0-or-later",
"dependencies": {
"axios": "^0.21.1",

2
packages/nocodb-sdk/package.json

@ -1,6 +1,6 @@
{
"name": "nocodb-sdk",
"version": "0.109.5",
"version": "0.109.6",
"description": "NocoDB SDK",
"main": "build/main/index.js",
"typings": "build/main/index.d.ts",

20
packages/nocodb/package-lock.json generated

@ -1,12 +1,12 @@
{
"name": "nocodb",
"version": "0.109.5",
"version": "0.109.6",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "nocodb",
"version": "0.109.5",
"version": "0.109.6",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@google-cloud/storage": "^5.7.2",
@ -80,7 +80,7 @@
"mysql2": "^3.2.0",
"nanoid": "^3.1.20",
"nc-help": "^0.2.87",
"nc-lib-gui": "0.109.5",
"nc-lib-gui": "0.109.6",
"nc-plugin": "^0.1.3",
"ncp": "^2.0.0",
"nocodb-sdk": "file:../nocodb-sdk",
@ -191,7 +191,7 @@
}
},
"../nocodb-sdk": {
"version": "0.109.5",
"version": "0.109.6",
"license": "AGPL-3.0-or-later",
"dependencies": {
"axios": "^0.21.1",
@ -13158,9 +13158,9 @@
}
},
"node_modules/nc-lib-gui": {
"version": "0.109.5",
"resolved": "https://registry.npmjs.org/nc-lib-gui/-/nc-lib-gui-0.109.5.tgz",
"integrity": "sha512-A6za8yFO167OVd2MnGrzEFAOQ5Nx9UJ5zpk1iH7FC427TEHJOAbvlOYMVa3s7Pb8wjYKhZ5Mr4sBldtY0cKaHA==",
"version": "0.109.6",
"resolved": "https://registry.npmjs.org/nc-lib-gui/-/nc-lib-gui-0.109.6.tgz",
"integrity": "sha512-HX0JuimDTgpliJowNDctjHGvsbkO8SZ1dq/i+WEIce1nlgxdBrOM+/79UlGMaeLWMb3RuU7zNLxpxYvvdoFs4w==",
"dependencies": {
"express": "^4.17.1"
}
@ -28474,9 +28474,9 @@
}
},
"nc-lib-gui": {
"version": "0.109.5",
"resolved": "https://registry.npmjs.org/nc-lib-gui/-/nc-lib-gui-0.109.5.tgz",
"integrity": "sha512-A6za8yFO167OVd2MnGrzEFAOQ5Nx9UJ5zpk1iH7FC427TEHJOAbvlOYMVa3s7Pb8wjYKhZ5Mr4sBldtY0cKaHA==",
"version": "0.109.6",
"resolved": "https://registry.npmjs.org/nc-lib-gui/-/nc-lib-gui-0.109.6.tgz",
"integrity": "sha512-HX0JuimDTgpliJowNDctjHGvsbkO8SZ1dq/i+WEIce1nlgxdBrOM+/79UlGMaeLWMb3RuU7zNLxpxYvvdoFs4w==",
"requires": {
"express": "^4.17.1"
}

4
packages/nocodb/package.json

@ -1,6 +1,6 @@
{
"name": "nocodb",
"version": "0.109.5",
"version": "0.109.6",
"description": "NocoDB Backend",
"main": "dist/bundle.js",
"author": {
@ -113,7 +113,7 @@
"mysql2": "^3.2.0",
"nanoid": "^3.1.20",
"nc-help": "^0.2.87",
"nc-lib-gui": "0.109.5",
"nc-lib-gui": "0.109.6",
"nc-plugin": "^0.1.3",
"ncp": "^2.0.0",
"nocodb-sdk": "file:../nocodb-sdk",

1
packages/nocodb/src/app.module.ts

@ -29,7 +29,6 @@ import type { MiddlewareConsumer } from '@nestjs/common';
JobsModule,
NestJsEventEmitter.forRoot(),
],
controllers: [],
providers: [
AuthService,
{

18
packages/nocodb/src/controllers/data-table.controller.spec.ts

@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { DataTableController } from './data-table.controller';
describe('DataTableController', () => {
let controller: DataTableController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [DataTableController],
}).compile();
controller = module.get<DataTableController>(DataTableController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

189
packages/nocodb/src/controllers/data-table.controller.ts

@ -0,0 +1,189 @@
import {
Body,
Controller,
Delete,
Get,
HttpCode,
Param,
Patch,
Post,
Query,
Request,
Response,
UseGuards,
} from '@nestjs/common';
import { GlobalGuard } from '../guards/global/global.guard';
import { parseHrtimeToSeconds } from '../helpers';
import {
Acl,
ExtractProjectIdMiddleware,
} from '../middlewares/extract-project-id/extract-project-id.middleware';
import { DataTableService } from '../services/data-table.service';
@Controller()
@UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
export class DataTableController {
constructor(private readonly dataTableService: DataTableService) {}
// todo: Handle the error case where view doesnt belong to model
@Get('/api/v1/tables/:modelId/rows')
@Acl('dataList')
async dataList(
@Request() req,
@Response() res,
@Param('modelId') modelId: string,
@Query('viewId') viewId: string,
) {
const startTime = process.hrtime();
const responseData = await this.dataTableService.dataList({
query: req.query,
modelId: modelId,
viewId: viewId,
});
const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime));
res.setHeader('xc-db-response', elapsedSeconds);
res.json(responseData);
}
@Get(['/api/v1/tables/:modelId/rows/count'])
@Acl('dataCount')
async dataCount(
@Request() req,
@Response() res,
@Param('modelId') modelId: string,
@Query('viewId') viewId: string,
) {
const countResult = await this.dataTableService.dataCount({
query: req.query,
modelId,
viewId,
});
res.json(countResult);
}
@Post(['/api/v1/tables/:modelId/rows'])
@HttpCode(200)
@Acl('dataInsert')
async dataInsert(
@Request() req,
@Param('modelId') modelId: string,
@Query('viewId') viewId: string,
@Body() body: any,
) {
return await this.dataTableService.dataInsert({
modelId: modelId,
body: body,
viewId,
cookie: req,
});
}
@Patch(['/api/v1/tables/:modelId/rows'])
@Acl('dataUpdate')
async dataUpdate(
@Request() req,
@Param('modelId') modelId: string,
@Query('viewId') viewId: string,
@Param('rowId') rowId: string,
) {
return await this.dataTableService.dataUpdate({
modelId: modelId,
body: req.body,
cookie: req,
viewId,
});
}
@Delete(['/api/v1/tables/:modelId/rows'])
@Acl('dataDelete')
async dataDelete(
@Request() req,
@Param('modelId') modelId: string,
@Query('viewId') viewId: string,
@Param('rowId') rowId: string,
) {
return await this.dataTableService.dataDelete({
modelId: modelId,
cookie: req,
viewId,
body: req.body,
});
}
@Get(['/api/v1/tables/:modelId/rows/:rowId'])
@Acl('dataRead')
async dataRead(
@Request() req,
@Param('modelId') modelId: string,
@Query('viewId') viewId: string,
@Param('rowId') rowId: string,
) {
return await this.dataTableService.dataRead({
modelId,
rowId: rowId,
query: req.query,
viewId,
});
}
@Get(['/api/v1/tables/:modelId/links/:columnId/rows/:rowId'])
@Acl('nestedDataList')
async nestedDataList(
@Request() req,
@Param('modelId') modelId: string,
@Query('viewId') viewId: string,
@Param('columnId') columnId: string,
@Param('rowId') rowId: string,
) {
return await this.dataTableService.nestedDataList({
modelId,
rowId: rowId,
query: req.query,
viewId,
columnId,
});
}
@Post(['/api/v1/tables/:modelId/links/:columnId/rows/:rowId'])
@Acl('nestedDataLink')
async nestedLink(
@Request() req,
@Param('modelId') modelId: string,
@Query('viewId') viewId: string,
@Param('columnId') columnId: string,
@Param('rowId') rowId: string,
@Body() refRowIds: string | string[] | number | number[],
) {
return await this.dataTableService.nestedLink({
modelId,
rowId: rowId,
query: req.query,
viewId,
columnId,
refRowIds,
cookie: req,
});
}
@Delete(['/api/v1/tables/:modelId/links/:columnId/rows/:rowId'])
@Acl('nestedDataUnlink')
async nestedUnlink(
@Request() req,
@Param('modelId') modelId: string,
@Query('viewId') viewId: string,
@Param('columnId') columnId: string,
@Param('rowId') rowId: string,
@Body() refRowIds: string | string[] | number | number[],
) {
return await this.dataTableService.nestedUnlink({
modelId,
rowId: rowId,
query: req.query,
viewId,
columnId,
refRowIds,
cookie: req,
});
}
}

2
packages/nocodb/src/controllers/test/TestResetService/index.ts

@ -1,6 +1,6 @@
import axios from 'axios';
import Project from '../../../models/Project';
import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
// import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
import Noco from '../../../Noco';
import User from '../../../models/User';
import NocoCache from '../../../cache/NocoCache';

648
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -18,6 +18,7 @@ import { customAlphabet } from 'nanoid';
import DOMPurify from 'isomorphic-dompurify';
import { v4 as uuidv4 } from 'uuid';
import { Knex } from 'knex';
import { extractLimitAndOffset } from '../helpers';
import { NcError } from '../helpers/catchError';
import getAst from '../helpers/getAst';
import {
@ -43,6 +44,7 @@ import genRollupSelectv2 from './genRollupSelectv2';
import conditionV2 from './conditionV2';
import sortV2 from './sortV2';
import { customValidators } from './util/customValidators';
import Transaction = Knex.Transaction;
import type { XKnex } from './CustomKnex';
import type {
XcFilter,
@ -58,7 +60,6 @@ import type {
SelectOption,
} from '../models';
import type { SortType } from 'nocodb-sdk';
import Transaction = Knex.Transaction;
dayjs.extend(utc);
dayjs.extend(timezone);
@ -1511,20 +1512,12 @@ class BaseModelSqlv2 {
}
_getListArgs(args: XcFilterWithAlias): XcFilter {
const obj: XcFilter = {};
obj.where = args.where || args.w || '';
const obj: XcFilter = extractLimitAndOffset(args);
obj.where = args.filter || args.where || args.w || '';
obj.having = args.having || args.h || '';
obj.shuffle = args.shuffle || args.r || '';
obj.condition = args.condition || args.c || {};
obj.conditionGraph = args.conditionGraph || {};
obj.limit = Math.max(
Math.min(
args.limit || args.l || this.config.limitDefault,
this.config.limitMax,
),
this.config.limitMin,
);
obj.offset = Math.max(+(args.offset || args.o) || 0, 0);
obj.fields = args.fields || args.f;
obj.sort = args.sort || args.s;
return obj;
@ -2244,12 +2237,14 @@ class BaseModelSqlv2 {
foreign_key_checks = true,
skip_hooks = false,
raw = false,
insertOneByOneAsFallback = false,
}: {
chunkSize?: number;
cookie?: any;
foreign_key_checks?: boolean;
skip_hooks?: boolean;
raw?: boolean;
insertOneByOneAsFallback?: boolean;
} = {},
) {
let trx;
@ -2403,12 +2398,28 @@ class BaseModelSqlv2 {
}
}
const response =
this.isPg || this.isMssql
? await trx
.batchInsert(this.tnPath, insertDatas, chunkSize)
.returning(this.model.primaryKey?.column_name)
: await trx.batchInsert(this.tnPath, insertDatas, chunkSize);
let response;
// insert one by one as fallback to get ids for sqlite and mysql
if (insertOneByOneAsFallback && (this.isSqlite || this.isMySQL)) {
// sqlite and mysql doesnt support returning, so insert one by one and return ids
response = [];
const aiPkCol = this.model.primaryKeys.find((pk) => pk.ai);
for (const insertData of insertDatas) {
const query = trx(this.tnPath).insert(insertData);
const id = (await query)[0];
response.push(aiPkCol ? { [aiPkCol.title]: id } : id);
}
} else {
response =
this.isPg || this.isMssql
? await trx
.batchInsert(this.tnPath, insertDatas, chunkSize)
.returning(this.model.primaryKey?.column_name)
: await trx.batchInsert(this.tnPath, insertDatas, chunkSize);
}
if (!foreign_key_checks) {
if (this.isPg) {
@ -2433,7 +2444,11 @@ class BaseModelSqlv2 {
async bulkUpdate(
datas: any[],
{ cookie, raw = false }: { cookie?: any; raw?: boolean } = {},
{
cookie,
raw = false,
throwExceptionIfNotExist = false,
}: { cookie?: any; raw?: boolean; throwExceptionIfNotExist?: boolean } = {},
) {
let transaction;
try {
@ -2476,9 +2491,12 @@ class BaseModelSqlv2 {
if (!raw) {
for (const pkValues of updatePkValues) {
newData.push(
await this.readByPk(pkValues, false, {}, { ignoreView: true }),
);
const oldRecord = await this.readByPk(pkValues);
if (!oldRecord && throwExceptionIfNotExist)
NcError.unprocessableEntity(
`Record with pk ${JSON.stringify(pkValues)} not found`,
);
newData.push(oldRecord);
}
}
@ -2553,7 +2571,13 @@ class BaseModelSqlv2 {
}
}
async bulkDelete(ids: any[], { cookie }: { cookie?: any } = {}) {
async bulkDelete(
ids: any[],
{
cookie,
throwExceptionIfNotExist = false,
}: { cookie?: any; throwExceptionIfNotExist?: boolean } = {},
) {
let transaction;
try {
const deleteIds = await Promise.all(
@ -2570,9 +2594,14 @@ class BaseModelSqlv2 {
// pk not specified - bypass
continue;
}
deleted.push(
await this.readByPk(pkValues, false, {}, { ignoreView: true }),
);
const oldRecord = await this.readByPk(pkValues);
if (!oldRecord && throwExceptionIfNotExist)
NcError.unprocessableEntity(
`Record with pk ${JSON.stringify(pkValues)} not found`,
);
deleted.push(oldRecord);
res.push(d);
}
@ -2635,12 +2664,6 @@ class BaseModelSqlv2 {
transaction = await this.dbDriver.transaction();
if (base.is_meta && execQueries.length > 0) {
for (const execQuery of execQueries) {
await execQuery(transaction, idsVals);
}
}
for (const d of res) {
await transaction(this.tnPath).del().where(d);
}
@ -2685,8 +2708,8 @@ class BaseModelSqlv2 {
qb,
this.dbDriver,
);
const execQueries: ((trx: Transaction, qb: any) => Promise<any>)[] = [];
// qb.del();
for (const column of this.model.columns) {
if (column.uidt !== UITypes.LinkToAnotherRecord) continue;
@ -2706,18 +2729,16 @@ class BaseModelSqlv2 {
await parentTable.getColumns();
const childTn = this.getTnPath(childTable);
const parentTn = this.getTnPath(parentTable);
switch (colOptions.type) {
case 'mm':
{
const vChildCol = await colOptions.getMMChildColumn();
const vParentCol = await colOptions.getMMParentColumn();
const vTable = await colOptions.getMMModel();
const vTn = this.getTnPath(vTable);
execQueries.push((trx, qb) =>
execQueries.push(() =>
this.dbDriver(vTn)
.where({
[vChildCol.column_name]: this.dbDriver(childTn)
@ -2781,7 +2802,6 @@ class BaseModelSqlv2 {
return count;
} catch (e) {
if (trx) await trx.rollback();
throw e;
}
}
@ -2977,104 +2997,12 @@ class BaseModelSqlv2 {
modelId: this.model.id,
tnPath: this.tnPath,
});
/*
const view = await View.get(this.viewId);
// handle form view data submission
if (
(hookName === 'after.insert' || hookName === 'after.bulkInsert') &&
view.type === ViewTypes.FORM
) {
try {
const formView = await view.getView<FormView>();
const { columns } = await FormView.getWithInfo(formView.fk_view_id);
const allColumns = await this.model.getColumns();
const fieldById = columns.reduce(
(o: Record<string, any>, f: Record<string, any>) => ({
...o,
[f.fk_column_id]: f,
}),
{},
);
let order = 1;
const filteredColumns = allColumns
?.map((c: Record<string, any>) => ({
...c,
fk_column_id: c.id,
fk_view_id: formView.fk_view_id,
...(fieldById[c.id] ? fieldById[c.id] : {}),
order: (fieldById[c.id] && fieldById[c.id].order) || order++,
id: fieldById[c.id] && fieldById[c.id].id,
}))
.sort(
(a: Record<string, any>, b: Record<string, any>) =>
a.order - b.order,
)
.filter(
(f: Record<string, any>) =>
f.show &&
f.uidt !== UITypes.Rollup &&
f.uidt !== UITypes.Lookup &&
f.uidt !== UITypes.Formula &&
f.uidt !== UITypes.QrCode &&
f.uidt !== UITypes.Barcode &&
f.uidt !== UITypes.SpecificDBType,
)
.sort(
(a: Record<string, any>, b: Record<string, any>) =>
a.order - b.order,
)
.map((c: Record<string, any>) => ({
...c,
required: !!(c.required || 0),
}));
const emails = Object.entries(JSON.parse(formView?.email) || {})
.filter((a) => a[1])
.map((a) => a[0]);
if (emails?.length) {
const transformedData = _transformSubmittedFormDataForEmail(
newData,
formView,
filteredColumns,
);
(await NcPluginMgrv2.emailAdapter(false))?.mailSend({
to: emails.join(','),
subject: 'NocoDB Form',
html: ejs.render(formSubmissionEmailTemplate, {
data: transformedData,
tn: this.tnPath,
_tn: this.model.title,
}),
});
}
} catch (e) {
console.log(e);
}
}
try {
const [event, operation] = hookName.split('.');
const hooks = await Hook.list({
fk_model_id: this.model.id,
event,
operation,
});
for (const hook of hooks) {
if (hook.active) {
invokeWebhook(hook, this.model, view, prevData, newData, req?.user);
}
}
} catch (e) {
console.log('hooks :: error', hookName, e);
}*/
}
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
protected async errorInsert(e, data, trx, cookie) {}
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
protected async errorUpdate(e, data, trx, cookie) {}
// todo: handle composite primary key
@ -3088,7 +3016,7 @@ class BaseModelSqlv2 {
);
}
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
protected async errorDelete(e, id, trx, cookie) {}
async validate(columns) {
@ -3809,6 +3737,464 @@ class BaseModelSqlv2 {
}
return data;
}
async addLinks({
cookie,
childIds,
colId,
rowId,
}: {
cookie: any;
childIds: (string | number)[];
colId: string;
rowId: string;
}) {
const columns = await this.model.getColumns();
const column = columns.find((c) => c.id === colId);
if (!column || column.uidt !== UITypes.LinkToAnotherRecord)
NcError.notFound(`Link column ${colId} not found`);
const row = await this.dbDriver(this.tnPath)
.where(await this._wherePk(rowId))
.first();
// validate rowId
if (!row) {
NcError.notFound(`Row with id '${rowId}' not found`);
}
if (!childIds.length) return;
const colOptions = await column.getColOptions<LinkToAnotherRecordColumn>();
const childColumn = await colOptions.getChildColumn();
const parentColumn = await colOptions.getParentColumn();
const parentTable = await parentColumn.getModel();
const childTable = await childColumn.getModel();
await childTable.getColumns();
await parentTable.getColumns();
const childTn = this.getTnPath(childTable);
const parentTn = this.getTnPath(parentTable);
switch (colOptions.type) {
case RelationTypes.MANY_TO_MANY:
{
const vChildCol = await colOptions.getMMChildColumn();
const vParentCol = await colOptions.getMMParentColumn();
const vTable = await colOptions.getMMModel();
const vTn = this.getTnPath(vTable);
let insertData: Record<string, any>[];
// validate Ids
{
const childRowsQb = this.dbDriver(parentTn)
.select(parentColumn.column_name)
.select(`${vTable.table_name}.${vChildCol.column_name}`)
.leftJoin(vTn, (qb) => {
qb.on(
`${vTable.table_name}.${vParentCol.column_name}`,
`${parentTable.table_name}.${parentColumn.column_name}`,
).andOn(
`${vTable.table_name}.${vChildCol.column_name}`,
row[childColumn.column_name],
);
});
// .where(_wherePk(parentTable.primaryKeys, childId))
if (parentTable.primaryKeys.length > 1) {
childRowsQb.where((qb) => {
for (const childId of childIds) {
qb.orWhere(_wherePk(parentTable.primaryKeys, childId));
}
});
} else {
childRowsQb.whereIn(parentTable.primaryKey.column_name, childIds);
}
if (parentTable.primaryKey.column_name !== parentColumn.column_name)
childRowsQb.select(parentTable.primaryKey.column_name);
const childRows = await childRowsQb;
if (childRows.length !== childIds.length) {
const missingIds = childIds.filter(
(id) =>
!childRows.find((r) => r[parentColumn.column_name] === id),
);
NcError.unprocessableEntity(
`Child record with id [${missingIds.join(', ')}] not found`,
);
}
insertData = childRows
// skip existing links
.filter((childRow) => !childRow[vChildCol.column_name])
// generate insert data for new links
.map((childRow) => ({
[vParentCol.column_name]: childRow[parentColumn.column_name],
[vChildCol.column_name]: row[childColumn.column_name],
}));
// if no new links, return true
if (!insertData.length) return true;
}
// if (this.isSnowflake) {
// const parentPK = this.dbDriver(parentTn)
// .select(parentColumn.column_name)
// // .where(_wherePk(parentTable.primaryKeys, childId))
// .whereIn(parentTable.primaryKey.column_name, childIds)
// .first();
//
// const childPK = this.dbDriver(childTn)
// .select(childColumn.column_name)
// .where(_wherePk(childTable.primaryKeys, rowId))
// .first();
//
// await this.dbDriver.raw(
// `INSERT INTO ?? (??, ??) SELECT (${parentPK.toQuery()}), (${childPK.toQuery()})`,
// [vTn, vParentCol.column_name, vChildCol.column_name],
// );
// } else {
// await this.dbDriver(vTn).insert({
// [vParentCol.column_name]: this.dbDriver(parentTn)
// .select(parentColumn.column_name)
// // .where(_wherePk(parentTable.primaryKeys, childId))
// .where(parentTable.primaryKey.column_name, childIds)
// .first(),
// [vChildCol.column_name]: this.dbDriver(childTn)
// .select(childColumn.column_name)
// .where(_wherePk(childTable.primaryKeys, rowId))
// .first(),
// });
// todo: use bulk insert
await this.dbDriver(vTn).insert(insertData);
// }
}
break;
case RelationTypes.HAS_MANY:
{
// validate Ids
{
const childRowsQb = this.dbDriver(childTn).select(
childTable.primaryKey.column_name,
);
if (childTable.primaryKeys.length > 1) {
childRowsQb.where((qb) => {
for (const childId of childIds) {
qb.orWhere(_wherePk(childTable.primaryKeys, childId));
}
});
} else {
childRowsQb.whereIn(parentTable.primaryKey.column_name, childIds);
}
const childRows = await childRowsQb;
if (childRows.length !== childIds.length) {
const missingIds = childIds.filter(
(id) =>
!childRows.find((r) => r[parentColumn.column_name] === id),
);
NcError.unprocessableEntity(
`Child record with id [${missingIds.join(', ')}] not found`,
);
}
}
await this.dbDriver(childTn)
.update({
[childColumn.column_name]: this.dbDriver.from(
this.dbDriver(parentTn)
.select(parentColumn.column_name)
.where(_wherePk(parentTable.primaryKeys, rowId))
.first()
.as('___cn_alias'),
),
})
// .where(_wherePk(childTable.primaryKeys, childId));
.whereIn(childTable.primaryKey.column_name, childIds);
}
break;
case RelationTypes.BELONGS_TO:
{
// validate Ids
{
const childRowsQb = this.dbDriver(parentTn)
.select(parentTable.primaryKey.column_name)
.where(_wherePk(parentTable.primaryKeys, childIds[0]))
.first();
const childRow = await childRowsQb;
if (!childRow) {
NcError.unprocessableEntity(
`Child record with id [${childIds[0]}] not found`,
);
}
}
await this.dbDriver(childTn)
.update({
[childColumn.column_name]: this.dbDriver.from(
this.dbDriver(parentTn)
.select(parentColumn.column_name)
.where(_wherePk(parentTable.primaryKeys, childIds[0]))
// .whereIn(parentTable.primaryKey.column_name, childIds)
.first()
.as('___cn_alias'),
),
})
.where(_wherePk(childTable.primaryKeys, rowId));
}
break;
}
// const response = await this.readByPk(rowId);
// await this.afterInsert(response, this.dbDriver, cookie);
// await this.afterAddChild(rowId, childId, cookie);
}
async removeLinks({
cookie,
childIds,
colId,
rowId,
}: {
cookie: any;
childIds: (string | number)[];
colId: string;
rowId: string;
}) {
const columns = await this.model.getColumns();
const column = columns.find((c) => c.id === colId);
if (!column || column.uidt !== UITypes.LinkToAnotherRecord)
NcError.notFound(`Link column ${colId} not found`);
const row = await this.dbDriver(this.tnPath)
.where(await this._wherePk(rowId))
.first();
// validate rowId
if (!row) {
NcError.notFound(`Row with id '${rowId}' not found`);
}
if (!childIds.length) return;
const colOptions = await column.getColOptions<LinkToAnotherRecordColumn>();
const childColumn = await colOptions.getChildColumn();
const parentColumn = await colOptions.getParentColumn();
const parentTable = await parentColumn.getModel();
const childTable = await childColumn.getModel();
await childTable.getColumns();
await parentTable.getColumns();
const childTn = this.getTnPath(childTable);
const parentTn = this.getTnPath(parentTable);
const prevData = await this.readByPk(rowId);
switch (colOptions.type) {
case RelationTypes.MANY_TO_MANY:
{
const vChildCol = await colOptions.getMMChildColumn();
const vParentCol = await colOptions.getMMParentColumn();
const vTable = await colOptions.getMMModel();
// validate Ids
{
const childRowsQb = this.dbDriver(parentTn)
.select(parentColumn.column_name)
// .where(_wherePk(parentTable.primaryKeys, childId))
.whereIn(parentTable.primaryKey.column_name, childIds);
if (parentTable.primaryKey.column_name !== parentColumn.column_name)
childRowsQb.select(parentTable.primaryKey.column_name);
const childRows = await childRowsQb;
if (childRows.length !== childIds.length) {
const missingIds = childIds.filter(
(id) =>
!childRows.find((r) => r[parentColumn.column_name] === id),
);
NcError.unprocessableEntity(
`Child record with id [${missingIds.join(', ')}] not found`,
);
}
}
const vTn = this.getTnPath(vTable);
await this.dbDriver(vTn)
.where({
[vChildCol.column_name]: this.dbDriver(childTn)
.select(childColumn.column_name)
.where(_wherePk(childTable.primaryKeys, rowId))
.first(),
})
.whereIn(
[vParentCol.column_name],
this.dbDriver(parentTn)
.select(parentColumn.column_name)
.whereIn(parentTable.primaryKey.column_name, childIds),
)
.delete();
}
break;
case RelationTypes.HAS_MANY:
{
// validate Ids
{
const childRowsQb = this.dbDriver(childTn)
.select(childTable.primaryKey.column_name)
.whereIn(childTable.primaryKey.column_name, childIds);
const childRows = await childRowsQb;
if (childRows.length !== childIds.length) {
const missingIds = childIds.filter(
(id) =>
!childRows.find((r) => r[parentColumn.column_name] === id),
);
NcError.unprocessableEntity(
`Child record with id [${missingIds.join(', ')}] not found`,
);
}
}
await this.dbDriver(childTn)
// .where({
// [childColumn.cn]: this.dbDriver(parentTable.tn)
// .select(parentColumn.cn)
// .where(parentTable.primaryKey.cn, rowId)
// .first()
// })
// .where(_wherePk(childTable.primaryKeys, childId))
.whereIn(childTable.primaryKey.column_name, childIds)
.update({ [childColumn.column_name]: null });
}
break;
case RelationTypes.BELONGS_TO:
{
// validate Ids
{
if (childIds.length > 1)
NcError.unprocessableEntity(
'Request must contain only one parent id',
);
const childRowsQb = this.dbDriver(parentTn)
.select(parentTable.primaryKey.column_name)
.where(_wherePk(parentTable.primaryKeys, childIds[0]))
.first();
const childRow = await childRowsQb;
if (!childRow) {
NcError.unprocessableEntity(
`Child record with id [${childIds[0]}] not found`,
);
}
}
await this.dbDriver(childTn)
// .where({
// [childColumn.cn]: this.dbDriver(parentTable.tn)
// .select(parentColumn.cn)
// .where(parentTable.primaryKey.cn, childId)
// .first()
// })
// .where(_wherePk(childTable.primaryKeys, rowId))
.where(childTable.primaryKey.column_name, rowId)
.update({ [childColumn.column_name]: null });
}
break;
}
// const newData = await this.readByPk(rowId);
// await this.afterUpdate(prevData, newData, this.dbDriver, cookie);
// await this.afterRemoveChild(rowId, childIds, cookie);
}
async btRead(
{ colId, id }: { colId; id },
args: { limit?; offset?; fieldSet?: Set<string> } = {},
) {
try {
const { where, sort } = this._getListArgs(args as any);
// todo: get only required fields
const relColumn = (await this.model.getColumns()).find(
(c) => c.id === colId,
);
const row = await this.dbDriver(this.tnPath)
.where(await this._wherePk(id))
.first();
// validate rowId
if (!row) {
NcError.notFound(`Row with id ${id} not found`);
}
const parentCol = await (
(await relColumn.getColOptions()) as LinkToAnotherRecordColumn
).getParentColumn();
const parentTable = await parentCol.getModel();
const chilCol = await (
(await relColumn.getColOptions()) as LinkToAnotherRecordColumn
).getChildColumn();
const childTable = await chilCol.getModel();
const parentModel = await Model.getBaseModelSQL({
model: parentTable,
dbDriver: this.dbDriver,
});
await childTable.getColumns();
const childTn = this.getTnPath(childTable);
const parentTn = this.getTnPath(parentTable);
const qb = this.dbDriver(parentTn);
await this.applySortAndFilter({ table: parentTable, where, qb, sort });
qb.where(
parentCol.column_name,
this.dbDriver(childTn)
.select(chilCol.column_name)
// .where(parentTable.primaryKey.cn, p)
.where(_wherePk(childTable.primaryKeys, id)),
);
await parentModel.selectObject({ qb, fieldsSet: args.fieldSet });
const parent = (await this.execAndParse(qb, childTable))?.[0];
const proto = await parentModel.getProto();
if (parent) {
parent.__proto__ = proto;
}
return parent;
} catch (e) {
console.log(e);
throw e;
}
}
}
function extractSortsObject(

41
packages/nocodb/src/db/sql-client/lib/mssql/MssqlClient.ts

@ -2101,7 +2101,7 @@ class MssqlClient extends KnexClient {
table = table.onUpdate(relation.ur);
}
if (relation.dr) {
table = table.onDelete(relation.dr);
table.onDelete(relation.dr);
}
})
.toQuery());
@ -2192,28 +2192,31 @@ class MssqlClient extends KnexClient {
table = table.onUpdate(args.onUpdate);
}
if (args.onDelete) {
table = table.onDelete(args.onDelete);
table.onDelete(args.onDelete);
}
},
);
const upStatement =
this.querySeparator() +
(await this.sqlClient.schema
.table(this.getTnPath(args.childTable), function (table) {
table = table
.foreign(args.childColumn, foreignKeyName)
.references(args.parentColumn)
.on(self.getTnPath(args.parentTable));
const upQb = this.sqlClient.schema.table(
this.getTnPath(args.childTable),
function (table) {
table = table
.foreign(args.childColumn, foreignKeyName)
.references(args.parentColumn)
.on(self.getTnPath(args.parentTable));
if (args.onUpdate) {
table = table.onUpdate(args.onUpdate);
}
if (args.onDelete) {
table = table.onDelete(args.onDelete);
}
})
.toQuery());
if (args.onUpdate) {
table = table.onUpdate(args.onUpdate);
}
if (args.onDelete) {
table.onDelete(args.onDelete);
}
},
);
await upQb;
const upStatement = this.querySeparator() + upQb.toQuery();
this.emit(`Success : ${upStatement}`);
@ -2221,7 +2224,7 @@ class MssqlClient extends KnexClient {
this.querySeparator() +
this.sqlClient.schema
.table(this.getTnPath(args.childTable), function (table) {
table = table.dropForeign(args.childColumn, foreignKeyName);
table.dropForeign(args.childColumn, foreignKeyName);
})
.toQuery();

2
packages/nocodb/src/db/sql-client/lib/oracle/OracleClient.ts

@ -1914,7 +1914,7 @@ class OracleClient extends KnexClient {
* @returns {String} message
*/
async totalRecords(_args: any = {}): Promise<Result> {
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const func = this.totalRecords.name;
throw new Error('Function not supported for oracle yet');
}

2
packages/nocodb/src/db/sql-client/lib/pg/PgClient.ts

@ -2359,7 +2359,7 @@ class PGClient extends KnexClient {
table = table.onUpdate(relation.ur);
}
if (relation.dr) {
table = table.onDelete(relation.dr);
table.onDelete(relation.dr);
}
})
.toQuery());

73
packages/nocodb/src/db/sql-client/lib/snowflake/SnowflakeClient.ts

@ -1970,23 +1970,24 @@ class SnowflakeClient extends KnexClient {
relationsList = relationsList.data.list;
for (const relation of relationsList) {
downQuery +=
this.querySeparator() +
(await this.sqlClient.schema
.table(relation.tn, function (table) {
table = table
.foreign(relation.cn, null)
.references(relation.rcn)
.on(relation.rtn);
if (relation.ur) {
table = table.onUpdate(relation.ur);
}
if (relation.dr) {
table = table.onDelete(relation.dr);
}
})
.toQuery());
const downQb = this.sqlClient.schema.table(
relation.tn,
function (table) {
table = table
.foreign(relation.cn, null)
.references(relation.rcn)
.on(relation.rtn);
if (relation.ur) {
table = table.onUpdate(relation.ur);
}
if (relation.dr) {
table.onDelete(relation.dr);
}
},
);
await downQb;
downQuery += this.querySeparator() + downQb.toQuery();
}
let indexList: any = await this.indexList(args);
@ -2060,8 +2061,6 @@ class SnowflakeClient extends KnexClient {
const foreignKeyName = args.foreignKeyName || null;
try {
// s = await this.sqlClient.schema.index(Object.keys(args.columns));
await this.sqlClient.schema.table(args.childTable, (table) => {
table = table
.foreign(args.childColumn, foreignKeyName)
@ -2072,27 +2071,27 @@ class SnowflakeClient extends KnexClient {
table = table.onUpdate(args.onUpdate);
}
if (args.onDelete) {
table = table.onDelete(args.onDelete);
table.onDelete(args.onDelete);
}
});
const upStatement =
this.querySeparator() +
(await this.sqlClient.schema
.table(args.childTable, (table) => {
table = table
.foreign(args.childColumn, foreignKeyName)
.references(args.parentColumn)
.on(this.getTnPath(args.parentTable));
const upQb = this.sqlClient.schema.table(args.childTable, (table) => {
table = table
.foreign(args.childColumn, foreignKeyName)
.references(args.parentColumn)
.on(this.getTnPath(args.parentTable));
if (args.onUpdate) {
table = table.onUpdate(args.onUpdate);
}
if (args.onDelete) {
table = table.onDelete(args.onDelete);
}
})
.toQuery());
if (args.onUpdate) {
table = table.onUpdate(args.onUpdate);
}
if (args.onDelete) {
table.onDelete(args.onDelete);
}
});
await upQb;
const upStatement = this.querySeparator() + upQb.toQuery();
this.emit(`Success : ${upStatement}`);
@ -2100,7 +2099,7 @@ class SnowflakeClient extends KnexClient {
this.querySeparator() +
this.sqlClient.schema
.table(args.childTable, (table) => {
table = table.dropForeign(args.childColumn, foreignKeyName);
table.dropForeign(args.childColumn, foreignKeyName);
})
.toQuery();

1
packages/nocodb/src/db/sql-client/lib/sqlite/SqliteClient.ts

@ -1866,6 +1866,7 @@ class SqliteClient extends KnexClient {
/* Filter relations for current table */
if (args.tn) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
relations = relations.filter(
(r) => r.tn === args.tn || r.rtn === args.tn,
);

44
packages/nocodb/src/db/sql-data-mapper/lib/BaseModel.ts

@ -321,7 +321,7 @@ abstract class BaseModel {
* @returns {Object} Table row data
*/
// @ts-ignore
async readByPk(id, { conditionGraph }) {
async readByPk(id) {
try {
return await this._run(
this.$db.select().where(this._wherePk(id)).first(),
@ -704,10 +704,7 @@ abstract class BaseModel {
*/
async exists(id, _) {
try {
return (
Object.keys(await this.readByPk(id, { conditionGraph: null }))
.length !== 0
);
return Object.keys(await this.readByPk(id)).length !== 0;
} catch (e) {
console.log(e);
throw e;
@ -1341,7 +1338,7 @@ abstract class BaseModel {
* @param {Object} data - insert data
* @param {Object} trx? - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async beforeInsert(data, trx?: any, cookie?: {}) {}
/**
@ -1350,7 +1347,7 @@ abstract class BaseModel {
* @param {Object} response - inserted data
* @param {Object} trx? - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async afterInsert(response, trx?: any, cookie?: {}) {}
/**
@ -1360,7 +1357,7 @@ abstract class BaseModel {
* @param {Object} data - insert data
* @param {Object} trx? - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async errorInsert(err, data, trx?: any, cookie?: {}) {}
/**
@ -1369,7 +1366,7 @@ abstract class BaseModel {
* @param {Object} data - update data
* @param {Object} trx? - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async beforeUpdate(data, trx?: any, cookie?: {}) {}
/**
@ -1378,7 +1375,7 @@ abstract class BaseModel {
* @param {Object} response - updated data
* @param {Object} trx? - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async afterUpdate(response, trx?: any, cookie?: {}) {}
/**
@ -1388,7 +1385,7 @@ abstract class BaseModel {
* @param {Object} data - update data
* @param {Object} trx? - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async errorUpdate(err, data, trx?: any, cookie?: {}) {}
/**
@ -1397,7 +1394,7 @@ abstract class BaseModel {
* @param {Object} data - delete data
* @param {Object} trx? - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async beforeDelete(data, trx?: any, cookie?: {}) {}
/**
@ -1406,7 +1403,7 @@ abstract class BaseModel {
* @param {Object} response - Deleted data
* @param {Object} trx? - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async afterDelete(response, trx?: any, cookie?: {}) {}
/**
@ -1416,7 +1413,7 @@ abstract class BaseModel {
* @param {Object} data - delete data
* @param {Object} trx? - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async errorDelete(err, data, trx?: any, cookie?: {}) {}
/**
@ -1425,7 +1422,7 @@ abstract class BaseModel {
* @param {Object[]} data - insert data
* @param {Object} trx - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async beforeInsertb(data, trx?: any) {}
/**
@ -1434,7 +1431,7 @@ abstract class BaseModel {
* @param {Object[]} response - inserted data
* @param {Object} trx - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async afterInsertb(response, trx?: any) {}
/**
@ -1444,7 +1441,7 @@ abstract class BaseModel {
* @param {Object} data - delete data
* @param {Object} trx - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async errorInsertb(err, data, trx?: any) {}
/**
@ -1453,7 +1450,7 @@ abstract class BaseModel {
* @param {Object[]} data - update data
* @param {Object} trx - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async beforeUpdateb(data, trx?: any) {}
/**
@ -1462,7 +1459,7 @@ abstract class BaseModel {
* @param {Object[]} response - updated data
* @param {Object} trx - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async afterUpdateb(response, trx?: any) {}
/**
@ -1472,7 +1469,7 @@ abstract class BaseModel {
* @param {Object[]} data - delete data
* @param {Object} trx - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async errorUpdateb(err, data, trx?: any) {}
/**
@ -1481,7 +1478,7 @@ abstract class BaseModel {
* @param {Object[]} data - delete data
* @param {Object} trx - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async beforeDeleteb(data, trx?: any) {}
/**
@ -1490,7 +1487,7 @@ abstract class BaseModel {
* @param {Object[]} response - deleted data
* @param {Object} trx - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async afterDeleteb(response, trx?: any) {}
/**
@ -1500,12 +1497,13 @@ abstract class BaseModel {
* @param {Object[]} data - delete data
* @param {Object} trx - knex transaction reference
*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async errorDeleteb(err, data, trx?: any) {}
}
export interface XcFilter {
where?: string;
filter?: string;
having?: string;
condition?: any;
conditionGraph?: any;

2
packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts

@ -137,6 +137,7 @@ export default class KnexMigrator extends SqlMigrator {
),
);
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const dirStat = await promisify(fs.stat)(
path.join(
this.toolDir,
@ -232,6 +233,7 @@ export default class KnexMigrator extends SqlMigrator {
);
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const metaStat = await promisify(fs.stat)(
path.join(
this.toolDir,

1
packages/nocodb/src/db/sql-migrator/lib/KnexMigratorv2.ts

@ -114,6 +114,7 @@ export default class KnexMigratorv2 {
}*/
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async _initDbOnFs(base: Base) {
// this.emit(
// 'Creating folder: ',

4
packages/nocodb/src/filters/global-exception/global-exception.filter.ts

@ -8,6 +8,7 @@ import {
NotFound,
NotImplemented,
Unauthorized,
UnprocessableEntity,
} from '../../helpers/catchError';
import type { ArgumentsHost, ExceptionFilter } from '@nestjs/common';
import type { Response } from 'express';
@ -15,6 +16,7 @@ import type { Response } from 'express';
@Catch()
export class GlobalExceptionFilter implements ExceptionFilter {
private logger = new Logger(GlobalExceptionFilter.name);
catch(exception: any, host: ArgumentsHost) {
const ctx = host.switchToHttp();
const response = ctx.getResponse<Response>();
@ -58,6 +60,8 @@ export class GlobalExceptionFilter implements ExceptionFilter {
return response
.status(400)
.json({ msg: exception.message, errors: exception.errors });
} else if (exception instanceof UnprocessableEntity) {
return response.status(422).json({ msg: exception.message });
}
// handle different types of exceptions

23
packages/nocodb/src/helpers/PagedResponse.ts

@ -1,11 +1,6 @@
import { extractLimitAndOffset } from '.';
import type { PaginatedType } from 'nocodb-sdk';
const config: any = {
limitDefault: Math.max(+process.env.DB_QUERY_LIMIT_DEFAULT || 25, 1),
limitMin: Math.max(+process.env.DB_QUERY_LIMIT_MIN || 1, 1),
limitMax: Math.max(+process.env.DB_QUERY_LIMIT_MAX || 1000, 1),
};
export class PagedResponseImpl<T> {
constructor(
list: T[],
@ -17,12 +12,7 @@ export class PagedResponseImpl<T> {
o?: number;
} = {},
) {
const limit = Math.max(
Math.min(args.limit || args.l || config.limitDefault, config.limitMax),
config.limitMin,
);
const offset = Math.max(+(args.offset || args.o) || 0, 0);
const { offset, limit } = extractLimitAndOffset(args);
let count = args.count ?? null;
@ -40,8 +30,17 @@ export class PagedResponseImpl<T> {
this.pageInfo.page ===
(Math.ceil(this.pageInfo.totalRows / this.pageInfo.pageSize) || 1);
}
if (offset && offset >= count) {
this.errors = [
{
message: 'Offset is beyond the total number of rows',
},
];
}
}
list: Array<T>;
pageInfo: PaginatedType;
errors?: any[];
}

2
packages/nocodb/src/helpers/apiMetrics.ts

@ -3,7 +3,7 @@ import type { Request } from 'express';
const countMap = {};
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const metrics = async (req: Request, c = 150) => {
if (!req?.route?.path) return;
const event = `a:api:${req.route.path}:${req.method}`;

8
packages/nocodb/src/helpers/catchError.ts

@ -413,6 +413,8 @@ export default function (
return res.status(501).json({ msg: e.message });
} else if (e instanceof AjvError) {
return res.status(400).json({ msg: e.message, errors: e.errors });
} else if (e instanceof UnprocessableEntity) {
return res.status(422).json({ msg: e.message });
}
next(e);
}
@ -431,6 +433,8 @@ export class InternalServerError extends Error {}
export class NotImplemented extends Error {}
export class UnprocessableEntity extends Error {}
export class AjvError extends Error {
constructor(param: { message: string; errors: ErrorObject[] }) {
super(param.message);
@ -468,4 +472,8 @@ export class NcError {
static ajvValidationError(param: { message: string; errors: ErrorObject[] }) {
throw new AjvError(param);
}
static unprocessableEntity(message = 'Unprocessable entity') {
throw new UnprocessableEntity(message);
}
}

39
packages/nocodb/src/helpers/extractLimitAndOffset.ts

@ -0,0 +1,39 @@
const config = {
limitDefault: Math.max(+process.env.DB_QUERY_LIMIT_DEFAULT || 25, 1),
limitMin: Math.max(+process.env.DB_QUERY_LIMIT_MIN || 1, 1),
limitMax: Math.max(+process.env.DB_QUERY_LIMIT_MAX || 1000, 1),
};
export function extractLimitAndOffset(
args: {
limit?: number | string;
offset?: number | string;
l?: number | string;
o?: number | string;
} = {},
) {
const obj: {
limit?: number;
offset?: number;
} = {};
// use default value if invalid limit
// for example, if limit is not a number, it will be ignored
// if limit is less than 1, it will be ignored
const limit = +(args.limit || args.l);
obj.limit = Math.max(
Math.min(
limit && limit > 0 && Number.isInteger(limit)
? limit
: config.limitDefault,
config.limitMax,
),
config.limitMin,
);
// skip any invalid offset, ignore negative and non-integer values
const offset = +(args.offset || args.o) || 0;
obj.offset = Math.max(Number.isInteger(offset) ? offset : 0, 0);
return obj;
}

1
packages/nocodb/src/helpers/index.ts

@ -2,5 +2,6 @@ import { populateMeta } from './populateMeta';
export * from './columnHelpers';
export * from './apiHelpers';
export * from './cacheHelpers';
export * from './extractLimitAndOffset';
export { populateMeta };

2
packages/nocodb/src/helpers/webhookHelpers.ts

@ -271,7 +271,7 @@ export async function invokeWebhook(
return;
}
if (hook.condition) {
if (hook.condition && !testHook) {
if (isBulkOperation) {
const filteredData = [];
for (const data of newData) {

2
packages/nocodb/src/meta/migrations/v1/nc_011_remove_old_ses_plugin.ts

@ -5,7 +5,7 @@ const up = async (knex: Knex) => {
await knex('nc_plugins').del().where({ title: 'SES' });
};
const down = async (knex: Knex) => {
const down = async (_: Knex) => {
// await knex('nc_plugins').insert([ses]);
};

1
packages/nocodb/src/models/Model.ts

@ -65,6 +65,7 @@ export default class Model implements TableType {
}
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async getViews(force = false, ncMeta = Noco.ncMeta): Promise<View[]> {
this.views = await View.listWithInfo(this.id, ncMeta);
return this.views;

1
packages/nocodb/src/models/View.ts

@ -1,4 +1,3 @@
import { title } from 'process';
import { isSystemColumn, UITypes, ViewTypes } from 'nocodb-sdk';
import Noco from '../Noco';
import {

4
packages/nocodb/src/modules/datas/datas.module.ts

@ -3,8 +3,10 @@ import { MulterModule } from '@nestjs/platform-express';
import multer from 'multer';
import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants';
import { DataAliasController } from '../../controllers/data-alias.controller';
import { DataTableController } from '../../controllers/data-table.controller';
import { PublicDatasExportController } from '../../controllers/public-datas-export.controller';
import { PublicDatasController } from '../../controllers/public-datas.controller';
import { DataTableService } from '../../services/data-table.service';
import { DatasService } from '../../services/datas.service';
import { DatasController } from '../../controllers/datas.controller';
import { BulkDataAliasController } from '../../controllers/bulk-data-alias.controller';
@ -29,6 +31,7 @@ import { PublicDatasService } from '../../services/public-datas.service';
controllers: [
...(process.env.NC_WORKER_CONTAINER !== 'true'
? [
DataTableController,
DatasController,
BulkDataAliasController,
DataAliasController,
@ -41,6 +44,7 @@ import { PublicDatasService } from '../../services/public-datas.service';
: []),
],
providers: [
DataTableService,
DatasService,
BulkDataAliasService,
DataAliasNestedService,

2
packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts

@ -65,7 +65,7 @@ export class AtImportController {
@Post('/api/v1/db/meta/syncs/:syncId/abort')
@HttpCode(200)
async abortImport(@Request() req) {
async abortImport(@Request() _) {
return {};
}
}

79
packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@ -348,32 +348,6 @@ export class AtImportProcessor {
}
};
const nc_DumpTableSchema = async () => {
console.log('[');
// const ncTblList = await api.base.tableList(
// ncCreatedProjectSchema.id,
// syncDB.baseId
// );
const ncTblList = { list: [] };
ncTblList['list'] = await this.tablesService.getAccessibleTables({
projectId: ncCreatedProjectSchema.id,
baseId: syncDB.baseId,
roles: userRole,
});
for (let i = 0; i < ncTblList.list.length; i++) {
// const ncTbl = await api.dbTable.read(ncTblList.list[i].id);
const ncTbl = await this.tablesService.getTableWithAccessibleViews({
tableId: ncTblList.list[i].id,
user: syncDB.user,
});
console.log(JSON.stringify(ncTbl, null, 2));
console.log(',');
}
console.log(']');
};
// retrieve nc column schema from using aTbl field ID as reference
//
const nc_getColumnSchema = async (aTblFieldId) => {
@ -1563,45 +1537,6 @@ export class AtImportProcessor {
return rec;
};
const nocoReadDataSelected = async (projName, table, callback, fields) => {
return new Promise((resolve, reject) => {
base(table.title)
.select({
pageSize: 100,
// maxRecords: 100,
fields: fields,
})
.eachPage(
async function page(records, fetchNextPage) {
// This function (`page`) will get called for each page of records.
// records.forEach(record => callback(table, record));
logBasic(
`:: ${table.title} / ${fields} : ${
recordCnt + 1
} ~ ${(recordCnt += 100)}`,
);
await Promise.all(
records.map((r) => callback(projName, table, r, fields)),
);
// To fetch the next page of records, call `fetchNextPage`.
// If there are more records, `page` will get called again.
// If there are no more records, `done` will get called.
fetchNextPage();
},
function done(err) {
if (err) {
console.error(err);
reject(err);
}
resolve(null);
},
);
});
};
//////////
const nc_isLinkExists = (airtableFieldId) => {
return !!ncLinkMappingTable.find(
(x) => x.aTbl.typeOptions.symmetricColumnId === airtableFieldId,
@ -1879,9 +1814,7 @@ export class AtImportProcessor {
req: { user: syncDB.user, clientIp: '' },
})
.catch((e) =>
e.response?.data?.msg
? logBasic(`NOTICE: ${e.response.data.msg}`)
: console.log(e),
e.message ? logBasic(`NOTICE: ${e.message}`) : console.log(e),
),
);
recordPerfStats(_perfStart, 'auth.projectUserAdd');
@ -2294,7 +2227,6 @@ export class AtImportProcessor {
};
///////////////////////////////////////////////////////////////////////////////
let recordCnt = 0;
try {
logBasic('SDK initialized');
logDetailed('Project initialization started');
@ -2376,7 +2308,6 @@ export class AtImportProcessor {
if (syncDB.options.syncData) {
try {
// await nc_DumpTableSchema();
const _perfStart = recordPerfStart();
const ncTblList = { list: [] };
ncTblList['list'] = await this.tablesService.getAccessibleTables({
@ -2406,8 +2337,6 @@ export class AtImportProcessor {
});
recordPerfStats(_perfStart, 'dbTable.read');
recordCnt = 0;
recordsMap[ncTbl.id] = await importData({
projectName: syncDB.projectName,
table: ncTbl,
@ -2471,12 +2400,12 @@ export class AtImportProcessor {
await generateMigrationStats(aTblSchema);
}
} catch (e) {
if (e.response?.data?.msg) {
if (e.message) {
T.event({
event: 'a:airtable-import:error',
data: { error: e.response.data.msg },
data: { error: e.message },
});
throw new Error(e.response.data.msg);
throw new Error(e.message);
}
throw e;
}

1
packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@ -1214,6 +1214,7 @@ export class ImportService {
storageAdapter as any
).fileReadByStream(linkFile);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
handledLinks = await this.importLinkFromCsvStream({
idMap,
linkStream: linkReadStream,

38
packages/nocodb/src/plugins/storage/Local.ts

@ -3,6 +3,7 @@ import path from 'path';
import { promisify } from 'util';
import mkdirp from 'mkdirp';
import axios from 'axios';
import { NcError } from '../../helpers/catchError';
import { getToolDir } from '../../utils/nc-config';
import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
import type { Readable } from 'stream';
@ -11,7 +12,7 @@ export default class Local implements IStorageAdapterV2 {
constructor() {}
public async fileCreate(key: string, file: XcFile): Promise<any> {
const destPath = path.join(getToolDir(), ...key.split('/'));
const destPath = this.validateAndNormalisePath(key);
try {
await mkdirp(path.dirname(destPath));
const data = await promisify(fs.readFile)(file.path);
@ -24,7 +25,7 @@ export default class Local implements IStorageAdapterV2 {
}
async fileCreateByUrl(key: string, url: string): Promise<any> {
const destPath = path.join(getToolDir(), ...key.split('/'));
const destPath = this.validateAndNormalisePath(key);
return new Promise((resolve, reject) => {
axios
.get(url, {
@ -71,7 +72,7 @@ export default class Local implements IStorageAdapterV2 {
stream: Readable,
): Promise<void> {
return new Promise((resolve, reject) => {
const destPath = path.join(getToolDir(), ...key.split('/'));
const destPath = this.validateAndNormalisePath(key);
try {
mkdirp(path.dirname(destPath)).then(() => {
const writableStream = fs.createWriteStream(destPath);
@ -86,12 +87,12 @@ export default class Local implements IStorageAdapterV2 {
}
public async fileReadByStream(key: string): Promise<Readable> {
const srcPath = path.join(getToolDir(), ...key.split('/'));
const srcPath = this.validateAndNormalisePath(key);
return fs.createReadStream(srcPath, { encoding: 'utf8' });
}
public async getDirectoryList(key: string): Promise<string[]> {
const destDir = path.join(getToolDir(), ...key.split('/'));
const destDir = this.validateAndNormalisePath(key);
return fs.promises.readdir(destDir);
}
@ -103,7 +104,7 @@ export default class Local implements IStorageAdapterV2 {
public async fileRead(filePath: string): Promise<any> {
try {
const fileData = await fs.promises.readFile(
path.join(getToolDir(), ...filePath.split('/')),
this.validateAndNormalisePath(filePath, true),
);
return fileData;
} catch (e) {
@ -118,4 +119,29 @@ export default class Local implements IStorageAdapterV2 {
test(): Promise<boolean> {
return Promise.resolve(false);
}
// method for validate/normalise the path for avoid path traversal attack
protected validateAndNormalisePath(
fileOrFolderPath: string,
throw404 = false,
): string {
// Get the absolute path to the base directory
const absoluteBasePath = path.resolve(getToolDir(), 'nc');
// Get the absolute path to the file
const absolutePath = path.resolve(
path.join(getToolDir(), ...fileOrFolderPath.split('/')),
);
// Check if the resolved path is within the intended directory
if (!absolutePath.startsWith(absoluteBasePath)) {
if (throw404) {
NcError.notFound();
} else {
NcError.badRequest('Invalid path');
}
}
return absolutePath;
}
}

4
packages/nocodb/src/services/auth.service.ts

@ -40,8 +40,8 @@ export class AuthService {
email: _email,
firstname,
lastname,
token,
ignore_subscribe,
// token,
// ignore_subscribe,
} = createUserDto as any;
let { password } = createUserDto;

18
packages/nocodb/src/services/data-table.service.spec.ts

@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { DataTableService } from './data-table.service';
describe('DataTableService', () => {
let service: DataTableService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [DataTableService],
}).compile();
service = module.get<DataTableService>(DataTableService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

434
packages/nocodb/src/services/data-table.service.ts

@ -0,0 +1,434 @@
import { Injectable } from '@nestjs/common';
import { RelationTypes, UITypes } from 'nocodb-sdk';
import { nocoExecute } from 'nc-help';
import { NcError } from '../helpers/catchError';
import getAst from '../helpers/getAst';
import { PagedResponseImpl } from '../helpers/PagedResponse';
import { Base, Column, Model, View } from '../models';
import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2';
import { DatasService } from './datas.service';
import type { LinkToAnotherRecordColumn } from '../models';
@Injectable()
export class DataTableService {
constructor(private datasService: DatasService) {}
async dataList(param: {
projectId?: string;
modelId: string;
query: any;
viewId?: string;
}) {
const { model, view } = await this.getModelAndView(param);
return await this.datasService.getDataList({
model,
view,
query: param.query,
});
}
async dataRead(param: {
projectId?: string;
modelId: string;
rowId: string;
viewId?: string;
query: any;
}) {
const { model, view } = await this.getModelAndView(param);
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(base),
});
const row = await baseModel.readByPk(param.rowId, false, param.query);
if (!row) {
NcError.notFound('Row not found');
}
return row;
}
async dataInsert(param: {
projectId?: string;
viewId?: string;
modelId: string;
body: any;
cookie: any;
}) {
const { model, view } = await this.getModelAndView(param);
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(base),
});
// if array then do bulk insert
const result = await baseModel.bulkInsert(
Array.isArray(param.body) ? param.body : [param.body],
{ cookie: param.cookie, insertOneByOneAsFallback: true },
);
return Array.isArray(param.body) ? result : result[0];
}
async dataUpdate(param: {
projectId?: string;
modelId: string;
viewId?: string;
// rowId: string;
body: any;
cookie: any;
}) {
const { model, view } = await this.getModelAndView(param);
await this.checkForDuplicateRow({ rows: param.body, model });
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(base),
});
const res = await baseModel.bulkUpdate(
Array.isArray(param.body) ? param.body : [param.body],
{ cookie: param.cookie, throwExceptionIfNotExist: true },
);
return this.extractIdObj({ body: param.body, model });
}
async dataDelete(param: {
projectId?: string;
modelId: string;
viewId?: string;
// rowId: string;
cookie: any;
body: any;
}) {
const { model, view } = await this.getModelAndView(param);
await this.checkForDuplicateRow({ rows: param.body, model });
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(base),
});
await baseModel.bulkDelete(
Array.isArray(param.body) ? param.body : [param.body],
{ cookie: param.cookie, throwExceptionIfNotExist: true },
);
return this.extractIdObj({ body: param.body, model });
}
async dataCount(param: {
projectId?: string;
viewId?: string;
modelId: string;
query: any;
}) {
const { model, view } = await this.getModelAndView(param);
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(base),
});
const countArgs: any = { ...param.query };
try {
countArgs.filterArr = JSON.parse(countArgs.filterArrJson);
} catch (e) {}
const count: number = await baseModel.count(countArgs);
return { count };
}
private async getModelAndView(param: {
projectId?: string;
viewId?: string;
modelId: string;
}) {
const model = await Model.get(param.modelId);
if (!model) {
NcError.notFound(`Table with id '${param.modelId}' not found`);
}
if (param.projectId && model.project_id !== param.projectId) {
throw new Error('Table not belong to project');
}
let view: View;
if (param.viewId) {
view = await View.get(param.viewId);
if (!view || (view.fk_model_id && view.fk_model_id !== param.modelId)) {
NcError.unprocessableEntity(`View with id '${param.viewId}' not found`);
}
}
return { model, view };
}
private async extractIdObj({
model,
body,
}: {
body: Record<string, any> | Record<string, any>[];
model: Model;
}) {
const pkColumns = await model
.getColumns()
.then((cols) => cols.filter((col) => col.pk));
const result = (Array.isArray(body) ? body : [body]).map((row) => {
return pkColumns.reduce((acc, col) => {
acc[col.title] = row[col.title];
return acc;
}, {});
});
return Array.isArray(body) ? result : result[0];
}
private async checkForDuplicateRow({
rows,
model,
}: {
rows: any[] | any;
model: Model;
}) {
if (!rows || !Array.isArray(rows) || rows.length === 1) {
return;
}
await model.getColumns();
const keys = new Set();
for (const row of rows) {
let pk;
// if only one primary key then extract the value
if (model.primaryKeys.length === 1) pk = row[model.primaryKey.title];
// if composite primary key then join the values with ___
else pk = model.primaryKeys.map((pk) => row[pk.title]).join('___');
// if duplicate then throw error
if (keys.has(pk)) {
NcError.unprocessableEntity('Duplicate row with id ' + pk);
}
keys.add(pk);
}
}
async nestedDataList(param: {
viewId: string;
modelId: string;
query: any;
rowId: string | string[] | number | number[];
columnId: string;
}) {
const { model, view } = await this.getModelAndView(param);
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(base),
});
if (!(await baseModel.exist(param.rowId))) {
NcError.notFound(`Row with id '${param.rowId}' not found`);
}
const column = await this.getColumn(param);
const colOptions = await column.getColOptions<LinkToAnotherRecordColumn>();
const relatedModel = await colOptions.getRelatedTable();
const { ast, dependencyFields } = await getAst({
model: relatedModel,
query: param.query,
extractOnlyPrimaries: !(param.query?.f || param.query?.fields),
});
const listArgs: any = dependencyFields;
try {
listArgs.filterArr = JSON.parse(listArgs.filterArrJson);
} catch (e) {}
try {
listArgs.sortArr = JSON.parse(listArgs.sortArrJson);
} catch (e) {}
let data: any[];
let count: number;
if (colOptions.type === RelationTypes.MANY_TO_MANY) {
data = await baseModel.mmList(
{
colId: column.id,
parentId: param.rowId,
},
listArgs as any,
);
count = (await baseModel.mmListCount({
colId: column.id,
parentId: param.rowId,
})) as number;
} else if (colOptions.type === RelationTypes.HAS_MANY) {
data = await baseModel.hmList(
{
colId: column.id,
id: param.rowId,
},
listArgs as any,
);
count = (await baseModel.hmListCount({
colId: column.id,
id: param.rowId,
})) as number;
} else {
data = await baseModel.btRead(
{
colId: column.id,
id: param.rowId,
},
param.query as any,
);
}
data = await nocoExecute(ast, data, {}, listArgs);
if (colOptions.type === RelationTypes.BELONGS_TO) return data;
return new PagedResponseImpl(data, {
count,
...param.query,
});
}
private async getColumn(param: { modelId: string; columnId: string }) {
const column = await Column.get({ colId: param.columnId });
if (!column)
NcError.notFound(`Column with id '${param.columnId}' not found`);
if (column.fk_model_id !== param.modelId)
NcError.badRequest('Column not belong to model');
if (column.uidt !== UITypes.LinkToAnotherRecord)
NcError.badRequest('Column is not LTAR');
return column;
}
async nestedLink(param: {
cookie: any;
viewId: string;
modelId: string;
columnId: string;
query: any;
refRowIds: string | string[] | number | number[];
rowId: string;
}) {
this.validateIds(param.refRowIds);
const { model, view } = await this.getModelAndView(param);
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(base),
});
const column = await this.getColumn(param);
await baseModel.addLinks({
colId: column.id,
childIds: Array.isArray(param.refRowIds)
? param.refRowIds
: [param.refRowIds],
rowId: param.rowId,
cookie: param.cookie,
});
return true;
}
async nestedUnlink(param: {
cookie: any;
viewId: string;
modelId: string;
columnId: string;
query: any;
refRowIds: string | string[] | number | number[];
rowId: string;
}) {
this.validateIds(param.refRowIds);
const { model, view } = await this.getModelAndView(param);
if (!model)
NcError.notFound('Table with id ' + param.modelId + ' not found');
const base = await Base.get(model.base_id);
const baseModel = await Model.getBaseModelSQL({
id: model.id,
viewId: view?.id,
dbDriver: await NcConnectionMgrv2.get(base),
});
const column = await this.getColumn(param);
await baseModel.removeLinks({
colId: column.id,
childIds: Array.isArray(param.refRowIds)
? param.refRowIds
: [param.refRowIds],
rowId: param.rowId,
cookie: param.cookie,
});
return true;
}
private validateIds(rowIds: any[] | any) {
if (Array.isArray(rowIds)) {
const map = new Map<string, boolean>();
const set = new Set<string>();
for (const rowId of rowIds) {
if (rowId === undefined || rowId === null)
NcError.unprocessableEntity('Invalid row id ' + rowId);
if (map.has(rowId)) {
set.add(rowId);
} else {
map.set(rowId, true);
}
}
if (set.size > 0)
NcError.unprocessableEntity(
'Child record with id [' + [...set].join(', ') + '] are duplicated',
);
} else if (rowIds === undefined || rowIds === null) {
NcError.unprocessableEntity('Invalid row id ' + rowIds);
}
}
}

2
packages/nocodb/src/services/datas.service.ts

@ -117,7 +117,7 @@ export class DatasService {
async getDataList(param: {
model: Model;
view: View;
view?: View;
query: any;
baseModel?: BaseModelSqlv2;
}) {

8
packages/nocodb/src/services/tables.service.ts

@ -14,13 +14,7 @@ import getColumnPropsFromUIDT from '../helpers/getColumnPropsFromUIDT';
import getColumnUiType from '../helpers/getColumnUiType';
import getTableNameAlias, { getColumnNameAlias } from '../helpers/getTableName';
import mapDefaultDisplayValue from '../helpers/mapDefaultDisplayValue';
import {
Audit,
Column,
Model,
ModelRoleVisibility,
Project,
} from '../models';
import { Audit, Column, Model, ModelRoleVisibility, Project } from '../models';
import Noco from '../Noco';
import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2';
import { validatePayload } from '../helpers';

1
packages/nocodb/src/utils/common/XcAudit.ts

@ -9,5 +9,6 @@ export default class XcAudit {
private static app: Noco;
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public static async log(data: { project }) {}
}

163
packages/nocodb/tests/unit/factory/column.ts

@ -1,13 +1,13 @@
import { UITypes } from 'nocodb-sdk';
import request from 'supertest';
import Column from '../../../src/models/Column';
import FormViewColumn from '../../../src/models/FormViewColumn';
import GalleryViewColumn from '../../../src/models/GalleryViewColumn';
import GridViewColumn from '../../../src/models/GridViewColumn';
import Model from '../../../src/models/Model';
import Project from '../../../src/models/Project';
import View from '../../../src/models/View';
import { isSqlite, isPg } from '../init/db';
import { isPg, isSqlite } from '../init/db';
import type Column from '../../../src/models/Column';
import type FormViewColumn from '../../../src/models/FormViewColumn';
import type GalleryViewColumn from '../../../src/models/GalleryViewColumn';
import type GridViewColumn from '../../../src/models/GridViewColumn';
import type Project from '../../../src/models/Project';
import type View from '../../../src/models/View';
const defaultColumns = function (context) {
return [
@ -46,6 +46,122 @@ const defaultColumns = function (context) {
];
};
const customColumns = function (type: string, options: any = {}) {
switch (type) {
case 'textBased':
return [
{
column_name: 'Id',
title: 'Id',
uidt: UITypes.ID,
},
{
column_name: 'SingleLineText',
title: 'SingleLineText',
uidt: UITypes.SingleLineText,
},
{
column_name: 'MultiLineText',
title: 'MultiLineText',
uidt: UITypes.LongText,
},
{
column_name: 'Email',
title: 'Email',
uidt: UITypes.Email,
},
{
column_name: 'Phone',
title: 'Phone',
uidt: UITypes.PhoneNumber,
},
{
column_name: 'Url',
title: 'Url',
uidt: UITypes.URL,
},
];
case 'numberBased':
return [
{
column_name: 'Id',
title: 'Id',
uidt: UITypes.ID,
},
{
column_name: 'Number',
title: 'Number',
uidt: UITypes.Number,
},
{
column_name: 'Decimal',
title: 'Decimal',
uidt: UITypes.Decimal,
},
{
column_name: 'Currency',
title: 'Currency',
uidt: UITypes.Currency,
},
{
column_name: 'Percent',
title: 'Percent',
uidt: UITypes.Percent,
},
{
column_name: 'Duration',
title: 'Duration',
uidt: UITypes.Duration,
},
{
column_name: 'Rating',
title: 'Rating',
uidt: UITypes.Rating,
},
];
case 'dateBased':
return [
{
column_name: 'Id',
title: 'Id',
uidt: UITypes.ID,
},
{
column_name: 'Date',
title: 'Date',
uidt: UITypes.Date,
},
{
column_name: 'DateTime',
title: 'DateTime',
uidt: UITypes.DateTime,
},
];
case 'selectBased':
return [
{
column_name: 'Id',
title: 'Id',
uidt: UITypes.ID,
},
{
column_name: 'SingleSelect',
title: 'SingleSelect',
uidt: UITypes.SingleSelect,
dtxp: "'jan','feb','mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'",
},
{
column_name: 'MultiSelect',
title: 'MultiSelect',
uidt: UITypes.MultiSelect,
dtxp: "'jan','feb','mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'",
},
];
case 'custom':
return [{ title: 'Id', column_name: 'Id', uidt: UITypes.ID }, ...options];
}
};
const createColumn = async (context, table, columnAttr) => {
await request(context.app)
.post(`/api/v1/db/meta/tables/${table.id}/columns`)
@ -55,7 +171,7 @@ const createColumn = async (context, table, columnAttr) => {
});
const column: Column = (await table.getColumns()).find(
(column) => column.title === columnAttr.title
(column) => column.title === columnAttr.title,
);
return column;
};
@ -76,7 +192,7 @@ const createRollupColumn = async (
table: Model;
relatedTableName: string;
relatedTableColumnTitle: string;
}
},
) => {
const childBases = await project.getBases();
const childTable = await Model.getByIdOrName({
@ -86,13 +202,13 @@ const createRollupColumn = async (
});
const childTableColumns = await childTable.getColumns();
const childTableColumn = await childTableColumns.find(
(column) => column.title === relatedTableColumnTitle
(column) => column.title === relatedTableColumnTitle,
);
const ltarColumn = (await table.getColumns()).find(
(column) =>
column.uidt === UITypes.LinkToAnotherRecord &&
column.colOptions?.fk_related_model_id === childTable.id
column.colOptions?.fk_related_model_id === childTable.id,
);
const rollupColumn = await createColumn(context, table, {
@ -122,7 +238,7 @@ const createLookupColumn = async (
table: Model;
relatedTableName: string;
relatedTableColumnTitle: string;
}
},
) => {
const childBases = await project.getBases();
const childTable = await Model.getByIdOrName({
@ -132,19 +248,19 @@ const createLookupColumn = async (
});
const childTableColumns = await childTable.getColumns();
const childTableColumn = await childTableColumns.find(
(column) => column.title === relatedTableColumnTitle
(column) => column.title === relatedTableColumnTitle,
);
if (!childTableColumn) {
throw new Error(
`Could not find column ${relatedTableColumnTitle} in ${relatedTableName}`
`Could not find column ${relatedTableColumnTitle} in ${relatedTableName}`,
);
}
const ltarColumn = (await table.getColumns()).find(
(column) =>
column.uidt === UITypes.LinkToAnotherRecord &&
column.colOptions?.fk_related_model_id === childTable.id
column.colOptions?.fk_related_model_id === childTable.id,
);
const lookupColumn = await createColumn(context, table, {
title: title,
@ -168,15 +284,15 @@ const createQrCodeColumn = async (
title: string;
table: Model;
referencedQrValueTableColumnTitle: string;
}
},
) => {
const referencedQrValueTableColumnId = await table
.getColumns()
.then(
(cols) =>
cols.find(
(column) => column.title == referencedQrValueTableColumnTitle
)['id']
(column) => column.title == referencedQrValueTableColumnTitle,
)['id'],
);
const qrCodeColumn = await createColumn(context, table, {
@ -198,15 +314,15 @@ const createBarcodeColumn = async (
title: string;
table: Model;
referencedBarcodeValueTableColumnTitle: string;
}
},
) => {
const referencedBarcodeValueTableColumnId = await table
.getColumns()
.then(
(cols) =>
cols.find(
(column) => column.title == referencedBarcodeValueTableColumnTitle
)['id']
(column) => column.title == referencedBarcodeValueTableColumnTitle,
)['id'],
);
const barcodeColumn = await createColumn(context, table, {
@ -230,7 +346,7 @@ const createLtarColumn = async (
parentTable: Model;
childTable: Model;
type: string;
}
},
) => {
const ltarColumn = await createColumn(context, parentTable, {
title: title,
@ -246,7 +362,7 @@ const createLtarColumn = async (
const updateViewColumn = async (
context,
{ view, column, attr }: { column: Column; view: View; attr: any }
{ view, column, attr }: { column: Column; view: View; attr: any },
) => {
const res = await request(context.app)
.patch(`/api/v1/db/meta/views/${view.id}/columns/${column.id}`)
@ -263,6 +379,7 @@ const updateViewColumn = async (
};
export {
customColumns,
defaultColumns,
createColumn,
createQrCodeColumn,

35
packages/nocodb/tests/unit/factory/row.ts

@ -1,11 +1,12 @@
import { ColumnType, UITypes } from 'nocodb-sdk';
import { UITypes } from 'nocodb-sdk';
import request from 'supertest';
import Column from '../../../src/models/Column';
import Filter from '../../../src/models/Filter';
import Model from '../../../src/models/Model';
import Project from '../../../src/models/Project';
import Sort from '../../../src/models/Sort';
import NcConnectionMgrv2 from '../../../src/utils/common/NcConnectionMgrv2';
import type { ColumnType } from 'nocodb-sdk';
import type Column from '../../../src/models/Column';
import type Filter from '../../../src/models/Filter';
import type Project from '../../../src/models/Project';
import type Sort from '../../../src/models/Sort';
const rowValue = (column: ColumnType, index: number) => {
switch (column.uidt) {
@ -175,9 +176,17 @@ const rowMixedValue = (column: ColumnType, index: number) => {
case UITypes.Date:
// set startDate as 400 days before today
// eslint-disable-next-line no-case-declarations
const result = new Date();
result.setDate(result.getDate() - 400 + index);
return result.toISOString().slice(0, 10);
const d1 = new Date();
d1.setDate(d1.getDate() - 400 + index);
return d1.toISOString().slice(0, 10);
case UITypes.DateTime:
// set startDate as 400 days before today
// eslint-disable-next-line no-case-declarations
const d2 = new Date();
d2.setDate(d2.getDate() - 400 + index);
// set time to 12:00:00
d2.setHours(12, 0, 0, 0);
return d2.toISOString();
case UITypes.URL:
return urls[index % urls.length];
case UITypes.SingleSelect:
@ -228,7 +237,7 @@ const listRow = async ({
const getOneRow = async (
context,
{ project, table }: { project: Project; table: Model }
{ project, table }: { project: Project; table: Model },
) => {
const response = await request(context.app)
.get(`/api/v1/db/data/noco/${project.id}/${table.id}/find-one`)
@ -266,7 +275,7 @@ const createRow = async (
project: Project;
table: Model;
index?: number;
}
},
) => {
const columns = await table.getColumns();
const rowData = generateDefaultRowAttributes({ columns, index });
@ -289,7 +298,7 @@ const createBulkRows = async (
project: Project;
table: Model;
values: any[];
}
},
) => {
await request(context.app)
.post(`/api/v1/db/data/bulk/noco/${project.id}/${table.id}`)
@ -317,7 +326,7 @@ const createChildRow = async (
rowId?: string;
childRowId?: string;
type: string;
}
},
) => {
if (!rowId) {
const row = await createRow(context, { project, table });
@ -331,7 +340,7 @@ const createChildRow = async (
await request(context.app)
.post(
`/api/v1/db/data/noco/${project.id}/${table.id}/${rowId}/${type}/${column.title}/${childRowId}`
`/api/v1/db/data/noco/${project.id}/${table.id}/${rowId}/${type}/${column.title}/${childRowId}`,
)
.set('xc-auth', context.token);

84
packages/nocodb/tests/unit/factory/view.ts

@ -1,9 +1,20 @@
import { ViewTypes } from 'nocodb-sdk';
import request from 'supertest';
import Model from '../../../src/models/Model';
import View from '../../../src/models/View';
import type Model from '../../../src/models/Model';
const createView = async (context, {title, table, type}: {title: string, table: Model, type: ViewTypes}) => {
const createView = async (
context,
{
title,
table,
type,
}: {
title: string;
table: Model;
type: ViewTypes;
},
) => {
const viewTypeStr = (type) => {
switch (type) {
case ViewTypes.GALLERY:
@ -26,13 +37,70 @@ const createView = async (context, {title, table, type}: {title: string, table:
title,
type,
});
if(response.status !== 200) {
throw new Error('createView',response.body.message);
if (response.status !== 200) {
throw new Error('createView', response.body.message);
}
const view = await View.getByTitleOrId({fk_model_id: table.id, titleOrId:title}) as View;
const view = (await View.getByTitleOrId({
fk_model_id: table.id,
titleOrId: title,
})) as View;
return view;
};
return view
}
const updateView = async (
context,
{
table,
view,
filter = [],
sort = [],
field = [],
}: {
table: Model;
view: View;
filter?: any[];
sort?: any[];
field?: any[];
},
) => {
if (filter.length) {
for (let i = 0; i < filter.length; i++) {
await request(context.app)
.post(`/api/v1/db/meta/views/${view.id}/filters`)
.set('xc-auth', context.token)
.send(filter[i])
.expect(200);
}
}
if (sort.length) {
for (let i = 0; i < sort.length; i++) {
await request(context.app)
.post(`/api/v1/db/meta/views/${view.id}/sorts`)
.set('xc-auth', context.token)
.send(sort[i])
.expect(200);
}
}
if (field.length) {
for (let i = 0; i < field.length; i++) {
const columns = await table.getColumns();
const viewColumns = await view.getColumns();
const columnId = columns.find((c) => c.title === field[i]).id;
const viewColumnId = viewColumns.find(
(c) => c.fk_column_id === columnId,
).id;
// configure view to hide selected fields
await request(context.app)
.patch(`/api/v1/db/meta/views/${view.id}/columns/${viewColumnId}`)
.set('xc-auth', context.token)
.send({ show: false })
.expect(200);
}
}
};
export {createView}
export { createView, updateView };

7
packages/nocodb/tests/unit/init/index.ts

@ -25,7 +25,7 @@ const serverInit = async () => {
const isFirstTimeRun = () => !server;
export default async function () {
export default async function (isSakila = true) {
const { default: TestDbMngr } = await import('../TestDbMngr');
if (isFirstTimeRun()) {
@ -33,7 +33,10 @@ export default async function () {
server = await serverInit();
}
await cleanUpSakila();
if (isSakila) {
await cleanUpSakila();
}
await cleanupMeta();
const { token } = await createUser({ app: server }, { roles: 'editor' });

2
packages/nocodb/tests/unit/rest/index.test.ts

@ -8,6 +8,7 @@ import tableRowTests from './tests/tableRow.test';
import viewRowTests from './tests/viewRow.test';
import attachmentTests from './tests/attachment.test';
import filterTest from './tests/filter.test';
import newDataApisTest from './tests/newDataApis.test';
function restTests() {
authTests();
@ -19,6 +20,7 @@ function restTests() {
columnTypeSpecificTests();
attachmentTests();
filterTest();
newDataApisTest();
}
export default function () {

2867
packages/nocodb/tests/unit/rest/tests/newDataApis.test.ts

File diff suppressed because it is too large Load Diff
Loading…
Cancel
Save