From 76deee749d7e21ae97765d79cc762b5d95f7bc7f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 3 Jun 2023 22:56:44 +0530 Subject: [PATCH 01/34] feat: hide advanced options in LTAR column creation for xcdb project Signed-off-by: Pranav C --- .../column/LinkedToAnotherRecordOptions.vue | 95 ++++++++++--------- .../composables/useColumnCreateStore.ts | 5 +- 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/column/LinkedToAnotherRecordOptions.vue b/packages/nc-gui/components/smartsheet/column/LinkedToAnotherRecordOptions.vue index b3d175f567..f285026379 100644 --- a/packages/nc-gui/components/smartsheet/column/LinkedToAnotherRecordOptions.vue +++ b/packages/nc-gui/components/smartsheet/column/LinkedToAnotherRecordOptions.vue @@ -16,7 +16,7 @@ const vModel = useVModel(props, 'value', emit) const meta = $(inject(MetaInj, ref())) -const { setAdditionalValidations, validateInfos, onDataTypeChange, sqlUi } = useColumnCreateStoreOrThrow() +const { setAdditionalValidations, validateInfos, onDataTypeChange, sqlUi, isXcdbBase } = useColumnCreateStoreOrThrow() const { tables } = $(storeToRefs(useProject())) @@ -86,54 +86,57 @@ const filterOption = (value: string, option: { key: string }) => option.key.toLo + diff --git a/packages/nc-gui/composables/useColumnCreateStore.ts b/packages/nc-gui/composables/useColumnCreateStore.ts index 978f71fdd4..a6b99847be 100644 --- a/packages/nc-gui/composables/useColumnCreateStore.ts +++ b/packages/nc-gui/composables/useColumnCreateStore.ts @@ -31,7 +31,7 @@ interface ValidationsObj { const [useProvideColumnCreateStore, useColumnCreateStore] = createInjectionState( (meta: Ref, column: Ref) => { const projectStore = useProject() - const { isMysql: isMysqlFunc, isPg: isPgFunc, isMssql: isMssqlFunc } = projectStore + const { isMysql: isMysqlFunc, isPg: isPgFunc, isMssql: isMssqlFunc, isXcdbBase: isXcdbBaseFunc } = projectStore const { project, sqlUis } = storeToRefs(projectStore) const { $api } = useNuxtApp() @@ -52,6 +52,8 @@ const [useProvideColumnCreateStore, useColumnCreateStore] = createInjectionState const isMssql = computed(() => isMssqlFunc(meta.value?.base_id ? meta.value?.base_id : Object.keys(sqlUis.value)[0])) + const isXcdbBase = computed(() => isXcdbBaseFunc(meta.value?.base_id ? meta.value?.base_id : Object.keys(sqlUis.value)[0])) + const idType = null const additionalValidations = ref({}) @@ -289,6 +291,7 @@ const [useProvideColumnCreateStore, useColumnCreateStore] = createInjectionState isMssql, isPg, isMysql, + isXcdbBase } }, ) From d9bd216bae5d167dc8d6b45389c5f55e6c4f2298 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sat, 3 Jun 2023 23:16:10 +0530 Subject: [PATCH 02/34] refactor: for xcdb make relation virtual Signed-off-by: Pranav C --- packages/nocodb/src/services/columns.service.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index aa4fd91da7..3d0bfe9c20 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -1553,6 +1553,11 @@ export class ColumnsService { childColumn = await Column.get({ colId: id }); + // if xcdb base then treat as virtual relation to avoid creating foreign key + if (param.base.is_meta) { + (param.column as LinkToAnotherColumnReqType).virtual = true; + } + // ignore relation creation if virtual if (!(param.column as LinkToAnotherColumnReqType).virtual) { foreignKeyName = generateFkName(parent, child); @@ -1570,7 +1575,7 @@ export class ColumnsService { } // todo: create index for virtual relations as well - // create index for foreign key in pg + // create index for foreign key in pg if ( param.base.type === 'pg' || (param.column as LinkToAnotherColumnReqType).virtual From 94262ac47f006192cdd031c80945320ade0b69e4 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sun, 4 Jun 2023 00:41:16 +0530 Subject: [PATCH 03/34] feat: on delete unlink rows (WIP) Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 60 +++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 86e2a96431..ec8d5efdde 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -17,6 +17,7 @@ import Validator from 'validator'; import { customAlphabet } from 'nanoid'; import DOMPurify from 'isomorphic-dompurify'; import { v4 as uuidv4 } from 'uuid'; +import { Knex } from 'knex'; import { NcError } from '../helpers/catchError'; import getAst from '../helpers/getAst'; import { Audit, Column, Filter, Model, Project, Sort, View } from '../models'; @@ -47,8 +48,8 @@ import type { RollupColumn, SelectOption, } from '../models'; -import type { Knex } from 'knex'; import type { SortType } from 'nocodb-sdk'; +import Transaction = Knex.Transaction; dayjs.extend(utc); dayjs.extend(timezone); @@ -1875,11 +1876,66 @@ class BaseModelSqlv2 { } } - async delByPk(id, trx?, cookie?) { + async delByPk(id, _trx?, cookie?) { + const trx: Transaction = _trx; try { // retrieve data for handling params in hook const data = await this.readByPk(id); await this.beforeDelete(id, trx, cookie); + + // todo: use transaction + // if (!trx) { + // trx = await this.dbDriver.transaction(); + // } + + // start a transaction if not already in one + for (const column of this.model.columns) { + if (column.uidt !== UITypes.LinkToAnotherRecord) continue; + + const colOptions = + await column.getColOptions(); + + switch (colOptions.type) { + case 'mm': + { + const mmTable = await Model.get(colOptions.fk_mm_model_id); + const mmParentColumn = await Column.get({ + colId: colOptions.fk_mm_child_column_id, + }); + + await this.dbDriver(mmTable.table_name) + .del() + .where(mmParentColumn.column_name, id); + } + break; + case 'hm': + { + const relatedTable = await Model.get( + colOptions.fk_related_model_id, + ); + const childColumn = await Column.get({ + colId: colOptions.fk_child_column_id, + }); + + await this.dbDriver(relatedTable.table_name) + .update({ + [childColumn.column_name]: null, + }) + .where(childColumn.column_name, id); + } + break; + case 'bt': + { + // nothing to do + } + break; + } + + await trx(this.model.table_name).where(column.column_name, id).del(); + } + + // iterate over all columns and unlink all LTAR data + const response = await this.dbDriver(this.tnPath) .del() .where(await this._wherePk(id)); From 5c0524b7ef050078d140bc9769caeac4129f5f7c Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 5 Jun 2023 00:55:24 +0530 Subject: [PATCH 04/34] feat: introduce transaction Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 38 ++++++++++++++---------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index ec8d5efdde..4fc86e7c4f 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -1877,16 +1877,13 @@ class BaseModelSqlv2 { } async delByPk(id, _trx?, cookie?) { - const trx: Transaction = _trx; + let trx: Transaction = _trx; try { // retrieve data for handling params in hook const data = await this.readByPk(id); await this.beforeDelete(id, trx, cookie); - // todo: use transaction - // if (!trx) { - // trx = await this.dbDriver.transaction(); - // } + const execQueries: ((trx: Transaction) => Promise)[] = []; // start a transaction if not already in one for (const column of this.model.columns) { @@ -1903,9 +1900,11 @@ class BaseModelSqlv2 { colId: colOptions.fk_mm_child_column_id, }); - await this.dbDriver(mmTable.table_name) - .del() - .where(mmParentColumn.column_name, id); + execQueries.push((trx) => + trx(mmTable.table_name) + .del() + .where(mmParentColumn.column_name, id), + ); } break; case 'hm': @@ -1917,11 +1916,13 @@ class BaseModelSqlv2 { colId: colOptions.fk_child_column_id, }); - await this.dbDriver(relatedTable.table_name) - .update({ - [childColumn.column_name]: null, - }) - .where(childColumn.column_name, id); + execQueries.push((trx) => + trx(relatedTable.table_name) + .update({ + [childColumn.column_name]: null, + }) + .where(childColumn.column_name, id), + ); } break; case 'bt': @@ -1931,14 +1932,19 @@ class BaseModelSqlv2 { break; } - await trx(this.model.table_name).where(column.column_name, id).del(); + // await trx(this.model.table_name).where(column.column_name, id).del(); } - // iterate over all columns and unlink all LTAR data + if (!trx) { + trx = await this.dbDriver.transaction(); + } - const response = await this.dbDriver(this.tnPath) + await Promise.all(execQueries.map((q) => q(trx))); + + const response = await trx(this.tnPath) .del() .where(await this._wherePk(id)); + await this.afterDelete(data, trx, cookie); return response; } catch (e) { From cca0724aa05cafd08ba44776998df0ac5240c0c0 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 5 Jun 2023 00:57:29 +0530 Subject: [PATCH 05/34] fix: add commit and rollback Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 4fc86e7c4f..c38f0ad51d 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -1945,10 +1945,13 @@ class BaseModelSqlv2 { .del() .where(await this._wherePk(id)); + if (!_trx) await trx.commit(); + await this.afterDelete(data, trx, cookie); return response; } catch (e) { console.log(e); + if (!_trx) await trx.rollback(); await this.errorDelete(e, id, trx, cookie); throw e; } From 37b1987cac8542a77f4047734fa4a67f9246c4c2 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 5 Jun 2023 12:45:03 +0530 Subject: [PATCH 06/34] fix: skip mm hasmany relations Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index c38f0ad51d..dae2103079 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -1909,9 +1909,12 @@ class BaseModelSqlv2 { break; case 'hm': { - const relatedTable = await Model.get( - colOptions.fk_related_model_id, - ); + // skip if it's an mm table column + const relatedTable = await colOptions.getRelatedTable(); + if (relatedTable.mm) { + break; + } + const childColumn = await Column.get({ colId: colOptions.fk_child_column_id, }); @@ -1931,8 +1934,6 @@ class BaseModelSqlv2 { } break; } - - // await trx(this.model.table_name).where(column.column_name, id).del(); } if (!trx) { From 671d87db5e2cfd6b4c630f064b2fbbac8a698ca1 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 5 Jun 2023 12:45:42 +0530 Subject: [PATCH 07/34] refactor: allow deleting rows with LTAR relation for matadb base Signed-off-by: Pranav C --- packages/nocodb/src/services/datas.service.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/nocodb/src/services/datas.service.ts b/packages/nocodb/src/services/datas.service.ts index c41609f6c0..cf82a9d585 100644 --- a/packages/nocodb/src/services/datas.service.ts +++ b/packages/nocodb/src/services/datas.service.ts @@ -103,11 +103,15 @@ export class DatasService { dbDriver: await NcConnectionMgrv2.get(base), }); - // todo: Should have error http status code - const message = await baseModel.hasLTARData(param.rowId, model); - if (message.length) { - return { message }; + // if xcdb project skip checking for LTAR + if (!base.is_meta) { + // todo: Should have error http status code + const message = await baseModel.hasLTARData(param.rowId, model); + if (message.length) { + return { message }; + } } + return await baseModel.delByPk(param.rowId, null, param.cookie); } From 6cba56bde1317959b5af8d64962e071f844067e0 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Mon, 5 Jun 2023 12:46:12 +0530 Subject: [PATCH 08/34] feat: table delete with relation (WIP) Signed-off-by: Pranav C --- packages/nocodb/src/models/Column.ts | 11 +- .../nocodb/src/services/columns.service.ts | 130 ++++++++++-------- .../nocodb/src/services/tables.service.ts | 94 +++++++++---- 3 files changed, 143 insertions(+), 92 deletions(-) diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts index 22ed522e37..e7e14b0f2b 100644 --- a/packages/nocodb/src/models/Column.ts +++ b/packages/nocodb/src/models/Column.ts @@ -65,10 +65,13 @@ export default class Column implements ColumnType { Object.assign(this, data); } - public async getModel(): Promise { - return Model.getByIdOrName({ - id: this.fk_model_id, - }); + public async getModel(ncMeta = Noco.ncMeta): Promise { + return Model.getByIdOrName( + { + id: this.fk_model_id, + }, + ncMeta, + ); } public static async insert( diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index 3d0bfe9c20..7fd2972fce 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -38,8 +38,8 @@ import { import Noco from '../Noco'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import { MetaTable } from '../utils/globals'; +import { MetaService } from '../meta/meta.service'; import type { LinkToAnotherRecordColumn, Project } from '../models'; -import type { MetaService } from '../meta/meta.service'; import type SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2'; import type { ColumnReqType, @@ -57,6 +57,8 @@ export enum Altered { @Injectable() export class ColumnsService { + constructor(private readonly metaService: MetaService) {} + async columnUpdate(param: { req?: any; columnId: string; @@ -1142,19 +1144,18 @@ export class ColumnsService { return table; } - async columnDelete(param: { req?: any; columnId: string }) { - const column = await Column.get({ colId: param.columnId }); - const table = await Model.getWithInfo({ - id: column.fk_model_id, - }); - const base = await Base.get(table.base_id); - - // const ncMeta = await Noco.ncMeta.startTransaction(); - // const sql-mgr = await ProjectMgrv2.getSqlMgrTrans( - // { id: base.project_id }, - // ncMeta, - // base - // ); + async columnDelete( + param: { req?: any; columnId: string }, + ncMeta = this.metaService, + ) { + const column = await Column.get({ colId: param.columnId }, ncMeta); + const table = await Model.getWithInfo( + { + id: column.fk_model_id, + }, + ncMeta, + ); + const base = await Base.get(table.base_id, ncMeta); const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: base.project_id }); @@ -1164,17 +1165,17 @@ export class ColumnsService { case UITypes.QrCode: case UITypes.Barcode: case UITypes.Formula: - await Column.delete(param.columnId); + await Column.delete(param.columnId, ncMeta); break; case UITypes.LinkToAnotherRecord: { const relationColOpt = - await column.getColOptions(); - const childColumn = await relationColOpt.getChildColumn(); - const childTable = await childColumn.getModel(); + await column.getColOptions(ncMeta); + const childColumn = await relationColOpt.getChildColumn(ncMeta); + const childTable = await childColumn.getModel(ncMeta); - const parentColumn = await relationColOpt.getParentColumn(); - const parentTable = await parentColumn.getModel(); + const parentColumn = await relationColOpt.getParentColumn(ncMeta); + const parentTable = await parentColumn.getModel(ncMeta); switch (relationColOpt.type) { case 'bt': @@ -1188,15 +1189,19 @@ export class ColumnsService { parentColumn, parentTable, sqlMgr, - // ncMeta + ncMeta, }); } break; case 'mm': { - const mmTable = await relationColOpt.getMMModel(); - const mmParentCol = await relationColOpt.getMMParentColumn(); - const mmChildCol = await relationColOpt.getMMChildColumn(); + const mmTable = await relationColOpt.getMMModel(ncMeta); + const mmParentCol = await relationColOpt.getMMParentColumn( + ncMeta, + ); + const mmChildCol = await relationColOpt.getMMChildColumn( + ncMeta, + ); await this.deleteHmOrBtRelation( { @@ -1207,7 +1212,7 @@ export class ColumnsService { parentTable: parentTable, childColumn: mmParentCol, base, - // ncMeta + ncMeta, }, true, ); @@ -1221,18 +1226,18 @@ export class ColumnsService { parentTable: childTable, childColumn: mmChildCol, base, - // ncMeta + ncMeta, }, true, ); const columnsInRelatedTable: Column[] = await relationColOpt - .getRelatedTable() - .then((m) => m.getColumns()); + .getRelatedTable(ncMeta) + .then((m) => m.getColumns(ncMeta)); for (const c of columnsInRelatedTable) { if (c.uidt !== UITypes.LinkToAnotherRecord) continue; const colOpt = - await c.getColOptions(); + await c.getColOptions(ncMeta); if ( colOpt.type === 'mm' && colOpt.fk_parent_column_id === childColumn.id && @@ -1241,55 +1246,55 @@ export class ColumnsService { colOpt.fk_mm_parent_column_id === mmChildCol.id && colOpt.fk_mm_child_column_id === mmParentCol.id ) { - await Column.delete(c.id); + await Column.delete(c.id, ncMeta); break; } } - await Column.delete(relationColOpt.fk_column_id); + await Column.delete(relationColOpt.fk_column_id, ncMeta); // delete bt columns in m2m table - await mmTable.getColumns(); + await mmTable.getColumns(ncMeta); for (const c of mmTable.columns) { if (c.uidt !== UITypes.LinkToAnotherRecord) continue; const colOpt = - await c.getColOptions(); + await c.getColOptions(ncMeta); if (colOpt.type === 'bt') { - await Column.delete(c.id); + await Column.delete(c.id, ncMeta); } } // delete hm columns in parent table - await parentTable.getColumns(); + await parentTable.getColumns(ncMeta); for (const c of parentTable.columns) { if (c.uidt !== UITypes.LinkToAnotherRecord) continue; const colOpt = - await c.getColOptions(); + await c.getColOptions(ncMeta); if (colOpt.fk_related_model_id === mmTable.id) { await Column.delete(c.id); } } // delete hm columns in child table - await childTable.getColumns(); + await childTable.getColumns(ncMeta); for (const c of childTable.columns) { if (c.uidt !== UITypes.LinkToAnotherRecord) continue; const colOpt = - await c.getColOptions(); + await c.getColOptions(ncMeta); if (colOpt.fk_related_model_id === mmTable.id) { - await Column.delete(c.id); + await Column.delete(c.id, ncMeta); } } // retrieve columns in m2m table again - await mmTable.getColumns(); + await mmTable.getColumns(ncMeta); // ignore deleting table if it has more than 2 columns // the expected 2 columns would be table1_id & table2_id if (mmTable.columns.length === 2) { (mmTable as any).tn = mmTable.table_name; await sqlMgr.sqlOpPlus(base, 'tableDelete', mmTable); - await mmTable.delete(); + await mmTable.delete(ncMeta); } } break; @@ -1337,26 +1342,30 @@ export class ColumnsService { await sqlMgr.sqlOpPlus(base, 'tableUpdate', tableUpdateBody); - await Column.delete(param.columnId); + await Column.delete(param.columnId, ncMeta); } } - await Audit.insert({ - project_id: base.project_id, - op_type: AuditOperationTypes.TABLE_COLUMN, - op_sub_type: AuditOperationSubTypes.DELETE, - user: param?.req?.user?.email, - description: `The column ${column.column_name} with alias ${column.title} from table ${table.table_name} has been deleted`, - ip: param?.req.clientIp, - }); - - await table.getColumns(); + await Audit.insert( + { + project_id: base.project_id, + op_type: AuditOperationTypes.TABLE_COLUMN, + op_sub_type: AuditOperationSubTypes.DELETE, + user: param?.req?.user?.email, + description: `The column ${column.column_name} with alias ${column.title} from table ${table.table_name} has been deleted`, + ip: param?.req.clientIp, + }, + ncMeta, + ); + + await table.getColumns(ncMeta); const displayValueColumn = mapDefaultDisplayValue(table.columns); if (displayValueColumn) { await Model.updatePrimaryColumn( displayValueColumn.fk_model_id, displayValueColumn.id, + ncMeta, ); } @@ -1394,7 +1403,7 @@ export class ColumnsService { if (!relationColOpt) { foreignKeyName = ( ( - await childTable.getColumns().then((cols) => { + await childTable.getColumns(ncMeta).then((cols) => { return cols?.find((c) => { return ( c.uidt === UITypes.LinkToAnotherRecord && @@ -1427,12 +1436,12 @@ export class ColumnsService { if (!relationColOpt) return; const columnsInRelatedTable: Column[] = await relationColOpt - .getRelatedTable() - .then((m) => m.getColumns()); + .getRelatedTable(ncMeta) + .then((m) => m.getColumns(ncMeta)); const relType = relationColOpt.type === 'bt' ? 'hm' : 'bt'; for (const c of columnsInRelatedTable) { if (c.uidt !== UITypes.LinkToAnotherRecord) continue; - const colOpt = await c.getColOptions(); + const colOpt = await c.getColOptions(ncMeta); if ( colOpt.fk_parent_column_id === parentColumn.id && colOpt.fk_child_column_id === childColumn.id && @@ -1447,9 +1456,12 @@ export class ColumnsService { await Column.delete(relationColOpt.fk_column_id, ncMeta); if (!ignoreFkDelete) { - const cTable = await Model.getWithInfo({ - id: childTable.id, - }); + const cTable = await Model.getWithInfo( + { + id: childTable.id, + }, + ncMeta, + ); const tableUpdateBody = { ...cTable, tn: cTable.table_name, diff --git a/packages/nocodb/src/services/tables.service.ts b/packages/nocodb/src/services/tables.service.ts index 7d86ce4d5d..4d9b21ca76 100644 --- a/packages/nocodb/src/services/tables.service.ts +++ b/packages/nocodb/src/services/tables.service.ts @@ -14,9 +14,19 @@ import getColumnPropsFromUIDT from '../helpers/getColumnPropsFromUIDT'; import getColumnUiType from '../helpers/getColumnUiType'; import getTableNameAlias, { getColumnNameAlias } from '../helpers/getTableName'; import mapDefaultDisplayValue from '../helpers/mapDefaultDisplayValue'; -import { Audit, Column, Model, ModelRoleVisibility, Project } from '../models'; +import { + Audit, + Base, + Column, + Model, + ModelRoleVisibility, + Project, +} from '../models'; +import Noco from '../Noco'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import { validatePayload } from '../helpers'; +import { ColumnsService } from './columns.service'; +import type { MetaService } from '../meta/meta.service'; import type { LinkToAnotherRecordColumn, User, View } from '../models'; import type { ColumnType, @@ -26,6 +36,8 @@ import type { @Injectable() export class TablesService { + constructor(private readonly columnsService: ColumnsService) {} + async tableUpdate(param: { tableId: any; table: TableReqType & { project_id?: string }; @@ -142,11 +154,14 @@ export class TablesService { const table = await Model.getByIdOrName({ id: param.tableId }); await table.getColumns(); + const project = await Project.getWithInfo(table.project_id); + const base = project.bases.find((b) => b.id === table.base_id); + const relationColumns = table.columns.filter( (c) => c.uidt === UITypes.LinkToAnotherRecord, ); - if (relationColumns?.length) { + if (relationColumns?.length && !base.is_meta) { const referredTables = await Promise.all( relationColumns.map(async (c) => c @@ -162,37 +177,58 @@ export class TablesService { ); } - const project = await Project.getWithInfo(table.project_id); - const base = project.bases.find((b) => b.id === table.base_id); - const sqlMgr = await ProjectMgrv2.getSqlMgr(project); - (table as any).tn = table.table_name; - table.columns = table.columns.filter((c) => !isVirtualCol(c)); - table.columns.forEach((c) => { - (c as any).cn = c.column_name; - }); + // start a transaction + const ncMeta = await (Noco.ncMeta as MetaService).startTransaction(); + let result; + try { + // delete all relations + await Promise.all( + relationColumns.map((c) => { + return this.columnsService.columnDelete( + { + req: param.req, + columnId: c.id, + }, + ncMeta, + ); + }), + ); - if (table.type === ModelTypes.TABLE) { - await sqlMgr.sqlOpPlus(base, 'tableDelete', table); - } else if (table.type === ModelTypes.VIEW) { - await sqlMgr.sqlOpPlus(base, 'viewDelete', { - ...table, - view_name: table.table_name, + const sqlMgr = await ProjectMgrv2.getSqlMgr(project); + (table as any).tn = table.table_name; + table.columns = table.columns.filter((c) => !isVirtualCol(c)); + table.columns.forEach((c) => { + (c as any).cn = c.column_name; }); - } - - await Audit.insert({ - project_id: project.id, - base_id: base.id, - op_type: AuditOperationTypes.TABLE, - op_sub_type: AuditOperationSubTypes.DELETE, - user: param.user?.email, - description: `Deleted ${table.type} ${table.table_name} with alias ${table.title} `, - ip: param.req?.clientIp, - }).then(() => {}); - T.emit('evt', { evt_type: 'table:deleted' }); + if (table.type === ModelTypes.TABLE) { + await sqlMgr.sqlOpPlus(base, 'tableDelete', table); + } else if (table.type === ModelTypes.VIEW) { + await sqlMgr.sqlOpPlus(base, 'viewDelete', { + ...table, + view_name: table.table_name, + }); + } - return table.delete(); + await Audit.insert({ + project_id: project.id, + base_id: base.id, + op_type: AuditOperationTypes.TABLE, + op_sub_type: AuditOperationSubTypes.DELETE, + user: param.user?.email, + description: `Deleted ${table.type} ${table.table_name} with alias ${table.title} `, + ip: param.req?.clientIp, + }).then(() => {}); + + T.emit('evt', { evt_type: 'table:deleted' }); + + result = await table.delete(ncMeta); + await ncMeta.commit(); + } catch (e) { + await ncMeta.rollback(); + throw e; + } + return result; } async getTableWithAccessibleViews(param: { tableId: string; user: User }) { From 1d45a501f320aed0a31822e22bad66ac485f9c05 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 6 Jun 2023 00:08:14 +0530 Subject: [PATCH 09/34] feat: table delete with relation - wrap in transaction Signed-off-by: Pranav C --- packages/nc-gui/composables/useTable.ts | 6 +- .../db/sql-client/lib/sqlite/SqliteClient.ts | 2 +- .../nocodb/src/db/sql-mgr/v2/ProjectMgrv2.ts | 23 ++--- packages/nocodb/src/db/sql-mgr/v2/SqlMgrv2.ts | 12 ++- packages/nocodb/src/meta/meta.service.ts | 1 - packages/nocodb/src/models/Model.ts | 6 +- packages/nocodb/src/models/View.ts | 10 +-- .../nocodb/src/services/columns.service.ts | 90 ++++++++++++------- .../nocodb/src/services/tables.service.ts | 28 +++--- 9 files changed, 113 insertions(+), 65 deletions(-) diff --git a/packages/nc-gui/composables/useTable.ts b/packages/nc-gui/composables/useTable.ts index cb4bccea52..4596c2ff2c 100644 --- a/packages/nc-gui/composables/useTable.ts +++ b/packages/nc-gui/composables/useTable.ts @@ -30,7 +30,7 @@ export function useTable(onTableCreate?: (tableMeta: TableType) => void, baseId? const { getMeta, removeMeta } = useMetas() - const { loadTables } = useProject() + const { loadTables, isXcdbBase } = useProject() const { closeTab } = useTabs() const projectStore = useProject() @@ -88,7 +88,9 @@ export function useTable(onTableCreate?: (tableMeta: TableType) => void, baseId? const meta = (await getMeta(table.id as string, true)) as TableType const relationColumns = meta?.columns?.filter((c) => c.uidt === UITypes.LinkToAnotherRecord && !isSystemColumn(c)) - if (relationColumns?.length) { + // Check if table has any relation columns and show notification + // skip for xcdb base + if (relationColumns?.length && !isXcdbBase(table.base_id)) { const refColMsgs = await Promise.all( relationColumns.map(async (c, i) => { const refMeta = (await getMeta( diff --git a/packages/nocodb/src/db/sql-client/lib/sqlite/SqliteClient.ts b/packages/nocodb/src/db/sql-client/lib/sqlite/SqliteClient.ts index 118423df7b..49e078a1b0 100644 --- a/packages/nocodb/src/db/sql-client/lib/sqlite/SqliteClient.ts +++ b/packages/nocodb/src/db/sql-client/lib/sqlite/SqliteClient.ts @@ -22,7 +22,7 @@ class SqliteClient extends KnexClient { // sqlite does not support inserting default values and knex fires a warning without this flag connectionConfig.connection.useNullAsDefault = true; super(connectionConfig); - this.sqlClient = knex(connectionConfig.connection); + this.sqlClient = connectionConfig?.knex || knex(connectionConfig.connection); this.queries = queries; this._version = {}; } diff --git a/packages/nocodb/src/db/sql-mgr/v2/ProjectMgrv2.ts b/packages/nocodb/src/db/sql-mgr/v2/ProjectMgrv2.ts index f8f884ea5f..730cbdda59 100644 --- a/packages/nocodb/src/db/sql-mgr/v2/ProjectMgrv2.ts +++ b/packages/nocodb/src/db/sql-mgr/v2/ProjectMgrv2.ts @@ -1,18 +1,21 @@ -import SqlMgrv2 from './SqlMgrv2'; -import SqlMgrv2Trans from './SqlMgrv2Trans'; +import { MetaService } from '../../../meta/meta.service' +import SqlMgrv2 from './SqlMgrv2' +import SqlMgrv2Trans from './SqlMgrv2Trans' // import type NcMetaIO from '../../../meta/NcMetaIO'; -import type Base from '../../../models/Base'; +import type Base from '../../../models/Base' export default class ProjectMgrv2 { private static sqlMgrMap: { [key: string]: SqlMgrv2; - } = {}; + } = {} + + public static getSqlMgr(project: { id: string }, ncMeta: MetaService = null): SqlMgrv2 { + if (ncMeta) return new SqlMgrv2(project, ncMeta) - public static getSqlMgr(project: { id: string }): SqlMgrv2 { if (!this.sqlMgrMap[project.id]) { - this.sqlMgrMap[project.id] = new SqlMgrv2(project); + this.sqlMgrMap[project.id] = new SqlMgrv2(project) } - return this.sqlMgrMap[project.id]; + return this.sqlMgrMap[project.id] } public static async getSqlMgrTrans( @@ -21,8 +24,8 @@ export default class ProjectMgrv2 { ncMeta: any, base: Base, ): Promise { - const sqlMgr = new SqlMgrv2Trans(project, ncMeta, base); - await sqlMgr.startTransaction(base); - return sqlMgr; + const sqlMgr = new SqlMgrv2Trans(project, ncMeta, base) + await sqlMgr.startTransaction(base) + return sqlMgr } } diff --git a/packages/nocodb/src/db/sql-mgr/v2/SqlMgrv2.ts b/packages/nocodb/src/db/sql-mgr/v2/SqlMgrv2.ts index 17d808e15e..437a79921f 100644 --- a/packages/nocodb/src/db/sql-mgr/v2/SqlMgrv2.ts +++ b/packages/nocodb/src/db/sql-mgr/v2/SqlMgrv2.ts @@ -5,12 +5,14 @@ import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; import SqlClientFactory from '../../sql-client/lib/SqlClientFactory'; import KnexMigratorv2 from '../../sql-migrator/lib/KnexMigratorv2'; import Debug from '../../util/Debug'; +import type { MetaService } from '../../../meta/meta.service'; import type Base from '../../../models/Base'; const log = new Debug('SqlMgr'); export default class SqlMgrv2 { protected _migrator: KnexMigratorv2; + protected ncMeta?: MetaService; // @ts-ignore private currentProjectFolder: any; @@ -20,18 +22,18 @@ export default class SqlMgrv2 { * @param {String} args.toolDbPath - path to sqlite file that sql mgr will use * @memberof SqlMgr */ - constructor(args: { id: string }) { + constructor(args: { id: string }, ncMeta = null) { const func = 'constructor'; log.api(`${func}:args:`, args); // this.metaDb = args.metaDb; this._migrator = new KnexMigratorv2(args); - - return this; + this.ncMeta = ncMeta; } public async migrator(_base: Base) { return this._migrator; } + public static async testConnection(args = {}) { const client = await SqlClientFactory.create(args); return client.testConnection(); @@ -119,6 +121,10 @@ export default class SqlMgrv2 { } protected async getSqlClient(base: Base) { + if (base.is_meta && this.ncMeta) { + return NcConnectionMgrv2.getSqlClient(base, this.ncMeta.knex); + } + return NcConnectionMgrv2.getSqlClient(base); } } diff --git a/packages/nocodb/src/meta/meta.service.ts b/packages/nocodb/src/meta/meta.service.ts index 5d30444980..565b921b42 100644 --- a/packages/nocodb/src/meta/meta.service.ts +++ b/packages/nocodb/src/meta/meta.service.ts @@ -532,7 +532,6 @@ export class MetaService { } else { query.where(idOrCondition); } - return query.first(); } diff --git a/packages/nocodb/src/models/Model.ts b/packages/nocodb/src/models/Model.ts index 0826fd538a..9fdbf44d4b 100644 --- a/packages/nocodb/src/models/Model.ts +++ b/packages/nocodb/src/models/Model.ts @@ -368,10 +368,10 @@ export default class Model implements TableType { } async delete(ncMeta = Noco.ncMeta, force = false): Promise { - await Audit.deleteRowComments(this.id); + await Audit.deleteRowComments(this.id, ncMeta); - for (const view of await this.getViews(true)) { - await view.delete(); + for (const view of await this.getViews(true, ncMeta)) { + await view.delete(ncMeta); } for (const col of await this.getColumns(ncMeta)) { diff --git a/packages/nocodb/src/models/View.ts b/packages/nocodb/src/models/View.ts index 86abcae743..e424200c64 100644 --- a/packages/nocodb/src/models/View.ts +++ b/packages/nocodb/src/models/View.ts @@ -986,9 +986,9 @@ export default class View implements ViewType { // @ts-ignore static async delete(viewId, ncMeta = Noco.ncMeta) { - const view = await this.get(viewId); - await Sort.deleteAll(viewId); - await Filter.deleteAll(viewId); + const view = await this.get(viewId, ncMeta); + await Sort.deleteAll(viewId, ncMeta); + await Filter.deleteAll(viewId, ncMeta); const table = this.extractViewTableName(view); const tableScope = this.extractViewTableNameScope(view); const columnTable = this.extractViewColumnsTableName(view); @@ -1258,8 +1258,8 @@ export default class View implements ViewType { ); } - async delete() { - await View.delete(this.id); + async delete(ncMeta = Noco.ncMeta){ + await View.delete(this.id, ncMeta); } static async shareViewList(tableId, ncMeta = Noco.ncMeta) { diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index 7fd2972fce..f1c8a81620 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -1157,7 +1157,10 @@ export class ColumnsService { ); const base = await Base.get(table.base_id, ncMeta); - const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: base.project_id }); + const sqlMgr = await ProjectMgrv2.getSqlMgr( + { id: base.project_id }, + ncMeta, + ); switch (column.uidt) { case UITypes.Lookup: @@ -1271,7 +1274,7 @@ export class ColumnsService { const colOpt = await c.getColOptions(ncMeta); if (colOpt.fk_related_model_id === mmTable.id) { - await Column.delete(c.id); + await Column.delete(c.id, ncMeta); } } @@ -1403,35 +1406,37 @@ export class ColumnsService { if (!relationColOpt) { foreignKeyName = ( ( - await childTable.getColumns(ncMeta).then((cols) => { - return cols?.find((c) => { - return ( - c.uidt === UITypes.LinkToAnotherRecord && - c.colOptions.fk_related_model_id === parentTable.id && - (c.colOptions as LinkToAnotherRecordType).fk_child_column_id === - childColumn.id && - (c.colOptions as LinkToAnotherRecordType) - .fk_parent_column_id === parentColumn.id - ); - }); + await childTable.getColumns(ncMeta).then(async (cols) => { + for (const col of cols) { + if (col.uidt === UITypes.LinkToAnotherRecord) { + const colOptions = + await col.getColOptions(ncMeta); + console.log(colOptions); + if (colOptions.fk_related_model_id === parentTable.id) { + return { colOptions }; + } + } + } }) - ).colOptions as LinkToAnotherRecordType + )?.colOptions as LinkToAnotherRecordType ).fk_index_name; } else { foreignKeyName = relationColOpt.fk_index_name; } - // todo: handle relation delete exception - try { - await sqlMgr.sqlOpPlus(base, 'relationDelete', { - childColumn: childColumn.column_name, - childTable: childTable.table_name, - parentTable: parentTable.table_name, - parentColumn: parentColumn.column_name, - foreignKeyName, - }); - } catch (e) { - console.log(e); + if (!relationColOpt?.virtual) { + // todo: handle relation delete exception + try { + await sqlMgr.sqlOpPlus(base, 'relationDelete', { + childColumn: childColumn.column_name, + childTable: childTable.table_name, + parentTable: parentTable.table_name, + parentColumn: parentColumn.column_name, + foreignKeyName, + }); + } catch (e) { + console.log(e); + } } if (!relationColOpt) return; @@ -1462,6 +1467,28 @@ export class ColumnsService { }, ncMeta, ); + + // if virtual column delete all index before deleting the column + if (relationColOpt?.virtual) { + const indexes = + ( + await sqlMgr.sqlOp(base, 'indexList', { + tn: cTable.table_name, + }) + )?.data?.list ?? []; + + for (const index of indexes) { + if (index.cn !== childColumn.column_name) continue; + + await sqlMgr.sqlOpPlus(base, 'indexDelete', { + ...index, + tn: cTable.table_name, + columns: [childColumn.column_name], + indexName: index.index_name, + }); + } + } + const tableUpdateBody = { ...cTable, tn: cTable.table_name, @@ -1511,6 +1538,12 @@ export class ColumnsService { const sqlMgr = await ProjectMgrv2.getSqlMgr({ id: param.base.project_id, }); + + // if xcdb base then treat as virtual relation to avoid creating foreign key + if (param.base.is_meta) { + (param.column as LinkToAnotherColumnReqType).virtual = true; + } + if ( (param.column as LinkToAnotherColumnReqType).type === 'hm' || (param.column as LinkToAnotherColumnReqType).type === 'bt' @@ -1565,11 +1598,6 @@ export class ColumnsService { childColumn = await Column.get({ colId: id }); - // if xcdb base then treat as virtual relation to avoid creating foreign key - if (param.base.is_meta) { - (param.column as LinkToAnotherColumnReqType).virtual = true; - } - // ignore relation creation if virtual if (!(param.column as LinkToAnotherColumnReqType).virtual) { foreignKeyName = generateFkName(parent, child); @@ -1746,6 +1774,7 @@ export class ColumnsService { fk_mm_child_column_id: childCol.id, fk_mm_parent_column_id: parentCol.id, fk_related_model_id: parent.id, + virtual: (param.column as LinkToAnotherColumnReqType).virtual, }); await Column.insert({ title: getUniqueColumnAliasName( @@ -1765,6 +1794,7 @@ export class ColumnsService { fk_mm_child_column_id: parentCol.id, fk_mm_parent_column_id: childCol.id, fk_related_model_id: child.id, + virtual: (param.column as LinkToAnotherColumnReqType).virtual, }); // todo: create index for virtual relations as well diff --git a/packages/nocodb/src/services/tables.service.ts b/packages/nocodb/src/services/tables.service.ts index 4d9b21ca76..cf005b04a9 100644 --- a/packages/nocodb/src/services/tables.service.ts +++ b/packages/nocodb/src/services/tables.service.ts @@ -184,6 +184,11 @@ export class TablesService { // delete all relations await Promise.all( relationColumns.map((c) => { + // skip if column is hasmany relation to mm table + if (c.system) { + return; + } + return this.columnsService.columnDelete( { req: param.req, @@ -194,7 +199,7 @@ export class TablesService { }), ); - const sqlMgr = await ProjectMgrv2.getSqlMgr(project); + const sqlMgr = await ProjectMgrv2.getSqlMgr(project, ncMeta); (table as any).tn = table.table_name; table.columns = table.columns.filter((c) => !isVirtualCol(c)); table.columns.forEach((c) => { @@ -210,15 +215,18 @@ export class TablesService { }); } - await Audit.insert({ - project_id: project.id, - base_id: base.id, - op_type: AuditOperationTypes.TABLE, - op_sub_type: AuditOperationSubTypes.DELETE, - user: param.user?.email, - description: `Deleted ${table.type} ${table.table_name} with alias ${table.title} `, - ip: param.req?.clientIp, - }).then(() => {}); + await Audit.insert( + { + project_id: project.id, + base_id: base.id, + op_type: AuditOperationTypes.TABLE, + op_sub_type: AuditOperationSubTypes.DELETE, + user: param.user?.email, + description: `Deleted ${table.type} ${table.table_name} with alias ${table.title} `, + ip: param.req?.clientIp, + }, + ncMeta, + ).then(() => {}); T.emit('evt', { evt_type: 'table:deleted' }); From 0d3e51793125f7aad9b15c8a5863390002d1abdc Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 6 Jun 2023 01:03:17 +0530 Subject: [PATCH 10/34] feat: delete related data in bulk delete Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 62 +++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index dae2103079..c3de3a4ffd 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -1885,7 +1885,6 @@ class BaseModelSqlv2 { const execQueries: ((trx: Transaction) => Promise)[] = []; - // start a transaction if not already in one for (const column of this.model.columns) { if (column.uidt !== UITypes.LinkToAnotherRecord) continue; @@ -2445,8 +2444,69 @@ class BaseModelSqlv2 { res.push(d); } + const execQueries: ((trx: Transaction, ids: any[]) => Promise)[] = + []; + + for (const column of this.model.columns) { + if (column.uidt !== UITypes.LinkToAnotherRecord) continue; + + const colOptions = + await column.getColOptions(); + + switch (colOptions.type) { + case 'mm': + { + const mmTable = await Model.get(colOptions.fk_mm_model_id); + const mmParentColumn = await Column.get({ + colId: colOptions.fk_mm_child_column_id, + }); + + execQueries.push((trx, ids) => + trx(mmTable.table_name) + .del() + .whereIn(mmParentColumn.column_name, ids), + ); + } + break; + case 'hm': + { + // skip if it's an mm table column + const relatedTable = await colOptions.getRelatedTable(); + if (relatedTable.mm) { + break; + } + + const childColumn = await Column.get({ + colId: colOptions.fk_child_column_id, + }); + + execQueries.push((trx, ids) => + trx(relatedTable.table_name) + .update({ + [childColumn.column_name]: null, + }) + .whereIn(childColumn.column_name, ids), + ); + } + break; + case 'bt': + { + // nothing to do + } + break; + } + } + + const idsVals = res.map((d) => d[this.model.primaryKey.column_name]); + transaction = await this.dbDriver.transaction(); + if (execQueries.length > 0) { + for (const execQuery of execQueries) { + await execQuery(transaction, idsVals); + } + } + for (const d of res) { await transaction(this.tnPath).del().where(d); } From d466c2342eebda4a21482fb608b02b2a38a98eea Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 6 Jun 2023 01:25:31 +0530 Subject: [PATCH 11/34] feat: implement upgrader to migrate existing xcdb relations Signed-off-by: Pranav C --- .../global/init-meta-service.provider.ts | 2 +- .../nocodb/src/version-upgrader/NcUpgrader.ts | 2 + .../ncFilterUpgrader_0104004.ts | 384 ++---------------- .../version-upgrader/ncXcdbLTARUpgrader.ts | 78 ++++ 4 files changed, 115 insertions(+), 351 deletions(-) create mode 100644 packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts diff --git a/packages/nocodb/src/modules/global/init-meta-service.provider.ts b/packages/nocodb/src/modules/global/init-meta-service.provider.ts index a86646b302..eab0293639 100644 --- a/packages/nocodb/src/modules/global/init-meta-service.provider.ts +++ b/packages/nocodb/src/modules/global/init-meta-service.provider.ts @@ -26,7 +26,7 @@ export const InitMetaServiceProvider: Provider = { const config = await NcConfig.createByEnv(); // set version - process.env.NC_VERSION = '0107004'; + process.env.NC_VERSION = '0108002'; // init cache await NocoCache.init(); diff --git a/packages/nocodb/src/version-upgrader/NcUpgrader.ts b/packages/nocodb/src/version-upgrader/NcUpgrader.ts index 951a93099f..a0969366a1 100644 --- a/packages/nocodb/src/version-upgrader/NcUpgrader.ts +++ b/packages/nocodb/src/version-upgrader/NcUpgrader.ts @@ -14,6 +14,7 @@ import ncProjectEnvUpgrader0011045 from './ncProjectEnvUpgrader0011045'; import ncProjectEnvUpgrader from './ncProjectEnvUpgrader'; import ncHookUpgrader from './ncHookUpgrader'; import ncProjectConfigUpgrader from './ncProjectConfigUpgrader'; +import ncXcdbLTARUpgrader from './ncXcdbLTARUpgrader'; import type { MetaService } from '../meta/meta.service'; import type { NcConfig } from '../interface/config'; @@ -50,6 +51,7 @@ export default class NcUpgrader { { name: '0105003', handler: ncFilterUpgrader_0105003 }, { name: '0105004', handler: ncHookUpgrader }, { name: '0107004', handler: ncProjectConfigUpgrader }, + { name: '0108002', handler: ncXcdbLTARUpgrader }, ]; if (!(await ctx.ncMeta.knexConnection?.schema?.hasTable?.('nc_store'))) { return; diff --git a/packages/nocodb/src/version-upgrader/ncFilterUpgrader_0104004.ts b/packages/nocodb/src/version-upgrader/ncFilterUpgrader_0104004.ts index 1aecb3eb32..f3e69083f8 100644 --- a/packages/nocodb/src/version-upgrader/ncFilterUpgrader_0104004.ts +++ b/packages/nocodb/src/version-upgrader/ncFilterUpgrader_0104004.ts @@ -1,359 +1,43 @@ -import { UITypes } from 'nocodb-sdk'; +import { OrgUserRoles } from 'nocodb-sdk'; +import { NC_APP_SETTINGS } from '../constants'; +import Store from '../models/Store'; import { MetaTable } from '../utils/globals'; -import Column from '../models/Column'; -import Filter from '../models/Filter'; -import Project from '../models/Project'; -import type { MetaService } from '../meta/meta.service'; import type { NcUpgraderCtx } from './NcUpgrader'; -import type { SelectOptionsType } from 'nocodb-sdk'; -// as of 0.104.3, almost all filter operators are available to all column types -// while some of them aren't supposed to be shown -// this upgrader is to remove those unsupported filters / migrate to the correct filter - -// Change Summary: -// - Text-based columns: -// - remove `>`, `<`, `>=`, `<=` -// - Numeric-based / SingleSelect columns: -// - remove `like` -// - migrate `null`, and `empty` to `blank` -// - Checkbox columns: -// - remove `equal` -// - migrate `empty` and `null` to `notchecked` -// - MultiSelect columns: -// - remove `like` -// - migrate `equal`, `null`, `empty` -// - Attachment columns: -// - remove `>`, `<`, `>=`, `<=`, `equal` -// - migrate `empty`, `null` to `blank` -// - LTAR columns: -// - remove `>`, `<`, `>=`, `<=` -// - migrate `empty`, `null` to `blank` -// - Lookup columns: -// - migrate `empty`, `null` to `blank` -// - Duration columns: -// - remove `like` -// - migrate `empty`, `null` to `blank` - -const removeEqualFilters = (filter, ncMeta) => { - const actions = []; - // remove `is equal`, `is not equal` - if (['eq', 'neq'].includes(filter.comparison_op)) { - actions.push(Filter.delete(filter.id, ncMeta)); - } - return actions; -}; - -const removeArithmeticFilters = (filter, ncMeta) => { - const actions = []; - // remove `>`, `<`, `>=`, `<=` - if (['gt', 'lt', 'gte', 'lte'].includes(filter.comparison_op)) { - actions.push(Filter.delete(filter.id, ncMeta)); - } - return actions; -}; - -const removeLikeFilters = (filter, ncMeta) => { - const actions = []; - // remove `is like`, `is not like` - if (['like', 'nlike'].includes(filter.comparison_op)) { - actions.push(Filter.delete(filter.id, ncMeta)); - } - return actions; -}; - -const migrateNullAndEmptyToBlankFilters = (filter, ncMeta) => { - const actions = []; - if (['empty', 'null'].includes(filter.comparison_op)) { - // migrate to blank - actions.push( - Filter.update( - filter.id, - { - comparison_op: 'blank', - }, - ncMeta, - ), - ); - } else if (['notempty', 'notnull'].includes(filter.comparison_op)) { - // migrate to not blank - actions.push( - Filter.update( - filter.id, - { - comparison_op: 'notblank', - }, - ncMeta, - ), - ); - } - return actions; -}; - -const migrateMultiSelectEq = async (filter, col: Column, ncMeta) => { - // only allow eq / neq - if (!['eq', 'neq'].includes(filter.comparison_op)) return; - // if there is no value -> delete this filter - if (!filter.value) { - return await Filter.delete(filter.id, ncMeta); - } - // options inputted from users - const options = filter.value.split(','); - // retrieve the possible col options - const colOptions = (await col.getColOptions()) as SelectOptionsType; - // only include valid options as the input value becomes dropdown type now - const validOptions = []; - for (const option of options) { - if (colOptions.options.includes(option)) { - validOptions.push(option); - } - } - const newFilterValue = validOptions.join(','); - // if all inputted options are invalid -> delete this filter - if (!newFilterValue) { - return await Filter.delete(filter.id, ncMeta); - } - const actions = []; - if (filter.comparison_op === 'eq') { - // migrate to `contains all of` - actions.push( - Filter.update( - filter.id, - { - comparison_op: 'anyof', - value: newFilterValue, - }, - ncMeta, - ), - ); - } else if (filter.comparison_op === 'neq') { - // migrate to `doesn't contain all of` - actions.push( - Filter.update( - filter.id, - { - comparison_op: 'nanyof', - value: newFilterValue, - }, - ncMeta, - ), - ); - } - return await Promise.all(actions); -}; - -const migrateToCheckboxFilter = (filter, ncMeta) => { - const actions = []; - const possibleTrueValues = ['true', 'True', '1', 'T', 'Y']; - const possibleFalseValues = ['false', 'False', '0', 'F', 'N']; - if (['empty', 'null'].includes(filter.comparison_op)) { - // migrate to not checked - actions.push( - Filter.update( - filter.id, - { - comparison_op: 'notchecked', - }, - ncMeta, - ), - ); - } else if (['notempty', 'notnull'].includes(filter.comparison_op)) { - // migrate to checked - actions.push( - Filter.update( - filter.id, - { - comparison_op: 'checked', - }, - ncMeta, - ), +/** Upgrader for upgrading roles */ +export default async function ({ ncMeta }: NcUpgraderCtx) { + const users = await ncMeta.metaList2(null, null, MetaTable.USERS); + + for (const user of users) { + user.roles = user.roles + .split(',') + .map((r) => { + // update old role names with new roles + if (r === 'user') { + return OrgUserRoles.CREATOR; + } else if (r === 'user-new') { + return OrgUserRoles.VIEWER; + } + return r; + }) + .join(','); + await ncMeta.metaUpdate( + null, + null, + MetaTable.USERS, + { roles: user.roles }, + user.id, ); - } else if (filter.comparison_op === 'eq') { - if (possibleTrueValues.includes(filter.value)) { - // migrate to checked - actions.push( - Filter.update( - filter.id, - { - comparison_op: 'checked', - value: '', - }, - ncMeta, - ), - ); - } else if (possibleFalseValues.includes(filter.value)) { - // migrate to notchecked - actions.push( - Filter.update( - filter.id, - { - comparison_op: 'notchecked', - value: '', - }, - ncMeta, - ), - ); - } else { - // invalid value - good to delete - actions.push(Filter.delete(filter.id, ncMeta)); - } - } else if (filter.comparison_op === 'neq') { - if (possibleFalseValues.includes(filter.value)) { - // migrate to checked - actions.push( - Filter.update( - filter.id, - { - comparison_op: 'checked', - value: '', - }, - ncMeta, - ), - ); - } else if (possibleTrueValues.includes(filter.value)) { - // migrate to not checked - actions.push( - Filter.update( - filter.id, - { - comparison_op: 'notchecked', - value: '', - }, - ncMeta, - ), - ); - } else { - // invalid value - good to delete - actions.push(Filter.delete(filter.id, ncMeta)); - } - } - return actions; -}; - -async function migrateFilters(ncMeta: MetaService) { - const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP); - for (const filter of filters) { - if (!filter.fk_column_id || filter.is_group) { - continue; - } - const col = await Column.get({ colId: filter.fk_column_id }, ncMeta); - if ( - [ - UITypes.SingleLineText, - UITypes.LongText, - UITypes.PhoneNumber, - UITypes.Email, - UITypes.URL, - ].includes(col.uidt) - ) { - await Promise.all(removeArithmeticFilters(filter, ncMeta)); - } else if ( - [ - // numeric fields - UITypes.Duration, - UITypes.Currency, - UITypes.Percent, - UITypes.Number, - UITypes.Decimal, - UITypes.Rating, - UITypes.Rollup, - // select fields - UITypes.SingleSelect, - ].includes(col.uidt) - ) { - await Promise.all([ - ...removeLikeFilters(filter, ncMeta), - ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), - ]); - } else if (col.uidt === UITypes.Checkbox) { - await Promise.all(migrateToCheckboxFilter(filter, ncMeta)); - } else if (col.uidt === UITypes.MultiSelect) { - await Promise.all([ - ...removeLikeFilters(filter, ncMeta), - ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), - ]); - await migrateMultiSelectEq(filter, col, ncMeta); - } else if (col.uidt === UITypes.Attachment) { - await Promise.all([ - ...removeArithmeticFilters(filter, ncMeta), - ...removeEqualFilters(filter, ncMeta), - ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), - ]); - } else if (col.uidt === UITypes.LinkToAnotherRecord) { - await Promise.all([ - ...removeArithmeticFilters(filter, ncMeta), - ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), - ]); - } else if (col.uidt === UITypes.Lookup) { - await Promise.all([ - ...removeArithmeticFilters(filter, ncMeta), - ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), - ]); - } else if (col.uidt === UITypes.Duration) { - await Promise.all([ - ...removeLikeFilters(filter, ncMeta), - ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), - ]); - } - } -} - -async function updateProjectMeta(ncMeta: MetaService) { - const projectHasEmptyOrFilters: Record = {}; - - const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP); - - const actions = []; - - for (const filter of filters) { - if ( - ['notempty', 'notnull', 'empty', 'null'].includes(filter.comparison_op) - ) { - projectHasEmptyOrFilters[filter.project_id] = true; - } } - const projects = await ncMeta.metaList2(null, null, MetaTable.PROJECT); - - const defaultProjectMeta = { - showNullAndEmptyInFilter: false, - }; - - for (const project of projects) { - const oldProjectMeta = project.meta; - let newProjectMeta = defaultProjectMeta; - try { - newProjectMeta = - (typeof oldProjectMeta === 'string' - ? JSON.parse(oldProjectMeta) - : oldProjectMeta) ?? defaultProjectMeta; - } catch {} - - newProjectMeta = { - ...newProjectMeta, - showNullAndEmptyInFilter: projectHasEmptyOrFilters[project.id] ?? false, - }; - - actions.push( - Project.update( - project.id, - { - meta: JSON.stringify(newProjectMeta), - }, - ncMeta, - ), + // set invite only signup if user have environment variable set + if (process.env.NC_INVITE_ONLY_SIGNUP) { + await Store.saveOrUpdate( + { + value: '{ "invite_only_signup": true }', + key: NC_APP_SETTINGS, + }, + ncMeta, ); } - await Promise.all(actions); -} - -export default async function ({ ncMeta }: NcUpgraderCtx) { - // fix the existing filter behaviours or - // migrate `null` or `empty` filters to `blank` - await migrateFilters(ncMeta); - // enrich `showNullAndEmptyInFilter` in project meta - // if there is empty / null filters in existing projects, - // then set `showNullAndEmptyInFilter` to true - // else set to false - await updateProjectMeta(ncMeta); } diff --git a/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts b/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts new file mode 100644 index 0000000000..81dd244797 --- /dev/null +++ b/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts @@ -0,0 +1,78 @@ +import { RelationTypes, UITypes } from 'nocodb-sdk'; +import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; +import type SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2'; +import type { MetaService } from '../meta/meta.service'; +import type { LinkToAnotherRecordColumn, Model } from '../models'; +import type { NcUpgraderCtx } from './NcUpgrader'; + +async function upgradeModelRelations({ + model, + sqlMgr, + ncMeta, +}: { + ncMeta: MetaService; + model: Model; + sqlMgr: SqlMgrv2; +}) { + // Iterate over each column and upgrade LTAR + for (const column of await model.getColumns()) { + if (column.uidt !== UITypes.LinkToAnotherRecord) { + continue; + } + + const colOptions = await column.getColOptions(); + + switch (colOptions.type) { + // case RelationTypes.MANY_TO_MANY: + // + // break; + case RelationTypes.HAS_MANY: + { + // delete the foreign key constraint if exists + // create a new index for the column + } + + break; + // case RelationTypes.BELONGS_TO: + // break; + } + } +} + +// An upgrader for upgrading any existing relation in xcdb +async function upgradeBaseRelations({ + ncMeta, + base, +}: { + ncMeta: MetaService; + base: any; +}) { + const sqlMgr = ProjectMgrv2.getSqlMgr({ id: base.project_id }, ncMeta); + + // get models for the base + const models = await ncMeta.metaList2(null, base.id, 'models'); + + // get all columns and filter out relations and upgrade + for (const model of models) { + await upgradeModelRelations({ ncMeta, model, sqlMgr }); + } +} + +// database to virtual relation and create an index for it +export default async function ({ ncMeta }: NcUpgraderCtx) { + // get all xcdb bases + const bases = await ncMeta.metaList2(null, null, 'bases', { + condition: { + is_meta: 1, + }, + orderBy: {}, + }); + + // iterate and upgrade each base + for (const base of bases) { + await upgradeBaseRelations({ + ncMeta, + base, + }); + } +} From f8ed2a397497f7aad601e2966d2d8b2d6b257de3 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 6 Jun 2023 12:58:05 +0530 Subject: [PATCH 12/34] feat: implement upgrader to migrate existing xcdb relations Signed-off-by: Pranav C --- .../version-upgrader/ncXcdbLTARUpgrader.ts | 84 +++++++++++++++---- 1 file changed, 70 insertions(+), 14 deletions(-) diff --git a/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts b/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts index 81dd244797..d247e84795 100644 --- a/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts +++ b/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts @@ -1,41 +1,92 @@ import { RelationTypes, UITypes } from 'nocodb-sdk'; -import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2'; -import type SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2'; +import { MetaTable } from '../meta/meta.service'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import type { MetaService } from '../meta/meta.service'; import type { LinkToAnotherRecordColumn, Model } from '../models'; import type { NcUpgraderCtx } from './NcUpgrader'; +// An upgrader for upgrading LTAR relations in XCDB bases +// it will delete all the foreign keys and create a new index +// and treat all the LTAR as virtual + async function upgradeModelRelations({ model, - sqlMgr, + relations, ncMeta, + sqlClient, }: { ncMeta: MetaService; model: Model; - sqlMgr: SqlMgrv2; + sqlClient: ReturnType< + (typeof NcConnectionMgrv2)['getSqlClient'] + > extends Promise + ? U + : ReturnType<(typeof NcConnectionMgrv2)['getSqlClient']>; + relations: { + tn: string; + rtn: string; + cn: string; + rcn: string; + }[]; }) { // Iterate over each column and upgrade LTAR - for (const column of await model.getColumns()) { + for (const column of await model.getColumns(ncMeta)) { if (column.uidt !== UITypes.LinkToAnotherRecord) { continue; } - const colOptions = await column.getColOptions(); + const colOptions = await column.getColOptions( + ncMeta, + ); switch (colOptions.type) { - // case RelationTypes.MANY_TO_MANY: - // - // break; case RelationTypes.HAS_MANY: { + // skip if virtual + if (colOptions.virtual) { + break; + } + + const parentCol = await colOptions.getParentColumn(ncMeta); + const childCol = await colOptions.getChildColumn(ncMeta); + + const parentModel = await parentCol.getModel(ncMeta); + const childModel = await childCol.getModel(ncMeta); + // delete the foreign key constraint if exists + const relation = relations.find((r) => { + return ( + parentCol.column_name === r.rcn && + childCol.column_name === r.cn && + parentModel.table_name === r.rtn && + childModel.table_name === r.tn + ); + }); + + // delete the relation + if (relation) { + await sqlClient.relationDelete(relation); + } + // create a new index for the column + const indexArgs = { + columns: [relation.cn], + tn: relation.tn, + non_unique: true, + }; + await sqlClient.indexCreate(indexArgs); } - break; - // case RelationTypes.BELONGS_TO: - // break; } + + // update the relation as virtual + await ncMeta.metaUpdate( + null, + null, + MetaTable.COL_RELATIONS, + { virtual: true }, + colOptions.id, + ); } } @@ -47,14 +98,19 @@ async function upgradeBaseRelations({ ncMeta: MetaService; base: any; }) { - const sqlMgr = ProjectMgrv2.getSqlMgr({ id: base.project_id }, ncMeta); + // const sqlMgr = ProjectMgrv2.getSqlMgr({ id: base.project_id }, ncMeta); + + const sqlClient = await NcConnectionMgrv2.getSqlClient(base, ncMeta.knex); + + // get all relations + const relations = (await sqlClient.relationListAll())?.data?.list; // get models for the base const models = await ncMeta.metaList2(null, base.id, 'models'); // get all columns and filter out relations and upgrade for (const model of models) { - await upgradeModelRelations({ ncMeta, model, sqlMgr }); + await upgradeModelRelations({ ncMeta, model, sqlClient, relations }); } } From 43f55877f2f4a044964f8aca72433f1021b12ce1 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Tue, 6 Jun 2023 13:38:51 +0530 Subject: [PATCH 13/34] test: metadb ltar ops (wip) Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../playwright/pages/Dashboard/Grid/index.ts | 13 +- tests/playwright/tests/db/metaLTAR.spec.ts | 351 ++++++++++++++++++ 2 files changed, 362 insertions(+), 2 deletions(-) create mode 100644 tests/playwright/tests/db/metaLTAR.spec.ts diff --git a/tests/playwright/pages/Dashboard/Grid/index.ts b/tests/playwright/pages/Dashboard/Grid/index.ts index 94b133b393..68ac87fcbc 100644 --- a/tests/playwright/pages/Dashboard/Grid/index.ts +++ b/tests/playwright/pages/Dashboard/Grid/index.ts @@ -182,6 +182,11 @@ export class GridPage extends BasePage { await (await this.rootPage.locator('.ant-drawer-body').elementHandle())?.waitForElementState('stable'); } + async selectRow(index: number) { + await this.get().locator(`td[data-testid="cell-Id-${index}"]`).locator('span.ant-checkbox').click(); + await this.rootPage.waitForTimeout(300); + } + async selectAll() { await this.get().locator('[data-testid="nc-check-all"]').hover(); @@ -198,8 +203,7 @@ export class GridPage extends BasePage { await this.rootPage.waitForTimeout(300); } - async deleteAll() { - await this.selectAll(); + async deleteSelectedRows() { await this.get().locator('[data-testid="nc-check-all"]').nth(0).click({ button: 'right', }); @@ -207,6 +211,11 @@ export class GridPage extends BasePage { await this.dashboard.waitForLoaderToDisappear(); } + async deleteAll() { + await this.selectAll(); + await this.deleteSelectedRows(); + } + async verifyTotalRowCount({ count }: { count: number }) { // wait for 100 ms and try again : 5 times let i = 0; diff --git a/tests/playwright/tests/db/metaLTAR.spec.ts b/tests/playwright/tests/db/metaLTAR.spec.ts new file mode 100644 index 0000000000..b8153edc7a --- /dev/null +++ b/tests/playwright/tests/db/metaLTAR.spec.ts @@ -0,0 +1,351 @@ +/* + * + * Meta projects, additional provision for deleting of rows, columns and tables with Link to another record field type + * + * Pre-requisite: + * TableA TableB TableC + * TableA TableD TableE + * TableA TableA : self relation + * TableA TableA : self relation + * Insert some records in TableA, TableB, TableC, TableD, TableE, add some links between them + * + * + * Tests: + * 1. Delete a row from TableA : Verify record status in adjacent tables + * 2. Delete hm link from TableA to TableB : Verify record status in adjacent tables + * 3. Delete mm link from TableA to TableD : Verify record status in adjacent tables + * 4. Delete a self link column from TableA : Verify + * 5. Delete TableA : Verify record status in adjacent tables + * + */ + +import { test } from '@playwright/test'; +import setup from '../../setup'; +import { Api, UITypes } from 'nocodb-sdk'; +import { DashboardPage } from '../../pages/Dashboard'; +import { GridPage } from '../../pages/Dashboard/Grid'; +let api: Api; +const recordCount = 10; + +test.describe('Test table', () => { + let context: any; + let dashboard: DashboardPage; + let grid: GridPage; + const tables = []; + + test.beforeEach(async ({ page }) => { + context = await setup({ page, isEmptyProject: true }); + dashboard = new DashboardPage(page, context.project); + grid = dashboard.grid; + + api = new Api({ + baseURL: `http://localhost:8080/`, + headers: { + 'xc-auth': context.token, + }, + }); + + const columns = [ + { + column_name: 'Id', + title: 'Id', + uidt: UITypes.ID, + }, + { + column_name: 'Title', + title: 'Title', + uidt: UITypes.SingleLineText, + pv: true, + }, + ]; + + const rows = []; + for (let i = 0; i < recordCount * 10; i++) { + rows.push({ + Id: i + 1, + SingleLineText: `${i + 1}`, + }); + } + + // Create tables + const project = await api.project.read(context.project.id); + + for (let i = 0; i < 5; i++) { + const table = await api.base.tableCreate(context.project.id, project.bases?.[0].id, { + table_name: `Table${i}`, + title: `Table${i}`, + columns: columns, + }); + tables.push(table); + await api.dbTableRow.bulkCreate('noco', context.project.id, tables[i].id, rows); + } + + // Create links + // TableA TableB TableC + await api.dbTableColumn.create(tables[0].id, { + uidt: UITypes.LinkToAnotherRecord, + title: `TableA:hm:TableB`, + column_name: `TableA:hm:TableB`, + parentId: tables[0].id, + childId: tables[1].id, + type: 'hm', + }); + await api.dbTableColumn.create(tables[1].id, { + uidt: UITypes.LinkToAnotherRecord, + title: `TableB:hm:TableC`, + column_name: `TableB:hm:TableC`, + parentId: tables[1].id, + childId: tables[2].id, + type: 'hm', + }); + + // TableA TableD TableE + await api.dbTableColumn.create(tables[0].id, { + uidt: UITypes.LinkToAnotherRecord, + title: `TableA:mm:TableD`, + column_name: `TableA:mm:TableD`, + parentId: tables[0].id, + childId: tables[3].id, + type: 'mm', + }); + await api.dbTableColumn.create(tables[3].id, { + uidt: UITypes.LinkToAnotherRecord, + title: `TableD:mm:TableE`, + column_name: `TableD:mm:TableE`, + parentId: tables[3].id, + childId: tables[4].id, + type: 'mm', + }); + + // TableA TableA : self relation + await api.dbTableColumn.create(tables[0].id, { + uidt: UITypes.LinkToAnotherRecord, + title: `TableA:hm:TableA`, + column_name: `TableA:hm:TableA`, + parentId: tables[0].id, + childId: tables[0].id, + type: 'hm', + }); + + // TableA TableA : self relation + await api.dbTableColumn.create(tables[0].id, { + uidt: UITypes.LinkToAnotherRecord, + title: `TableA:mm:TableA`, + column_name: `TableA:mm:TableA`, + parentId: tables[0].id, + childId: tables[0].id, + type: 'mm', + }); + + // Add links + // TableA TableB TableC + // Link every record in tableA to 3 records in tableB + for (let i = 1; i <= recordCount; i++) { + await api.dbTableRow.nestedAdd( + 'noco', + context.project.id, + tables[0].id, + i, + 'hm', + 'TableA:hm:TableB', + `${i * 3 - 2}` + ); + await api.dbTableRow.nestedAdd( + 'noco', + context.project.id, + tables[0].id, + i, + 'hm', + 'TableA:hm:TableB', + `${i * 3 - 1}` + ); + await api.dbTableRow.nestedAdd( + 'noco', + context.project.id, + tables[0].id, + i, + 'hm', + 'TableA:hm:TableB', + `${i * 3 - 0}` + ); + } + // Link every record in tableB to 3 records in tableC + for (let i = 1; i <= recordCount; i++) { + await api.dbTableRow.nestedAdd( + 'noco', + context.project.id, + tables[1].id, + i, + 'hm', + 'TableB:hm:TableC', + `${i * 3 - 2}` + ); + await api.dbTableRow.nestedAdd( + 'noco', + context.project.id, + tables[1].id, + i, + 'hm', + 'TableB:hm:TableC', + `${i * 3 - 1}` + ); + await api.dbTableRow.nestedAdd( + 'noco', + context.project.id, + tables[1].id, + i, + 'hm', + 'TableB:hm:TableC', + `${i * 3 - 0}` + ); + } + + // TableA TableD TableE + // Link every record in tableA to 5 records in tableD + for (let i = 1; i <= recordCount; i++) { + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 1}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 2}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 3}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 4}`); + } + // Link every record in tableD to 5 records in tableE + for (let i = 1; i <= recordCount; i++) { + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 1}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 2}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 3}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 4}`); + } + + // TableA TableA : self relation + // Link every record in tableA to 3 records in tableA + for (let i = 1; i <= recordCount; i++) { + await api.dbTableRow.nestedAdd( + 'noco', + context.project.id, + tables[0].id, + i, + 'hm', + 'TableA:hm:TableA', + `${i * 3 - 2}` + ); + await api.dbTableRow.nestedAdd( + 'noco', + context.project.id, + tables[0].id, + i, + 'hm', + 'TableA:hm:TableA', + `${i * 3 - 1}` + ); + await api.dbTableRow.nestedAdd( + 'noco', + context.project.id, + tables[0].id, + i, + 'hm', + 'TableA:hm:TableA', + `${i * 3 - 0}` + ); + } + + // TableA TableA : self relation + // Link every record in tableA to 5 records in tableA + for (let i = 1; i <= recordCount; i++) { + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 1}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 2}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 3}`); + await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 4}`); + } + + // refresh page + await page.reload(); + }); + + test('Delete record - single, over UI', async () => { + await dashboard.treeView.openTable({ title: 'Table0' }); + await grid.deleteRow(0); + + // verify row count + await dashboard.grid.verifyTotalRowCount({ count: 99 }); + + // verify row count in all tables + for (let i = 1; i <= 4; i++) { + await dashboard.treeView.openTable({ title: `Table${i}` }); + await dashboard.grid.verifyTotalRowCount({ count: 100 }); + } + }); + + test('Delete record - bulk, over UI', async () => { + await dashboard.treeView.openTable({ title: 'Table0' }); + await grid.selectRow(0); + await grid.selectRow(1); + await grid.selectRow(2); + await grid.deleteSelectedRows(); + + // verify row count + await dashboard.grid.verifyTotalRowCount({ count: 97 }); + + // verify row count in all tables + for (let i = 1; i <= 4; i++) { + await dashboard.treeView.openTable({ title: `Table${i}` }); + await dashboard.grid.verifyTotalRowCount({ count: 100 }); + } + }); + + test('Delete column', async () => { + // has-many + await dashboard.treeView.openTable({ title: 'Table0' }); + await dashboard.grid.column.delete({ title: 'TableA:hm:TableB' }); + + // verify + await dashboard.treeView.openTable({ title: 'Table1' }); + await dashboard.grid.column.verify({ title: 'Table0', isVisible: false }); + await dashboard.grid.column.verify({ title: 'TableB:hm:TableC', isVisible: true }); + + /////////////////////////////////////////////////////////////////////////////////////////////// + + // many-many + await dashboard.treeView.openTable({ title: 'Table0' }); + await dashboard.grid.column.delete({ title: 'TableA:mm:TableD' }); + + // verify + await dashboard.treeView.openTable({ title: 'Table3' }); + await dashboard.grid.column.verify({ title: 'Table0 List', isVisible: false }); + await dashboard.grid.column.verify({ title: 'TableD:mm:TableE', isVisible: true }); + + /////////////////////////////////////////////////////////////////////////////////////////////// + + // has-many self relation + await dashboard.treeView.openTable({ title: 'Table0' }); + await dashboard.grid.column.delete({ title: 'TableA:hm:TableA' }); + + // verify + await dashboard.grid.column.verify({ title: 'TableA:hm:TableA', isVisible: false }); + await dashboard.grid.column.verify({ title: 'Table0', isVisible: false }); + + /////////////////////////////////////////////////////////////////////////////////////////////// + + // many-many self relation + await dashboard.treeView.openTable({ title: 'Table0' }); + await dashboard.grid.column.delete({ title: 'TableA:mm:TableA' }); + + // verify + await dashboard.grid.column.verify({ title: 'Table0 List', isVisible: false }); + await dashboard.grid.column.verify({ title: 'TableA:mm:TableA', isVisible: false }); + }); + + test('Delete table', async () => { + await dashboard.treeView.deleteTable({ title: 'Table0' }); + await dashboard.treeView.verifyTable({ title: 'Table0', exists: false }); + + // verify + await dashboard.treeView.openTable({ title: 'Table1' }); + await dashboard.grid.column.verify({ title: 'Table0', isVisible: false }); + + await dashboard.treeView.openTable({ title: 'Table3' }); + await dashboard.grid.column.verify({ title: 'Table0 List', isVisible: false }); + }); +}); From a4e15b46d57e53c24f415f1f687f8c302957d7b9 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 6 Jun 2023 14:05:58 +0530 Subject: [PATCH 14/34] fix: update the cache Signed-off-by: Pranav C --- .../src/version-upgrader/ncXcdbLTARUpgrader.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts b/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts index d247e84795..53bd8a2425 100644 --- a/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts +++ b/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts @@ -1,6 +1,8 @@ import { RelationTypes, UITypes } from 'nocodb-sdk'; +import NocoCache from '../cache/NocoCache'; import { MetaTable } from '../meta/meta.service'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import { CacheGetType, CacheScope } from '../utils/globals'; import type { MetaService } from '../meta/meta.service'; import type { LinkToAnotherRecordColumn, Model } from '../models'; import type { NcUpgraderCtx } from './NcUpgrader'; @@ -87,6 +89,19 @@ async function upgradeModelRelations({ { virtual: true }, colOptions.id, ); + + // update the cache as well + const cachedData = await NocoCache.get( + `${CacheScope.COL_RELATION}:${colOptions.fk_column_id}`, + CacheGetType.TYPE_OBJECT, + ); + if (cachedData) { + cachedData.virtual = true; + await NocoCache.set( + `${CacheScope.COL_RELATION}:${colOptions.fk_column_id}`, + cachedData, + ); + } } } From ad18b55b5f3411f1328ac85e48acf4b9f1595138 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Tue, 6 Jun 2023 14:39:43 +0530 Subject: [PATCH 15/34] test: typo fix Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- tests/playwright/tests/db/metaLTAR.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playwright/tests/db/metaLTAR.spec.ts b/tests/playwright/tests/db/metaLTAR.spec.ts index b8153edc7a..9e1260e75a 100644 --- a/tests/playwright/tests/db/metaLTAR.spec.ts +++ b/tests/playwright/tests/db/metaLTAR.spec.ts @@ -63,7 +63,7 @@ test.describe('Test table', () => { for (let i = 0; i < recordCount * 10; i++) { rows.push({ Id: i + 1, - SingleLineText: `${i + 1}`, + Title: `${i + 1}`, }); } From ee1b831fe60379b5b6bad382c62aaf3bdd12ca4a Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 6 Jun 2023 15:14:28 +0530 Subject: [PATCH 16/34] fix: upgrader correction Signed-off-by: Pranav C --- .../version-upgrader/ncXcdbLTARUpgrader.ts | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts b/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts index 53bd8a2425..a936d84cf7 100644 --- a/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts +++ b/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts @@ -1,10 +1,12 @@ import { RelationTypes, UITypes } from 'nocodb-sdk'; import NocoCache from '../cache/NocoCache'; import { MetaTable } from '../meta/meta.service'; +import { Base } from '../models'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import { CacheGetType, CacheScope } from '../utils/globals'; +import { Model } from '../models'; +import type { LinkToAnotherRecordColumn } from '../models'; import type { MetaService } from '../meta/meta.service'; -import type { LinkToAnotherRecordColumn, Model } from '../models'; import type { NcUpgraderCtx } from './NcUpgrader'; // An upgrader for upgrading LTAR relations in XCDB bases @@ -67,7 +69,12 @@ async function upgradeModelRelations({ // delete the relation if (relation) { - await sqlClient.relationDelete(relation); + await sqlClient.relationDelete({ + parentColumn: relation.rcn, + childColumn: relation.cn, + parentTable: relation.rtn, + childTable: relation.tn, + }); } // create a new index for the column @@ -121,18 +128,23 @@ async function upgradeBaseRelations({ const relations = (await sqlClient.relationListAll())?.data?.list; // get models for the base - const models = await ncMeta.metaList2(null, base.id, 'models'); + const models = await ncMeta.metaList2(null, base.id, MetaTable.MODELS); // get all columns and filter out relations and upgrade for (const model of models) { - await upgradeModelRelations({ ncMeta, model, sqlClient, relations }); + await upgradeModelRelations({ + ncMeta, + model: new Model(model), + sqlClient, + relations, + }); } } // database to virtual relation and create an index for it export default async function ({ ncMeta }: NcUpgraderCtx) { // get all xcdb bases - const bases = await ncMeta.metaList2(null, null, 'bases', { + const bases = await ncMeta.metaList2(null, null, MetaTable.BASES, { condition: { is_meta: 1, }, @@ -143,7 +155,7 @@ export default async function ({ ncMeta }: NcUpgraderCtx) { for (const base of bases) { await upgradeBaseRelations({ ncMeta, - base, + base: new Base(base), }); } } From 1e3743a43ad705983f9762fe0138aeef7438eb0b Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 6 Jun 2023 15:15:10 +0530 Subject: [PATCH 17/34] feat: unlink on condition based bulk delete Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 103 ++++++++++++++++++++++- 1 file changed, 100 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index c3de3a4ffd..43221a2a12 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -20,7 +20,16 @@ import { v4 as uuidv4 } from 'uuid'; import { Knex } from 'knex'; import { NcError } from '../helpers/catchError'; import getAst from '../helpers/getAst'; -import { Audit, Column, Filter, Model, Project, Sort, View } from '../models'; +import { + Audit, + Base, + Column, + Filter, + Model, + Project, + Sort, + View, +} from '../models'; import { sanitize, unsanitize } from '../helpers/sqlSanitize'; import { COMPARISON_OPS, @@ -2527,6 +2536,7 @@ class BaseModelSqlv2 { args: { where?: string; filterArr?: Filter[] } = {}, { cookie }: { cookie?: any } = {}, ) { + let trx: Transaction; try { await this.model.getColumns(); const { where } = this._getListArgs(args); @@ -2551,14 +2561,101 @@ class BaseModelSqlv2 { this.dbDriver, ); - qb.del(); + const execQueries: ((trx: Transaction, qb: any) => Promise)[] = []; + + for (const column of this.model.columns) { + if (column.uidt !== UITypes.LinkToAnotherRecord) continue; + + const colOptions = + await column.getColOptions(); + + if (colOptions.type === 'bt') { + continue; + } + + const childColumn = await colOptions.getChildColumn(); + const parentColumn = await colOptions.getParentColumn(); + const parentTable = await parentColumn.getModel(); + const childTable = await childColumn.getModel(); + await childTable.getColumns(); + await parentTable.getColumns(); + + const childTn = this.getTnPath(childTable); + const parentTn = this.getTnPath(parentTable); + + switch (colOptions.type) { + case 'mm': + { + const vChildCol = await colOptions.getMMChildColumn(); + const vParentCol = await colOptions.getMMParentColumn(); + const vTable = await colOptions.getMMModel(); + + const vTn = this.getTnPath(vTable); + + execQueries.push((trx, qb) => + this.dbDriver(vTn) + .where({ + [vChildCol.column_name]: this.dbDriver(childTn) + .select(childColumn.column_name) + .first(), + }) + .delete(), + ); + } + break; + case 'hm': + { + // skip if it's an mm table column + const relatedTable = await colOptions.getRelatedTable(); + if (relatedTable.mm) { + break; + } + + const childColumn = await Column.get({ + colId: colOptions.fk_child_column_id, + }); - const count = (await qb) as any; + execQueries.push((trx, qb) => + trx(childTn) + .where({ + [childColumn.column_name]: this.dbDriver.from( + qb + .select(parentColumn.column_name) + // .where(_wherePk(parentTable.primaryKeys, rowId)) + .first() + .as('___cn_alias'), + ), + }) + .update({ + [childColumn.column_name]: null, + }), + ); + } + break; + } + } + + trx = await this.dbDriver.transaction(); + + const base = await Base.get(this.model.base_id); + // unlink LTAR data + if (base.is_meta) { + for (const execQuery of execQueries) { + await execQuery(trx, qb.clone()); + } + } + + const deleteQb = qb.clone().transacting(trx).del(); + + const count = (await deleteQb) as any; + + await trx.commit(); await this.afterBulkDelete(count, this.dbDriver, cookie, true); return count; } catch (e) { + if (trx) await trx.rollback(); throw e; } } From 4f13be442940fcb2b204652b53dc71feb546bf69 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 6 Jun 2023 23:10:38 +0530 Subject: [PATCH 18/34] fix: transaction related correction in delete handler Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 43221a2a12..34b77f319d 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -1943,16 +1943,14 @@ class BaseModelSqlv2 { break; } } - + const where = await this._wherePk(id); if (!trx) { trx = await this.dbDriver.transaction(); } await Promise.all(execQueries.map((q) => q(trx))); - const response = await trx(this.tnPath) - .del() - .where(await this._wherePk(id)); + const response = await trx(this.tnPath).del().where(where); if (!_trx) await trx.commit(); From 6b43d475122fe729bdeb092e74095c61bf1a1fd8 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 7 Jun 2023 00:56:58 +0530 Subject: [PATCH 19/34] fix: get base data before starting the transaction Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 34b77f319d..cb353dfc6b 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2633,9 +2633,10 @@ class BaseModelSqlv2 { } } + const base = await Base.get(this.model.base_id); + trx = await this.dbDriver.transaction(); - const base = await Base.get(this.model.base_id); // unlink LTAR data if (base.is_meta) { for (const execQuery of execQueries) { From a7014aef5e2356e8e8b197fba58240a9bc86b5a5 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 7 Jun 2023 01:15:57 +0530 Subject: [PATCH 20/34] fix: do the LTAR unlink only if it's metadb project Signed-off-by: Pranav C --- packages/nocodb/src/db/BaseModelSqlv2.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index cb353dfc6b..52d202a69c 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -2454,6 +2454,8 @@ class BaseModelSqlv2 { const execQueries: ((trx: Transaction, ids: any[]) => Promise)[] = []; + const base = await Base.get(this.model.base_id); + for (const column of this.model.columns) { if (column.uidt !== UITypes.LinkToAnotherRecord) continue; @@ -2508,7 +2510,7 @@ class BaseModelSqlv2 { transaction = await this.dbDriver.transaction(); - if (execQueries.length > 0) { + if (base.is_meta && execQueries.length > 0) { for (const execQuery of execQueries) { await execQuery(transaction, idsVals); } From 40b638ef0478223150a5143c8d573e346aa293ef Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Wed, 7 Jun 2023 16:16:48 +0530 Subject: [PATCH 21/34] test: virtual cell verification after row delete Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../pages/Dashboard/common/Cell/index.ts | 2 +- tests/playwright/setup/xcdbProject.ts | 40 ++++ tests/playwright/tests/db/metaLTAR.spec.ts | 173 ++++++++---------- 3 files changed, 113 insertions(+), 102 deletions(-) create mode 100644 tests/playwright/setup/xcdbProject.ts diff --git a/tests/playwright/pages/Dashboard/common/Cell/index.ts b/tests/playwright/pages/Dashboard/common/Cell/index.ts index 37d20dc3f7..c14bf062b8 100644 --- a/tests/playwright/pages/Dashboard/common/Cell/index.ts +++ b/tests/playwright/pages/Dashboard/common/Cell/index.ts @@ -282,7 +282,7 @@ export class CellPageObject extends BasePage { await this.get({ index, columnHeader }).scrollIntoViewIfNeeded(); // verify chip count & contents - if (count) await expect(chips).toHaveCount(count); + await expect(chips).toHaveCount(count); // verify only the elements that are passed in for (let i = 0; i < value.length; ++i) { diff --git a/tests/playwright/setup/xcdbProject.ts b/tests/playwright/setup/xcdbProject.ts new file mode 100644 index 0000000000..7123e80b2c --- /dev/null +++ b/tests/playwright/setup/xcdbProject.ts @@ -0,0 +1,40 @@ +import { Api } from 'nocodb-sdk'; +let api: Api; +async function createXcdb(token?: string) { + api = new Api({ + baseURL: `http://localhost:8080/`, + headers: { + 'xc-auth': token, + }, + }); + + const projectList = await api.project.list(); + for (const project of projectList.list) { + // delete project with title 'xcdb' if it exists + if (project.title === 'xcdb') { + await api.project.delete(project.id); + } + } + + const project = await api.project.create({ title: 'xcdb' }); + return project; +} + +async function deleteXcdb(token?: string) { + api = new Api({ + baseURL: `http://localhost:8080/`, + headers: { + 'xc-auth': token, + }, + }); + + const projectList = await api.project.list(); + for (const project of projectList.list) { + // delete project with title 'xcdb' if it exists + if (project.title === 'xcdb') { + await api.project.delete(project.id); + } + } +} + +export { createXcdb, deleteXcdb }; diff --git a/tests/playwright/tests/db/metaLTAR.spec.ts b/tests/playwright/tests/db/metaLTAR.spec.ts index 9e1260e75a..180089032c 100644 --- a/tests/playwright/tests/db/metaLTAR.spec.ts +++ b/tests/playwright/tests/db/metaLTAR.spec.ts @@ -24,6 +24,9 @@ import setup from '../../setup'; import { Api, UITypes } from 'nocodb-sdk'; import { DashboardPage } from '../../pages/Dashboard'; import { GridPage } from '../../pages/Dashboard/Grid'; +import { createXcdb, deleteXcdb } from '../../setup/xcdbProject'; +import { ProjectsPage } from '../../pages/ProjectsPage'; +import { isSqlite } from '../../setup/db'; let api: Api; const recordCount = 10; @@ -33,11 +36,30 @@ test.describe('Test table', () => { let grid: GridPage; const tables = []; + test.afterEach(async () => { + try { + if (context) { + await deleteXcdb(context.token); + } + } catch (e) { + console.log(e); + } + + // reset tables array + tables.length = 0; + }); + test.beforeEach(async ({ page }) => { context = await setup({ page, isEmptyProject: true }); dashboard = new DashboardPage(page, context.project); grid = dashboard.grid; + // create a new xcdb project + const xcdb = await createXcdb(context.token); + await dashboard.clickHome(); + const projectsPage = new ProjectsPage(dashboard.rootPage); + await projectsPage.openProject({ title: 'xcdb', withoutPrefix: true }); + api = new Api({ baseURL: `http://localhost:8080/`, headers: { @@ -67,17 +89,14 @@ test.describe('Test table', () => { }); } - // Create tables - const project = await api.project.read(context.project.id); - for (let i = 0; i < 5; i++) { - const table = await api.base.tableCreate(context.project.id, project.bases?.[0].id, { + const table = await api.base.tableCreate(xcdb.id, xcdb.bases?.[0].id, { table_name: `Table${i}`, title: `Table${i}`, columns: columns, }); tables.push(table); - await api.dbTableRow.bulkCreate('noco', context.project.id, tables[i].id, rows); + await api.dbTableRow.bulkCreate('noco', xcdb.id, tables[i].id, rows); } // Create links @@ -141,123 +160,51 @@ test.describe('Test table', () => { // TableA TableB TableC // Link every record in tableA to 3 records in tableB for (let i = 1; i <= recordCount; i++) { - await api.dbTableRow.nestedAdd( - 'noco', - context.project.id, - tables[0].id, - i, - 'hm', - 'TableA:hm:TableB', - `${i * 3 - 2}` - ); - await api.dbTableRow.nestedAdd( - 'noco', - context.project.id, - tables[0].id, - i, - 'hm', - 'TableA:hm:TableB', - `${i * 3 - 1}` - ); - await api.dbTableRow.nestedAdd( - 'noco', - context.project.id, - tables[0].id, - i, - 'hm', - 'TableA:hm:TableB', - `${i * 3 - 0}` - ); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'hm', 'TableA:hm:TableB', `${i * 3 - 2}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'hm', 'TableA:hm:TableB', `${i * 3 - 1}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'hm', 'TableA:hm:TableB', `${i * 3 - 0}`); } // Link every record in tableB to 3 records in tableC for (let i = 1; i <= recordCount; i++) { - await api.dbTableRow.nestedAdd( - 'noco', - context.project.id, - tables[1].id, - i, - 'hm', - 'TableB:hm:TableC', - `${i * 3 - 2}` - ); - await api.dbTableRow.nestedAdd( - 'noco', - context.project.id, - tables[1].id, - i, - 'hm', - 'TableB:hm:TableC', - `${i * 3 - 1}` - ); - await api.dbTableRow.nestedAdd( - 'noco', - context.project.id, - tables[1].id, - i, - 'hm', - 'TableB:hm:TableC', - `${i * 3 - 0}` - ); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[1].id, i, 'hm', 'TableB:hm:TableC', `${i * 3 - 2}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[1].id, i, 'hm', 'TableB:hm:TableC', `${i * 3 - 1}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[1].id, i, 'hm', 'TableB:hm:TableC', `${i * 3 - 0}`); } // TableA TableD TableE // Link every record in tableA to 5 records in tableD for (let i = 1; i <= recordCount; i++) { - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 1}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 2}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 3}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 4}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 1}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 2}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 3}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'mm', 'TableA:mm:TableD', `${i + 4}`); } // Link every record in tableD to 5 records in tableE for (let i = 1; i <= recordCount; i++) { - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 1}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 2}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 3}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 4}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 1}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 2}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 3}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[3].id, i, 'mm', 'TableD:mm:TableE', `${i + 4}`); } // TableA TableA : self relation // Link every record in tableA to 3 records in tableA for (let i = 1; i <= recordCount; i++) { - await api.dbTableRow.nestedAdd( - 'noco', - context.project.id, - tables[0].id, - i, - 'hm', - 'TableA:hm:TableA', - `${i * 3 - 2}` - ); - await api.dbTableRow.nestedAdd( - 'noco', - context.project.id, - tables[0].id, - i, - 'hm', - 'TableA:hm:TableA', - `${i * 3 - 1}` - ); - await api.dbTableRow.nestedAdd( - 'noco', - context.project.id, - tables[0].id, - i, - 'hm', - 'TableA:hm:TableA', - `${i * 3 - 0}` - ); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'hm', 'TableA:hm:TableA', `${i * 3 - 2}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'hm', 'TableA:hm:TableA', `${i * 3 - 1}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'hm', 'TableA:hm:TableA', `${i * 3 - 0}`); } // TableA TableA : self relation // Link every record in tableA to 5 records in tableA for (let i = 1; i <= recordCount; i++) { - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 1}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 2}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 3}`); - await api.dbTableRow.nestedAdd('noco', context.project.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 4}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 1}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 2}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 3}`); + await api.dbTableRow.nestedAdd('noco', xcdb.id, tables[0].id, i, 'mm', 'TableA:mm:TableA', `${i + 4}`); } // refresh page @@ -276,6 +223,30 @@ test.describe('Test table', () => { await dashboard.treeView.openTable({ title: `Table${i}` }); await dashboard.grid.verifyTotalRowCount({ count: 100 }); } + + // has-many removal verification + await dashboard.treeView.openTable({ title: 'Table1' }); + await dashboard.grid.cell.verifyVirtualCell({ index: 0, columnHeader: 'Table0', count: 0, value: [] }); + await dashboard.grid.cell.verifyVirtualCell({ index: 1, columnHeader: 'Table0', count: 0, value: [] }); + await dashboard.grid.cell.verifyVirtualCell({ index: 2, columnHeader: 'Table0', count: 0, value: [] }); + + // many-many removal verification + await dashboard.treeView.openTable({ title: 'Table3' }); + await dashboard.grid.cell.verifyVirtualCell({ index: 0, columnHeader: 'Table0 List', count: 0, value: [] }); + await dashboard.grid.cell.verifyVirtualCell({ index: 1, columnHeader: 'Table0 List', count: 1, value: ['2'] }); + await dashboard.grid.cell.verifyVirtualCell({ index: 2, columnHeader: 'Table0 List', count: 2, value: ['2', '3'] }); + await dashboard.grid.cell.verifyVirtualCell({ + index: 3, + columnHeader: 'Table0 List', + count: 3, + value: ['2', '3', '4'], + }); + await dashboard.grid.cell.verifyVirtualCell({ + index: 4, + columnHeader: 'Table0 List', + count: 4, + value: ['2', '3', '4', '5'], + }); }); test('Delete record - bulk, over UI', async () => { From 2e06521a69c849fa75d55c1aba18228288e23a7a Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Wed, 7 Jun 2023 18:37:09 +0530 Subject: [PATCH 22/34] test: select row function fix Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- tests/playwright/pages/Dashboard/Grid/index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/playwright/pages/Dashboard/Grid/index.ts b/tests/playwright/pages/Dashboard/Grid/index.ts index 68ac87fcbc..f085b732e6 100644 --- a/tests/playwright/pages/Dashboard/Grid/index.ts +++ b/tests/playwright/pages/Dashboard/Grid/index.ts @@ -183,8 +183,9 @@ export class GridPage extends BasePage { } async selectRow(index: number) { - await this.get().locator(`td[data-testid="cell-Id-${index}"]`).locator('span.ant-checkbox').click(); - await this.rootPage.waitForTimeout(300); + const cell: Locator = await this.get().locator(`td[data-testid="cell-Id-${index}"]`); + await cell.hover(); + await cell.locator('input[type="checkbox"]').check({ force: true }); } async selectAll() { From ad9eb9ff988b7f0faa72ab29f61de4763803b4fb Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 7 Jun 2023 18:56:50 +0530 Subject: [PATCH 23/34] fix: if column not found return Signed-off-by: Pranav C --- packages/nocodb/src/models/Column.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts index e7e14b0f2b..78ae4edc9f 100644 --- a/packages/nocodb/src/models/Column.ts +++ b/packages/nocodb/src/models/Column.ts @@ -601,6 +601,11 @@ export default class Column implements ColumnType { static async delete(id, ncMeta = Noco.ncMeta) { const col = await this.get({ colId: id }, ncMeta); + // if column is not found, return + if (!col) { + return; + } + // todo: or instead of delete reset related foreign key value to null and handle in BaseModel // get qr code columns and delete From 0da392e4501317f47b5d2b88875481554d69dce6 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 7 Jun 2023 19:31:59 +0530 Subject: [PATCH 24/34] fix: handle if column missing Signed-off-by: Pranav C --- .../nocodb/src/services/tables.service.ts | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/nocodb/src/services/tables.service.ts b/packages/nocodb/src/services/tables.service.ts index cf005b04a9..a0ef5488ca 100644 --- a/packages/nocodb/src/services/tables.service.ts +++ b/packages/nocodb/src/services/tables.service.ts @@ -182,22 +182,25 @@ export class TablesService { let result; try { // delete all relations - await Promise.all( - relationColumns.map((c) => { - // skip if column is hasmany relation to mm table - if (c.system) { - return; - } - - return this.columnsService.columnDelete( - { - req: param.req, - columnId: c.id, - }, - ncMeta, - ); - }), - ); + for (const c of relationColumns) { + // skip if column is hasmany relation to mm table + if (c.system) { + continue; + } + + // verify column exist or not and based on that delete the column + if (!(await Column.get({ colId: c.id }))) { + continue; + } + + await this.columnsService.columnDelete( + { + req: param.req, + columnId: c.id, + }, + ncMeta, + ); + } const sqlMgr = await ProjectMgrv2.getSqlMgr(project, ncMeta); (table as any).tn = table.table_name; From 6be94cafaaf24df311012df8f1c88ba4c896b82c Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Wed, 7 Jun 2023 21:57:18 +0530 Subject: [PATCH 25/34] test: serialize xcdb tests Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- tests/playwright/pages/Dashboard/common/Cell/index.ts | 2 +- tests/playwright/tests/db/metaLTAR.spec.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/playwright/pages/Dashboard/common/Cell/index.ts b/tests/playwright/pages/Dashboard/common/Cell/index.ts index 4e441f313c..a081a39509 100644 --- a/tests/playwright/pages/Dashboard/common/Cell/index.ts +++ b/tests/playwright/pages/Dashboard/common/Cell/index.ts @@ -282,7 +282,7 @@ export class CellPageObject extends BasePage { await this.get({ index, columnHeader }).scrollIntoViewIfNeeded(); // verify chip count & contents - await expect(chips).toHaveCount(count); + if (count) await expect(chips).toHaveCount(count); // verify only the elements that are passed in for (let i = 0; i < value.length; ++i) { diff --git a/tests/playwright/tests/db/metaLTAR.spec.ts b/tests/playwright/tests/db/metaLTAR.spec.ts index 180089032c..7205c37c33 100644 --- a/tests/playwright/tests/db/metaLTAR.spec.ts +++ b/tests/playwright/tests/db/metaLTAR.spec.ts @@ -30,7 +30,9 @@ import { isSqlite } from '../../setup/db'; let api: Api; const recordCount = 10; -test.describe('Test table', () => { +// serial as all projects end up creating xcdb using same name +// fix me : use worker ID logic for creating unique project name +test.describe.serial('Test table', () => { let context: any; let dashboard: DashboardPage; let grid: GridPage; From 4378516352dfad13653cf248a93a883b4a7f20a1 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 7 Jun 2023 22:37:14 +0530 Subject: [PATCH 26/34] fix: pass transaction reference Signed-off-by: Pranav C --- packages/nocodb/src/services/tables.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb/src/services/tables.service.ts b/packages/nocodb/src/services/tables.service.ts index a0ef5488ca..ae89adb87b 100644 --- a/packages/nocodb/src/services/tables.service.ts +++ b/packages/nocodb/src/services/tables.service.ts @@ -189,7 +189,7 @@ export class TablesService { } // verify column exist or not and based on that delete the column - if (!(await Column.get({ colId: c.id }))) { + if (!(await Column.get({ colId: c.id }, ncMeta))) { continue; } From b77b43148402a5478319da4ea6b0a1a7736dff76 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Thu, 8 Jun 2023 09:38:07 +0530 Subject: [PATCH 27/34] test: place file to the end Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../playwright/tests/db/{metaLTAR.spec.ts => zMetaLTAR.spec.ts} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/playwright/tests/db/{metaLTAR.spec.ts => zMetaLTAR.spec.ts} (99%) diff --git a/tests/playwright/tests/db/metaLTAR.spec.ts b/tests/playwright/tests/db/zMetaLTAR.spec.ts similarity index 99% rename from tests/playwright/tests/db/metaLTAR.spec.ts rename to tests/playwright/tests/db/zMetaLTAR.spec.ts index 7205c37c33..e25b971deb 100644 --- a/tests/playwright/tests/db/metaLTAR.spec.ts +++ b/tests/playwright/tests/db/zMetaLTAR.spec.ts @@ -32,7 +32,7 @@ const recordCount = 10; // serial as all projects end up creating xcdb using same name // fix me : use worker ID logic for creating unique project name -test.describe.serial('Test table', () => { +test.describe.serial.only('Test table', () => { let context: any; let dashboard: DashboardPage; let grid: GridPage; From d85170c0995f0e6f99cd0b54b5b29e093c6d9590 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Thu, 8 Jun 2023 09:38:39 +0530 Subject: [PATCH 28/34] test: remove .only Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- tests/playwright/tests/db/zMetaLTAR.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playwright/tests/db/zMetaLTAR.spec.ts b/tests/playwright/tests/db/zMetaLTAR.spec.ts index e25b971deb..7205c37c33 100644 --- a/tests/playwright/tests/db/zMetaLTAR.spec.ts +++ b/tests/playwright/tests/db/zMetaLTAR.spec.ts @@ -32,7 +32,7 @@ const recordCount = 10; // serial as all projects end up creating xcdb using same name // fix me : use worker ID logic for creating unique project name -test.describe.serial.only('Test table', () => { +test.describe.serial('Test table', () => { let context: any; let dashboard: DashboardPage; let grid: GridPage; From 30a2adc2a01cb6e6b850c4c1ddee619f6c9f270f Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Thu, 8 Jun 2023 10:55:00 +0530 Subject: [PATCH 29/34] test: UT fix as per new behaviour --- .../tests/unit/rest/tests/tableRow.test.ts | 15 +- .../tests/unit/rest/tests/viewRow.test.ts | 148 +++++++++--------- 2 files changed, 75 insertions(+), 88 deletions(-) diff --git a/packages/nocodb/tests/unit/rest/tests/tableRow.test.ts b/packages/nocodb/tests/unit/rest/tests/tableRow.test.ts index 488bb50493..aec23a2a0a 100644 --- a/packages/nocodb/tests/unit/rest/tests/tableRow.test.ts +++ b/packages/nocodb/tests/unit/rest/tests/tableRow.test.ts @@ -1422,22 +1422,15 @@ function tableTest() { rowId: row['Id'], }); - const response = await request(context.app) + await request(context.app) .delete(`/api/v1/db/data/noco/${project.id}/${table.id}/${row['Id']}`) .set('xc-auth', context.token) .expect(200); const deleteRow = await getRow(context, { project, table, id: row['Id'] }); - if (!deleteRow) { - throw new Error('Should not delete'); - } - - if ( - !(response.body.message[0] as string).includes( - 'is a LinkToAnotherRecord of', - ) - ) { - throw new Error('Should give ltar foreign key error'); + if (deleteRow && Object.keys(deleteRow).length > 0) { + console.log(deleteRow); + throw new Error('Wrong delete'); } }); diff --git a/packages/nocodb/tests/unit/rest/tests/viewRow.test.ts b/packages/nocodb/tests/unit/rest/tests/viewRow.test.ts index 1cf5501855..481eaafa33 100644 --- a/packages/nocodb/tests/unit/rest/tests/viewRow.test.ts +++ b/packages/nocodb/tests/unit/rest/tests/viewRow.test.ts @@ -1,12 +1,11 @@ import 'mocha'; +import { isString } from 'util'; +import request from 'supertest'; +import { UITypes, ViewTypes } from 'nocodb-sdk'; +import { expect } from 'chai'; import init from '../../init'; import { createProject, createSakilaProject } from '../../factory/project'; -import request from 'supertest'; -import Project from '../../../../src/models/Project'; -import Model from '../../../../src/models/Model'; import { createTable, getTable } from '../../factory/table'; -import View from '../../../../src/models/View'; -import { ColumnType, UITypes, ViewTypes } from 'nocodb-sdk'; import { createView } from '../../factory/view'; import { createColumn, @@ -21,9 +20,11 @@ import { getOneRow, getRow, } from '../../factory/row'; -import { expect } from 'chai'; import { isPg } from '../../init/db'; -import { isString } from 'util'; +import type { ColumnType } from 'nocodb-sdk'; +import type View from '../../../../src/models/View'; +import type Model from '../../../../src/models/Model'; +import type Project from '../../../../src/models/Project'; // Test case list // 1. Get view row list g @@ -160,7 +161,7 @@ function viewRowTests() { const testGetViewRowList = async (view: View) => { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .expect(200); @@ -179,7 +180,7 @@ function viewRowTests() { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/views/${view.id}/group/${ratingColumn.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/views/${view.id}/group/${ratingColumn.id}`, ) .set('xc-auth', context.token) .expect(200); @@ -225,7 +226,7 @@ function viewRowTests() { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .query({ @@ -244,7 +245,7 @@ function viewRowTests() { requiredColumns.map((c: ColumnType) => ({ title: c.title, uidt: c.uidt, - })) + })), ); throw new Error('Wrong columns'); } @@ -271,7 +272,7 @@ function viewRowTests() { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/views/${view.id}/group/${ratingColumn.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/views/${view.id}/group/${ratingColumn.id}`, ) .set('xc-auth', context.token) .query({ @@ -287,7 +288,7 @@ function viewRowTests() { expect( Object.keys(response.body.find((e) => e.key === 'NC-17').value.list[0]) .sort() - .join(',') + .join(','), ).to.equal('FilmId,Title'); }; @@ -297,14 +298,14 @@ function viewRowTests() { const testDescSortedViewDataList = async (view: View) => { const firstNameColumn = customerColumns.find( - (col) => col.title === 'FirstName' + (col) => col.title === 'FirstName', ); const visibleColumns = [firstNameColumn]; const sortInfo = [{ fk_column_id: firstNameColumn.id, direction: 'desc' }]; const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .query({ @@ -332,7 +333,7 @@ function viewRowTests() { Math.trunc(pageInfo.totalRows / pageInfo.pageSize) * pageInfo.pageSize; const lastPageResponse = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .query({ @@ -375,7 +376,7 @@ function viewRowTests() { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/views/${view.id}/group/${ratingColumn.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/views/${view.id}/group/${ratingColumn.id}`, ) .set('xc-auth', context.token) .query({ @@ -390,7 +391,7 @@ function viewRowTests() { expect(response.body).to.be.have.length(6); expect( - response.body.find((e) => e.key === 'PG').value.list[0].Title + response.body.find((e) => e.key === 'PG').value.list[0].Title, ).to.equal('WORST BANGER'); }; @@ -400,14 +401,14 @@ function viewRowTests() { const testAscSortedViewDataList = async (view: View) => { const firstNameColumn = customerColumns.find( - (col) => col.title === 'FirstName' + (col) => col.title === 'FirstName', ); const visibleColumns = [firstNameColumn]; const sortInfo = [{ fk_column_id: firstNameColumn.id, direction: 'asc' }]; const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .query({ @@ -435,7 +436,7 @@ function viewRowTests() { Math.trunc(pageInfo.totalRows / pageInfo.pageSize) * pageInfo.pageSize; const lastPageResponse = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .query({ @@ -478,7 +479,7 @@ function viewRowTests() { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/views/${view.id}/group/${ratingColumn.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${filmTable.id}/views/${view.id}/group/${ratingColumn.id}`, ) .set('xc-auth', context.token) .query({ @@ -493,7 +494,7 @@ function viewRowTests() { expect(response.body).to.be.have.length(6); expect( - response.body.find((e) => e.key === 'PG').value.list[0].Title + response.body.find((e) => e.key === 'PG').value.list[0].Title, ).to.equal('ACADEMY DINOSAUR'); }; @@ -502,7 +503,7 @@ function viewRowTests() { }); const testGetViewDataListWithRequiredColumnsAndFilter = async ( - viewType: ViewTypes + viewType: ViewTypes, ) => { const rentalTable = await getTable({ project: sakilaProject, @@ -523,7 +524,7 @@ function viewRowTests() { }); const paymentListColumn = (await rentalTable.getColumns()).find( - (c) => c.title === 'Payment List' + (c) => c.title === 'Payment List', ); const nestedFilter = { @@ -549,7 +550,7 @@ function viewRowTests() { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${rentalTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${rentalTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .query({ @@ -560,7 +561,7 @@ function viewRowTests() { const ascResponse = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${rentalTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${rentalTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .query({ @@ -579,7 +580,7 @@ function viewRowTests() { const descResponse = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${rentalTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${rentalTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .query({ @@ -606,7 +607,7 @@ function viewRowTests() { }); const testGetNestedSortedFilteredTableDataListWithLookupColumn = async ( - viewType: ViewTypes + viewType: ViewTypes, ) => { const view = await createView(context, { title: 'View', @@ -624,11 +625,11 @@ function viewRowTests() { }); const paymentListColumn = (await customerTable.getColumns()).find( - (c) => c.title === 'Payment List' + (c) => c.title === 'Payment List', ); const activeColumn = (await customerTable.getColumns()).find( - (c) => c.title === 'Active' + (c) => c.title === 'Active', ); const nestedFields = { @@ -681,7 +682,7 @@ function viewRowTests() { const ascResponse = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .query({ @@ -703,7 +704,7 @@ function viewRowTests() { } const nestedRentalResponse = Object.keys( - ascResponse.body.list[0]['Rental List'][0] + ascResponse.body.list[0]['Rental List'][0], ); if ( @@ -719,7 +720,7 @@ function viewRowTests() { it('Get nested sorted filtered table with nested fields data list with a rollup column in customer table view grid', async () => { await testGetNestedSortedFilteredTableDataListWithLookupColumn( - ViewTypes.GRID + ViewTypes.GRID, ); }); @@ -774,7 +775,7 @@ function viewRowTests() { await request(context.app) .post( - `/api/v1/db/data/noco/${project.id}/${table.id}/views/${nonRelatedView.id}` + `/api/v1/db/data/noco/${project.id}/${table.id}/views/${nonRelatedView.id}`, ) .set('xc-auth', context.token) .send({ @@ -802,7 +803,7 @@ function viewRowTests() { // todo: Test that all the columns needed to be shown in the view are returned const testFindOneSortedDataWithRequiredColumns = async ( - viewType: ViewTypes + viewType: ViewTypes, ) => { const view = await createView(context, { title: 'View', @@ -810,13 +811,13 @@ function viewRowTests() { type: viewType, }); const firstNameColumn = customerColumns.find( - (col) => col.title === 'FirstName' + (col) => col.title === 'FirstName', ); const visibleColumns = [firstNameColumn]; let response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/find-one` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/find-one`, ) .set('xc-auth', context.token) .query({ @@ -837,7 +838,7 @@ function viewRowTests() { response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/find-one` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/find-one`, ) .set('xc-auth', context.token) .query({ @@ -870,7 +871,7 @@ function viewRowTests() { }); const testFindOneSortedFilteredNestedFieldsDataWithRollup = async ( - viewType: ViewTypes + viewType: ViewTypes, ) => { const rollupColumn = await createRollupColumn(context, { project: sakilaProject, @@ -893,11 +894,11 @@ function viewRowTests() { }); const paymentListColumn = (await customerTable.getColumns()).find( - (c) => c.title === 'Payment List' + (c) => c.title === 'Payment List', ); const activeColumn = (await customerTable.getColumns()).find( - (c) => c.title === 'Active' + (c) => c.title === 'Active', ); const nestedFields = { @@ -950,7 +951,7 @@ function viewRowTests() { const ascResponse = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/find-one` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/find-one`, ) .set('xc-auth', context.token) .query({ @@ -995,7 +996,7 @@ function viewRowTests() { type: viewType, }); const firstNameColumn = customerColumns.find( - (col) => col.title === 'FirstName' + (col) => col.title === 'FirstName', ); const rollupColumn = await createRollupColumn(context, { @@ -1012,7 +1013,7 @@ function viewRowTests() { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/groupby` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/groupby`, ) .set('xc-auth', context.token) .query({ @@ -1049,7 +1050,7 @@ function viewRowTests() { }); const firstNameColumn = customerColumns.find( - (col) => col.title === 'FirstName' + (col) => col.title === 'FirstName', ); const rollupColumn = await createRollupColumn(context, { @@ -1066,7 +1067,7 @@ function viewRowTests() { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/groupby` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/groupby`, ) .set('xc-auth', context.token) .query({ @@ -1105,7 +1106,7 @@ function viewRowTests() { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/count` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/count`, ) .set('xc-auth', context.token) .expect(200); @@ -1136,7 +1137,7 @@ function viewRowTests() { const listResponse = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}`, ) .set('xc-auth', context.token) .expect(200); @@ -1145,7 +1146,7 @@ function viewRowTests() { const readResponse = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/${row['CustomerId']}` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/${row['CustomerId']}`, ) .set('xc-auth', context.token) .expect(200); @@ -1181,7 +1182,7 @@ function viewRowTests() { const updateResponse = await request(context.app) .patch( - `/api/v1/db/data/noco/${project.id}/${table.id}/views/${view.id}/${row['Id']}` + `/api/v1/db/data/noco/${project.id}/${table.id}/views/${view.id}/${row['Id']}`, ) .set('xc-auth', context.token) .send({ @@ -1207,7 +1208,7 @@ function viewRowTests() { }); const testUpdateViewRowWithValidationAndInvalidData = async ( - viewType: ViewTypes + viewType: ViewTypes, ) => { const table = await createTable(context, project); const emailColumn = await createColumn(context, table, { @@ -1228,7 +1229,7 @@ function viewRowTests() { await request(context.app) .patch( - `/api/v1/db/data/noco/${project.id}/${table.id}/views/${view.id}/${row['Id']}` + `/api/v1/db/data/noco/${project.id}/${table.id}/views/${view.id}/${row['Id']}`, ) .set('xc-auth', context.token) .send({ @@ -1253,7 +1254,7 @@ function viewRowTests() { // todo: Test with form view const testUpdateViewRowWithValidationAndValidData = async ( - viewType: ViewTypes + viewType: ViewTypes, ) => { const table = await createTable(context, project); const emailColumn = await createColumn(context, table, { @@ -1273,7 +1274,7 @@ function viewRowTests() { const response = await request(context.app) .patch( - `/api/v1/db/data/noco/${project.id}/${table.id}/views/${view.id}/${row['Id']}` + `/api/v1/db/data/noco/${project.id}/${table.id}/views/${view.id}/${row['Id']}`, ) .set('xc-auth', context.token) .send({ @@ -1314,7 +1315,7 @@ function viewRowTests() { await request(context.app) .delete( - `/api/v1/db/data/noco/${project.id}/${table.id}/views/${view.id}/${row['Id']}` + `/api/v1/db/data/noco/${project.id}/${table.id}/views/${view.id}/${row['Id']}`, ) .set('xc-auth', context.token) .expect(200); @@ -1338,8 +1339,8 @@ function viewRowTests() { await testDeleteViewRow(ViewTypes.FORM); }); - const testDeleteViewRowWithForiegnKeyConstraint = async ( - viewType: ViewTypes + const testDeleteViewRowWithForeignKeyConstraint = async ( + viewType: ViewTypes, ) => { const table = await createTable(context, project); const relatedTable = await createTable(context, project, { @@ -1369,37 +1370,30 @@ function viewRowTests() { rowId: row['Id'], }); - const response = await request(context.app) + await request(context.app) .delete( - `/api/v1/db/data/noco/${project.id}/${table.id}/views/${view.id}/${row['Id']}` + `/api/v1/db/data/noco/${project.id}/${table.id}/views/${view.id}/${row['Id']}`, ) .set('xc-auth', context.token) .expect(200); const deleteRow = await getRow(context, { project, table, id: row['Id'] }); - if (!deleteRow) { - throw new Error('Should not delete'); - } - - if ( - !(response.body.message[0] as string).includes( - 'is a LinkToAnotherRecord of' - ) - ) { - throw new Error('Should give ltar foreign key error'); + if (deleteRow && Object.keys(deleteRow).length > 0) { + console.log(deleteRow); + throw new Error('Wrong delete'); } }; it('Delete view row with ltar foreign key constraint GALLERY', async function () { - await testDeleteViewRowWithForiegnKeyConstraint(ViewTypes.GALLERY); + await testDeleteViewRowWithForeignKeyConstraint(ViewTypes.GALLERY); }); it('Delete view row with ltar foreign key constraint GRID', async function () { - await testDeleteViewRowWithForiegnKeyConstraint(ViewTypes.GRID); + await testDeleteViewRowWithForeignKeyConstraint(ViewTypes.GRID); }); it('Delete view row with ltar foreign key constraint FORM', async function () { - await testDeleteViewRowWithForiegnKeyConstraint(ViewTypes.FORM); + await testDeleteViewRowWithForeignKeyConstraint(ViewTypes.FORM); }); const testViewRowExists = async (viewType: ViewTypes) => { @@ -1415,7 +1409,7 @@ function viewRowTests() { const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/${row['CustomerId']}/exist` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/${row['CustomerId']}/exist`, ) .set('xc-auth', context.token) .expect(200); @@ -1445,7 +1439,7 @@ function viewRowTests() { }); const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/999999/exist` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.id}/views/${view.id}/999999/exist`, ) .set('xc-auth', context.token) .expect(200); @@ -1475,7 +1469,7 @@ function viewRowTests() { }); const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.title}/views/${view.id}/export/csv` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.title}/views/${view.id}/export/csv`, ) .set('xc-auth', context.token) .expect(200); @@ -1499,7 +1493,7 @@ function viewRowTests() { }); const response = await request(context.app) .get( - `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.title}/views/${view.id}/export/excel` + `/api/v1/db/data/noco/${sakilaProject.id}/${customerTable.title}/views/${view.id}/export/excel`, ) .set('xc-auth', context.token) .expect(200); From 3b65bfde61034ebf88db055b21b5c5d92959f004 Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Thu, 8 Jun 2023 10:55:51 +0530 Subject: [PATCH 30/34] refactor: rename file --- tests/playwright/tests/db/{zMetaLTAR.spec.ts => metaLTAR.spec.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/playwright/tests/db/{zMetaLTAR.spec.ts => metaLTAR.spec.ts} (100%) diff --git a/tests/playwright/tests/db/zMetaLTAR.spec.ts b/tests/playwright/tests/db/metaLTAR.spec.ts similarity index 100% rename from tests/playwright/tests/db/zMetaLTAR.spec.ts rename to tests/playwright/tests/db/metaLTAR.spec.ts From 4e67941de0a359b5e999db17b20784543ea55dc5 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 8 Jun 2023 11:24:47 +0530 Subject: [PATCH 31/34] fix: if project not found throw error Signed-off-by: Pranav C --- .../extract-project-id/extract-project-id.middleware.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/nocodb/src/middlewares/extract-project-id/extract-project-id.middleware.ts b/packages/nocodb/src/middlewares/extract-project-id/extract-project-id.middleware.ts index d797e146c7..d26858f077 100644 --- a/packages/nocodb/src/middlewares/extract-project-id/extract-project-id.middleware.ts +++ b/packages/nocodb/src/middlewares/extract-project-id/extract-project-id.middleware.ts @@ -37,6 +37,7 @@ export class ExtractProjectIdMiddleware implements NestMiddleware, CanActivate { // extract project id based on request path params if (params.projectName) { const project = await Project.getByTitleOrId(params.projectName); + if (!project) NcError.notFound('Project not found'); req.ncProjectId = project.id; res.locals.project = project; } From 32c7138d8cf740056d4e58bfe8a71f56d5f44763 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 8 Jun 2023 16:47:26 +0530 Subject: [PATCH 32/34] fix: delete audit data on project delete Signed-off-by: Pranav C --- packages/nocodb/src/models/Project.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/models/Project.ts b/packages/nocodb/src/models/Project.ts index b062222f69..e08d5e036a 100644 --- a/packages/nocodb/src/models/Project.ts +++ b/packages/nocodb/src/models/Project.ts @@ -10,7 +10,7 @@ import NocoCache from '../cache/NocoCache'; import Base from './/Base'; import { ProjectUser } from './index'; import type { BoolType, MetaType, ProjectType } from 'nocodb-sdk'; -import type { DB_TYPES } from './/Base'; +import type { DB_TYPES } from './Base'; export default class Project implements ProjectType { public id: string; @@ -309,6 +309,11 @@ export default class Project implements ProjectType { `${CacheScope.PROJECT}:${projectId}`, CacheDelDirection.CHILD_TO_PARENT, ); + + await ncMeta.metaDelete(null, null, MetaTable.AUDIT, { + project_id: projectId, + }); + return await ncMeta.metaDelete(null, null, MetaTable.PROJECT, projectId); } From a5951dc2ce6fdf2bca9091379f66238369c69297 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 8 Jun 2023 16:49:59 +0530 Subject: [PATCH 33/34] fix: disable/enable foreign key check only if it's enabled Signed-off-by: Pranav C --- .../src/db/sql-client/lib/sqlite/SqliteClient.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/db/sql-client/lib/sqlite/SqliteClient.ts b/packages/nocodb/src/db/sql-client/lib/sqlite/SqliteClient.ts index 49e078a1b0..00ecca611d 100644 --- a/packages/nocodb/src/db/sql-client/lib/sqlite/SqliteClient.ts +++ b/packages/nocodb/src/db/sql-client/lib/sqlite/SqliteClient.ts @@ -22,7 +22,8 @@ class SqliteClient extends KnexClient { // sqlite does not support inserting default values and knex fires a warning without this flag connectionConfig.connection.useNullAsDefault = true; super(connectionConfig); - this.sqlClient = connectionConfig?.knex || knex(connectionConfig.connection); + this.sqlClient = + connectionConfig?.knex || knex(connectionConfig.connection); this.queries = queries; this._version = {}; } @@ -1557,7 +1558,13 @@ class SqliteClient extends KnexClient { upQuery, ); - await this.sqlClient.raw('PRAGMA foreign_keys = OFF;'); + const fkCheckEnabled = ( + await this.sqlClient.raw('PRAGMA foreign_keys;') + )?.[0]?.foreign_keys; + + if (fkCheckEnabled) + await this.sqlClient.raw('PRAGMA foreign_keys = OFF;'); + await this.sqlClient.raw('PRAGMA legacy_alter_table = ON;'); const trx = await this.sqlClient.transaction(); @@ -1594,7 +1601,8 @@ class SqliteClient extends KnexClient { log.ppe(e, _func); throw e; } finally { - await this.sqlClient.raw('PRAGMA foreign_keys = ON;'); + if (fkCheckEnabled) + await this.sqlClient.raw('PRAGMA foreign_keys = ON;'); await this.sqlClient.raw('PRAGMA legacy_alter_table = OFF;'); } From 4707611093ff99471150a8c75412087f2010cf77 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 8 Jun 2023 22:53:28 +0530 Subject: [PATCH 34/34] fix: skip index creation for postgres Signed-off-by: Pranav C --- .../src/version-upgrader/ncXcdbLTARUpgrader.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts b/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts index a936d84cf7..19d2f94a94 100644 --- a/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts +++ b/packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts @@ -77,13 +77,16 @@ async function upgradeModelRelations({ }); } - // create a new index for the column - const indexArgs = { - columns: [relation.cn], - tn: relation.tn, - non_unique: true, - }; - await sqlClient.indexCreate(indexArgs); + // skip postgres since we were already creating the index while creating the relation + if (ncMeta.knex.clientType() !== 'pg') { + // create a new index for the column + const indexArgs = { + columns: [relation.cn], + tn: relation.tn, + non_unique: true, + }; + await sqlClient.indexCreate(indexArgs); + } } break; }