From f80a154d51f4a15fa2cebd0410d0fe3b5a6af867 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Sun, 9 Apr 2023 15:21:30 +0530 Subject: [PATCH] feat: meta diff apis Signed-off-by: Pranav C --- packages/nocodb-nest/src/app.module.ts | 3 +- .../meta-diffs/meta-diffs.controller.spec.ts | 20 + .../meta-diffs/meta-diffs.controller.ts | 51 + .../modules/meta-diffs/meta-diffs.module.ts | 9 + .../meta-diffs/meta-diffs.service.spec.ts | 18 + .../modules/meta-diffs/meta-diffs.service.ts | 1113 +++++++++++++++++ 6 files changed, 1213 insertions(+), 1 deletion(-) create mode 100644 packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.controller.spec.ts create mode 100644 packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.controller.ts create mode 100644 packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.module.ts create mode 100644 packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.service.spec.ts create mode 100644 packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.service.ts diff --git a/packages/nocodb-nest/src/app.module.ts b/packages/nocodb-nest/src/app.module.ts index 467aa47c34..f66049e14b 100644 --- a/packages/nocodb-nest/src/app.module.ts +++ b/packages/nocodb-nest/src/app.module.ts @@ -33,9 +33,10 @@ import { AttachmentsModule } from './modules/attachments/attachments.module'; import { OrgLcenseModule } from './modules/org-lcense/org-lcense.module'; import { OrgTokensModule } from './modules/org-tokens/org-tokens.module'; import { OrgUsersModule } from './modules/org-users/org-users.module'; +import { MetaDiffsModule } from './modules/meta-diffs/meta-diffs.module'; @Module({ - imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule], + imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule, OrgLcenseModule, OrgTokensModule, OrgUsersModule, MetaDiffsModule], controllers: [], providers: [Connection, MetaService, JwtStrategy, ExtractProjectIdMiddleware], exports: [Connection, MetaService], diff --git a/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.controller.spec.ts b/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.controller.spec.ts new file mode 100644 index 0000000000..56a42a1e8a --- /dev/null +++ b/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { MetaDiffsController } from './meta-diffs.controller'; +import { MetaDiffsService } from './meta-diffs.service'; + +describe('MetaDiffsController', () => { + let controller: MetaDiffsController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [MetaDiffsController], + providers: [MetaDiffsService], + }).compile(); + + controller = module.get(MetaDiffsController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.controller.ts b/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.controller.ts new file mode 100644 index 0000000000..a3a9d016ba --- /dev/null +++ b/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.controller.ts @@ -0,0 +1,51 @@ +import { Controller, Get, Param, Post, UseGuards } from '@nestjs/common'; +import { + Acl, + ExtractProjectIdMiddleware, +} from '../../middlewares/extract-project-id/extract-project-id.middleware'; +import { MetaDiffsService } from './meta-diffs.service'; +import { AuthGuard } from '@nestjs/passport'; + +@Controller('meta-diffs') +@UseGuards(ExtractProjectIdMiddleware, AuthGuard('jwt')) +export class MetaDiffsController { + constructor(private readonly metaDiffsService: MetaDiffsService) {} + + @Get('/api/v1/db/meta/projects/:projectId/meta-diff') + @Acl('metaDiff') + async metaDiff(@Param('projectId') projectId: string) { + return await this.metaDiffsService.metaDiff({ projectId }); + } + + @Get('/api/v1/db/meta/projects/:projectId/meta-diff/:baseId') + async baseMetaDiff( + @Param('projectId') projectId: string, + @Param('baseId') baseId: string, + ) { + return await this.metaDiffsService.baseMetaDiff({ + baseId, + projectId, + }); + } + + @Post('/api/v1/db/meta/projects/:projectId/meta-diff') + @Acl('metaDiffSync') + async metaDiffSync(@Param('projectId') projectId: string) { + await this.metaDiffsService.metaDiffSync({ projectId }); + return { msg: 'The meta has been synchronized successfully' }; + } + + @Post('/api/v1/db/meta/projects/:projectId/meta-diff/:baseId') + @Acl('baseMetaDiffSync') + async baseMetaDiffSync( + @Param('projectId') projectId: string, + @Param('baseId') baseId: string, + ) { + await this.metaDiffsService.baseMetaDiffSync({ + projectId, + baseId, + }); + + return { msg: 'The base meta has been synchronized successfully' }; + } +} diff --git a/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.module.ts b/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.module.ts new file mode 100644 index 0000000000..cc73587ea6 --- /dev/null +++ b/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { MetaDiffsService } from './meta-diffs.service'; +import { MetaDiffsController } from './meta-diffs.controller'; + +@Module({ + controllers: [MetaDiffsController], + providers: [MetaDiffsService] +}) +export class MetaDiffsModule {} diff --git a/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.service.spec.ts b/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.service.spec.ts new file mode 100644 index 0000000000..dd9c5165ca --- /dev/null +++ b/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { MetaDiffsService } from './meta-diffs.service'; + +describe('MetaDiffsService', () => { + let service: MetaDiffsService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [MetaDiffsService], + }).compile(); + + service = module.get(MetaDiffsService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.service.ts b/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.service.ts new file mode 100644 index 0000000000..af465dc14b --- /dev/null +++ b/packages/nocodb-nest/src/modules/meta-diffs/meta-diffs.service.ts @@ -0,0 +1,1113 @@ +import { Injectable } from '@nestjs/common'; +import { isVirtualCol, ModelTypes, RelationTypes, UITypes } from 'nocodb-sdk'; +import { + Model, + Column, + Project, + LinkToAnotherRecordColumn, + Base, +} from 'src/models'; +import ModelXcMetaFactory from '../../db/sql-mgr/code/models/xc/ModelXcMetaFactory'; +import getColumnUiType from '../../helpers/getColumnUiType'; +import getTableNameAlias, { + getColumnNameAlias, +} from '../../helpers/getTableName'; +import { getUniqueColumnAliasName } from '../../helpers/getUniqueName'; +import mapDefaultDisplayValue from '../../helpers/mapDefaultDisplayValue'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; +import NcHelp from '../../utils/NcHelp'; +import { T } from 'nc-help'; + +// todo:move enum and types +export enum MetaDiffType { + TABLE_NEW = 'TABLE_NEW', + TABLE_REMOVE = 'TABLE_REMOVE', + TABLE_COLUMN_ADD = 'TABLE_COLUMN_ADD', + TABLE_COLUMN_TYPE_CHANGE = 'TABLE_COLUMN_TYPE_CHANGE', + TABLE_COLUMN_REMOVE = 'TABLE_COLUMN_REMOVE', + VIEW_NEW = 'VIEW_NEW', + VIEW_REMOVE = 'VIEW_REMOVE', + VIEW_COLUMN_ADD = 'VIEW_COLUMN_ADD', + VIEW_COLUMN_TYPE_CHANGE = 'VIEW_COLUMN_TYPE_CHANGE', + VIEW_COLUMN_REMOVE = 'VIEW_COLUMN_REMOVE', + TABLE_RELATION_ADD = 'TABLE_RELATION_ADD', + TABLE_RELATION_REMOVE = 'TABLE_RELATION_REMOVE', + TABLE_VIRTUAL_M2M_REMOVE = 'TABLE_VIRTUAL_M2M_REMOVE', +} + +const applyChangesPriorityOrder = [ + MetaDiffType.VIEW_COLUMN_REMOVE, + MetaDiffType.TABLE_RELATION_REMOVE, +]; + +type MetaDiff = { + title?: string; + table_name: string; + base_id: string; + type: ModelTypes; + meta?: any; + detectedChanges: Array; +}; + +type MetaDiffChange = { + msg?: string; + // type: MetaDiffType; +} & ( + | { + type: MetaDiffType.TABLE_NEW | MetaDiffType.VIEW_NEW; + tn?: string; + } + | { + type: MetaDiffType.TABLE_REMOVE | MetaDiffType.VIEW_REMOVE; + tn?: string; + model?: Model; + id?: string; + } + | { + type: MetaDiffType.TABLE_COLUMN_ADD | MetaDiffType.VIEW_COLUMN_ADD; + tn?: string; + model?: Model; + id?: string; + cn: string; + } + | { + type: + | MetaDiffType.TABLE_COLUMN_TYPE_CHANGE + | MetaDiffType.VIEW_COLUMN_TYPE_CHANGE + | MetaDiffType.TABLE_COLUMN_REMOVE + | MetaDiffType.VIEW_COLUMN_REMOVE; + tn?: string; + model?: Model; + id?: string; + cn: string; + column: Column; + colId?: string; + } + | { + type: MetaDiffType.TABLE_RELATION_REMOVE; + tn?: string; + rtn?: string; + cn?: string; + rcn?: string; + colId: string; + column: Column; + } + | { + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE; + tn?: string; + rtn?: string; + cn?: string; + rcn?: string; + colId: string; + column: Column; + } + | { + type: MetaDiffType.TABLE_RELATION_ADD; + tn?: string; + rtn?: string; + cn?: string; + rcn?: string; + relationType: RelationTypes; + cstn?: string; + } +); + +@Injectable() +export class MetaDiffsService { + async getMetaDiff( + sqlClient, + project: Project, + base: Base, + ): Promise> { + const changes: Array = []; + const virtualRelationColumns: Column[] = []; + + // @ts-ignore + const tableList: Array<{ tn: string }> = ( + await sqlClient.tableList() + )?.data?.list?.filter((t) => { + if (project?.prefix && base.is_meta) { + return t.tn?.startsWith(project?.prefix); + } + return true; + }); + + const colListRef = {}; + const oldMetas = await base.getModels(); + // @ts-ignore + const oldTableMetas: Model[] = []; + const oldViewMetas: Model[] = []; + + for (const model of oldMetas) { + if (model.type === ModelTypes.TABLE) oldTableMetas.push(model); + else if (model.type === ModelTypes.VIEW) oldViewMetas.push(model); + } + + // @ts-ignore + const relationList: Array<{ + tn: string; + rtn: string; + cn: string; + rcn: string; + found?: any; + cstn?: string; + }> = (await sqlClient.relationListAll())?.data?.list; + + for (const table of tableList) { + if (table.tn === 'nc_evolutions') continue; + + const oldMetaIdx = oldTableMetas.findIndex( + (m) => m.table_name === table.tn, + ); + + // new table + if (oldMetaIdx === -1) { + changes.push({ + table_name: table.tn, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [ + { + type: MetaDiffType.TABLE_NEW, + msg: `New table`, + }, + ], + }); + continue; + } + + const oldMeta = oldTableMetas[oldMetaIdx]; + + oldTableMetas.splice(oldMetaIdx, 1); + + const tableProp: MetaDiff = { + title: oldMeta.title, + meta: oldMeta.meta, + table_name: table.tn, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [], + }; + changes.push(tableProp); + + // check for column change + colListRef[table.tn] = ( + await sqlClient.columnList({ tn: table.tn }) + )?.data?.list; + + await oldMeta.getColumns(); + + for (const column of colListRef[table.tn]) { + const oldColIdx = oldMeta.columns.findIndex( + (c) => c.column_name === column.cn, + ); + + // new table + if (oldColIdx === -1) { + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_ADD, + msg: `New column(${column.cn})`, + cn: column.cn, + id: oldMeta.id, + }); + continue; + } + + const [oldCol] = oldMeta.columns.splice(oldColIdx, 1); + + if (oldCol.dt !== column.dt) { + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_TYPE_CHANGE, + msg: `Column type changed(${column.cn})`, + cn: oldCol.column_name, + id: oldMeta.id, + column: oldCol, + }); + } + } + for (const column of oldMeta.columns) { + if ( + [ + UITypes.LinkToAnotherRecord, + UITypes.Rollup, + UITypes.Lookup, + UITypes.Formula, + ].includes(column.uidt) + ) { + if (column.uidt === UITypes.LinkToAnotherRecord) { + virtualRelationColumns.push(column); + } + + continue; + } + + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_REMOVE, + msg: `Column removed(${column.column_name})`, + cn: column.column_name, + id: oldMeta.id, + column: column, + colId: column.id, + }); + } + } + + for (const model of oldTableMetas) { + changes.push({ + table_name: model.table_name, + meta: model.meta, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [ + { + type: MetaDiffType.TABLE_REMOVE, + msg: `Table removed`, + tn: model.table_name, + id: model.id, + model, + }, + ], + }); + } + + for (const relationCol of virtualRelationColumns) { + const colOpt = + await relationCol.getColOptions(); + const parentCol = await colOpt.getParentColumn(); + const childCol = await colOpt.getChildColumn(); + const parentModel = await parentCol.getModel(); + const childModel = await childCol.getModel(); + + // many to many relation + if (colOpt.type === RelationTypes.MANY_TO_MANY) { + const m2mModel = await colOpt.getMMModel(); + + const relatedTable = tableList.find( + (t) => t.tn === parentModel.table_name, + ); + const m2mTable = tableList.find((t) => t.tn === m2mModel.table_name); + + if (!relatedTable) { + changes + .find((t) => t.table_name === childModel.table_name) + .detectedChanges.push({ + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, + msg: `Many to many removed(${relatedTable.tn} removed)`, + colId: relationCol.id, + column: relationCol, + }); + continue; + } + if (!m2mTable) { + changes + .find((t) => t.table_name === childModel.table_name) + .detectedChanges.push({ + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, + msg: `Many to many removed(${m2mModel.table_name} removed)`, + colId: relationCol.id, + column: relationCol, + }); + continue; + } + + // verify columns + + const cColumns = (colListRef[childModel.table_name] = + colListRef[childModel.table_name] || + (await sqlClient.columnList({ tn: childModel.table_name }))?.data + ?.list); + + const pColumns = (colListRef[parentModel.table_name] = + colListRef[parentModel.table_name] || + (await sqlClient.columnList({ tn: parentModel.table_name }))?.data + ?.list); + + const vColumns = (colListRef[m2mTable.tn] = + colListRef[m2mTable.tn] || + (await sqlClient.columnList({ tn: m2mTable.tn }))?.data?.list); + + const m2mChildCol = await colOpt.getMMChildColumn(); + const m2mParentCol = await colOpt.getMMParentColumn(); + + if ( + pColumns.every((c) => c.cn !== parentCol.column_name) || + cColumns.every((c) => c.cn !== childCol.column_name) || + vColumns.every((c) => c.cn !== m2mChildCol.column_name) || + vColumns.every((c) => c.cn !== m2mParentCol.column_name) + ) { + changes + .find((t) => t.table_name === childModel.table_name) + .detectedChanges.push({ + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, + msg: `Many to many removed(One of the relation column removed)`, + colId: relationCol.id, + column: relationCol, + }); + } + + continue; + } + + if (relationCol.colOptions.virtual) continue; + + const dbRelation = relationList.find( + (r) => + r.cn === childCol.column_name && + r.tn === childModel.table_name && + r.rcn === parentCol.column_name && + r.rtn === parentModel.table_name, + ); + + if (dbRelation) { + dbRelation.found = dbRelation.found || {}; + + if (dbRelation.found[colOpt.type]) { + // todo: handle duplicate + } else { + dbRelation.found[colOpt.type] = true; + } + } else { + changes + .find( + (t) => + t.table_name === + (colOpt.type === RelationTypes.BELONGS_TO + ? childModel.table_name + : parentModel.table_name), + ) + .detectedChanges.push({ + type: MetaDiffType.TABLE_RELATION_REMOVE, + tn: childModel.table_name, + rtn: parentModel.table_name, + cn: childCol.column_name, + rcn: parentCol.column_name, + msg: `Relation removed`, + colId: relationCol.id, + column: relationCol, + }); + } + } + + for (const relation of relationList) { + if (!relation?.found?.[RelationTypes.BELONGS_TO]) { + changes + .find((t) => t.table_name === relation.tn) + ?.detectedChanges.push({ + type: MetaDiffType.TABLE_RELATION_ADD, + tn: relation.tn, + rtn: relation.rtn, + cn: relation.cn, + rcn: relation.rcn, + msg: `New relation added`, + relationType: RelationTypes.BELONGS_TO, + cstn: relation.cstn, + }); + } + if (!relation?.found?.[RelationTypes.HAS_MANY]) { + changes + .find((t) => t.table_name === relation.rtn) + ?.detectedChanges.push({ + type: MetaDiffType.TABLE_RELATION_ADD, + tn: relation.tn, + rtn: relation.rtn, + cn: relation.cn, + rcn: relation.rcn, + msg: `New relation added`, + relationType: RelationTypes.HAS_MANY, + }); + } + } + + // views + // @ts-ignore + const viewList: Array<{ + view_name: string; + tn: string; + type: 'view'; + }> = (await sqlClient.viewList())?.data?.list + ?.map((v) => { + v.type = 'view'; + v.tn = v.view_name; + return v; + }) + .filter((t) => { + if (project?.prefix && base.is_meta) { + return t.tn?.startsWith(project?.prefix); + } + return true; + }); // @ts-ignore + + for (const view of viewList) { + const oldMetaIdx = oldViewMetas.findIndex( + (m) => m.table_name === view.tn, + ); + + // new table + if (oldMetaIdx === -1) { + changes.push({ + table_name: view.tn, + base_id: base.id, + type: ModelTypes.VIEW, + detectedChanges: [ + { + type: MetaDiffType.VIEW_NEW, + msg: `New view`, + }, + ], + }); + continue; + } + + const oldMeta = oldViewMetas[oldMetaIdx]; + + oldViewMetas.splice(oldMetaIdx, 1); + + const tableProp: MetaDiff = { + title: oldMeta.title, + meta: oldMeta.meta, + table_name: view.tn, + base_id: base.id, + type: ModelTypes.VIEW, + detectedChanges: [], + }; + changes.push(tableProp); + + // check for column change + colListRef[view.tn] = ( + await sqlClient.columnList({ tn: view.tn }) + )?.data?.list; + + await oldMeta.getColumns(); + + for (const column of colListRef[view.tn]) { + const oldColIdx = oldMeta.columns.findIndex( + (c) => c.column_name === column.cn, + ); + + // new table + if (oldColIdx === -1) { + tableProp.detectedChanges.push({ + type: MetaDiffType.VIEW_COLUMN_ADD, + msg: `New column(${column.cn})`, + cn: column.cn, + id: oldMeta.id, + }); + continue; + } + + const [oldCol] = oldMeta.columns.splice(oldColIdx, 1); + + if (oldCol.dt !== column.dt) { + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_TYPE_CHANGE, + msg: `Column type changed(${column.cn})`, + cn: oldCol.column_name, + id: oldMeta.id, + column: oldCol, + }); + } + } + for (const column of oldMeta.columns) { + if ( + [ + UITypes.LinkToAnotherRecord, + UITypes.Rollup, + UITypes.Lookup, + UITypes.Formula, + ].includes(column.uidt) + ) { + continue; + } + + tableProp.detectedChanges.push({ + type: MetaDiffType.VIEW_COLUMN_REMOVE, + msg: `Column removed(${column.column_name})`, + cn: column.column_name, + id: oldMeta.id, + column: column, + colId: column.id, + }); + } + } + + for (const model of oldViewMetas) { + changes.push({ + table_name: model.table_name, + meta: model.meta, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [ + { + type: MetaDiffType.VIEW_REMOVE, + msg: `Table removed`, + tn: model.table_name, + id: model.id, + model, + }, + ], + }); + } + + return changes; + } + + async metaDiff(param: { projectId: string }) { + const project = await Project.getWithInfo(param.projectId); + let changes = []; + for (const base of project.bases) { + try { + // @ts-ignore + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + changes = changes.concat( + await this.getMetaDiff(sqlClient, project, base), + ); + } catch (e) { + console.log(e); + } + } + + return changes; + } + + async baseMetaDiff(param: { projectId: string; baseId: string }) { + const project = await Project.getWithInfo(param.projectId); + const base = await Base.get(param.baseId); + let changes = []; + + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + changes = await this.getMetaDiff(sqlClient, project, base); + + return changes; + } + + async metaDiffSync(param: { projectId: string }) { + const project = await Project.getWithInfo(param.projectId); + for (const base of project.bases) { + const virtualColumnInsert: Array<() => Promise> = []; + + // @ts-ignore + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + const changes = await this.getMetaDiff(sqlClient, project, base); + + /* Get all relations */ + // const relations = (await sqlClient.relationListAll())?.data?.list; + + for (const { table_name, detectedChanges } of changes) { + // reorder changes to apply relation remove changes + // before column remove to avoid foreign key constraint error + detectedChanges.sort((a, b) => { + return ( + applyChangesPriorityOrder.indexOf(b.type) - + applyChangesPriorityOrder.indexOf(a.type) + ); + }); + + for (const change of detectedChanges) { + switch (change.type) { + case MetaDiffType.TABLE_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias( + table_name, + base.is_meta ? project.prefix : '', + base, + ), + type: ModelTypes.TABLE, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.VIEW_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias(table_name, project.prefix, base), + type: ModelTypes.VIEW, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.TABLE_REMOVE: + case MetaDiffType.VIEW_REMOVE: + { + await change.model.delete(); + } + break; + case MetaDiffType.TABLE_COLUMN_ADD: + case MetaDiffType.VIEW_COLUMN_ADD: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + column.uidt = getColumnUiType(base, column); + //todo: inflection + column.title = getColumnNameAlias(column.cn, base); + await Column.insert({ fk_model_id: change.id, ...column }); + } + // update old + // populateParams.tableNames.push({ tn }); + // populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn); + + break; + case MetaDiffType.TABLE_COLUMN_TYPE_CHANGE: + case MetaDiffType.VIEW_COLUMN_TYPE_CHANGE: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + const metaFact = ModelXcMetaFactory.create( + { client: base.type }, + {}, + ); + column.uidt = metaFact.getUIDataType(column); + column.title = change.column.title; + await Column.update(change.column.id, column); + } + break; + case MetaDiffType.TABLE_COLUMN_REMOVE: + case MetaDiffType.VIEW_COLUMN_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_REMOVE: + case MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_ADD: + { + virtualColumnInsert.push(async () => { + const parentModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.rtn, + }); + const childModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.tn, + }); + const parentCol = await parentModel + .getColumns() + .then((cols) => + cols.find((c) => c.column_name === change.rcn), + ); + const childCol = await childModel + .getColumns() + .then((cols) => + cols.find((c) => c.column_name === change.cn), + ); + + await Column.update(childCol.id, { + ...childCol, + uidt: UITypes.ForeignKey, + system: true, + }); + + if (change.relationType === RelationTypes.BELONGS_TO) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${parentModel.title || parentModel.table_name}`, + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: childModel.id, + fk_related_model_id: parentModel.id, + type: RelationTypes.BELONGS_TO, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + fk_index_name: change.cstn, + }); + } else if (change.relationType === RelationTypes.HAS_MANY) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${childModel.title || childModel.table_name} List`, + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: parentModel.id, + fk_related_model_id: childModel.id, + type: RelationTypes.HAS_MANY, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + fk_index_name: change.cstn, + }); + } + }); + } + break; + } + } + } + + await NcHelp.executeOperations(virtualColumnInsert, base.type); + + // populate m2m relations + await this.extractAndGenerateManyToManyRelations(await base.getModels()); + } + + T.emit('evt', { evt_type: 'metaDiff:synced' }); + + return true; + } + + async baseMetaDiffSync(param: { projectId: string; baseId: string }) { + const project = await Project.getWithInfo(param.projectId); + const base = await Base.get(param.baseId); + + const virtualColumnInsert: Array<() => Promise> = []; + + // @ts-ignore + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + const changes = await this.getMetaDiff(sqlClient, project, base); + + /* Get all relations */ + // const relations = (await sqlClient.relationListAll())?.data?.list; + + for (const { table_name, detectedChanges } of changes) { + for (const change of detectedChanges) { + switch (change.type) { + case MetaDiffType.TABLE_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias( + table_name, + base.is_meta ? project.prefix : '', + base, + ), + type: ModelTypes.TABLE, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.VIEW_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias(table_name, project.prefix, base), + type: ModelTypes.VIEW, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.TABLE_REMOVE: + case MetaDiffType.VIEW_REMOVE: + { + await change.model.delete(); + } + break; + case MetaDiffType.TABLE_COLUMN_ADD: + case MetaDiffType.VIEW_COLUMN_ADD: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + column.uidt = getColumnUiType(base, column); + //todo: inflection + column.title = getColumnNameAlias(column.cn, base); + await Column.insert({ fk_model_id: change.id, ...column }); + } + // update old + // populateParams.tableNames.push({ tn }); + // populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn); + + break; + case MetaDiffType.TABLE_COLUMN_TYPE_CHANGE: + case MetaDiffType.VIEW_COLUMN_TYPE_CHANGE: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + const metaFact = ModelXcMetaFactory.create( + { client: base.type }, + {}, + ); + column.uidt = metaFact.getUIDataType(column); + column.title = change.column.title; + await Column.update(change.column.id, column); + } + break; + case MetaDiffType.TABLE_COLUMN_REMOVE: + case MetaDiffType.VIEW_COLUMN_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_REMOVE: + case MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_ADD: + { + virtualColumnInsert.push(async () => { + const parentModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.rtn, + }); + const childModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.tn, + }); + const parentCol = await parentModel + .getColumns() + .then((cols) => + cols.find((c) => c.column_name === change.rcn), + ); + const childCol = await childModel + .getColumns() + .then((cols) => + cols.find((c) => c.column_name === change.cn), + ); + + await Column.update(childCol.id, { + ...childCol, + uidt: UITypes.ForeignKey, + system: true, + }); + + if (change.relationType === RelationTypes.BELONGS_TO) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${parentModel.title || parentModel.table_name}`, + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: childModel.id, + fk_related_model_id: parentModel.id, + type: RelationTypes.BELONGS_TO, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + }); + } else if (change.relationType === RelationTypes.HAS_MANY) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${childModel.title || childModel.table_name} List`, + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: parentModel.id, + fk_related_model_id: childModel.id, + type: RelationTypes.HAS_MANY, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + }); + } + }); + } + break; + } + } + } + + await NcHelp.executeOperations(virtualColumnInsert, base.type); + + // populate m2m relations + await this.extractAndGenerateManyToManyRelations(await base.getModels()); + + T.emit('evt', { evt_type: 'baseMetaDiff:synced' }); + + return true; + } + + async isMMRelationExist( + model: Model, + assocModel: Model, + belongsToCol: Column, + ) { + let isExist = false; + const colChildOpt = + await belongsToCol.getColOptions(); + for (const col of await model.getColumns()) { + if (col.uidt === UITypes.LinkToAnotherRecord) { + const colOpt = await col.getColOptions(); + if ( + colOpt && + colOpt.type === RelationTypes.MANY_TO_MANY && + colOpt.fk_mm_model_id === assocModel.id && + colOpt.fk_child_column_id === colChildOpt.fk_parent_column_id && + colOpt.fk_mm_child_column_id === colChildOpt.fk_child_column_id + ) { + isExist = true; + break; + } + } + } + return isExist; + } + + // @ts-ignore + async extractAndGenerateManyToManyRelations(modelsArr: Array) { + for (const assocModel of modelsArr) { + await assocModel.getColumns(); + // check if table is a Bridge table(or Associative Table) by checking + // number of foreign keys and columns + + const normalColumns = assocModel.columns.filter((c) => !isVirtualCol(c)); + const belongsToCols: Column[] = []; + for (const col of assocModel.columns) { + if (col.uidt == UITypes.LinkToAnotherRecord) { + const colOpt = await col.getColOptions(); + if (colOpt?.type === RelationTypes.BELONGS_TO) + belongsToCols.push(col); + } + } + + // todo: impl better method to identify m2m relation + if (belongsToCols?.length === 2 && normalColumns.length < 5) { + const modelA = await belongsToCols[0].colOptions.getRelatedTable(); + const modelB = await belongsToCols[1].colOptions.getRelatedTable(); + + await modelA.getColumns(); + await modelB.getColumns(); + + // check tableA already have the relation or not + const isRelationAvailInA = await this.isMMRelationExist( + modelA, + assocModel, + belongsToCols[0], + ); + const isRelationAvailInB = await this.isMMRelationExist( + modelB, + assocModel, + belongsToCols[1], + ); + + if (!isRelationAvailInA) { + await Column.insert({ + title: getUniqueColumnAliasName( + modelA.columns, + `${modelB.title} List`, + ), + fk_model_id: modelA.id, + fk_related_model_id: modelB.id, + fk_mm_model_id: assocModel.id, + fk_child_column_id: belongsToCols[0].colOptions.fk_parent_column_id, + fk_parent_column_id: + belongsToCols[1].colOptions.fk_parent_column_id, + fk_mm_child_column_id: + belongsToCols[0].colOptions.fk_child_column_id, + fk_mm_parent_column_id: + belongsToCols[1].colOptions.fk_child_column_id, + type: RelationTypes.MANY_TO_MANY, + uidt: UITypes.LinkToAnotherRecord, + }); + } + if (!isRelationAvailInB) { + await Column.insert({ + title: getUniqueColumnAliasName( + modelB.columns, + `${modelA.title} List`, + ), + fk_model_id: modelB.id, + fk_related_model_id: modelA.id, + fk_mm_model_id: assocModel.id, + fk_child_column_id: belongsToCols[1].colOptions.fk_parent_column_id, + fk_parent_column_id: + belongsToCols[0].colOptions.fk_parent_column_id, + fk_mm_child_column_id: + belongsToCols[1].colOptions.fk_child_column_id, + fk_mm_parent_column_id: + belongsToCols[0].colOptions.fk_child_column_id, + type: RelationTypes.MANY_TO_MANY, + uidt: UITypes.LinkToAnotherRecord, + }); + } + + await Model.markAsMmTable(assocModel.id, true); + + // mark has many relation associated with mm as system field in both table + for (const btCol of [belongsToCols[0], belongsToCols[1]]) { + const colOpt = await btCol.colOptions; + const model = await colOpt.getRelatedTable(); + + for (const col of await model.getColumns()) { + if (col.uidt !== UITypes.LinkToAnotherRecord) continue; + + const colOpt1 = + await col.getColOptions(); + if (!colOpt1 || colOpt1.type !== RelationTypes.HAS_MANY) continue; + + if ( + colOpt1.fk_child_column_id !== colOpt.fk_child_column_id || + colOpt1.fk_parent_column_id !== colOpt.fk_parent_column_id + ) + continue; + + await Column.markAsSystemField(col.id); + break; + } + } + } else { + if (assocModel.mm) await Model.markAsMmTable(assocModel.id, false); + } + } + } +}