From 3af09aacb2221a7598d200065a71de1f2d58f375 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Mar 2023 01:06:49 +0530 Subject: [PATCH] refcator: handler to service-controller - metadiff and map view (WIP) Signed-off-by: Pranav C --- .../src/lib/controllers/mapViewController.ts | 27 +- .../src/lib/controllers/metaDiffController.ts | 1091 +--------------- .../src/lib/meta/api/helpers/populateMeta.ts | 2 +- packages/nocodb/src/lib/models/Model.ts | 578 ++++----- packages/nocodb/src/lib/services/index.ts | 2 + .../nocodb/src/lib/services/mapViewService.ts | 34 + .../src/lib/services/metaDiffService.ts | 1105 +++++++++++++++++ 7 files changed, 1459 insertions(+), 1380 deletions(-) create mode 100644 packages/nocodb/src/lib/services/mapViewService.ts create mode 100644 packages/nocodb/src/lib/services/metaDiffService.ts diff --git a/packages/nocodb/src/lib/controllers/mapViewController.ts b/packages/nocodb/src/lib/controllers/mapViewController.ts index 9c5123f98a..3a982d1b03 100644 --- a/packages/nocodb/src/lib/controllers/mapViewController.ts +++ b/packages/nocodb/src/lib/controllers/mapViewController.ts @@ -1,29 +1,30 @@ import { Request, Response, Router } from 'express'; -import { MapType, ViewTypes } from 'nocodb-sdk'; -import View from '../models/View'; +import { MapType } from 'nocodb-sdk'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import { Tele } from 'nc-help'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; -import MapView from '../models/MapView'; +import { mapViewService } from '../services'; export async function mapViewGet(req: Request, res: Response) { - res.json(await MapView.get(req.params.mapViewId)); + res.json( + await mapViewService.mapViewGet({ mapViewId: req.params.mapViewId }) + ); } export async function mapViewCreate(req: Request, res) { - Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'map' }); - const view = await View.insert({ - ...req.body, - // todo: sanitize - fk_model_id: req.params.tableId, - type: ViewTypes.MAP, + const view = await mapViewService.mapViewCreate({ + tableId: req.params.tableId, + map: req.body, }); res.json(view); } export async function mapViewUpdate(req, res) { - Tele.emit('evt', { evt_type: 'view:updated', type: 'map' }); - res.json(await MapView.update(req.params.mapViewId, req.body)); + res.json( + await mapViewService.mapViewUpdate({ + mapViewId: req.params.mapViewId, + map: req.body, + }) + ); } const router = Router({ mergeParams: true }); diff --git a/packages/nocodb/src/lib/controllers/metaDiffController.ts b/packages/nocodb/src/lib/controllers/metaDiffController.ts index 551846074d..45ac703a0d 100644 --- a/packages/nocodb/src/lib/controllers/metaDiffController.ts +++ b/packages/nocodb/src/lib/controllers/metaDiffController.ts @@ -1,1100 +1,35 @@ -// // Project CRUD - -import { Tele } from 'nc-help'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; -import Model from '../models/Model'; -import Project from '../models/Project'; -import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; -import { isVirtualCol, ModelTypes, RelationTypes, UITypes } from 'nocodb-sdk'; import { Router } from 'express'; -import Base from '../models/Base'; -import ModelXcMetaFactory from '../db/sql-mgr/code/models/xc/ModelXcMetaFactory'; -import Column from '../models/Column'; -import LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; -import { getUniqueColumnAliasName } from '../meta/helpers/getUniqueName'; -import NcHelp from '../utils/NcHelp'; -import getTableNameAlias, { getColumnNameAlias } from '../meta/helpers/getTableName'; -import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; -import getColumnUiType from '../meta/helpers/getColumnUiType'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; - -export enum MetaDiffType { - TABLE_NEW = 'TABLE_NEW', - TABLE_REMOVE = 'TABLE_REMOVE', - TABLE_COLUMN_ADD = 'TABLE_COLUMN_ADD', - TABLE_COLUMN_TYPE_CHANGE = 'TABLE_COLUMN_TYPE_CHANGE', - TABLE_COLUMN_REMOVE = 'TABLE_COLUMN_REMOVE', - VIEW_NEW = 'VIEW_NEW', - VIEW_REMOVE = 'VIEW_REMOVE', - VIEW_COLUMN_ADD = 'VIEW_COLUMN_ADD', - VIEW_COLUMN_TYPE_CHANGE = 'VIEW_COLUMN_TYPE_CHANGE', - VIEW_COLUMN_REMOVE = 'VIEW_COLUMN_REMOVE', - TABLE_RELATION_ADD = 'TABLE_RELATION_ADD', - TABLE_RELATION_REMOVE = 'TABLE_RELATION_REMOVE', - TABLE_VIRTUAL_M2M_REMOVE = 'TABLE_VIRTUAL_M2M_REMOVE', -} - -const applyChangesPriorityOrder = [ - MetaDiffType.VIEW_COLUMN_REMOVE, - MetaDiffType.TABLE_RELATION_REMOVE, -]; - -type MetaDiff = { - title?: string; - table_name: string; - base_id: string; - type: ModelTypes; - meta?: any; - detectedChanges: Array; -}; - -type MetaDiffChange = { - msg?: string; - // type: MetaDiffType; -} & ( - | { - type: MetaDiffType.TABLE_NEW | MetaDiffType.VIEW_NEW; - tn?: string; - } - | { - type: MetaDiffType.TABLE_REMOVE | MetaDiffType.VIEW_REMOVE; - tn?: string; - model?: Model; - id?: string; - } - | { - type: MetaDiffType.TABLE_COLUMN_ADD | MetaDiffType.VIEW_COLUMN_ADD; - tn?: string; - model?: Model; - id?: string; - cn: string; - } - | { - type: - | MetaDiffType.TABLE_COLUMN_TYPE_CHANGE - | MetaDiffType.VIEW_COLUMN_TYPE_CHANGE - | MetaDiffType.TABLE_COLUMN_REMOVE - | MetaDiffType.VIEW_COLUMN_REMOVE; - tn?: string; - model?: Model; - id?: string; - cn: string; - column: Column; - colId?: string; - } - | { - type: MetaDiffType.TABLE_RELATION_REMOVE; - tn?: string; - rtn?: string; - cn?: string; - rcn?: string; - colId: string; - column: Column; - } - | { - type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE; - tn?: string; - rtn?: string; - cn?: string; - rcn?: string; - colId: string; - column: Column; - } - | { - type: MetaDiffType.TABLE_RELATION_ADD; - tn?: string; - rtn?: string; - cn?: string; - rcn?: string; - relationType: RelationTypes; - cstn?: string; - } -); - -async function getMetaDiff( - sqlClient, - project: Project, - base: Base -): Promise> { - const changes: Array = []; - const virtualRelationColumns: Column[] = []; - - // @ts-ignore - const tableList: Array<{ tn: string }> = ( - await sqlClient.tableList() - )?.data?.list?.filter((t) => { - if (project?.prefix && base.is_meta) { - return t.tn?.startsWith(project?.prefix); - } - return true; - }); - - const colListRef = {}; - const oldMetas = await base.getModels(); - // @ts-ignore - const oldTableMetas: Model[] = []; - const oldViewMetas: Model[] = []; - - for (const model of oldMetas) { - if (model.type === ModelTypes.TABLE) oldTableMetas.push(model); - else if (model.type === ModelTypes.VIEW) oldViewMetas.push(model); - } - - // @ts-ignore - const relationList: Array<{ - tn: string; - rtn: string; - cn: string; - rcn: string; - found?: any; - cstn?: string; - }> = (await sqlClient.relationListAll())?.data?.list; - - for (const table of tableList) { - if (table.tn === 'nc_evolutions') continue; - - const oldMetaIdx = oldTableMetas.findIndex( - (m) => m.table_name === table.tn - ); - - // new table - if (oldMetaIdx === -1) { - changes.push({ - table_name: table.tn, - base_id: base.id, - type: ModelTypes.TABLE, - detectedChanges: [ - { - type: MetaDiffType.TABLE_NEW, - msg: `New table`, - }, - ], - }); - continue; - } - - const oldMeta = oldTableMetas[oldMetaIdx]; - - oldTableMetas.splice(oldMetaIdx, 1); - - const tableProp: MetaDiff = { - title: oldMeta.title, - meta: oldMeta.meta, - table_name: table.tn, - base_id: base.id, - type: ModelTypes.TABLE, - detectedChanges: [], - }; - changes.push(tableProp); - - // check for column change - colListRef[table.tn] = ( - await sqlClient.columnList({ tn: table.tn }) - )?.data?.list; - - await oldMeta.getColumns(); - - for (const column of colListRef[table.tn]) { - const oldColIdx = oldMeta.columns.findIndex( - (c) => c.column_name === column.cn - ); - - // new table - if (oldColIdx === -1) { - tableProp.detectedChanges.push({ - type: MetaDiffType.TABLE_COLUMN_ADD, - msg: `New column(${column.cn})`, - cn: column.cn, - id: oldMeta.id, - }); - continue; - } - - const [oldCol] = oldMeta.columns.splice(oldColIdx, 1); - - if (oldCol.dt !== column.dt) { - tableProp.detectedChanges.push({ - type: MetaDiffType.TABLE_COLUMN_TYPE_CHANGE, - msg: `Column type changed(${column.cn})`, - cn: oldCol.column_name, - id: oldMeta.id, - column: oldCol, - }); - } - } - for (const column of oldMeta.columns) { - if ( - [ - UITypes.LinkToAnotherRecord, - UITypes.Rollup, - UITypes.Lookup, - UITypes.Formula, - ].includes(column.uidt) - ) { - if (column.uidt === UITypes.LinkToAnotherRecord) { - virtualRelationColumns.push(column); - } - - continue; - } - - tableProp.detectedChanges.push({ - type: MetaDiffType.TABLE_COLUMN_REMOVE, - msg: `Column removed(${column.column_name})`, - cn: column.column_name, - id: oldMeta.id, - column: column, - colId: column.id, - }); - } - } - - for (const model of oldTableMetas) { - changes.push({ - table_name: model.table_name, - meta: model.meta, - base_id: base.id, - type: ModelTypes.TABLE, - detectedChanges: [ - { - type: MetaDiffType.TABLE_REMOVE, - msg: `Table removed`, - tn: model.table_name, - id: model.id, - model, - }, - ], - }); - } - - for (const relationCol of virtualRelationColumns) { - const colOpt = await relationCol.getColOptions(); - const parentCol = await colOpt.getParentColumn(); - const childCol = await colOpt.getChildColumn(); - const parentModel = await parentCol.getModel(); - const childModel = await childCol.getModel(); - - // many to many relation - if (colOpt.type === RelationTypes.MANY_TO_MANY) { - const m2mModel = await colOpt.getMMModel(); - - const relatedTable = tableList.find( - (t) => t.tn === parentModel.table_name - ); - const m2mTable = tableList.find((t) => t.tn === m2mModel.table_name); - - if (!relatedTable) { - changes - .find((t) => t.table_name === childModel.table_name) - .detectedChanges.push({ - type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, - msg: `Many to many removed(${relatedTable.tn} removed)`, - colId: relationCol.id, - column: relationCol, - }); - continue; - } - if (!m2mTable) { - changes - .find((t) => t.table_name === childModel.table_name) - .detectedChanges.push({ - type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, - msg: `Many to many removed(${m2mModel.table_name} removed)`, - colId: relationCol.id, - column: relationCol, - }); - continue; - } - - // verify columns - - const cColumns = (colListRef[childModel.table_name] = - colListRef[childModel.table_name] || - (await sqlClient.columnList({ tn: childModel.table_name }))?.data - ?.list); - - const pColumns = (colListRef[parentModel.table_name] = - colListRef[parentModel.table_name] || - (await sqlClient.columnList({ tn: parentModel.table_name }))?.data - ?.list); - - const vColumns = (colListRef[m2mTable.tn] = - colListRef[m2mTable.tn] || - (await sqlClient.columnList({ tn: m2mTable.tn }))?.data?.list); - - const m2mChildCol = await colOpt.getMMChildColumn(); - const m2mParentCol = await colOpt.getMMParentColumn(); - - if ( - pColumns.every((c) => c.cn !== parentCol.column_name) || - cColumns.every((c) => c.cn !== childCol.column_name) || - vColumns.every((c) => c.cn !== m2mChildCol.column_name) || - vColumns.every((c) => c.cn !== m2mParentCol.column_name) - ) { - changes - .find((t) => t.table_name === childModel.table_name) - .detectedChanges.push({ - type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, - msg: `Many to many removed(One of the relation column removed)`, - colId: relationCol.id, - column: relationCol, - }); - } - - continue; - } - - if (relationCol.colOptions.virtual) continue; - - const dbRelation = relationList.find( - (r) => - r.cn === childCol.column_name && - r.tn === childModel.table_name && - r.rcn === parentCol.column_name && - r.rtn === parentModel.table_name - ); - - if (dbRelation) { - dbRelation.found = dbRelation.found || {}; - - if (dbRelation.found[colOpt.type]) { - // todo: handle duplicate - } else { - dbRelation.found[colOpt.type] = true; - } - } else { - changes - .find( - (t) => - t.table_name === - (colOpt.type === RelationTypes.BELONGS_TO - ? childModel.table_name - : parentModel.table_name) - ) - .detectedChanges.push({ - type: MetaDiffType.TABLE_RELATION_REMOVE, - tn: childModel.table_name, - rtn: parentModel.table_name, - cn: childCol.column_name, - rcn: parentCol.column_name, - msg: `Relation removed`, - colId: relationCol.id, - column: relationCol, - }); - } - } - - for (const relation of relationList) { - if (!relation?.found?.[RelationTypes.BELONGS_TO]) { - changes - .find((t) => t.table_name === relation.tn) - ?.detectedChanges.push({ - type: MetaDiffType.TABLE_RELATION_ADD, - tn: relation.tn, - rtn: relation.rtn, - cn: relation.cn, - rcn: relation.rcn, - msg: `New relation added`, - relationType: RelationTypes.BELONGS_TO, - cstn: relation.cstn, - }); - } - if (!relation?.found?.[RelationTypes.HAS_MANY]) { - changes - .find((t) => t.table_name === relation.rtn) - ?.detectedChanges.push({ - type: MetaDiffType.TABLE_RELATION_ADD, - tn: relation.tn, - rtn: relation.rtn, - cn: relation.cn, - rcn: relation.rcn, - msg: `New relation added`, - relationType: RelationTypes.HAS_MANY, - }); - } - } - - // views - // @ts-ignore - const viewList: Array<{ - view_name: string; - tn: string; - type: 'view'; - }> = (await sqlClient.viewList())?.data?.list - ?.map((v) => { - v.type = 'view'; - v.tn = v.view_name; - return v; - }) - .filter((t) => { - if (project?.prefix && base.is_meta) { - return t.tn?.startsWith(project?.prefix); - } - return true; - }); // @ts-ignore - - for (const view of viewList) { - const oldMetaIdx = oldViewMetas.findIndex((m) => m.table_name === view.tn); - - // new table - if (oldMetaIdx === -1) { - changes.push({ - table_name: view.tn, - base_id: base.id, - type: ModelTypes.VIEW, - detectedChanges: [ - { - type: MetaDiffType.VIEW_NEW, - msg: `New view`, - }, - ], - }); - continue; - } - - const oldMeta = oldViewMetas[oldMetaIdx]; - - oldViewMetas.splice(oldMetaIdx, 1); - - const tableProp: MetaDiff = { - title: oldMeta.title, - meta: oldMeta.meta, - table_name: view.tn, - base_id: base.id, - type: ModelTypes.VIEW, - detectedChanges: [], - }; - changes.push(tableProp); - - // check for column change - colListRef[view.tn] = ( - await sqlClient.columnList({ tn: view.tn }) - )?.data?.list; - - await oldMeta.getColumns(); - - for (const column of colListRef[view.tn]) { - const oldColIdx = oldMeta.columns.findIndex( - (c) => c.column_name === column.cn - ); - - // new table - if (oldColIdx === -1) { - tableProp.detectedChanges.push({ - type: MetaDiffType.VIEW_COLUMN_ADD, - msg: `New column(${column.cn})`, - cn: column.cn, - id: oldMeta.id, - }); - continue; - } - - const [oldCol] = oldMeta.columns.splice(oldColIdx, 1); - - if (oldCol.dt !== column.dt) { - tableProp.detectedChanges.push({ - type: MetaDiffType.TABLE_COLUMN_TYPE_CHANGE, - msg: `Column type changed(${column.cn})`, - cn: oldCol.column_name, - id: oldMeta.id, - column: oldCol, - }); - } - } - for (const column of oldMeta.columns) { - if ( - [ - UITypes.LinkToAnotherRecord, - UITypes.Rollup, - UITypes.Lookup, - UITypes.Formula, - ].includes(column.uidt) - ) { - continue; - } - - tableProp.detectedChanges.push({ - type: MetaDiffType.VIEW_COLUMN_REMOVE, - msg: `Column removed(${column.column_name})`, - cn: column.column_name, - id: oldMeta.id, - column: column, - colId: column.id, - }); - } - } - - for (const model of oldViewMetas) { - changes.push({ - table_name: model.table_name, - meta: model.meta, - base_id: base.id, - type: ModelTypes.TABLE, - detectedChanges: [ - { - type: MetaDiffType.VIEW_REMOVE, - msg: `Table removed`, - tn: model.table_name, - id: model.id, - model, - }, - ], - }); - } - - return changes; -} +import { metaDiffService } from '../services'; export async function metaDiff(req, res) { - const project = await Project.getWithInfo(req.params.projectId); - let changes = []; - for (const base of project.bases) { - try { - // @ts-ignore - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - changes = changes.concat(await getMetaDiff(sqlClient, project, base)); - } catch (e) { - console.log(e); - } - } - - res.json(changes); + res.json(await metaDiffService.metaDiff({ projectId: req.params.projectId })); } export async function baseMetaDiff(req, res) { - const project = await Project.getWithInfo(req.params.projectId); - const base = await Base.get(req.params.baseId); - let changes = []; - - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - changes = await getMetaDiff(sqlClient, project, base); - - res.json(changes); + res.json( + await metaDiffService.baseMetaDiff({ + baseId: req.params.baseId, + projectId: req.params.projectId, + }) + ); } export async function metaDiffSync(req, res) { - const project = await Project.getWithInfo(req.params.projectId); - for (const base of project.bases) { - const virtualColumnInsert: Array<() => Promise> = []; - - // @ts-ignore - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - const changes = await getMetaDiff(sqlClient, project, base); - - /* Get all relations */ - // const relations = (await sqlClient.relationListAll())?.data?.list; - - for (const { table_name, detectedChanges } of changes) { - // reorder changes to apply relation remove changes - // before column remove to avoid foreign key constraint error - detectedChanges.sort((a, b) => { - return ( - applyChangesPriorityOrder.indexOf(b.type) - - applyChangesPriorityOrder.indexOf(a.type) - ); - }); - - for (const change of detectedChanges) { - switch (change.type) { - case MetaDiffType.TABLE_NEW: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - - mapDefaultDisplayValue(columns); - - const model = await Model.insert(project.id, base.id, { - table_name: table_name, - title: getTableNameAlias( - table_name, - base.is_meta ? project.prefix : '', - base - ), - type: ModelTypes.TABLE, - }); - - for (const column of columns) { - await Column.insert({ - uidt: getColumnUiType(base, column), - fk_model_id: model.id, - ...column, - title: getColumnNameAlias(column.column_name, base), - }); - } - } - break; - case MetaDiffType.VIEW_NEW: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - - mapDefaultDisplayValue(columns); - - const model = await Model.insert(project.id, base.id, { - table_name: table_name, - title: getTableNameAlias(table_name, project.prefix, base), - type: ModelTypes.VIEW, - }); - - for (const column of columns) { - await Column.insert({ - uidt: getColumnUiType(base, column), - fk_model_id: model.id, - ...column, - title: getColumnNameAlias(column.column_name, base), - }); - } - } - break; - case MetaDiffType.TABLE_REMOVE: - case MetaDiffType.VIEW_REMOVE: - { - await change.model.delete(); - } - break; - case MetaDiffType.TABLE_COLUMN_ADD: - case MetaDiffType.VIEW_COLUMN_ADD: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - const column = columns.find((c) => c.cn === change.cn); - column.uidt = getColumnUiType(base, column); - //todo: inflection - column.title = getColumnNameAlias(column.cn, base); - await Column.insert({ fk_model_id: change.id, ...column }); - } - // update old - // populateParams.tableNames.push({ tn }); - // populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn); - - break; - case MetaDiffType.TABLE_COLUMN_TYPE_CHANGE: - case MetaDiffType.VIEW_COLUMN_TYPE_CHANGE: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - const column = columns.find((c) => c.cn === change.cn); - const metaFact = ModelXcMetaFactory.create( - { client: base.type }, - {} - ); - column.uidt = metaFact.getUIDataType(column); - column.title = change.column.title; - await Column.update(change.column.id, column); - } - break; - case MetaDiffType.TABLE_COLUMN_REMOVE: - case MetaDiffType.VIEW_COLUMN_REMOVE: - await change.column.delete(); - break; - case MetaDiffType.TABLE_RELATION_REMOVE: - case MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE: - await change.column.delete(); - break; - case MetaDiffType.TABLE_RELATION_ADD: - { - virtualColumnInsert.push(async () => { - const parentModel = await Model.getByIdOrName({ - project_id: base.project_id, - base_id: base.id, - table_name: change.rtn, - }); - const childModel = await Model.getByIdOrName({ - project_id: base.project_id, - base_id: base.id, - table_name: change.tn, - }); - const parentCol = await parentModel - .getColumns() - .then((cols) => - cols.find((c) => c.column_name === change.rcn) - ); - const childCol = await childModel - .getColumns() - .then((cols) => - cols.find((c) => c.column_name === change.cn) - ); - - await Column.update(childCol.id, { - ...childCol, - uidt: UITypes.ForeignKey, - system: true, - }); - - if (change.relationType === RelationTypes.BELONGS_TO) { - const title = getUniqueColumnAliasName( - childModel.columns, - `${parentModel.title || parentModel.table_name}` - ); - await Column.insert({ - uidt: UITypes.LinkToAnotherRecord, - title, - fk_model_id: childModel.id, - fk_related_model_id: parentModel.id, - type: RelationTypes.BELONGS_TO, - fk_parent_column_id: parentCol.id, - fk_child_column_id: childCol.id, - virtual: false, - fk_index_name: change.cstn, - }); - } else if (change.relationType === RelationTypes.HAS_MANY) { - const title = getUniqueColumnAliasName( - childModel.columns, - `${childModel.title || childModel.table_name} List` - ); - await Column.insert({ - uidt: UITypes.LinkToAnotherRecord, - title, - fk_model_id: parentModel.id, - fk_related_model_id: childModel.id, - type: RelationTypes.HAS_MANY, - fk_parent_column_id: parentCol.id, - fk_child_column_id: childCol.id, - virtual: false, - fk_index_name: change.cstn, - }); - } - }); - } - break; - } - } - } - - await NcHelp.executeOperations(virtualColumnInsert, base.type); - - // populate m2m relations - await extractAndGenerateManyToManyRelations(await base.getModels()); - } - - Tele.emit('evt', { evt_type: 'metaDiff:synced' }); - + await metaDiffService.metaDiffSync({ projectId: req.params.projectId }); res.json({ msg: 'success' }); } export async function baseMetaDiffSync(req, res) { - const project = await Project.getWithInfo(req.params.projectId); - const base = await Base.get(req.params.baseId); - - const virtualColumnInsert: Array<() => Promise> = []; - - // @ts-ignore - const sqlClient = await NcConnectionMgrv2.getSqlClient(base); - const changes = await getMetaDiff(sqlClient, project, base); - - /* Get all relations */ - // const relations = (await sqlClient.relationListAll())?.data?.list; - - for (const { table_name, detectedChanges } of changes) { - for (const change of detectedChanges) { - switch (change.type) { - case MetaDiffType.TABLE_NEW: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - - mapDefaultDisplayValue(columns); - - const model = await Model.insert(project.id, base.id, { - table_name: table_name, - title: getTableNameAlias( - table_name, - base.is_meta ? project.prefix : '', - base - ), - type: ModelTypes.TABLE, - }); - - for (const column of columns) { - await Column.insert({ - uidt: getColumnUiType(base, column), - fk_model_id: model.id, - ...column, - title: getColumnNameAlias(column.column_name, base), - }); - } - } - break; - case MetaDiffType.VIEW_NEW: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - - mapDefaultDisplayValue(columns); - - const model = await Model.insert(project.id, base.id, { - table_name: table_name, - title: getTableNameAlias(table_name, project.prefix, base), - type: ModelTypes.VIEW, - }); - - for (const column of columns) { - await Column.insert({ - uidt: getColumnUiType(base, column), - fk_model_id: model.id, - ...column, - title: getColumnNameAlias(column.column_name, base), - }); - } - } - break; - case MetaDiffType.TABLE_REMOVE: - case MetaDiffType.VIEW_REMOVE: - { - await change.model.delete(); - } - break; - case MetaDiffType.TABLE_COLUMN_ADD: - case MetaDiffType.VIEW_COLUMN_ADD: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - const column = columns.find((c) => c.cn === change.cn); - column.uidt = getColumnUiType(base, column); - //todo: inflection - column.title = getColumnNameAlias(column.cn, base); - await Column.insert({ fk_model_id: change.id, ...column }); - } - // update old - // populateParams.tableNames.push({ tn }); - // populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn); - - break; - case MetaDiffType.TABLE_COLUMN_TYPE_CHANGE: - case MetaDiffType.VIEW_COLUMN_TYPE_CHANGE: - { - const columns = ( - await sqlClient.columnList({ tn: table_name }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); - const column = columns.find((c) => c.cn === change.cn); - const metaFact = ModelXcMetaFactory.create( - { client: base.type }, - {} - ); - column.uidt = metaFact.getUIDataType(column); - column.title = change.column.title; - await Column.update(change.column.id, column); - } - break; - case MetaDiffType.TABLE_COLUMN_REMOVE: - case MetaDiffType.VIEW_COLUMN_REMOVE: - await change.column.delete(); - break; - case MetaDiffType.TABLE_RELATION_REMOVE: - case MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE: - await change.column.delete(); - break; - case MetaDiffType.TABLE_RELATION_ADD: - { - virtualColumnInsert.push(async () => { - const parentModel = await Model.getByIdOrName({ - project_id: base.project_id, - base_id: base.id, - table_name: change.rtn, - }); - const childModel = await Model.getByIdOrName({ - project_id: base.project_id, - base_id: base.id, - table_name: change.tn, - }); - const parentCol = await parentModel - .getColumns() - .then((cols) => cols.find((c) => c.column_name === change.rcn)); - const childCol = await childModel - .getColumns() - .then((cols) => cols.find((c) => c.column_name === change.cn)); - - await Column.update(childCol.id, { - ...childCol, - uidt: UITypes.ForeignKey, - system: true, - }); - - if (change.relationType === RelationTypes.BELONGS_TO) { - const title = getUniqueColumnAliasName( - childModel.columns, - `${parentModel.title || parentModel.table_name}` - ); - await Column.insert({ - uidt: UITypes.LinkToAnotherRecord, - title, - fk_model_id: childModel.id, - fk_related_model_id: parentModel.id, - type: RelationTypes.BELONGS_TO, - fk_parent_column_id: parentCol.id, - fk_child_column_id: childCol.id, - virtual: false, - }); - } else if (change.relationType === RelationTypes.HAS_MANY) { - const title = getUniqueColumnAliasName( - childModel.columns, - `${childModel.title || childModel.table_name} List` - ); - await Column.insert({ - uidt: UITypes.LinkToAnotherRecord, - title, - fk_model_id: parentModel.id, - fk_related_model_id: childModel.id, - type: RelationTypes.HAS_MANY, - fk_parent_column_id: parentCol.id, - fk_child_column_id: childCol.id, - virtual: false, - }); - } - }); - } - break; - } - } - } - - await NcHelp.executeOperations(virtualColumnInsert, base.type); - - // populate m2m relations - await extractAndGenerateManyToManyRelations(await base.getModels()); - - Tele.emit('evt', { evt_type: 'baseMetaDiff:synced' }); + await metaDiffService.baseMetaDiffSync({ + projectId: req.params.projectId, + baseId: req.params.baseId, + }); res.json({ msg: 'success' }); } -async function isMMRelationExist( - model: Model, - assocModel: Model, - belongsToCol: Column -) { - let isExist = false; - const colChildOpt = - await belongsToCol.getColOptions(); - for (const col of await model.getColumns()) { - if (col.uidt === UITypes.LinkToAnotherRecord) { - const colOpt = await col.getColOptions(); - if ( - colOpt && - colOpt.type === RelationTypes.MANY_TO_MANY && - colOpt.fk_mm_model_id === assocModel.id && - colOpt.fk_child_column_id === colChildOpt.fk_parent_column_id && - colOpt.fk_mm_child_column_id === colChildOpt.fk_child_column_id - ) { - isExist = true; - break; - } - } - } - return isExist; -} - -// @ts-ignore -export async function extractAndGenerateManyToManyRelations( - modelsArr: Array -) { - for (const assocModel of modelsArr) { - await assocModel.getColumns(); - // check if table is a Bridge table(or Associative Table) by checking - // number of foreign keys and columns - - const normalColumns = assocModel.columns.filter((c) => !isVirtualCol(c)); - const belongsToCols: Column[] = []; - for (const col of assocModel.columns) { - if (col.uidt == UITypes.LinkToAnotherRecord) { - const colOpt = await col.getColOptions(); - if (colOpt?.type === RelationTypes.BELONGS_TO) belongsToCols.push(col); - } - } - - // todo: impl better method to identify m2m relation - if (belongsToCols?.length === 2 && normalColumns.length < 5) { - const modelA = await belongsToCols[0].colOptions.getRelatedTable(); - const modelB = await belongsToCols[1].colOptions.getRelatedTable(); - - await modelA.getColumns(); - await modelB.getColumns(); - - // check tableA already have the relation or not - const isRelationAvailInA = await isMMRelationExist( - modelA, - assocModel, - belongsToCols[0] - ); - const isRelationAvailInB = await isMMRelationExist( - modelB, - assocModel, - belongsToCols[1] - ); - - if (!isRelationAvailInA) { - await Column.insert({ - title: getUniqueColumnAliasName( - modelA.columns, - `${modelB.title} List` - ), - fk_model_id: modelA.id, - fk_related_model_id: modelB.id, - fk_mm_model_id: assocModel.id, - fk_child_column_id: belongsToCols[0].colOptions.fk_parent_column_id, - fk_parent_column_id: belongsToCols[1].colOptions.fk_parent_column_id, - fk_mm_child_column_id: belongsToCols[0].colOptions.fk_child_column_id, - fk_mm_parent_column_id: - belongsToCols[1].colOptions.fk_child_column_id, - type: RelationTypes.MANY_TO_MANY, - uidt: UITypes.LinkToAnotherRecord, - }); - } - if (!isRelationAvailInB) { - await Column.insert({ - title: getUniqueColumnAliasName( - modelB.columns, - `${modelA.title} List` - ), - fk_model_id: modelB.id, - fk_related_model_id: modelA.id, - fk_mm_model_id: assocModel.id, - fk_child_column_id: belongsToCols[1].colOptions.fk_parent_column_id, - fk_parent_column_id: belongsToCols[0].colOptions.fk_parent_column_id, - fk_mm_child_column_id: belongsToCols[1].colOptions.fk_child_column_id, - fk_mm_parent_column_id: - belongsToCols[0].colOptions.fk_child_column_id, - type: RelationTypes.MANY_TO_MANY, - uidt: UITypes.LinkToAnotherRecord, - }); - } - - await Model.markAsMmTable(assocModel.id, true); - - // mark has many relation associated with mm as system field in both table - for (const btCol of [belongsToCols[0], belongsToCols[1]]) { - const colOpt = await btCol.colOptions; - const model = await colOpt.getRelatedTable(); - - for (const col of await model.getColumns()) { - if (col.uidt !== UITypes.LinkToAnotherRecord) continue; - - const colOpt1 = await col.getColOptions(); - if (!colOpt1 || colOpt1.type !== RelationTypes.HAS_MANY) continue; - - if ( - colOpt1.fk_child_column_id !== colOpt.fk_child_column_id || - colOpt1.fk_parent_column_id !== colOpt.fk_parent_column_id - ) - continue; - - await Column.markAsSystemField(col.id); - break; - } - } - } else { - if (assocModel.mm) await Model.markAsMmTable(assocModel.id, false); - } - } -} - const router = Router(); router.get( '/api/v1/db/meta/projects/:projectId/meta-diff', diff --git a/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts b/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts index b4a7098da0..1f71c96bde 100644 --- a/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts +++ b/packages/nocodb/src/lib/meta/api/helpers/populateMeta.ts @@ -11,7 +11,7 @@ import getTableNameAlias, { import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; import getColumnUiType from '../../helpers/getColumnUiType'; import mapDefaultDisplayValue from '../../helpers/mapDefaultDisplayValue'; -import { extractAndGenerateManyToManyRelations } from '../../../controllers/metaDiffController'; +import { extractAndGenerateManyToManyRelations } from '../../../services/metaDiffService'; import { ModelTypes, UITypes, ViewTypes } from 'nocodb-sdk'; import { IGNORE_TABLES } from '../../../utils/common/BaseApiBuilder'; diff --git a/packages/nocodb/src/lib/models/Model.ts b/packages/nocodb/src/lib/models/Model.ts index 834e8025e4..eb32a3cc7b 100644 --- a/packages/nocodb/src/lib/models/Model.ts +++ b/packages/nocodb/src/lib/models/Model.ts @@ -1,9 +1,9 @@ -import Noco from '../Noco'; -import { parseMetaProp } from '../utils/modelUtils'; -import Column from './Column'; -import NocoCache from '../cache/NocoCache'; -import { XKnex } from '../db/sql-data-mapper'; -import { BaseModelSqlv2 } from '../db/sql-data-mapper/lib/sql/BaseModelSqlv2'; +import Noco from '../Noco' +import { parseMetaProp } from '../utils/modelUtils' +import Column from './Column' +import NocoCache from '../cache/NocoCache' +import { XKnex } from '../db/sql-data-mapper' +import { BaseModelSqlv2 } from '../db/sql-data-mapper/lib/sql/BaseModelSqlv2' import { isVirtualCol, ModelTypes, @@ -12,52 +12,52 @@ import { TableType, UITypes, ViewTypes, -} from 'nocodb-sdk'; +} from 'nocodb-sdk' import { CacheDelDirection, CacheGetType, CacheScope, MetaTable, -} from '../utils/globals'; -import View from './View'; -import { NcError } from '../meta/helpers/catchError'; -import Audit from './Audit'; -import { sanitize } from '../db/sql-data-mapper/lib/sql/helpers/sanitize'; -import { extractProps } from '../meta/helpers/extractProps'; +} from '../utils/globals' +import View from './View' +import { NcError } from '../meta/helpers/catchError' +import Audit from './Audit' +import { sanitize } from '../db/sql-data-mapper/lib/sql/helpers/sanitize' +import { extractProps } from '../meta/helpers/extractProps' export default class Model implements TableType { - copy_enabled: BoolType; - created_at: Date | number | string; - base_id: 'db' | string; - deleted: BoolType; - enabled: BoolType; - export_enabled: BoolType; - id: string; - order: number; - parent_id: string; - password: string; - pin: BoolType; - project_id: string; - schema: any; - show_all_fields: boolean; - tags: string; - type: ModelTypes; - updated_at: Date | number | string; - - table_name: string; - title: string; - - mm: BoolType; - - uuid: string; - - columns?: Column[]; - columnsById?: { [id: string]: Column }; - views?: View[]; - meta?: Record | string; + copy_enabled: BoolType + created_at: Date | number | string + base_id: 'db' | string + deleted: BoolType + enabled: BoolType + export_enabled: BoolType + id: string + order: number + parent_id: string + password: string + pin: BoolType + project_id: string + schema: any + show_all_fields: boolean + tags: string + type: ModelTypes + updated_at: Date | number | string + + table_name: string + title: string + + mm: BoolType + + uuid: string + + columns?: Column[] + columnsById?: { [id: string]: Column } + views?: View[] + meta?: Record | string constructor(data: Partial) { - Object.assign(this, data); + Object.assign(this, data) } public async getColumns(ncMeta = Noco.ncMeta): Promise { @@ -65,34 +65,34 @@ export default class Model implements TableType { { fk_model_id: this.id, }, - ncMeta - ); - return this.columns; + ncMeta, + ) + return this.columns } // @ts-ignore public async getViews(force = false, ncMeta = Noco.ncMeta): Promise { - this.views = await View.listWithInfo(this.id, ncMeta); - return this.views; + this.views = await View.listWithInfo(this.id, ncMeta) + return this.views } public get primaryKey(): Column { - if (!this.columns) return null; - return this.columns?.find((c) => c.pk); + if (!this.columns) return null + return this.columns?.find((c) => c.pk) } public get primaryKeys(): Column[] { - if (!this.columns) return null; - return this.columns?.filter((c) => c.pk); + if (!this.columns) return null + return this.columns?.filter((c) => c.pk) } public get displayValue(): Column { - if (!this.columns) return null; - const pCol = this.columns?.find((c) => c.pv); - if (pCol) return pCol; - const pkIndex = this.columns.indexOf(this.primaryKey); - if (pkIndex < this.columns.length - 1) return this.columns[pkIndex + 1]; - return this.columns[0]; + if (!this.columns) return null + const pCol = this.columns?.find((c) => c.pv) + if (pCol) return pCol + const pkIndex = this.columns.indexOf(this.primaryKey) + if (pkIndex < this.columns.length - 1) return this.columns[pkIndex + 1] + return this.columns[0] } public static async insert( @@ -102,8 +102,9 @@ export default class Model implements TableType { mm?: BoolType; created_at?: any; updated_at?: any; + type?: ModelTypes; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { const insertObj = extractProps(model, [ 'table_name', @@ -114,9 +115,9 @@ export default class Model implements TableType { 'created_at', 'updated_at', 'id', - ]); + ]) - insertObj.mm = !!insertObj.mm; + insertObj.mm = !!insertObj.mm if (!insertObj.order) { insertObj.order = await ncMeta.metaGetNextOrder( @@ -124,26 +125,26 @@ export default class Model implements TableType { { project_id: projectId, base_id: baseId, - } - ); + }, + ) } if (!insertObj.type) { - insertObj.type = ModelTypes.TABLE; + insertObj.type = ModelTypes.TABLE } const { id } = await ncMeta.metaInsert2( projectId, baseId, MetaTable.MODELS, - insertObj - ); + insertObj, + ) await NocoCache.appendToList( CacheScope.MODEL, [projectId], - `${CacheScope.MODEL}:${id}` - ); + `${CacheScope.MODEL}:${id}`, + ) const view = await View.insert( { @@ -154,14 +155,14 @@ export default class Model implements TableType { created_at: model.created_at, updated_at: model.updated_at, }, - ncMeta - ); + ncMeta, + ) for (const column of model?.columns || []) { - await Column.insert({ ...column, fk_model_id: id, view } as any, ncMeta); + await Column.insert({ ...column, fk_model_id: id, view } as any, ncMeta) } - return this.getWithInfo({ id }, ncMeta); + return this.getWithInfo({ id }, ncMeta) } public static async list( @@ -172,13 +173,13 @@ export default class Model implements TableType { project_id: string; base_id: string; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ): Promise { - let modelList = []; + let modelList = [] if (base_id) { - await NocoCache.getList(CacheScope.MODEL, [project_id, base_id]); + await NocoCache.getList(CacheScope.MODEL, [project_id, base_id]) } else { - await NocoCache.getList(CacheScope.MODEL, [project_id]); + await NocoCache.getList(CacheScope.MODEL, [project_id]) } if (!modelList.length) { modelList = await ncMeta.metaList2( @@ -189,30 +190,30 @@ export default class Model implements TableType { orderBy: { order: 'asc', }, - } - ); + }, + ) // parse meta of each model for (const model of modelList) { - model.meta = parseMetaProp(model); + model.meta = parseMetaProp(model) } if (base_id) { await NocoCache.setList( CacheScope.MODEL, [project_id, base_id], - modelList - ); + modelList, + ) } else { - await NocoCache.setList(CacheScope.MODEL, [project_id], modelList); + await NocoCache.setList(CacheScope.MODEL, [project_id], modelList) } } modelList.sort( (a, b) => (a.order != null ? a.order : Infinity) - - (b.order != null ? b.order : Infinity) - ); - return modelList.map((m) => new Model(m)); + (b.order != null ? b.order : Infinity), + ) + return modelList.map((m) => new Model(m)) } public static async listWithInfo( @@ -223,33 +224,33 @@ export default class Model implements TableType { project_id: string; db_alias: string; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ): Promise { let modelList = await NocoCache.getList(CacheScope.MODEL, [ project_id, db_alias, - ]); + ]) if (!modelList.length) { modelList = await ncMeta.metaList2( project_id, db_alias, - MetaTable.MODELS - ); + MetaTable.MODELS, + ) // parse meta of each model for (const model of modelList) { - model.meta = parseMetaProp(model); + model.meta = parseMetaProp(model) } - await NocoCache.setList(CacheScope.MODEL, [project_id], modelList); + await NocoCache.setList(CacheScope.MODEL, [project_id], modelList) } - return modelList.map((m) => new Model(m)); + return modelList.map((m) => new Model(m)) } public static async clear({ id }: { id: string }): Promise { - await NocoCache.delAll(CacheScope.MODEL, `*${id}*`); - await Column.clearList({ fk_model_id: id }); + await NocoCache.delAll(CacheScope.MODEL, `*${id}*`) + await Column.clearList({ fk_model_id: id }) } public static async get(id: string, ncMeta = Noco.ncMeta): Promise { @@ -257,47 +258,47 @@ export default class Model implements TableType { id && (await NocoCache.get( `${CacheScope.MODEL}:${id}`, - CacheGetType.TYPE_OBJECT - )); + CacheGetType.TYPE_OBJECT, + )) if (!modelData) { - modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, id); + modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, id) if (modelData) { - modelData.meta = parseMetaProp(modelData); - await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData); + modelData.meta = parseMetaProp(modelData) + await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData) } } - return modelData && new Model(modelData); + return modelData && new Model(modelData) } public static async getByIdOrName( args: | { - project_id: string; - base_id: string; - table_name: string; - } + project_id: string; + base_id: string; + table_name: string; + } | { - id?: string; - }, - ncMeta = Noco.ncMeta + id?: string; + }, + ncMeta = Noco.ncMeta, ): Promise { - const k = 'id' in args ? args?.id : args; + const k = 'id' in args ? args?.id : args let modelData = k && (await NocoCache.get( `${CacheScope.MODEL}:${k}`, - CacheGetType.TYPE_OBJECT - )); + CacheGetType.TYPE_OBJECT, + )) if (!modelData) { - modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, k); - modelData.meta = parseMetaProp(modelData); + modelData = await ncMeta.metaGet2(null, null, MetaTable.MODELS, k) + modelData.meta = parseMetaProp(modelData) } if (modelData) { - await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData); - return new Model(modelData); + await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData) + return new Model(modelData) } - return null; + return null } public static async getWithInfo( @@ -308,14 +309,14 @@ export default class Model implements TableType { table_name?: string; id?: string; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ): Promise { let modelData = id && (await NocoCache.get( `${CacheScope.MODEL}:${id}`, - CacheGetType.TYPE_OBJECT - )); + CacheGetType.TYPE_OBJECT, + )) if (!modelData) { modelData = await ncMeta.metaGet2( null, @@ -323,23 +324,23 @@ export default class Model implements TableType { MetaTable.MODELS, id || { table_name, - } - ); - modelData.meta = parseMetaProp(modelData); - await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData); + }, + ) + modelData.meta = parseMetaProp(modelData) + await NocoCache.set(`${CacheScope.MODEL}:${modelData.id}`, modelData) // modelData.filters = await Filter.getFilterObject({ // viewId: modelData.id // }); // modelData.sorts = await Sort.list({ modelId: modelData.id }); } if (modelData) { - const m = new Model(modelData); - const columns = await m.getColumns(ncMeta); - await m.getViews(false, ncMeta); - m.columnsById = columns.reduce((agg, c) => ({ ...agg, [c.id]: c }), {}); - return m; + const m = new Model(modelData) + const columns = await m.getColumns(ncMeta) + await m.getViews(false, ncMeta) + m.columnsById = columns.reduce((agg, c) => ({ ...agg, [c.id]: c }), {}) + return m } - return null; + return null } public static async getBaseModelSQL( @@ -349,60 +350,60 @@ export default class Model implements TableType { dbDriver: XKnex; model?: Model; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ): Promise { - const model = args?.model || (await this.get(args.id, ncMeta)); + const model = args?.model || (await this.get(args.id, ncMeta)) return new BaseModelSqlv2({ dbDriver: args.dbDriver, viewId: args.viewId, model, - }); + }) } async delete(ncMeta = Noco.ncMeta, force = false): Promise { - await Audit.deleteRowComments(this.id); + await Audit.deleteRowComments(this.id) for (const view of await this.getViews(true)) { - await view.delete(); + await view.delete() } for (const col of await this.getColumns(ncMeta)) { - let colOptionTableName = null; - let cacheScopeName = null; + let colOptionTableName = null + let cacheScopeName = null switch (col.uidt) { case UITypes.Rollup: - colOptionTableName = MetaTable.COL_ROLLUP; - cacheScopeName = CacheScope.COL_ROLLUP; - break; + colOptionTableName = MetaTable.COL_ROLLUP + cacheScopeName = CacheScope.COL_ROLLUP + break case UITypes.Lookup: - colOptionTableName = MetaTable.COL_LOOKUP; - cacheScopeName = CacheScope.COL_LOOKUP; - break; + colOptionTableName = MetaTable.COL_LOOKUP + cacheScopeName = CacheScope.COL_LOOKUP + break case UITypes.ForeignKey: case UITypes.LinkToAnotherRecord: - colOptionTableName = MetaTable.COL_RELATIONS; - cacheScopeName = CacheScope.COL_RELATION; - break; + colOptionTableName = MetaTable.COL_RELATIONS + cacheScopeName = CacheScope.COL_RELATION + break case UITypes.MultiSelect: case UITypes.SingleSelect: - colOptionTableName = MetaTable.COL_SELECT_OPTIONS; - cacheScopeName = CacheScope.COL_SELECT_OPTION; - break; + colOptionTableName = MetaTable.COL_SELECT_OPTIONS + cacheScopeName = CacheScope.COL_SELECT_OPTION + break case UITypes.Formula: - colOptionTableName = MetaTable.COL_FORMULA; - cacheScopeName = CacheScope.COL_FORMULA; - break; + colOptionTableName = MetaTable.COL_FORMULA + cacheScopeName = CacheScope.COL_FORMULA + break } if (colOptionTableName && cacheScopeName) { await ncMeta.metaDelete(null, null, colOptionTableName, { fk_column_id: col.id, - }); + }) await NocoCache.deepDel( cacheScopeName, `${cacheScopeName}:${col.id}`, - CacheDelDirection.CHILD_TO_PARENT - ); + CacheDelDirection.CHILD_TO_PARENT, + ) } } @@ -415,82 +416,82 @@ export default class Model implements TableType { condition: { fk_related_model_id: this.id, }, - } - ); + }, + ) for (const col of leftOverColumns) { await NocoCache.deepDel( CacheScope.COL_RELATION, `${CacheScope.COL_RELATION}:${col.fk_column_id}`, - CacheDelDirection.CHILD_TO_PARENT - ); + CacheDelDirection.CHILD_TO_PARENT, + ) } await ncMeta.metaDelete(null, null, MetaTable.COL_RELATIONS, { fk_related_model_id: this.id, - }); + }) } await NocoCache.deepDel( CacheScope.COLUMN, `${CacheScope.COLUMN}:${this.id}`, - CacheDelDirection.CHILD_TO_PARENT - ); + CacheDelDirection.CHILD_TO_PARENT, + ) await ncMeta.metaDelete(null, null, MetaTable.COLUMNS, { fk_model_id: this.id, - }); + }) await NocoCache.deepDel( CacheScope.MODEL, `${CacheScope.MODEL}:${this.id}`, - CacheDelDirection.CHILD_TO_PARENT - ); - await ncMeta.metaDelete(null, null, MetaTable.MODELS, this.id); + CacheDelDirection.CHILD_TO_PARENT, + ) + await ncMeta.metaDelete(null, null, MetaTable.MODELS, this.id) - await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.id}`); - await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.title}`); - return true; + await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.id}`) + await NocoCache.del(`${CacheScope.MODEL}:${this.project_id}:${this.title}`) + return true } async mapAliasToColumn(data) { - const insertObj = {}; + const insertObj = {} for (const col of await this.getColumns()) { - if (isVirtualCol(col)) continue; + if (isVirtualCol(col)) continue let val = data?.[col.column_name] !== undefined ? data?.[col.column_name] - : data?.[col.title]; + : data?.[col.title] if (val !== undefined) { if (col.uidt === UITypes.Attachment && typeof val !== 'string') { - val = JSON.stringify(val); + val = JSON.stringify(val) } - insertObj[sanitize(col.column_name)] = val; + insertObj[sanitize(col.column_name)] = val } } - return insertObj; + return insertObj } static async updateAliasAndTableName( tableId, title: string, table_name: string, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { if (!title) { - NcError.badRequest("Missing 'title' property in body"); + NcError.badRequest('Missing \'title\' property in body') } if (!table_name) { - NcError.badRequest("Missing 'table_name' property in body"); + NcError.badRequest('Missing \'table_name\' property in body') } // get existing cache - const key = `${CacheScope.MODEL}:${tableId}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.MODEL}:${tableId}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) // update alias if (o) { - o.title = title; - o.table_name = table_name; + o.title = title + o.table_name = table_name // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta return await ncMeta.metaUpdate( @@ -501,19 +502,19 @@ export default class Model implements TableType { title, table_name, }, - tableId - ); + tableId, + ) } static async markAsMmTable(tableId, isMm = true, ncMeta = Noco.ncMeta) { // get existing cache - const key = `${CacheScope.MODEL}:${tableId}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.MODEL}:${tableId}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) // update alias if (o) { - o.mm = isMm; + o.mm = isMm // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta return await ncMeta.metaUpdate( @@ -523,40 +524,40 @@ export default class Model implements TableType { { mm: isMm, }, - tableId - ); + tableId, + ) } async getAliasColMapping() { return (await this.getColumns()).reduce((o, c) => { if (c.column_name) { - o[c.title] = c.column_name; + o[c.title] = c.column_name } - return o; - }, {}); + return o + }, {}) } async getColAliasMapping() { return (await this.getColumns()).reduce((o, c) => { if (c.column_name) { - o[c.column_name] = c.title; + o[c.column_name] = c.title } - return o; - }, {}); + return o + }, {}) } static async updateOrder( tableId: string, order: number, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { // get existing cache - const key = `${CacheScope.MODEL}:${tableId}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.MODEL}:${tableId}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (o) { - o.order = order; + o.order = order // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta return await ncMeta.metaUpdate( @@ -566,29 +567,29 @@ export default class Model implements TableType { { order, }, - tableId - ); + tableId, + ) } static async updatePrimaryColumn( tableId: string, columnId: string, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { - const model = await this.getWithInfo({ id: tableId }); - const newPvCol = model.columns.find((c) => c.id === columnId); + const model = await this.getWithInfo({ id: tableId }) + const newPvCol = model.columns.find((c) => c.id === columnId) - if (!newPvCol) NcError.badRequest('Column not found'); + if (!newPvCol) NcError.badRequest('Column not found') // drop existing primary column/s for (const col of model.columns?.filter((c) => c.pv) || []) { // get existing cache - const key = `${CacheScope.COLUMN}:${col.id}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.COLUMN}:${col.id}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (o) { - o.pv = false; + o.pv = false // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta await ncMeta.metaUpdate( @@ -598,17 +599,17 @@ export default class Model implements TableType { { pv: false, }, - col.id - ); + col.id, + ) } // get existing cache - const key = `${CacheScope.COLUMN}:${newPvCol.id}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.COLUMN}:${newPvCol.id}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (o) { - o.pv = true; + o.pv = true // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta await ncMeta.metaUpdate( @@ -618,8 +619,8 @@ export default class Model implements TableType { { pv: true, }, - newPvCol.id - ); + newPvCol.id, + ) const grid_views_with_column = await ncMeta.metaList2( null, @@ -629,26 +630,26 @@ export default class Model implements TableType { condition: { fk_column_id: newPvCol.id, }, - } - ); + }, + ) if (grid_views_with_column.length) { for (const gv of grid_views_with_column) { - await View.fixPVColumnForView(gv.fk_view_id, ncMeta); + await View.fixPVColumnForView(gv.fk_view_id, ncMeta) } } - return true; + return true } static async setAsMm(id: any, ncMeta = Noco.ncMeta) { // get existing cache - const key = `${CacheScope.MODEL}:${id}`; - const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.MODEL}:${id}` + const o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (o) { - o.mm = true; + o.mm = true // set cache - await NocoCache.set(key, o); + await NocoCache.set(key, o) } // set meta await ncMeta.metaUpdate( @@ -658,8 +659,8 @@ export default class Model implements TableType { { mm: true, }, - id - ); + id, + ) } static async getByAliasOrId( @@ -672,69 +673,69 @@ export default class Model implements TableType { base_id?: string; aliasOrId: string; }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { const modelId = project_id && aliasOrId && (await NocoCache.get( `${CacheScope.MODEL}:${project_id}:${aliasOrId}`, - CacheGetType.TYPE_OBJECT - )); + CacheGetType.TYPE_OBJECT, + )) if (!modelId) { const model = base_id ? await ncMeta.metaGet2( - null, - null, - MetaTable.MODELS, - { project_id, base_id }, - null, - { - _or: [ - { - id: { - eq: aliasOrId, - }, + null, + null, + MetaTable.MODELS, + { project_id, base_id }, + null, + { + _or: [ + { + id: { + eq: aliasOrId, }, - { - title: { - eq: aliasOrId, - }, + }, + { + title: { + eq: aliasOrId, }, - ], - } - ) + }, + ], + }, + ) : await ncMeta.metaGet2( - null, - null, - MetaTable.MODELS, - { project_id }, - null, - { - _or: [ - { - id: { - eq: aliasOrId, - }, + null, + null, + MetaTable.MODELS, + { project_id }, + null, + { + _or: [ + { + id: { + eq: aliasOrId, }, - { - title: { - eq: aliasOrId, - }, + }, + { + title: { + eq: aliasOrId, }, - ], - } - ); + }, + ], + }, + ) if (model) { await NocoCache.set( `${CacheScope.MODEL}:${project_id}:${aliasOrId}`, - model.id - ); - await NocoCache.set(`${CacheScope.MODEL}:${model.id}`, model); + model.id, + ) + await NocoCache.set(`${CacheScope.MODEL}:${model.id}`, model) } - return model && new Model(model); + return model && new Model(model) } - return modelId && this.get(modelId); + return modelId && this.get(modelId) } static async checkTitleAvailable( @@ -744,7 +745,7 @@ export default class Model implements TableType { base_id, exclude_id, }: { table_name; project_id; base_id; exclude_id? }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { return !(await ncMeta.metaGet2( project_id, @@ -754,8 +755,8 @@ export default class Model implements TableType { table_name, }, null, - exclude_id && { id: { neq: exclude_id } } - )); + exclude_id && { id: { neq: exclude_id } }, + )) } static async checkAliasAvailable( @@ -765,7 +766,7 @@ export default class Model implements TableType { base_id, exclude_id, }: { title; project_id; base_id; exclude_id? }, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { return !(await ncMeta.metaGet2( project_id, @@ -775,32 +776,33 @@ export default class Model implements TableType { title, }, null, - exclude_id && { id: { neq: exclude_id } } - )); + exclude_id && { id: { neq: exclude_id } }, + )) } async getAliasColObjMap() { return (await this.getColumns()).reduce( (sortAgg, c) => ({ ...sortAgg, [c.title]: c }), - {} - ); + {}, + ) } // For updating table meta static async updateMeta( tableId: string, meta: string | Record, - ncMeta = Noco.ncMeta + ncMeta = Noco.ncMeta, ) { // get existing cache - const key = `${CacheScope.MODEL}:${tableId}`; - const existingCache = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); + const key = `${CacheScope.MODEL}:${tableId}` + const existingCache = await NocoCache.get(key, CacheGetType.TYPE_OBJECT) if (existingCache) { try { - existingCache.meta = typeof meta === 'string' ? JSON.parse(meta) : meta; + existingCache.meta = typeof meta === 'string' ? JSON.parse(meta) : meta // set cache - await NocoCache.set(key, existingCache); - } catch {} + await NocoCache.set(key, existingCache) + } catch { + } } // set meta return await ncMeta.metaUpdate( @@ -810,7 +812,7 @@ export default class Model implements TableType { { meta: typeof meta === 'object' ? JSON.stringify(meta) : meta, }, - tableId - ); + tableId, + ) } } diff --git a/packages/nocodb/src/lib/services/index.ts b/packages/nocodb/src/lib/services/index.ts index d4998f493b..614dfd5ee2 100644 --- a/packages/nocodb/src/lib/services/index.ts +++ b/packages/nocodb/src/lib/services/index.ts @@ -16,3 +16,5 @@ export * as galleryViewService from './galleryViewService'; export * as kanbanViewService from './kanbanViewService'; export * as gridViewColumnService from './gridViewColumnService'; export * as viewColumnService from './viewColumnService'; +export * as metaDiffService from './metaDiffService'; +export * as mapViewService from './mapViewService'; diff --git a/packages/nocodb/src/lib/services/mapViewService.ts b/packages/nocodb/src/lib/services/mapViewService.ts new file mode 100644 index 0000000000..ba21af1b6c --- /dev/null +++ b/packages/nocodb/src/lib/services/mapViewService.ts @@ -0,0 +1,34 @@ +import { MapType, ViewTypes } from 'nocodb-sdk'; +import View from '../models/View'; +import { Tele } from 'nc-help'; +import MapView from '../models/MapView'; + +export async function mapViewGet(param:{mapViewId: string}) { + return await MapView.get(param.mapViewId) +} + +export async function mapViewCreate(param:{ + tableId: string, + // todo: add MapReq in schema + map: MapType +}) { + Tele.emit('evt', { evt_type: 'vtable:created', show_as: 'map' }); + const view = await View.insert({ + ...param.map, + // todo: sanitize + fk_model_id: param.tableId, + type: ViewTypes.MAP, + }); + return view; +} + +export async function mapViewUpdate(param:{ + mapViewId: string, + // todo: add MapReq in schema + map: MapType + +}) { + Tele.emit('evt', { evt_type: 'view:updated', type: 'map' }); + // todo: type correction + return await MapView.update(param.mapViewId, param.map as any) +} diff --git a/packages/nocodb/src/lib/services/metaDiffService.ts b/packages/nocodb/src/lib/services/metaDiffService.ts new file mode 100644 index 0000000000..0786a0c19a --- /dev/null +++ b/packages/nocodb/src/lib/services/metaDiffService.ts @@ -0,0 +1,1105 @@ +// // Project CRUD + +import { Tele } from 'nc-help'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import { isVirtualCol, ModelTypes, RelationTypes, UITypes } from 'nocodb-sdk'; +import { + Base, + Column, + LinkToAnotherRecordColumn, + Model, + Project, +} from '../models'; +import ModelXcMetaFactory from '../db/sql-mgr/code/models/xc/ModelXcMetaFactory'; +import { getUniqueColumnAliasName } from '../meta/helpers/getUniqueName'; +import NcHelp from '../utils/NcHelp'; +import getTableNameAlias, { + getColumnNameAlias, +} from '../meta/helpers/getTableName'; +import mapDefaultDisplayValue from '../meta/helpers/mapDefaultDisplayValue'; +import getColumnUiType from '../meta/helpers/getColumnUiType'; + +export enum MetaDiffType { + TABLE_NEW = 'TABLE_NEW', + TABLE_REMOVE = 'TABLE_REMOVE', + TABLE_COLUMN_ADD = 'TABLE_COLUMN_ADD', + TABLE_COLUMN_TYPE_CHANGE = 'TABLE_COLUMN_TYPE_CHANGE', + TABLE_COLUMN_REMOVE = 'TABLE_COLUMN_REMOVE', + VIEW_NEW = 'VIEW_NEW', + VIEW_REMOVE = 'VIEW_REMOVE', + VIEW_COLUMN_ADD = 'VIEW_COLUMN_ADD', + VIEW_COLUMN_TYPE_CHANGE = 'VIEW_COLUMN_TYPE_CHANGE', + VIEW_COLUMN_REMOVE = 'VIEW_COLUMN_REMOVE', + TABLE_RELATION_ADD = 'TABLE_RELATION_ADD', + TABLE_RELATION_REMOVE = 'TABLE_RELATION_REMOVE', + TABLE_VIRTUAL_M2M_REMOVE = 'TABLE_VIRTUAL_M2M_REMOVE', +} + +const applyChangesPriorityOrder = [ + MetaDiffType.VIEW_COLUMN_REMOVE, + MetaDiffType.TABLE_RELATION_REMOVE, +]; + +type MetaDiff = { + title?: string; + table_name: string; + base_id: string; + type: ModelTypes; + meta?: any; + detectedChanges: Array; +}; + +type MetaDiffChange = { + msg?: string; + // type: MetaDiffType; +} & ( + | { + type: MetaDiffType.TABLE_NEW | MetaDiffType.VIEW_NEW; + tn?: string; + } + | { + type: MetaDiffType.TABLE_REMOVE | MetaDiffType.VIEW_REMOVE; + tn?: string; + model?: Model; + id?: string; + } + | { + type: MetaDiffType.TABLE_COLUMN_ADD | MetaDiffType.VIEW_COLUMN_ADD; + tn?: string; + model?: Model; + id?: string; + cn: string; + } + | { + type: + | MetaDiffType.TABLE_COLUMN_TYPE_CHANGE + | MetaDiffType.VIEW_COLUMN_TYPE_CHANGE + | MetaDiffType.TABLE_COLUMN_REMOVE + | MetaDiffType.VIEW_COLUMN_REMOVE; + tn?: string; + model?: Model; + id?: string; + cn: string; + column: Column; + colId?: string; + } + | { + type: MetaDiffType.TABLE_RELATION_REMOVE; + tn?: string; + rtn?: string; + cn?: string; + rcn?: string; + colId: string; + column: Column; + } + | { + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE; + tn?: string; + rtn?: string; + cn?: string; + rcn?: string; + colId: string; + column: Column; + } + | { + type: MetaDiffType.TABLE_RELATION_ADD; + tn?: string; + rtn?: string; + cn?: string; + rcn?: string; + relationType: RelationTypes; + cstn?: string; + } +); + +async function getMetaDiff( + sqlClient, + project: Project, + base: Base +): Promise> { + const changes: Array = []; + const virtualRelationColumns: Column[] = []; + + // @ts-ignore + const tableList: Array<{ tn: string }> = ( + await sqlClient.tableList() + )?.data?.list?.filter((t) => { + if (project?.prefix && base.is_meta) { + return t.tn?.startsWith(project?.prefix); + } + return true; + }); + + const colListRef = {}; + const oldMetas = await base.getModels(); + // @ts-ignore + const oldTableMetas: Model[] = []; + const oldViewMetas: Model[] = []; + + for (const model of oldMetas) { + if (model.type === ModelTypes.TABLE) oldTableMetas.push(model); + else if (model.type === ModelTypes.VIEW) oldViewMetas.push(model); + } + + // @ts-ignore + const relationList: Array<{ + tn: string; + rtn: string; + cn: string; + rcn: string; + found?: any; + cstn?: string; + }> = (await sqlClient.relationListAll())?.data?.list; + + for (const table of tableList) { + if (table.tn === 'nc_evolutions') continue; + + const oldMetaIdx = oldTableMetas.findIndex( + (m) => m.table_name === table.tn + ); + + // new table + if (oldMetaIdx === -1) { + changes.push({ + table_name: table.tn, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [ + { + type: MetaDiffType.TABLE_NEW, + msg: `New table`, + }, + ], + }); + continue; + } + + const oldMeta = oldTableMetas[oldMetaIdx]; + + oldTableMetas.splice(oldMetaIdx, 1); + + const tableProp: MetaDiff = { + title: oldMeta.title, + meta: oldMeta.meta, + table_name: table.tn, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [], + }; + changes.push(tableProp); + + // check for column change + colListRef[table.tn] = ( + await sqlClient.columnList({ tn: table.tn }) + )?.data?.list; + + await oldMeta.getColumns(); + + for (const column of colListRef[table.tn]) { + const oldColIdx = oldMeta.columns.findIndex( + (c) => c.column_name === column.cn + ); + + // new table + if (oldColIdx === -1) { + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_ADD, + msg: `New column(${column.cn})`, + cn: column.cn, + id: oldMeta.id, + }); + continue; + } + + const [oldCol] = oldMeta.columns.splice(oldColIdx, 1); + + if (oldCol.dt !== column.dt) { + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_TYPE_CHANGE, + msg: `Column type changed(${column.cn})`, + cn: oldCol.column_name, + id: oldMeta.id, + column: oldCol, + }); + } + } + for (const column of oldMeta.columns) { + if ( + [ + UITypes.LinkToAnotherRecord, + UITypes.Rollup, + UITypes.Lookup, + UITypes.Formula, + ].includes(column.uidt) + ) { + if (column.uidt === UITypes.LinkToAnotherRecord) { + virtualRelationColumns.push(column); + } + + continue; + } + + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_REMOVE, + msg: `Column removed(${column.column_name})`, + cn: column.column_name, + id: oldMeta.id, + column: column, + colId: column.id, + }); + } + } + + for (const model of oldTableMetas) { + changes.push({ + table_name: model.table_name, + meta: model.meta, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [ + { + type: MetaDiffType.TABLE_REMOVE, + msg: `Table removed`, + tn: model.table_name, + id: model.id, + model, + }, + ], + }); + } + + for (const relationCol of virtualRelationColumns) { + const colOpt = await relationCol.getColOptions(); + const parentCol = await colOpt.getParentColumn(); + const childCol = await colOpt.getChildColumn(); + const parentModel = await parentCol.getModel(); + const childModel = await childCol.getModel(); + + // many to many relation + if (colOpt.type === RelationTypes.MANY_TO_MANY) { + const m2mModel = await colOpt.getMMModel(); + + const relatedTable = tableList.find( + (t) => t.tn === parentModel.table_name + ); + const m2mTable = tableList.find((t) => t.tn === m2mModel.table_name); + + if (!relatedTable) { + changes + .find((t) => t.table_name === childModel.table_name) + .detectedChanges.push({ + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, + msg: `Many to many removed(${relatedTable.tn} removed)`, + colId: relationCol.id, + column: relationCol, + }); + continue; + } + if (!m2mTable) { + changes + .find((t) => t.table_name === childModel.table_name) + .detectedChanges.push({ + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, + msg: `Many to many removed(${m2mModel.table_name} removed)`, + colId: relationCol.id, + column: relationCol, + }); + continue; + } + + // verify columns + + const cColumns = (colListRef[childModel.table_name] = + colListRef[childModel.table_name] || + (await sqlClient.columnList({ tn: childModel.table_name }))?.data + ?.list); + + const pColumns = (colListRef[parentModel.table_name] = + colListRef[parentModel.table_name] || + (await sqlClient.columnList({ tn: parentModel.table_name }))?.data + ?.list); + + const vColumns = (colListRef[m2mTable.tn] = + colListRef[m2mTable.tn] || + (await sqlClient.columnList({ tn: m2mTable.tn }))?.data?.list); + + const m2mChildCol = await colOpt.getMMChildColumn(); + const m2mParentCol = await colOpt.getMMParentColumn(); + + if ( + pColumns.every((c) => c.cn !== parentCol.column_name) || + cColumns.every((c) => c.cn !== childCol.column_name) || + vColumns.every((c) => c.cn !== m2mChildCol.column_name) || + vColumns.every((c) => c.cn !== m2mParentCol.column_name) + ) { + changes + .find((t) => t.table_name === childModel.table_name) + .detectedChanges.push({ + type: MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE, + msg: `Many to many removed(One of the relation column removed)`, + colId: relationCol.id, + column: relationCol, + }); + } + + continue; + } + + if (relationCol.colOptions.virtual) continue; + + const dbRelation = relationList.find( + (r) => + r.cn === childCol.column_name && + r.tn === childModel.table_name && + r.rcn === parentCol.column_name && + r.rtn === parentModel.table_name + ); + + if (dbRelation) { + dbRelation.found = dbRelation.found || {}; + + if (dbRelation.found[colOpt.type]) { + // todo: handle duplicate + } else { + dbRelation.found[colOpt.type] = true; + } + } else { + changes + .find( + (t) => + t.table_name === + (colOpt.type === RelationTypes.BELONGS_TO + ? childModel.table_name + : parentModel.table_name) + ) + .detectedChanges.push({ + type: MetaDiffType.TABLE_RELATION_REMOVE, + tn: childModel.table_name, + rtn: parentModel.table_name, + cn: childCol.column_name, + rcn: parentCol.column_name, + msg: `Relation removed`, + colId: relationCol.id, + column: relationCol, + }); + } + } + + for (const relation of relationList) { + if (!relation?.found?.[RelationTypes.BELONGS_TO]) { + changes + .find((t) => t.table_name === relation.tn) + ?.detectedChanges.push({ + type: MetaDiffType.TABLE_RELATION_ADD, + tn: relation.tn, + rtn: relation.rtn, + cn: relation.cn, + rcn: relation.rcn, + msg: `New relation added`, + relationType: RelationTypes.BELONGS_TO, + cstn: relation.cstn, + }); + } + if (!relation?.found?.[RelationTypes.HAS_MANY]) { + changes + .find((t) => t.table_name === relation.rtn) + ?.detectedChanges.push({ + type: MetaDiffType.TABLE_RELATION_ADD, + tn: relation.tn, + rtn: relation.rtn, + cn: relation.cn, + rcn: relation.rcn, + msg: `New relation added`, + relationType: RelationTypes.HAS_MANY, + }); + } + } + + // views + // @ts-ignore + const viewList: Array<{ + view_name: string; + tn: string; + type: 'view'; + }> = (await sqlClient.viewList())?.data?.list + ?.map((v) => { + v.type = 'view'; + v.tn = v.view_name; + return v; + }) + .filter((t) => { + if (project?.prefix && base.is_meta) { + return t.tn?.startsWith(project?.prefix); + } + return true; + }); // @ts-ignore + + for (const view of viewList) { + const oldMetaIdx = oldViewMetas.findIndex((m) => m.table_name === view.tn); + + // new table + if (oldMetaIdx === -1) { + changes.push({ + table_name: view.tn, + base_id: base.id, + type: ModelTypes.VIEW, + detectedChanges: [ + { + type: MetaDiffType.VIEW_NEW, + msg: `New view`, + }, + ], + }); + continue; + } + + const oldMeta = oldViewMetas[oldMetaIdx]; + + oldViewMetas.splice(oldMetaIdx, 1); + + const tableProp: MetaDiff = { + title: oldMeta.title, + meta: oldMeta.meta, + table_name: view.tn, + base_id: base.id, + type: ModelTypes.VIEW, + detectedChanges: [], + }; + changes.push(tableProp); + + // check for column change + colListRef[view.tn] = ( + await sqlClient.columnList({ tn: view.tn }) + )?.data?.list; + + await oldMeta.getColumns(); + + for (const column of colListRef[view.tn]) { + const oldColIdx = oldMeta.columns.findIndex( + (c) => c.column_name === column.cn + ); + + // new table + if (oldColIdx === -1) { + tableProp.detectedChanges.push({ + type: MetaDiffType.VIEW_COLUMN_ADD, + msg: `New column(${column.cn})`, + cn: column.cn, + id: oldMeta.id, + }); + continue; + } + + const [oldCol] = oldMeta.columns.splice(oldColIdx, 1); + + if (oldCol.dt !== column.dt) { + tableProp.detectedChanges.push({ + type: MetaDiffType.TABLE_COLUMN_TYPE_CHANGE, + msg: `Column type changed(${column.cn})`, + cn: oldCol.column_name, + id: oldMeta.id, + column: oldCol, + }); + } + } + for (const column of oldMeta.columns) { + if ( + [ + UITypes.LinkToAnotherRecord, + UITypes.Rollup, + UITypes.Lookup, + UITypes.Formula, + ].includes(column.uidt) + ) { + continue; + } + + tableProp.detectedChanges.push({ + type: MetaDiffType.VIEW_COLUMN_REMOVE, + msg: `Column removed(${column.column_name})`, + cn: column.column_name, + id: oldMeta.id, + column: column, + colId: column.id, + }); + } + } + + for (const model of oldViewMetas) { + changes.push({ + table_name: model.table_name, + meta: model.meta, + base_id: base.id, + type: ModelTypes.TABLE, + detectedChanges: [ + { + type: MetaDiffType.VIEW_REMOVE, + msg: `Table removed`, + tn: model.table_name, + id: model.id, + model, + }, + ], + }); + } + + return changes; +} + +export async function metaDiff(param: { projectId: string }) { + const project = await Project.getWithInfo(param.projectId); + let changes = []; + for (const base of project.bases) { + try { + // @ts-ignore + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + changes = changes.concat(await getMetaDiff(sqlClient, project, base)); + } catch (e) { + console.log(e); + } + } + + return changes; +} + +export async function baseMetaDiff(param: { + projectId: string; + baseId: string; +}) { + const project = await Project.getWithInfo(param.projectId); + const base = await Base.get(param.baseId); + let changes = []; + + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + changes = await getMetaDiff(sqlClient, project, base); + + return changes; +} + +export async function metaDiffSync(param: { + projectId: string; +}) { + const project = await Project.getWithInfo(param.projectId); + for (const base of project.bases) { + const virtualColumnInsert: Array<() => Promise> = []; + + // @ts-ignore + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + const changes = await getMetaDiff(sqlClient, project, base); + + /* Get all relations */ + // const relations = (await sqlClient.relationListAll())?.data?.list; + + for (const { table_name, detectedChanges } of changes) { + // reorder changes to apply relation remove changes + // before column remove to avoid foreign key constraint error + detectedChanges.sort((a, b) => { + return ( + applyChangesPriorityOrder.indexOf(b.type) - + applyChangesPriorityOrder.indexOf(a.type) + ); + }); + + for (const change of detectedChanges) { + switch (change.type) { + case MetaDiffType.TABLE_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias( + table_name, + base.is_meta ? project.prefix : '', + base + ), + type: ModelTypes.TABLE, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.VIEW_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias(table_name, project.prefix, base), + type: ModelTypes.VIEW, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.TABLE_REMOVE: + case MetaDiffType.VIEW_REMOVE: + { + await change.model.delete(); + } + break; + case MetaDiffType.TABLE_COLUMN_ADD: + case MetaDiffType.VIEW_COLUMN_ADD: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + column.uidt = getColumnUiType(base, column); + //todo: inflection + column.title = getColumnNameAlias(column.cn, base); + await Column.insert({ fk_model_id: change.id, ...column }); + } + // update old + // populateParams.tableNames.push({ tn }); + // populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn); + + break; + case MetaDiffType.TABLE_COLUMN_TYPE_CHANGE: + case MetaDiffType.VIEW_COLUMN_TYPE_CHANGE: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + const metaFact = ModelXcMetaFactory.create( + { client: base.type }, + {} + ); + column.uidt = metaFact.getUIDataType(column); + column.title = change.column.title; + await Column.update(change.column.id, column); + } + break; + case MetaDiffType.TABLE_COLUMN_REMOVE: + case MetaDiffType.VIEW_COLUMN_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_REMOVE: + case MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_ADD: + { + virtualColumnInsert.push(async () => { + const parentModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.rtn, + }); + const childModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.tn, + }); + const parentCol = await parentModel + .getColumns() + .then((cols) => + cols.find((c) => c.column_name === change.rcn) + ); + const childCol = await childModel + .getColumns() + .then((cols) => + cols.find((c) => c.column_name === change.cn) + ); + + await Column.update(childCol.id, { + ...childCol, + uidt: UITypes.ForeignKey, + system: true, + }); + + if (change.relationType === RelationTypes.BELONGS_TO) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${parentModel.title || parentModel.table_name}` + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: childModel.id, + fk_related_model_id: parentModel.id, + type: RelationTypes.BELONGS_TO, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + fk_index_name: change.cstn, + }); + } else if (change.relationType === RelationTypes.HAS_MANY) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${childModel.title || childModel.table_name} List` + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: parentModel.id, + fk_related_model_id: childModel.id, + type: RelationTypes.HAS_MANY, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + fk_index_name: change.cstn, + }); + } + }); + } + break; + } + } + } + + await NcHelp.executeOperations(virtualColumnInsert, base.type); + + // populate m2m relations + await extractAndGenerateManyToManyRelations(await base.getModels()); + } + + Tele.emit('evt', { evt_type: 'metaDiff:synced' }); + + return true; +} + +export async function baseMetaDiffSync(param: { + projectId: string; + baseId: string; +}) { + const project = await Project.getWithInfo(param.projectId); + const base = await Base.get(param.baseId); + + const virtualColumnInsert: Array<() => Promise> = []; + + // @ts-ignore + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + const changes = await getMetaDiff(sqlClient, project, base); + + /* Get all relations */ + // const relations = (await sqlClient.relationListAll())?.data?.list; + + for (const { table_name, detectedChanges } of changes) { + for (const change of detectedChanges) { + switch (change.type) { + case MetaDiffType.TABLE_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias( + table_name, + base.is_meta ? project.prefix : '', + base + ), + type: ModelTypes.TABLE, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.VIEW_NEW: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + + mapDefaultDisplayValue(columns); + + const model = await Model.insert(project.id, base.id, { + table_name: table_name, + title: getTableNameAlias(table_name, project.prefix, base), + type: ModelTypes.VIEW, + }); + + for (const column of columns) { + await Column.insert({ + uidt: getColumnUiType(base, column), + fk_model_id: model.id, + ...column, + title: getColumnNameAlias(column.column_name, base), + }); + } + } + break; + case MetaDiffType.TABLE_REMOVE: + case MetaDiffType.VIEW_REMOVE: + { + await change.model.delete(); + } + break; + case MetaDiffType.TABLE_COLUMN_ADD: + case MetaDiffType.VIEW_COLUMN_ADD: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + column.uidt = getColumnUiType(base, column); + //todo: inflection + column.title = getColumnNameAlias(column.cn, base); + await Column.insert({ fk_model_id: change.id, ...column }); + } + // update old + // populateParams.tableNames.push({ tn }); + // populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn); + + break; + case MetaDiffType.TABLE_COLUMN_TYPE_CHANGE: + case MetaDiffType.VIEW_COLUMN_TYPE_CHANGE: + { + const columns = ( + await sqlClient.columnList({ tn: table_name }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })); + const column = columns.find((c) => c.cn === change.cn); + const metaFact = ModelXcMetaFactory.create( + { client: base.type }, + {} + ); + column.uidt = metaFact.getUIDataType(column); + column.title = change.column.title; + await Column.update(change.column.id, column); + } + break; + case MetaDiffType.TABLE_COLUMN_REMOVE: + case MetaDiffType.VIEW_COLUMN_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_REMOVE: + case MetaDiffType.TABLE_VIRTUAL_M2M_REMOVE: + await change.column.delete(); + break; + case MetaDiffType.TABLE_RELATION_ADD: + { + virtualColumnInsert.push(async () => { + const parentModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.rtn, + }); + const childModel = await Model.getByIdOrName({ + project_id: base.project_id, + base_id: base.id, + table_name: change.tn, + }); + const parentCol = await parentModel + .getColumns() + .then((cols) => cols.find((c) => c.column_name === change.rcn)); + const childCol = await childModel + .getColumns() + .then((cols) => cols.find((c) => c.column_name === change.cn)); + + await Column.update(childCol.id, { + ...childCol, + uidt: UITypes.ForeignKey, + system: true, + }); + + if (change.relationType === RelationTypes.BELONGS_TO) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${parentModel.title || parentModel.table_name}` + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: childModel.id, + fk_related_model_id: parentModel.id, + type: RelationTypes.BELONGS_TO, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + }); + } else if (change.relationType === RelationTypes.HAS_MANY) { + const title = getUniqueColumnAliasName( + childModel.columns, + `${childModel.title || childModel.table_name} List` + ); + await Column.insert({ + uidt: UITypes.LinkToAnotherRecord, + title, + fk_model_id: parentModel.id, + fk_related_model_id: childModel.id, + type: RelationTypes.HAS_MANY, + fk_parent_column_id: parentCol.id, + fk_child_column_id: childCol.id, + virtual: false, + }); + } + }); + } + break; + } + } + } + + await NcHelp.executeOperations(virtualColumnInsert, base.type); + + // populate m2m relations + await extractAndGenerateManyToManyRelations(await base.getModels()); + + Tele.emit('evt', { evt_type: 'baseMetaDiff:synced' }); + + return true; +} + +async function isMMRelationExist( + model: Model, + assocModel: Model, + belongsToCol: Column +) { + let isExist = false; + const colChildOpt = + await belongsToCol.getColOptions(); + for (const col of await model.getColumns()) { + if (col.uidt === UITypes.LinkToAnotherRecord) { + const colOpt = await col.getColOptions(); + if ( + colOpt && + colOpt.type === RelationTypes.MANY_TO_MANY && + colOpt.fk_mm_model_id === assocModel.id && + colOpt.fk_child_column_id === colChildOpt.fk_parent_column_id && + colOpt.fk_mm_child_column_id === colChildOpt.fk_child_column_id + ) { + isExist = true; + break; + } + } + } + return isExist; +} + +// @ts-ignore +export async function extractAndGenerateManyToManyRelations( + modelsArr: Array +) { + for (const assocModel of modelsArr) { + await assocModel.getColumns(); + // check if table is a Bridge table(or Associative Table) by checking + // number of foreign keys and columns + + const normalColumns = assocModel.columns.filter((c) => !isVirtualCol(c)); + const belongsToCols: Column[] = []; + for (const col of assocModel.columns) { + if (col.uidt == UITypes.LinkToAnotherRecord) { + const colOpt = await col.getColOptions(); + if (colOpt?.type === RelationTypes.BELONGS_TO) belongsToCols.push(col); + } + } + + // todo: impl better method to identify m2m relation + if (belongsToCols?.length === 2 && normalColumns.length < 5) { + const modelA = await belongsToCols[0].colOptions.getRelatedTable(); + const modelB = await belongsToCols[1].colOptions.getRelatedTable(); + + await modelA.getColumns(); + await modelB.getColumns(); + + // check tableA already have the relation or not + const isRelationAvailInA = await isMMRelationExist( + modelA, + assocModel, + belongsToCols[0] + ); + const isRelationAvailInB = await isMMRelationExist( + modelB, + assocModel, + belongsToCols[1] + ); + + if (!isRelationAvailInA) { + await Column.insert({ + title: getUniqueColumnAliasName( + modelA.columns, + `${modelB.title} List` + ), + fk_model_id: modelA.id, + fk_related_model_id: modelB.id, + fk_mm_model_id: assocModel.id, + fk_child_column_id: belongsToCols[0].colOptions.fk_parent_column_id, + fk_parent_column_id: belongsToCols[1].colOptions.fk_parent_column_id, + fk_mm_child_column_id: belongsToCols[0].colOptions.fk_child_column_id, + fk_mm_parent_column_id: + belongsToCols[1].colOptions.fk_child_column_id, + type: RelationTypes.MANY_TO_MANY, + uidt: UITypes.LinkToAnotherRecord, + }); + } + if (!isRelationAvailInB) { + await Column.insert({ + title: getUniqueColumnAliasName( + modelB.columns, + `${modelA.title} List` + ), + fk_model_id: modelB.id, + fk_related_model_id: modelA.id, + fk_mm_model_id: assocModel.id, + fk_child_column_id: belongsToCols[1].colOptions.fk_parent_column_id, + fk_parent_column_id: belongsToCols[0].colOptions.fk_parent_column_id, + fk_mm_child_column_id: belongsToCols[1].colOptions.fk_child_column_id, + fk_mm_parent_column_id: + belongsToCols[0].colOptions.fk_child_column_id, + type: RelationTypes.MANY_TO_MANY, + uidt: UITypes.LinkToAnotherRecord, + }); + } + + await Model.markAsMmTable(assocModel.id, true); + + // mark has many relation associated with mm as system field in both table + for (const btCol of [belongsToCols[0], belongsToCols[1]]) { + const colOpt = await btCol.colOptions; + const model = await colOpt.getRelatedTable(); + + for (const col of await model.getColumns()) { + if (col.uidt !== UITypes.LinkToAnotherRecord) continue; + + const colOpt1 = await col.getColOptions(); + if (!colOpt1 || colOpt1.type !== RelationTypes.HAS_MANY) continue; + + if ( + colOpt1.fk_child_column_id !== colOpt.fk_child_column_id || + colOpt1.fk_parent_column_id !== colOpt.fk_parent_column_id + ) + continue; + + await Column.markAsSystemField(col.id); + break; + } + } + } else { + if (assocModel.mm) await Model.markAsMmTable(assocModel.id, false); + } + } +}