From 232a9e2328b1f2b6dc81b0b2dd8e5769819bf898 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 14 Dec 2021 17:56:22 +0530 Subject: [PATCH] feat: metasync(wip) Signed-off-by: Pranav C --- .../nocodb/src/lib/noco/NcProjectBuilderEE.ts | 4 +- .../src/lib/noco/common/BaseApiBuilder.ts | 126 ++++++++++++++---- .../nocodb/src/lib/noco/gql/GqlApiBuilder.ts | 7 +- .../src/lib/noco/rest/RestApiBuilder.ts | 27 +++- .../sqlMgr/code/models/xc/BaseModelXcMeta.ts | 78 ++++++++++- .../sqlMgr/code/models/xc/ModelXcMetaMssql.ts | 4 +- .../sqlMgr/code/models/xc/ModelXcMetaMysql.ts | 4 +- .../code/models/xc/ModelXcMetaOracle.ts | 4 +- .../sqlMgr/code/models/xc/ModelXcMetaPg.ts | 4 +- .../code/models/xc/ModelXcMetaSqlite.ts | 6 +- 10 files changed, 221 insertions(+), 43 deletions(-) diff --git a/packages/nocodb/src/lib/noco/NcProjectBuilderEE.ts b/packages/nocodb/src/lib/noco/NcProjectBuilderEE.ts index 46e9b7fae2..e6d1221e07 100644 --- a/packages/nocodb/src/lib/noco/NcProjectBuilderEE.ts +++ b/packages/nocodb/src/lib/noco/NcProjectBuilderEE.ts @@ -10,8 +10,8 @@ export default class NcProjectBuilderEE extends NcProjectBuilder { switch (data?.req?.api) { case 'tableMetaRecreate': - await curBuilder.onTableDelete(data.req.args.tn); - await curBuilder.onTableCreate(data.req.args.tn, {}); + await curBuilder.onTableMetaRecreate(data.req.args.tn); + break; case 'viewMetaRecreate': diff --git a/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts b/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts index 096ee68099..52455e8678 100644 --- a/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts +++ b/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts @@ -133,7 +133,7 @@ export default abstract class BaseApiBuilder protected models: { [key: string]: BaseModelSql }; - protected metas: { [key: string]: any }; + protected metas: { [key: string]: NcMetaData }; protected sqlClient: MysqlClient | PgClient | SqlClient | any; @@ -206,29 +206,33 @@ export default abstract class BaseApiBuilder public abstract onToggleModelRelation(relationInModels: any): Promise; - public async onTableDelete(tn: string): Promise { + public async onTableDelete( + tn: string, + extras?: { ignoreRelations?: boolean } + ): Promise { this.baseLog(`onTableDelete : '%s'`, tn); XcCache.del([this.projectId, this.dbAlias, 'table', tn].join('::')); - await this.xcMeta.metaDelete( - this.projectId, - this.dbAlias, - 'nc_relations', - null, - { - _or: [ - { - tn: { - eq: tn - } - }, - { - rtn: { - eq: tn + if (!extras?.ignoreRelations) + await this.xcMeta.metaDelete( + this.projectId, + this.dbAlias, + 'nc_relations', + null, + { + _or: [ + { + tn: { + eq: tn + } + }, + { + rtn: { + eq: tn + } } - } - ] - } - ); + ] + } + ); await this.deleteTableNameInACL(tn); await this.xcMeta.metaDelete( @@ -2948,10 +2952,69 @@ export default abstract class BaseApiBuilder await this.loadFormViews(); } - public getMeta(tableName: string): any { + public getMeta(tableName: string): NcMetaData { return this.metas?.[tableName]; } + public async onTableMetaRecreate(tableName: string): Promise { + const meta = this.getMeta(tableName); + + const virtualRelations = await this.xcMeta.metaList( + this.projectId, + this.dbAlias, + 'nc_relations', + + { + xcCondition: { + virtual: true, + _or: [ + { + tn: { + eq: tableName + } + }, + { + rtn: { + eq: tableName + } + } + ] + } + } + ); + + const tableList = (await this.getSqlClient().tableList()?.data?.list) || []; + for (const rel of virtualRelations) { + const tnColList = await this.getColumnList(rel.tn); + const rtnColList = await this.getColumnList(rel.rtn); + + if ( + !( + tableList.find(t => t.tn === rel.rtn) && + tableList.find(t => t.tn === rel.tn) && + tnColList.find(t => t.cn === rel.cn) && + rtnColList.find(t => t.cn === rel.rcn) + ) + ) + await this.xcMeta.metaDelete( + this.projectId, + this.dbAlias, + 'nc_relations', + rel.id + ); + } + + await this.onTableDelete(tableName, { + ignoreRelations: true, + ignoreViews: true + } as any); + + // todo : handle query params + // todo : handle query para + + await this.onTableCreate(tableName, { oldMeta: meta }); + } + protected async getOrderVal(): Promise { const order = ( @@ -2975,7 +3038,24 @@ interface NcBuilderUpgraderCtx { dbAlias: string; } -export { IGNORE_TABLES, NcBuilderUpgraderCtx }; +interface NcMetaData { + tn: string; + _tn?: string; + v: Array<{ + _cn?: string; + [key: string]: any; + }>; + columns: Array<{ + _cn?: string; + cn?: string; + uidt?: string; + [key: string]: any; + }>; + + [key: string]: any; +} + +export { IGNORE_TABLES, NcBuilderUpgraderCtx, NcMetaData }; /** * @copyright Copyright (c) 2021, Xgene Cloud Ltd diff --git a/packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts b/packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts index 203afa44f5..8918508ef7 100644 --- a/packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts +++ b/packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts @@ -199,8 +199,11 @@ export class GqlApiBuilder extends BaseApiBuilder implements XcMetaMgr { await this.reInitializeGraphqlEndpoint(); } - public async onTableDelete(tn: string): Promise { - await super.onTableDelete(tn); + public async onTableDelete( + tn: string, + extras?: { ignoreRelations?: boolean } + ): Promise { + await super.onTableDelete(tn, extras); this.log(`onTableDelete : '%s' `, tn); delete this.models[tn]; await this.xcTablesRowDelete(tn); diff --git a/packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts b/packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts index 4ce3548f56..78c8a37f7b 100644 --- a/packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts +++ b/packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts @@ -19,7 +19,10 @@ import ExpressXcTsRoutesHm from '../../sqlMgr/code/routes/xc-ts/ExpressXcTsRoute import NcHelp from '../../utils/NcHelp'; import NcProjectBuilder from '../NcProjectBuilder'; import Noco from '../Noco'; -import BaseApiBuilder, { IGNORE_TABLES } from '../common/BaseApiBuilder'; +import BaseApiBuilder, { + IGNORE_TABLES, + NcMetaData +} from '../common/BaseApiBuilder'; import NcMetaIO from '../meta/NcMetaIO'; import { RestCtrl } from './RestCtrl'; @@ -346,6 +349,9 @@ export class RestApiBuilder extends BaseApiBuilder { columns?: { [tn: string]: any; }; + oldMetas?: { + [tn: string]: NcMetaData; + }; }): Promise { this.log( `xcTablesPopulate : tables - %o , type - %s`, @@ -499,6 +505,8 @@ export class RestApiBuilder extends BaseApiBuilder { args.tableNames?.find(t => t.tn === table.tn)?._tn ); + ctx.oldMeta = args?.oldMetas?.[table.tn]; + // ctx._tn = args.tableNames?.find(t => t.tn === table.tn)?._tn || ctx._tn; /* create models from table metadata */ @@ -854,7 +862,10 @@ export class RestApiBuilder extends BaseApiBuilder { }); } - public async onTableCreate(tn: string, args?: any): Promise { + public async onTableCreate( + tn: string, + args?: { _tn?: string; columns?: any; oldMeta?: NcMetaData } + ): Promise { await super.onTableCreate(tn, args); const columns = args.columns @@ -865,12 +876,18 @@ export class RestApiBuilder extends BaseApiBuilder { await this.xcTablesPopulate({ tableNames: [{ tn, _tn: args._tn }], - columns + columns, + oldMetas: { + [tn]: args.oldMeta + } }); } - public async onTableDelete(tn: string): Promise { - await super.onTableDelete(tn); + public async onTableDelete( + tn: string, + extras?: { ignoreRelations?: boolean } + ): Promise { + await super.onTableDelete(tn, extras); this.log("onTableDelete : '%s'", tn); try { const ctrlIndex = this.router.stack.findIndex(r => { diff --git a/packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts b/packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts index 6ba21252a0..00a161d6ac 100644 --- a/packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts +++ b/packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts @@ -1,7 +1,83 @@ import BaseRender from '../../BaseRender'; +import UITypes from '../../../../sqlUi/UITypes'; abstract class BaseModelXcMeta extends BaseRender { - public abstract getXcColumnsObject(context: any): any[]; + protected abstract _getAbstractType(column: any): any; + + protected abstract _getUIDataType(column: any): any; + + public getXcColumnsObject(args) { + const columnsArr = []; + + for (const column of args.columns) { + const columnObj = { + validate: { + func: [], + args: [], + msg: [] + }, + cn: column.cn, + _cn: column._cn || column.cn, + type: this._getAbstractType(column), + dt: column.dt, + + uidt: column.uidt || this._getUIDataType(column), + uip: column.uip, + uicn: column.uicn, + ...column + }; + + if (column.rqd) { + columnObj.rqd = column.rqd; + } + + if (column.cdf) { + columnObj.default = column.cdf; + columnObj.columnDefault = column.cdf; + } + + if (column.un) { + columnObj.un = column.un; + } + + if (column.pk) { + columnObj.pk = column.pk; + } + + if (column.ai) { + columnObj.ai = column.ai; + } + + if (column.dtxp) { + columnObj.dtxp = column.dtxp; + } + + if (column.dtxs) { + columnObj.dtxs = column.dtxs; + } + + const oldColMeta = this.ctx?.oldMeta?.columns?.find(c => { + return columnObj.cn == c.cn && columnObj.tpe == c.type; + }); + + if (oldColMeta) { + columnObj._cn = oldColMeta._cn || columnObj._cn; + columnObj.uidt = oldColMeta.uidt; + if ( + (columnObj.dtxp === UITypes.MultiSelect || + columnObj.dtxp === UITypes.SingleSelect) && + columnObj.dt !== 'set' && + columnObj.dt !== 'enum' + ) { + columnObj.dtxp = columnObj.dtxp || oldColMeta.dtxp; + } + } + + columnsArr.push(columnObj); + } + this.mapDefaultPrimaryValue(columnsArr); + return columnsArr; + } public getObject() { return { diff --git a/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaMssql.ts b/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaMssql.ts index 1fbd4a868c..5263da9dc8 100644 --- a/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaMssql.ts +++ b/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaMssql.ts @@ -492,7 +492,7 @@ class ModelXcMetaMssql extends BaseModelXcMeta { return str; } - getXcColumnsObject(args) { + /* getXcColumnsObject(args) { const columnsArr = []; for (const column of args.columns) { @@ -547,7 +547,7 @@ class ModelXcMetaMssql extends BaseModelXcMeta { this.mapDefaultPrimaryValue(columnsArr); return columnsArr; - } + }*/ /* getObject() { diff --git a/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaMysql.ts b/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaMysql.ts index f96c2734fb..ee60e12f3f 100644 --- a/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaMysql.ts +++ b/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaMysql.ts @@ -111,7 +111,7 @@ class ModelXcMetaMysql extends BaseModelXcMeta { return str; } - getXcColumnsObject(args) { + /* getXcColumnsObject(args) { const columnsArr = []; for (const column of args.columns) { @@ -166,7 +166,7 @@ class ModelXcMetaMysql extends BaseModelXcMeta { this.mapDefaultPrimaryValue(columnsArr); return columnsArr; } - +*/ _getAbstractType(column) { let str = ''; switch (column.dt) { diff --git a/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaOracle.ts b/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaOracle.ts index db68d90c03..58d2ea3bf9 100644 --- a/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaOracle.ts +++ b/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaOracle.ts @@ -111,7 +111,7 @@ class ModelXcMetaOracle extends BaseModelXcMeta { return str; } - getXcColumnsObject(args) { + /* getXcColumnsObject(args) { const columnsArr = []; for (const column of args.columns) { @@ -166,7 +166,7 @@ class ModelXcMetaOracle extends BaseModelXcMeta { this.mapDefaultPrimaryValue(columnsArr); return columnsArr; } - +*/ _getAbstractType(column) { let str = ''; switch (column.dt) { diff --git a/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaPg.ts b/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaPg.ts index 307910cc9a..9700883e2d 100644 --- a/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaPg.ts +++ b/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaPg.ts @@ -902,7 +902,7 @@ class ModelXcMetaPg extends BaseModelXcMeta { return str; } - getXcColumnsObject(args) { + /* getXcColumnsObject(args) { const columnsArr = []; for (const column of args.columns) { @@ -956,7 +956,7 @@ class ModelXcMetaPg extends BaseModelXcMeta { this.mapDefaultPrimaryValue(columnsArr); return columnsArr; - } + }*/ /* getObject() { return { diff --git a/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaSqlite.ts b/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaSqlite.ts index 6016c45cde..68dda6f8b7 100644 --- a/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaSqlite.ts +++ b/packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaSqlite.ts @@ -110,7 +110,7 @@ class ModelXcMetaSqlite extends BaseModelXcMeta { return str; } - _getAbstractType(column) { + protected _getAbstractType(column) { let str = ''; switch (column.dt) { case 'int': @@ -452,6 +452,7 @@ class ModelXcMetaSqlite extends BaseModelXcMeta { return str; } + /* public getXcColumnsObject(args): any { const columnsArr = []; @@ -506,6 +507,7 @@ class ModelXcMetaSqlite extends BaseModelXcMeta { this.mapDefaultPrimaryValue(columnsArr); return columnsArr; } +*/ /* public getObject(): any { return { @@ -521,7 +523,7 @@ class ModelXcMetaSqlite extends BaseModelXcMeta { }*/ - private _getUIDataType(col): any { + protected _getUIDataType(col): any { switch (this.getAbstractType(col)) { case 'integer': return 'Number';