From 0af8ab30cadc31205a56c28ec18013c8a4a70459 Mon Sep 17 00:00:00 2001 From: Pranav C <61551451+pranavxc@users.noreply.github.com> Date: Mon, 5 Jul 2021 18:22:43 +0530 Subject: [PATCH] feat: upgrade migrations Signed-off-by: Pranav C <61551451+pranavxc@users.noreply.github.com> --- .../nocodb/src/lib/noco/NcProjectBuilder.ts | 4 +- .../src/lib/noco/common/BaseApiBuilder.ts | 295 ++++++++++-------- .../nocodb/src/lib/noco/gql/GqlApiBuilder.ts | 6 + .../src/lib/noco/rest/RestApiBuilder.ts | 4 + .../sqlMgr/code/models/xc/BaseModelXcMeta.ts | 28 +- 5 files changed, 189 insertions(+), 148 deletions(-) diff --git a/packages/nocodb/src/lib/noco/NcProjectBuilder.ts b/packages/nocodb/src/lib/noco/NcProjectBuilder.ts index 18c4e19890..01079f4669 100644 --- a/packages/nocodb/src/lib/noco/NcProjectBuilder.ts +++ b/packages/nocodb/src/lib/noco/NcProjectBuilder.ts @@ -60,10 +60,10 @@ export default class NcProjectBuilder { let routeInfo; if (meta instanceof RestApiBuilder) { console.log(`Creating REST APIs ${meta.getDbType()} - > ${meta.getDbName()}`); - routeInfo = await (meta as RestApiBuilder).loadRoutes(null); + routeInfo = await (meta as RestApiBuilder).init(); } else if (meta instanceof GqlApiBuilder) { console.log(`Creating GraphQL APIs ${meta.getDbType()} - > ${meta.getDbName()}`); - routeInfo = await (meta as GqlApiBuilder).loadResolvers(null); + routeInfo = await (meta as GqlApiBuilder).init(); } allRoutesInfo.push(routeInfo); this.progress(routeInfo, allRoutesInfo, isFirstTime); diff --git a/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts b/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts index fa313cf6b3..a4d75a3ada 100644 --- a/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts +++ b/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts @@ -94,6 +94,10 @@ export default abstract class BaseApiBuilder implements XcDynami return this.models; } + public get client() { + return this.connectionConfig?.client; + } + public readonly app: T; public hooks: { @@ -663,40 +667,45 @@ export default abstract class BaseApiBuilder implements XcDynami const NC_VERSIONS = [ - // {name: '0.6', handler: null}, - // {name: '0.7', handler: this.xcUpZeroPointSeven}, - // {name: '0.8', handler: this.xcUpZeroPointEight}, - // {name: '0.9', handler: this.xcUpZeroPointNine}, + {name: '0009000', handler: null}, + {name: '0009044', handler: this.xcUpManyToMany} ] - const knex = this.getDbDriver(); - if (!await knex.schema.hasTable('nc_store')) { + if (!await this.xcMeta?.knex?.schema?.hasTable?.('nc_store')) { return; } this.baseLog(`xcUpgrade : Getting configuration from meta database`,) - const config = await knex('nc_store').where({key: 'NC_CONFIG'}).first(); + const config = await this.xcMeta.metaGet(this.projectId, this.dbAlias, 'nc_store', {key: 'NC_CONFIG'}); + if (config) { const configObj: NcConfig = JSON.parse(config.value); if (configObj.version !== process.env.NC_VERSION) { - let start = false; for (const version of NC_VERSIONS) { - if (start) { - await version.handler() - } else if (version.name === configObj.version) { - start = true; - // todo: take backup of current version + // compare current version and old version + if (version.name > configObj.version) { + this.baseLog(`xcUpgrade : Upgrading '%s' => '%s'`, configObj.version, version.name) + await version?.handler?.(); + + // update version in meta after each upgrade + configObj.version = version.name; + await this.xcMeta.metaInsert(this.projectId, this.dbAlias, 'nc_store', { + key: 'NC_CONFIG', + value: JSON.stringify(config) + }); + + // todo: backup data + } if (version.name === process.env.NC_VERSION) { break; - // todo: } } } } else { - this.baseLog(`xcUpgrade : Inserting config to meta database`,) const configObj: NcConfig = JSON.parse(JSON.stringify(this.config)); delete configObj.envs; + configObj.version = process.env.NC_VERSION; await this.xcMeta.metaInsert(this.projectId, this.dbAlias, 'nc_store', { key: 'NC_CONFIG', value: JSON.stringify(configObj) @@ -770,6 +779,14 @@ export default abstract class BaseApiBuilder implements XcDynami } } + public getProjectId(): string { + return this.projectId; + } + + public async init(): Promise { + await this.xcUpgrade(); + } + protected async loadCommon(): Promise { this.baseLog(`loadCommon :`); @@ -841,7 +858,7 @@ export default abstract class BaseApiBuilder implements XcDynami protected async getColumnList(tn: string): Promise { this.baseLog(`getColumnList : '%s'`, tn); - let columns = await this.sqlClient.columnList({tn: tn}); + let columns = await this.sqlClient.columnList({tn}); columns = columns.data.list; return columns; } @@ -903,11 +920,6 @@ export default abstract class BaseApiBuilder implements XcDynami return ctx; } - - private getColumnNameAlias(col, tableName?: string) { - return this.metas?.[tableName]?.columns?.find(c => c.cn === col.cn)?._cn || col._cn || this.getInflectedName(col.cn, this.connectionConfig?.meta?.inflection?.cn); - } - protected getTableNameAlias(tn: string) { if (this.metas?.[tn]?._tn) { return this.metas?.[tn]?._tn; @@ -922,7 +934,7 @@ export default abstract class BaseApiBuilder implements XcDynami ...ctx, _tn: this.metas[ctx.tn]._tn, _ctn: this.metas[tnc]._tn, - tnc: tnc, + tnc, project_id: this.projectId }; } @@ -1151,6 +1163,120 @@ export default abstract class BaseApiBuilder implements XcDynami }, {rtn: oldTableName}); } + + protected async getManyToManyRelations({parent = null, child = null} = {}) { + const metas = new Set(); + const assocMetas = new Set(); + + for (const meta of Object.values(this.metas)) { + + // check if table is a Bridge table(or Associative Table) by checking + // number of foreign keys and columns + if (meta.belongsTo?.length === 2 && meta.columns.length < 5) { + + if (parent && child && !([parent, child].includes(meta.belongsTo[0].rtn) && [parent, child].includes(meta.belongsTo[1].rtn))) { + continue; + } + + const tableMetaA = this.metas[meta.belongsTo[0].rtn]; + const tableMetaB = this.metas[meta.belongsTo[1].rtn]; + + /* // remove hasmany relation with associate table from tables + tableMetaA.hasMany.splice(tableMetaA.hasMany.findIndex(hm => hm.tn === meta.tn), 1) + tableMetaB.hasMany.splice(tableMetaB.hasMany.findIndex(hm => hm.tn === meta.tn), 1)*/ + + // add manytomany data under metadata of both related columns + tableMetaA.manyToMany = tableMetaA.manyToMany || []; + if (tableMetaA.manyToMany.every(mm => mm.vtn === meta.vtn)) { + tableMetaA.manyToMany.push({ + "tn": tableMetaA.tn, + "cn": meta.belongsTo[0].rcn, + "vtn": meta.tn, + "vcn": meta.belongsTo[0].cn, + "vrcn": meta.belongsTo[1].cn, + "rtn": meta.belongsTo[1].rtn, + "rcn": meta.belongsTo[1].rcn, + "_tn": tableMetaA._tn, + "_cn": meta.belongsTo[0]._rcn, + "_rtn": meta.belongsTo[1]._rtn, + "_rcn": meta.belongsTo[1]._rcn + }) + metas.add(tableMetaA) + } + tableMetaB.manyToMany = tableMetaB.manyToMany || []; + if (tableMetaB.manyToMany.every(mm => mm.vtn === meta.vtn)) { + tableMetaB.manyToMany.push({ + "tn": tableMetaB.tn, + "cn": meta.belongsTo[1].rcn, + "vtn": meta.tn, + "vcn": meta.belongsTo[1].cn, + "vrcn": meta.belongsTo[0].cn, + "rtn": meta.belongsTo[0].rtn, + "rcn": meta.belongsTo[0].rcn, + "_tn": tableMetaB._tn, + "_cn": meta.belongsTo[1]._rcn, + "_rtn": meta.belongsTo[0]._rtn, + "_rcn": meta.belongsTo[0]._rcn + }) + metas.add(tableMetaB) + } + assocMetas.add(meta) + } + } + + // Update metadata of tables which have manytomany relation + // and recreate basemodel with new meta information + for (const meta of metas) { + + let queryParams; + + // update showfields on new many to many relation create + if (parent && child) { + try { + queryParams = JSON.parse((await this.xcMeta.metaGet(this.projectId, this.dbAlias, 'nc_models', {title: meta.tn})).query_params) + } catch (e) { + // ignore + } + } + + meta.v = [ + ...meta.v.filter(vc => !(vc.hm && meta.manyToMany.some(mm => vc.hm.tn === mm.vtn))), + // todo: ignore existing m2m relations + ...meta.manyToMany.map(mm => { + + if (queryParams?.showFields && !(`${mm._tn} <=> ${mm._rtn}` in queryParams.showFields)) { + queryParams.showFields[`${mm._tn} <=> ${mm._rtn}`] = true; + } + + return { + mm, + _cn: `${mm._tn} <=> ${mm._rtn}` + } + + })] + await this.xcMeta.metaUpdate(this.projectId, this.dbAlias, 'nc_models', { + meta: JSON.stringify(meta), + ...(queryParams ? {query_params: JSON.stringify(queryParams)} : {}) + }, {title: meta.tn}) + XcCache.del([this.projectId, this.dbAlias, 'table', meta.tn].join('::')); + this.models[meta.tn] = this.getBaseModel(meta) + } + + // Update metadata of associative table + for (const meta of assocMetas) { + await this.xcMeta.metaUpdate(this.projectId, this.dbAlias, 'nc_models', { + mm: 1, + }, {title: meta.tn}) + XcCache.del([this.projectId, this.dbAlias, 'table', meta.tn].join('::')); + this.models[meta.tn] = this.getBaseModel(meta) + } + } + + + private getColumnNameAlias(col, tableName?: string) { + return this.metas?.[tableName]?.columns?.find(c => c.cn === col.cn)?._cn || col._cn || this.getInflectedName(col.cn, this.connectionConfig?.meta?.inflection?.cn); + } + private baseLog(str, ...args): void { log(`${this.dbAlias} : ${str}`, ...args); } @@ -1533,123 +1659,24 @@ export default abstract class BaseApiBuilder implements XcDynami await this.loadXcAcl() } - public get client() { - return this.connectionConfig?.client; - } - - public getProjectId() { - return this.projectId; - } - - - protected async getManyToManyRelations({parent = null, child = null} = {}) { - const metas = new Set(); - const assocMetas = new Set(); - - for (const meta of Object.values(this.metas)) { - - // check if table is a Bridge table(or Associative Table) by checking - // number of foreign keys and columns - if (meta.belongsTo?.length === 2 && meta.columns.length < 5) { - - if (parent && child && !([parent, child].includes(meta.belongsTo[0].rtn) && [parent, child].includes(meta.belongsTo[1].rtn))) { - continue; - } - - const tableMetaA = this.metas[meta.belongsTo[0].rtn]; - const tableMetaB = this.metas[meta.belongsTo[1].rtn]; - - /* // remove hasmany relation with associate table from tables - tableMetaA.hasMany.splice(tableMetaA.hasMany.findIndex(hm => hm.tn === meta.tn), 1) - tableMetaB.hasMany.splice(tableMetaB.hasMany.findIndex(hm => hm.tn === meta.tn), 1)*/ - // add manytomany data under metadata of both related columns - tableMetaA.manyToMany = tableMetaA.manyToMany || []; - if (tableMetaA.manyToMany.every(mm => mm.vtn === meta.vtn)) { - tableMetaA.manyToMany.push({ - "tn": tableMetaA.tn, - "cn": meta.belongsTo[0].rcn, - "vtn": meta.tn, - "vcn": meta.belongsTo[0].cn, - "vrcn": meta.belongsTo[1].cn, - "rtn": meta.belongsTo[1].rtn, - "rcn": meta.belongsTo[1].rcn, - "_tn": tableMetaA._tn, - "_cn": meta.belongsTo[0]._rcn, - "_rtn": meta.belongsTo[1]._rtn, - "_rcn": meta.belongsTo[1]._rcn - }) - metas.add(tableMetaA) - } - tableMetaB.manyToMany = tableMetaB.manyToMany || []; - if (tableMetaB.manyToMany.every(mm => mm.vtn === meta.vtn)) { - tableMetaB.manyToMany.push({ - "tn": tableMetaB.tn, - "cn": meta.belongsTo[1].rcn, - "vtn": meta.tn, - "vcn": meta.belongsTo[1].cn, - "vrcn": meta.belongsTo[0].cn, - "rtn": meta.belongsTo[0].rtn, - "rcn": meta.belongsTo[0].rcn, - "_tn": tableMetaB._tn, - "_cn": meta.belongsTo[1]._rcn, - "_rtn": meta.belongsTo[0]._rtn, - "_rcn": meta.belongsTo[0]._rcn - }) - metas.add(tableMetaB) - } - assocMetas.add(meta) - } - } - - // Update metadata of tables which have manytomany relation - // and recreate basemodel with new meta information - for (const meta of metas) { - - let queryParams; - - // update showfields on new many to many relation create - if (parent && child) { - try { - queryParams = JSON.parse((await this.xcMeta.metaGet(this.projectId, this.dbAlias, 'nc_models', {title: meta.tn})).query_params) - } catch (e) { - // ignore - } - } - - meta.v = [ - ...meta.v.filter(vc => !(vc.hm && meta.manyToMany.some(mm => vc.hm.tn === mm.vtn))), - // todo: ignore existing m2m relations - ...meta.manyToMany.map(mm => { - - if (queryParams?.showFields && !(`${mm._tn} <=> ${mm._rtn}` in queryParams.showFields)) { - queryParams.showFields[`${mm._tn} <=> ${mm._rtn}`] = true; - } - - return { - mm, - _cn: `${mm._tn} <=> ${mm._rtn}` - } - - })] - await this.xcMeta.metaUpdate(this.projectId, this.dbAlias, 'nc_models', { - meta: JSON.stringify(meta), - ...(queryParams ? {query_params: JSON.stringify(queryParams)} : {}) + private async xcUpManyToMany(): Promise { + const metas = await this.xcMeta.metaList(this.projectId, this.dbAlias, 'xc_models', { + fields: ['meta'] + }); + // add virtual columns for relations + for (const {meta: metaJson} of metas) { + const meta = JSON.parse(metaJson.meta); + const ctx = this.generateContextForTable(meta.tn, meta.columns, [], meta.hasMany, meta.belongsTo, meta.type, meta._tn); + meta.v = ModelXcMetaFactory.create(this.connectionConfig, {dir: '', ctx, filename: ''}).getVitualColumns(); + await this.xcMeta.metaUpdate(this.projectId, this.dbAlias, 'xc_models', { + meta: JSON.stringify(meta) }, {title: meta.tn}) - XcCache.del([this.projectId, this.dbAlias, 'table', meta.tn].join('::')); - this.models[meta.tn] = this.getBaseModel(meta) } - // Update metadata of associative table - for (const meta of assocMetas) { - await this.xcMeta.metaUpdate(this.projectId, this.dbAlias, 'nc_models', { - mm: 1, - }, {title: meta.tn}) - XcCache.del([this.projectId, this.dbAlias, 'table', meta.tn].join('::')); - this.models[meta.tn] = this.getBaseModel(meta) - } + // generate many to many relations an columns + await this.getManyToManyRelations(); } - } export {IGNORE_TABLES}; diff --git a/packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts b/packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts index 1b305176cc..754ced0f4b 100644 --- a/packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts +++ b/packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts @@ -94,6 +94,12 @@ export class GqlApiBuilder extends BaseApiBuilder implements XcMetaMgr { this.xcMeta = xcMeta; } + + public async init(): Promise { + await super.init(); + await this.loadResolvers(null); + } + public async onToggleModelRelation(relationInModels: any): Promise { this.log(`onToggleModelRelation: Within ToggleModelRelation event handler`) diff --git a/packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts b/packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts index c35b9d2c77..dc50ff82dc 100644 --- a/packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts +++ b/packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts @@ -59,6 +59,10 @@ export class RestApiBuilder extends BaseApiBuilder { this.xcMeta = xcMeta; } + public async init():Promise{ + await super.init(); + await this.loadRoutes(null); + } public async loadRoutes(customRoutes: any): Promise { this.customRoutes = customRoutes; diff --git a/packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts b/packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts index 2442dd714b..8aaddee873 100644 --- a/packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts +++ b/packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts @@ -2,7 +2,7 @@ import BaseRender from "../../BaseRender"; abstract class BaseModelXcMeta extends BaseRender { - public abstract getXcColumnsObject(context:any):any[]; + public abstract getXcColumnsObject(context: any): any[]; public getObject() { return { @@ -15,21 +15,25 @@ abstract class BaseModelXcMeta extends BaseRender { db_type: this.ctx.db_type, type: this.ctx.type, - v: [ - ...(this.ctx.hasMany || []).map(hm => ({ - hm, - _cn:`${hm._rtn} => ${hm._tn}` - })), - ...(this.ctx.belongsTo || []).map(bt => ({ - bt, - _cn:`${bt._rtn} <= ${bt._tn}` - })), - ] + v: this.getVitualColumns() } } - protected mapDefaultPrimaryValue(columnsArr: any[]):void { + public getVitualColumns(): any[] { + return [ + ...(this.ctx.hasMany || []).map(hm => ({ + hm, + _cn: `${hm._rtn} => ${hm._tn}` + })), + ...(this.ctx.belongsTo || []).map(bt => ({ + bt, + _cn: `${bt._rtn} <= ${bt._tn}` + })), + ]; + } + + protected mapDefaultPrimaryValue(columnsArr: any[]): void { // pk can be at the end //