feat: metasync(wip)

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb/src/lib/noco/NcProjectBuilderEE.ts

@@ -10,8 +10,8 @@ export default class NcProjectBuilderEE extends NcProjectBuilder {
 
     switch (data?.req?.api) {
       case 'tableMetaRecreate':
-        await curBuilder.onTableDelete(data.req.args.tn);
+        await curBuilder.onTableMetaRecreate(data.req.args.tn);
-        await curBuilder.onTableCreate(data.req.args.tn, {});
+
         break;
 
       case 'viewMetaRecreate':

packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts

@@ -133,7 +133,7 @@ export default abstract class BaseApiBuilder<T extends Noco>
 
   protected models: { [key: string]: BaseModelSql };
 
-  protected metas: { [key: string]: any };
+  protected metas: { [key: string]: NcMetaData };
 
   protected sqlClient: MysqlClient | PgClient | SqlClient | any;
 
@@ -206,29 +206,33 @@ export default abstract class BaseApiBuilder<T extends Noco>
 
   public abstract onToggleModelRelation(relationInModels: any): Promise<void>;
 
-  public async onTableDelete(tn: string): Promise<void> {
+  public async onTableDelete(
+    tn: string,
+    extras?: { ignoreRelations?: boolean }
+  ): Promise<void> {
     this.baseLog(`onTableDelete : '%s'`, tn);
     XcCache.del([this.projectId, this.dbAlias, 'table', tn].join('::'));
-    await this.xcMeta.metaDelete(
+    if (!extras?.ignoreRelations)
-      this.projectId,
+      await this.xcMeta.metaDelete(
-      this.dbAlias,
+        this.projectId,
-      'nc_relations',
+        this.dbAlias,
-      null,
+        'nc_relations',
-      {
+        null,
-        _or: [
+        {
-          {
+          _or: [
-            tn: {
+            {
-              eq: tn
+              tn: {
-            }
+                eq: tn
-          },
+              }
-          {
+            },
-            rtn: {
+            {
-              eq: tn
+              rtn: {
+                eq: tn
+              }
             }
-          }
+          ]
-        ]
+        }
-      }
+      );
-    );
     await this.deleteTableNameInACL(tn);
 
     await this.xcMeta.metaDelete(
@@ -2948,10 +2952,69 @@ export default abstract class BaseApiBuilder<T extends Noco>
     await this.loadFormViews();
   }
 
-  public getMeta(tableName: string): any {
+  public getMeta(tableName: string): NcMetaData {
     return this.metas?.[tableName];
   }
 
+  public async onTableMetaRecreate(tableName: string): Promise<void> {
+    const meta = this.getMeta(tableName);
+
+    const virtualRelations = await this.xcMeta.metaList(
+      this.projectId,
+      this.dbAlias,
+      'nc_relations',
+
+      {
+        xcCondition: {
+          virtual: true,
+          _or: [
+            {
+              tn: {
+                eq: tableName
+              }
+            },
+            {
+              rtn: {
+                eq: tableName
+              }
+            }
+          ]
+        }
+      }
+    );
+
+    const tableList = (await this.getSqlClient().tableList()?.data?.list) || [];
+    for (const rel of virtualRelations) {
+      const tnColList = await this.getColumnList(rel.tn);
+      const rtnColList = await this.getColumnList(rel.rtn);
+
+      if (
+        !(
+          tableList.find(t => t.tn === rel.rtn) &&
+          tableList.find(t => t.tn === rel.tn) &&
+          tnColList.find(t => t.cn === rel.cn) &&
+          rtnColList.find(t => t.cn === rel.rcn)
+        )
+      )
+        await this.xcMeta.metaDelete(
+          this.projectId,
+          this.dbAlias,
+          'nc_relations',
+          rel.id
+        );
+    }
+
+    await this.onTableDelete(tableName, {
+      ignoreRelations: true,
+      ignoreViews: true
+    } as any);
+
+    // todo : handle query params
+    // todo : handle query para
+
+    await this.onTableCreate(tableName, { oldMeta: meta });
+  }
+
   protected async getOrderVal(): Promise<number> {
     const order =
       (
@@ -2975,7 +3038,24 @@ interface NcBuilderUpgraderCtx {
   dbAlias: string;
 }
 
-export { IGNORE_TABLES, NcBuilderUpgraderCtx };
+interface NcMetaData {
+  tn: string;
+  _tn?: string;
+  v: Array<{
+    _cn?: string;
+    [key: string]: any;
+  }>;
+  columns: Array<{
+    _cn?: string;
+    cn?: string;
+    uidt?: string;
+    [key: string]: any;
+  }>;
+
+  [key: string]: any;
+}
+
+export { IGNORE_TABLES, NcBuilderUpgraderCtx, NcMetaData };
 
 /**
  * @copyright Copyright (c) 2021, Xgene Cloud Ltd

packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts

@@ -199,8 +199,11 @@ export class GqlApiBuilder extends BaseApiBuilder<Noco> implements XcMetaMgr {
     await this.reInitializeGraphqlEndpoint();
   }
 
-  public async onTableDelete(tn: string): Promise<void> {
+  public async onTableDelete(
-    await super.onTableDelete(tn);
+    tn: string,
+    extras?: { ignoreRelations?: boolean }
+  ): Promise<void> {
+    await super.onTableDelete(tn, extras);
     this.log(`onTableDelete : '%s' `, tn);
     delete this.models[tn];
     await this.xcTablesRowDelete(tn);

packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts

@@ -19,7 +19,10 @@ import ExpressXcTsRoutesHm from '../../sqlMgr/code/routes/xc-ts/ExpressXcTsRoute
 import NcHelp from '../../utils/NcHelp';
 import NcProjectBuilder from '../NcProjectBuilder';
 import Noco from '../Noco';
-import BaseApiBuilder, { IGNORE_TABLES } from '../common/BaseApiBuilder';
+import BaseApiBuilder, {
+  IGNORE_TABLES,
+  NcMetaData
+} from '../common/BaseApiBuilder';
 import NcMetaIO from '../meta/NcMetaIO';
 
 import { RestCtrl } from './RestCtrl';
@@ -346,6 +349,9 @@ export class RestApiBuilder extends BaseApiBuilder<Noco> {
     columns?: {
       [tn: string]: any;
     };
+    oldMetas?: {
+      [tn: string]: NcMetaData;
+    };
   }): Promise<any> {
     this.log(
       `xcTablesPopulate : tables - %o , type - %s`,
@@ -499,6 +505,8 @@ export class RestApiBuilder extends BaseApiBuilder<Noco> {
           args.tableNames?.find(t => t.tn === table.tn)?._tn
         );
 
+        ctx.oldMeta = args?.oldMetas?.[table.tn];
+
         // ctx._tn = args.tableNames?.find(t => t.tn === table.tn)?._tn || ctx._tn;
 
         /* create models from table metadata */
@@ -854,7 +862,10 @@ export class RestApiBuilder extends BaseApiBuilder<Noco> {
     });
   }
 
-  public async onTableCreate(tn: string, args?: any): Promise<void> {
+  public async onTableCreate(
+    tn: string,
+    args?: { _tn?: string; columns?: any; oldMeta?: NcMetaData }
+  ): Promise<void> {
     await super.onTableCreate(tn, args);
 
     const columns = args.columns
@@ -865,12 +876,18 @@ export class RestApiBuilder extends BaseApiBuilder<Noco> {
 
     await this.xcTablesPopulate({
       tableNames: [{ tn, _tn: args._tn }],
-      columns
+      columns,
+      oldMetas: {
+        [tn]: args.oldMeta
+      }
     });
   }
 
-  public async onTableDelete(tn: string): Promise<void> {
+  public async onTableDelete(
-    await super.onTableDelete(tn);
+    tn: string,
+    extras?: { ignoreRelations?: boolean }
+  ): Promise<void> {
+    await super.onTableDelete(tn, extras);
     this.log("onTableDelete : '%s'", tn);
     try {
       const ctrlIndex = this.router.stack.findIndex(r => {

packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts

@@ -1,7 +1,83 @@
 import BaseRender from '../../BaseRender';
+import UITypes from '../../../../sqlUi/UITypes';
 
 abstract class BaseModelXcMeta extends BaseRender {
-  public abstract getXcColumnsObject(context: any): any[];
+  protected abstract _getAbstractType(column: any): any;
+
+  protected abstract _getUIDataType(column: any): any;
+
+  public getXcColumnsObject(args) {
+    const columnsArr = [];
+
+    for (const column of args.columns) {
+      const columnObj = {
+        validate: {
+          func: [],
+          args: [],
+          msg: []
+        },
+        cn: column.cn,
+        _cn: column._cn || column.cn,
+        type: this._getAbstractType(column),
+        dt: column.dt,
+
+        uidt: column.uidt || this._getUIDataType(column),
+        uip: column.uip,
+        uicn: column.uicn,
+        ...column
+      };
+
+      if (column.rqd) {
+        columnObj.rqd = column.rqd;
+      }
+
+      if (column.cdf) {
+        columnObj.default = column.cdf;
+        columnObj.columnDefault = column.cdf;
+      }
+
+      if (column.un) {
+        columnObj.un = column.un;
+      }
+
+      if (column.pk) {
+        columnObj.pk = column.pk;
+      }
+
+      if (column.ai) {
+        columnObj.ai = column.ai;
+      }
+
+      if (column.dtxp) {
+        columnObj.dtxp = column.dtxp;
+      }
+
+      if (column.dtxs) {
+        columnObj.dtxs = column.dtxs;
+      }
+
+      const oldColMeta = this.ctx?.oldMeta?.columns?.find(c => {
+        return columnObj.cn == c.cn && columnObj.tpe == c.type;
+      });
+
+      if (oldColMeta) {
+        columnObj._cn = oldColMeta._cn || columnObj._cn;
+        columnObj.uidt = oldColMeta.uidt;
+        if (
+          (columnObj.dtxp === UITypes.MultiSelect ||
+            columnObj.dtxp === UITypes.SingleSelect) &&
+          columnObj.dt !== 'set' &&
+          columnObj.dt !== 'enum'
+        ) {
+          columnObj.dtxp = columnObj.dtxp || oldColMeta.dtxp;
+        }
+      }
+
+      columnsArr.push(columnObj);
+    }
+    this.mapDefaultPrimaryValue(columnsArr);
+    return columnsArr;
+  }
 
   public getObject() {
     return {

packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaMssql.ts

@@ -492,7 +492,7 @@ class ModelXcMetaMssql extends BaseModelXcMeta {
     return str;
   }
 
-  getXcColumnsObject(args) {
+  /*  getXcColumnsObject(args) {
     const columnsArr = [];
 
     for (const column of args.columns) {
@@ -547,7 +547,7 @@ class ModelXcMetaMssql extends BaseModelXcMeta {
     this.mapDefaultPrimaryValue(columnsArr);
 
     return columnsArr;
-  }
+  }*/
 
   /*
   getObject() {

packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaMysql.ts

@@ -111,7 +111,7 @@ class ModelXcMetaMysql extends BaseModelXcMeta {
     return str;
   }
 
-  getXcColumnsObject(args) {
+  /* getXcColumnsObject(args) {
     const columnsArr = [];
 
     for (const column of args.columns) {
@@ -166,7 +166,7 @@ class ModelXcMetaMysql extends BaseModelXcMeta {
     this.mapDefaultPrimaryValue(columnsArr);
     return columnsArr;
   }
-
+*/
   _getAbstractType(column) {
     let str = '';
     switch (column.dt) {

packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaOracle.ts

@@ -111,7 +111,7 @@ class ModelXcMetaOracle extends BaseModelXcMeta {
     return str;
   }
 
-  getXcColumnsObject(args) {
+  /* getXcColumnsObject(args) {
     const columnsArr = [];
 
     for (const column of args.columns) {
@@ -166,7 +166,7 @@ class ModelXcMetaOracle extends BaseModelXcMeta {
     this.mapDefaultPrimaryValue(columnsArr);
     return columnsArr;
   }
-
+*/
   _getAbstractType(column) {
     let str = '';
     switch (column.dt) {

packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaPg.ts

@@ -902,7 +902,7 @@ class ModelXcMetaPg extends BaseModelXcMeta {
     return str;
   }
 
-  getXcColumnsObject(args) {
+  /*  getXcColumnsObject(args) {
     const columnsArr = [];
 
     for (const column of args.columns) {
@@ -956,7 +956,7 @@ class ModelXcMetaPg extends BaseModelXcMeta {
 
     this.mapDefaultPrimaryValue(columnsArr);
     return columnsArr;
-  }
+  }*/
 
   /*  getObject() {
     return {

packages/nocodb/src/lib/sqlMgr/code/models/xc/ModelXcMetaSqlite.ts

@@ -110,7 +110,7 @@ class ModelXcMetaSqlite extends BaseModelXcMeta {
     return str;
   }
 
-  _getAbstractType(column) {
+  protected _getAbstractType(column) {
     let str = '';
     switch (column.dt) {
       case 'int':
@@ -452,6 +452,7 @@ class ModelXcMetaSqlite extends BaseModelXcMeta {
     return str;
   }
 
+  /*
   public getXcColumnsObject(args): any {
     const columnsArr = [];
 
@@ -506,6 +507,7 @@ class ModelXcMetaSqlite extends BaseModelXcMeta {
     this.mapDefaultPrimaryValue(columnsArr);
     return columnsArr;
   }
+*/
 
   /*  public getObject(): any {
     return {
@@ -521,7 +523,7 @@ class ModelXcMetaSqlite extends BaseModelXcMeta {
 
   }*/
 
-  private _getUIDataType(col): any {
+  protected _getUIDataType(col): any {
     switch (this.getAbstractType(col)) {
       case 'integer':
         return 'Number';