feat: meta sync

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb/src/lib/noco/common/handlers/xcMetaDiffSync.ts

@@ -38,16 +38,20 @@ export default async function(this: BaseApiBuilder<any> | any) {
     { condition: { type: 'table' } }
   );
 
+  const oldMetasRef = {};
   // @ts-ignore
   const oldMetas = oldModels.map(m => {
     const meta = JSON.parse(m.meta);
     XcCache.del([this.projectId, this.dbAlias, 'table', meta.tn].join('::'));
     meta.id = m.id;
+    populateParams.oldMetas[meta.tn] = meta;
+    oldMetasRef[meta.tn] = meta;
     return meta;
   });
+
   const oldQueryParams = oldModels.map(m => JSON.parse(m.query_params));
 
-  const relationTableMetas = new Set();
+  const relationTableMetas = new Set<any>();
   for (const { tn, detectedChanges } of changes) {
     if (!detectedChanges?.length) continue;
 
@@ -80,8 +84,16 @@ export default async function(this: BaseApiBuilder<any> | any) {
               this.projectId,
               this.dbAlias,
               'nc_models',
+              null,
+              {
+                _or: [
+                  {
+                    title: { eq: tn }
+                  },
                   {
-                title: tn
+                    parent_model_title: { eq: tn }
+                  }
+                ]
               }
             );
             if (delete this.metas[tn]) delete this.metas[tn];
@@ -204,9 +216,9 @@ export default async function(this: BaseApiBuilder<any> | any) {
             if (oldMeta.belongsTo?.length) {
               for (const bt of oldMeta.belongsTo) {
                 // filter out lookup columns which maps to current col
-                this.metas[bt.rtn].v = this.metas[bt.rtn].v?.filter(v => {
+                oldMetasRef[bt.rtn].v = oldMetasRef[bt.rtn].v?.filter(v => {
                   if (v.lk && v.lk.ltn === tn && v.lk.lcn === oldColumn.cn) {
-                    relationTableMetas.add(this.metas[bt.rtn]);
+                    relationTableMetas.add(oldMetasRef[bt.rtn]);
                     return false;
                   }
                   return true;
@@ -218,9 +230,9 @@ export default async function(this: BaseApiBuilder<any> | any) {
             if (oldMeta.hasMany?.length) {
               for (const hm of oldMeta.hasMany) {
                 // filter out lookup columns which maps to current col
-                this.metas[hm.tn].v = this.metas[hm.tn].v?.filter(v => {
+                oldMetasRef[hm.tn].v = oldMetasRef[hm.tn].v?.filter(v => {
                   if (v.lk && v.lk.ltn === tn && v.lk.lcn === change.cn) {
-                    relationTableMetas.add(this.metas[hm.tn]);
+                    relationTableMetas.add(oldMetasRef[hm.tn]);
                     return false;
                   }
                   return true;
@@ -232,9 +244,9 @@ export default async function(this: BaseApiBuilder<any> | any) {
             if (oldMeta.manyToMany?.length) {
               for (const mm of oldMeta.manyToMany) {
                 // filter out lookup columns which maps to current col
-                this.metas[mm.rtn].v = this.metas[mm.rtn].v?.filter(v => {
+                oldMetasRef[mm.rtn].v = oldMetasRef[mm.rtn].v?.filter(v => {
                   if (v.lk && v.lk.ltn === tn && v.lk.lcn === change.cn) {
-                    relationTableMetas.add(this.metas[mm.rtn]);
+                    relationTableMetas.add(oldMetasRef[mm.rtn]);
                     return false;
                   }
                   return true;
@@ -245,7 +257,7 @@ export default async function(this: BaseApiBuilder<any> | any) {
           break;
         case XcMetaDiffType.TABLE_RELATION_ADD:
           {
-            if (change.tn === tn)
+            if (change.tn === tn) {
               // todo : enable
               // ignore duplicate
               await this.xcMeta.metaInsert(
@@ -267,9 +279,112 @@ export default async function(this: BaseApiBuilder<any> | any) {
               fkn*/
                 }
               );
+              populateParams.tableNames.push({ tn: change.tn });
+              populateParams.tableNames.push({ tn: change.rtn });
+            }
+          }
+          break;
+        case XcMetaDiffType.TABLE_VIRTUAL_M2M_REMOVE:
+          {
+            for (const tn of [change.mm.tn, change.mm.rtn]) {
+              const {
+                // @ts-ignore
+                virtualViews,
+                virtualViewsParamsArr
+              } = await this.extractSharedAndVirtualViewsParams(tn);
+
+              const oldMeta = oldMetas.find(m => m.tn === tn);
+              populateParams.oldMetas[tn] = oldMeta;
+              populateParams.tableNames.push({
+                tn,
+                _tn: populateParams.oldMetas[tn]?._tn
+              });
+
+              // extract alias of relation virtual column
+              const alias = oldMeta?.v?.find(
+                v =>
+                  v?.mm?.tn === change.mm.tn &&
+                  v?.mm?.vtn === change.mm.vtn &&
+                  v?.mm?.rtn === change.mm.rtn
+              )?._cn;
+
+              // virtual views param update
+              for (const qp of virtualViewsParamsArr) {
+                // @ts-ignore
+                const {
+                  showFields = {},
+                  fieldsOrder,
+                  extraViewParams = {}
+                } = qp;
+
+                /* update show field */
+                if (alias in showFields) {
+                  delete showFields[alias];
+                }
+
+                /* update fieldsOrder */
+                const index = fieldsOrder.indexOf(alias);
+                if (index > -1) {
+                  fieldsOrder.splice(index, 1);
+                }
+
+                /* update formView params */
+                if (extraViewParams?.formParams?.fields?.[alias]) {
+                  delete extraViewParams.formParams.fields[alias];
+                }
+              }
+              // todo: enable
+              await this.updateSharedAndVirtualViewsParams(
+                virtualViewsParamsArr,
+                virtualViews
+              );
+            }
+
+            const parentMeta = oldMetas.find(m => m.tn === change.mm.tn);
+
+            Object.assign(parentMeta, {
+              v: parentMeta.v.filter(
+                ({ mm, lk, rl }) =>
+                  (!mm || mm.rtn !== change.mm.rtn || mm.tn !== change.mm.tn) &&
+                  !(
+                    lk &&
+                    lk.type === 'hm' &&
+                    lk.rtn === change.mm.rtn &&
+                    lk.tn === change.mm.tn
+                  ) &&
+                  !(
+                    rl &&
+                    rl.type === 'hm' &&
+                    rl.rtn === change.mm.rtn &&
+                    rl.tn === change.mm.tn
+                  )
+              )
+            });
+
+            const childMeta = oldMetas.find(m => m.tn === change.mm.rtn);
+
+            Object.assign(childMeta, {
+              v: childMeta.v.filter(
+                ({ mm, lk, rl }) =>
+                  (!mm || mm.rtn !== change.mm.tn || mm.tn !== change.mm.rtn) &&
+                  !(
+                    lk &&
+                    lk.type === 'hm' &&
+                    lk.rtn === change.mm.tn &&
+                    lk.tn === change.mm.rtn
+                  ) &&
+                  !(
+                    rl &&
+                    rl.type === 'hm' &&
+                    rl.rtn === change.mm.tn &&
+                    rl.tn === change.mm.rtn
+                  )
+              )
+            });
           }
           break;
         case XcMetaDiffType.TABLE_RELATION_REMOVE:
+        case XcMetaDiffType.TABLE_VIRTUAL_RELATION_REMOVE:
           {
             // todo: remove from  nc_relations
             // todo:enable
@@ -282,7 +397,10 @@ export default async function(this: BaseApiBuilder<any> | any) {
                 cn: change.cn,
                 rtn: change.rtn,
                 rcn: change.rcn,
-                type: 'real'
+                type:
+                  XcMetaDiffType.TABLE_RELATION_REMOVE === change.type
+                    ? 'real'
+                    : 'virtual'
                 // db_type: this.connectionConfig?.client
               }
             );
@@ -383,12 +501,20 @@ export default async function(this: BaseApiBuilder<any> | any) {
     }
   }
 
+  // update relation tables metadata
+  for (const relMeta of relationTableMetas) {
+    populateParams.tableNames.push({
+      tn: relMeta.tn,
+      _tn: relMeta._tn
+    });
+    populateParams.oldMetas[relMeta.tn] = relMeta;
+  }
+
   // todo: optimize
   // remove duplicate from list
   populateParams.tableNames = populateParams.tableNames?.filter(t => {
     return t === populateParams.tableNames.find(t1 => t1.tn === t.tn);
   });
-
   await this.xcTablesPopulate(populateParams);
 
   return populateParams;

packages/nocodb/src/lib/noco/meta/handlers/xcMetaDiff.ts

@@ -9,7 +9,8 @@ enum XcMetaDiffType {
   TABLE_RELATION_ADD = 'TABLE_RELATION_ADD',
   TABLE_RELATION_REMOVE = 'TABLE_RELATION_REMOVE',
   TABLE_VIRTUAL_RELATION_ADD = 'TABLE_VIRTUAL_RELATION_ADD',
-  TABLE_VIRTUAL_RELATION_DELETE = 'TABLE_VIRTUAL_RELATION_DELETE'
+  TABLE_VIRTUAL_RELATION_REMOVE = 'TABLE_VIRTUAL_RELATION_REMOVE',
+  TABLE_VIRTUAL_M2M_REMOVE = 'TABLE_VIRTUAL_M2M_REMOVE'
 }
 
 interface NcMetaDiff {
@@ -33,7 +34,7 @@ export default async function(
 
   // @ts-ignore
   const tableList = (await sqlClient.tableList())?.data?.list;
-
+  const colListRef = {};
   // @ts-ignore
   const oldMetas = (
     await this.xcMeta.metaList(
@@ -44,6 +45,9 @@ export default async function(
     )
   ).map(m => JSON.parse(m.meta));
 
+  // @ts-ignore
+  const relationList = (await sqlClient.relationListAll())?.data?.list;
+
   for (const table of tableList) {
     if (table.tn === 'nc_evolutions') continue;
 
@@ -73,10 +77,11 @@ export default async function(
     changes.push(tableProp);
 
     // check for column change
-    const columnList = (await sqlClient.columnList({ tn: table.tn }))?.data
+    colListRef[table.tn] = (
-      ?.list;
+      await sqlClient.columnList({ tn: table.tn })
+    )?.data?.list;
 
-    for (const column of columnList) {
+    for (const column of colListRef[table.tn]) {
       const oldColIdx = oldMeta.columns.findIndex(c => c.cn === column.cn);
 
       // new table
@@ -107,6 +112,59 @@ export default async function(
         cn
       });
     }
+    for (const vCol of oldMeta.v) {
+      if (!vCol.mm) continue;
+
+      // check related tables & columns
+
+      const rTable = tableList.find(t => t.tn === vCol.mm?.rtn);
+      const m2mTable = tableList.find(t => t.tn === vCol.mm?.vtn);
+
+      if (!rTable) {
+        tableProp.detectedChanges.push({
+          ...vCol,
+          type: XcMetaDiffType.TABLE_VIRTUAL_M2M_REMOVE,
+          msg: `Many to many removed(${vCol.mm?.rtn} removed)`
+        });
+        continue;
+      }
+      if (!m2mTable) {
+        tableProp.detectedChanges.push({
+          ...vCol,
+          type: XcMetaDiffType.TABLE_VIRTUAL_M2M_REMOVE,
+          msg: `Many to many removed(${vCol.mm?.vtn} removed)`
+        });
+        continue;
+      }
+
+      // verify columns
+
+      const pColumns = (colListRef[vCol.mm.tn] =
+        colListRef[vCol.mm.tn] ||
+        (await sqlClient.columnList({ tn: vCol.mm.tn }))?.data?.list);
+
+      const cColumns = (colListRef[vCol.mm.rtn] =
+        colListRef[vCol.mm.rtn] ||
+        (await sqlClient.columnList({ tn: vCol.mm.rtn }))?.data?.list);
+
+      const vColumns = (colListRef[vCol.mm.vtn] =
+        colListRef[vCol.mm.vtn] ||
+        (await sqlClient.columnList({ tn: vCol.mm.vtn }))?.data?.list);
+
+      if (
+        pColumns.every(c => c.cn !== vCol.mm.cn) ||
+        cColumns.every(c => c.cn !== vCol.mm.rcn) ||
+        vColumns.every(c => c.cn !== vCol.mm.vcn) ||
+        vColumns.every(c => c.cn !== vCol.mm.vrcn)
+      ) {
+        tableProp.detectedChanges.push({
+          ...vCol,
+          type: XcMetaDiffType.TABLE_VIRTUAL_M2M_REMOVE,
+          msg: `Many to many removed(One of the relation column removed)`
+        });
+        continue;
+      }
+    }
   }
 
   for (const { tn } of oldMetas) {
@@ -121,11 +179,9 @@ export default async function(
     });
   }
 
-  // @ts-ignore
+  // extract unique relations
-  const relationList = (await sqlClient.relationListAll())?.data?.list;
+  const oldRelations = (
-
+    await this.xcMeta.metaList(
-  // todo: handle virtual
-  const oldRelations = await this.xcMeta.metaList(
       this.getProjectId(args),
       this.getDbAlias(args),
       'nc_relations',
@@ -134,7 +190,19 @@ export default async function(
           type: 'real'
         }
       }
+    )
+  ).filter((r, i, arr) => {
+    return (
+      i ===
+      arr.findIndex(
+        r1 =>
+          r1.tn === r.tn &&
+          r1.rtn === r.rtn &&
+          r1.cn === r.cn &&
+          r1.rcn === r.rcn
+      )
     );
+  });
 
   // check relations
   for (const rel of relationList) {
@@ -197,6 +265,80 @@ export default async function(
       });
   }
 
+  const oldVirtualRelations = await this.xcMeta.metaList(
+    this.getProjectId(args),
+    this.getDbAlias(args),
+    'nc_relations',
+    {
+      condition: {
+        type: 'virtual'
+      }
+    }
+  );
+
+  // check relations
+  for (const vRel of oldVirtualRelations) {
+    if (tableList.every(t => t.tn !== vRel.tn && t.tn !== vRel.rtn)) {
+      changes
+        .find(t => t.tn === vRel.tn)
+        ?.detectedChanges.push({
+          type: XcMetaDiffType.TABLE_VIRTUAL_RELATION_REMOVE,
+          tn: vRel.tn,
+          rtn: vRel.rtn,
+          cn: vRel.cn,
+          rcn: vRel.rcn,
+          msg: `Virtual relation removed`
+        });
+      changes
+        .find(t => t.tn === vRel.rtn)
+        ?.detectedChanges.push({
+          type: XcMetaDiffType.TABLE_VIRTUAL_RELATION_REMOVE,
+          tn: vRel.tn,
+          rtn: vRel.rtn,
+          cn: vRel.cn,
+          rcn: vRel.rcn,
+          msg: `Virtual relation removed`
+        });
+      continue;
+    }
+
+    colListRef[vRel.tn] = (
+      await sqlClient.columnList({ tn: vRel.tn })
+    )?.data?.list;
+    colListRef[vRel.rtn] = (
+      await sqlClient.columnList({ tn: vRel.rtn })
+    )?.data?.list;
+
+    if (
+      colListRef[vRel.tn].every(c => c.cn !== vRel.cn) ||
+      colListRef[vRel.rtn].every(c => c.cn !== vRel.rcn)
+    ) {
+      changes
+        .find(t => t.tn === vRel.tn)
+        ?.detectedChanges.push({
+          type: XcMetaDiffType.TABLE_VIRTUAL_RELATION_REMOVE,
+          tn: vRel.tn,
+          rtn: vRel.rtn,
+          cn: vRel.cn,
+          rcn: vRel.rcn,
+          msg: `Virtual relation column missing`
+        });
+      changes
+        .find(t => t.tn === vRel.rtn)
+        ?.detectedChanges.push({
+          type: XcMetaDiffType.TABLE_VIRTUAL_RELATION_REMOVE,
+          tn: vRel.tn,
+          rtn: vRel.rtn,
+          cn: vRel.cn,
+          rcn: vRel.rcn,
+          msg: `Virtual relation column missing`
+        });
+    }
+
+    // colListRef[table.tn]= (await sqlClient.columnList({ tn: table.tn }))?.data
+    //   ?.list;
+  }
+
   return changes;
 }
 export { XcMetaDiffType, NcMetaDiff };

packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts

@@ -376,6 +376,7 @@ export class RestApiBuilder extends BaseApiBuilder<Noco> {
     if (args?.tableNames?.length) {
       const relatedTableList = [];
 
+      if (!args?.oldMetas)
         // extract tables which have relation with the tables in list
         for (const r of relations) {
           if (args.tableNames.some(t => t.tn === r.tn)) {

packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts

@@ -109,7 +109,7 @@ abstract class BaseModelXcMeta extends BaseRender {
     const oldVirtualCols = this.ctx?.oldMeta?.v || [];
 
     for (const oldVCol of oldVirtualCols) {
-      console.log(oldVCol);
+      if (oldVCol.lk || oldVCol.rl || oldVCol.formula)
         virtualColumns.push(oldVCol);
     }
 

scripts/metaSync/queries.js

@@ -1,54 +0,0 @@
-const init = `CREATE TABLE \`table_col_delete\` (
-  \`id\` int NOT NULL AUTO_INCREMENT,
-  \`title\` varchar(45) DEFAULT NULL,
-  PRIMARY KEY (\`id\`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-
-CREATE TABLE \`table_to_col_add\` (
-  \`id\` int NOT NULL AUTO_INCREMENT,
-  PRIMARY KEY (\`id\`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-
-CREATE TABLE \`table_to_rel_add_child\` (
-  \`id\` int NOT NULL AUTO_INCREMENT,
-  PRIMARY KEY (\`id\`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-
-CREATE TABLE \`table_to_rel_add_parent\` (
-  \`id\` int NOT NULL AUTO_INCREMENT,
-  PRIMARY KEY (\`id\`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-
-CREATE TABLE \`table_to_rel_remove_child\` (
-  \`id\` int NOT NULL AUTO_INCREMENT,
-  \`table_to_rel_remove_parent_id\` int DEFAULT NULL,
-  PRIMARY KEY (\`id\`),
-  KEY \`table_to_rel_remove_parent_fk_idx\` (\`table_to_rel_remove_parent_id\`),
-  CONSTRAINT \`table_to_rel_remove_parent_fk\` FOREIGN KEY (\`table_to_rel_remove_parent_id\`) REFERENCES \`table_to_rel_remove_parent\` (\`id\`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-
-CREATE TABLE \`table_to_rel_remove_parent\` (
-  \`id\` int NOT NULL AUTO_INCREMENT,
-  PRIMARY KEY (\`id\`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-
-CREATE TABLE \`table_to_remove\` (
-  \`id\` int NOT NULL AUTO_INCREMENT,
-  \`table_to_removecol\` varchar(45) NOT NULL,
-  PRIMARY KEY (\`id\`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-`
-
-
-
-const colDel = `ALTER TABLE \`sakila\`.\`table_col_delete\` 
-DROP COLUMN \`title\`;`
-
-const relDel =`ALTER TABLE \`sakila\`.\`table_to_rel_remove_child\` 
-DROP FOREIGN KEY \`table_to_rel_remove_parent_fk\`;
-ALTER TABLE \`sakila\`.\`table_to_rel_remove_child\` 
-DROP COLUMN \`table_to_rel_remove_parent_id\`,
-DROP INDEX \`table_to_rel_remove_parent_fk_idx\` ;
-;
-`
-const tableDel =`DROP TABLE \`sakila\`.\`table_to_remove\`;`

scripts/metaSync/queries.md

@@ -0,0 +1,93 @@
+
+
+### Init
+
+```sql
+CREATE TABLE `table_col_delete` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `title` varchar(45) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE `table_to_col_add` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE `table_to_rel_add_child` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE `table_to_rel_add_parent` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE `table_to_rel_remove_parent` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE `table_to_rel_remove_child` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `table_to_rel_remove_parent_id` int DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  KEY `table_to_rel_remove_parent_fk_idx` (`table_to_rel_remove_parent_id`),
+  CONSTRAINT `table_to_rel_remove_parent_fk` FOREIGN KEY (`table_to_rel_remove_parent_id`) REFERENCES `table_to_rel_remove_parent` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE `table_to_remove` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `table_to_removecol` varchar(45) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+```
+
+
+## Col delete
+
+```sql
+ALTER TABLE `table_col_delete` 
+DROP COLUMN `title`;
+```
+
+## Rel delete
+
+```sql
+ALTER TABLE `table_to_rel_remove_child` 
+DROP FOREIGN KEY `table_to_rel_remove_parent_fk`;
+ALTER TABLE `table_to_rel_remove_child` 
+DROP COLUMN `table_to_rel_remove_parent_id`,
+DROP INDEX `table_to_rel_remove_parent_fk_idx` ;
+```
+
+## table delete
+
+```sql
+DROP TABLE `table_to_remove`;`
+```
+
+## column add
+
+```sql
+ALTER TABLE `table_to_col_add` 
+ADD COLUMN `new_col_1` VARCHAR(45) NULL AFTER `new_col`;
+```
+
+
+## Add rel
+
+```sql
+ALTER TABLE `table_to_rel_add_child` 
+ADD COLUMN `parent_id` INT NULL AFTER `id`,
+ADD INDEX `_p_fk_idx` (`parent_id` ASC) VISIBLE;
+;
+ALTER TABLE `table_to_rel_add_child` 
+ADD CONSTRAINT `_p_fk`
+  FOREIGN KEY (`parent_id`)
+  REFERENCES `table_to_rel_add_parent` (`id`)
+  ON DELETE NO ACTION
+  ON UPDATE NO ACTION;
+```