Browse Source

wip: xcdb meta sync

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/894/head
Pranav C 3 years ago
parent
commit
aea7eb8220
  1. 29
      packages/nc-gui/components/project/projectMetadata/disableOrEnableModels.vue
  2. 18
      packages/nc-gui/components/project/projectMetadata/sync/disableOrEnableTables.vue
  3. 4
      packages/nc-gui/store/tabs.js
  4. 2
      packages/nocodb/src/example/dockerRunMysql.ts
  5. 4
      packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts
  6. 254
      packages/nocodb/src/lib/noco/common/handlers/xcMetaDiffSync.ts
  7. 172
      packages/nocodb/src/lib/noco/meta/handlers/xcMetaDiff.ts

29
packages/nc-gui/components/project/projectMetadata/disableOrEnableModels.vue

@ -19,7 +19,12 @@
Metadata
</v-tab>
<v-tab-item :key="db.meta.dbAlias + 't' + i" :value=" db.meta.dbAlias">
<v-tabs color="x-active" height="28">
<disable-or-enable-tables
:nodes="nodes"
:db="db"
:db-alias="db.meta.dbAlias"
/>
<!-- <v-tabs color="x-active" height="28">
<v-tab class="text-capitalize caption">
Tables
</v-tab>
@ -30,7 +35,7 @@
:db-alias="db.meta.dbAlias"
/>
</v-tab-item>
<!-- enable extra -->
&lt;!&ndash; enable extra &ndash;&gt;
<v-tab class="text-capitalize caption">
Views
</v-tab>
@ -41,7 +46,7 @@
:db-alias="db.meta.dbAlias"
/>
</v-tab-item>
<!-- <v-tab class="text-capitalize caption">Functions</v-tab>
&lt;!&ndash; <v-tab class="text-capitalize caption">Functions</v-tab>
<v-tab-item>
<disable-or-enable-functions :nodes="nodes" :db="db"
:db-alias="db.meta.dbAlias"></disable-or-enable-functions>
@ -50,7 +55,7 @@
<v-tab-item>
<disable-or-enable-procedures :nodes="nodes" :db="db"
:db-alias="db.meta.dbAlias"></disable-or-enable-procedures>
</v-tab-item>-->
</v-tab-item>&ndash;&gt;
<v-tab class="text-capitalize caption">
Relations
@ -58,7 +63,7 @@
<v-tab-item>
<disable-or-enable-relations :nodes="nodes" :db-alias="db.meta.dbAlias" />
</v-tab-item>
</v-tabs>
</v-tabs>-->
</v-tab-item>
<template v-if="uiacl">
<v-tab :key="db.meta.dbAlias + 'acl'" :href="'#' + db.meta.dbAlias + 'acl'" class="text-capitalize caption nc-ui-acl-tab">
@ -116,22 +121,22 @@
<script>
import { mapGetters } from 'vuex'
import XcMeta from '../settings/xcMeta'
import DisableOrEnableRelations from './sync/disableOrEnableRelations'
// import DisableOrEnableRelations from './sync/disableOrEnableRelations'
import { isMetaTable } from '@/helpers/xutils'
import DisableOrEnableTables from '@/components/project/projectMetadata/sync/disableOrEnableTables'
import ToggleTableUiAcl from '@/components/project/projectMetadata/uiAcl/toggleTableUIAcl'
import ToggleRelationsUiAcl from '@/components/project/projectMetadata/uiAcl/toggleRelationsUIAcl'
import DisableOrEnableViews from '~/components/project/projectMetadata/sync/disableOrEnableViews'
// import ToggleRelationsUiAcl from '@/components/project/projectMetadata/uiAcl/toggleRelationsUIAcl'
// import DisableOrEnableViews from '~/components/project/projectMetadata/sync/disableOrEnableViews'
export default {
name: 'DisableOrEnableModels',
components: {
DisableOrEnableViews,
ToggleRelationsUiAcl,
// DisableOrEnableViews,
// ToggleRelationsUiAcl,
ToggleTableUiAcl,
DisableOrEnableTables,
XcMeta,
DisableOrEnableRelations
XcMeta
// DisableOrEnableRelations
},
props: ['nodes'],
data: () => ({

18
packages/nc-gui/components/project/projectMetadata/sync/disableOrEnableTables.vue

@ -71,7 +71,12 @@
</tr>
</thead>
<tbody>
<tr v-for="model in diff" v-show="!filter.trim() || (model.tn || model.title || '').toLowerCase().includes(filter.toLowerCase())" :key="model.title" :class="`nc-metasync-row-${model.tn}`">
<tr
v-for="model in diff"
v-show="!filter.trim() || (model.tn || model.title || '').toLowerCase().includes(filter.toLowerCase())"
:key="model.title"
:class="`nc-metasync-row-${model.tn}`"
>
<!-- v-if="model.alias.toLowerCase().indexOf(filter.toLowerCase()) > -1">-->
<td>
<v-tooltip bottom>
@ -345,6 +350,17 @@ export default {
}, 'xcMetaDiffSync', {}])
this.$toast.success('Table metadata recreated successfully').goAway(3000)
await this.loadXcDiff()
await this.$store.dispatch('project/_loadTables', {
dbKey: '0.projectJson.envs._noco.db.0',
key: '0.projectJson.envs._noco.db.0.tables',
_nodes: {
dbAlias: 'db',
env: '_noco',
type: 'tableDir'
}
})
await this.$store.commit('meta/MutClear')
} catch (e) {
this.$toast[e.response?.status === 402 ? 'info' : 'error'](e.message).goAway(3000)
}

4
packages/nc-gui/store/tabs.js

@ -18,6 +18,10 @@ export const mutations = {
remove(state, index) {
state.list.splice(index, 1)
},
removeTableOrViewTabs(state) {
debugger
// state.list.splice(index, 1)
},
clear(state, index) {
state.list = []
},

2
packages/nocodb/src/example/dockerRunMysql.ts

@ -17,7 +17,7 @@ const date = new Date();
process.env[
`NC_DB`
] = `mysql2://localhost:3306?u=root&p=password&d=meta_${date.getFullYear()}_${date.getMonth() +
1}_${date.getDate() - 1}`;
1}_${date.getDate()}`;
// process.env[`NC_DB`] = `pg://localhost:3306?u=root&p=password&d=mar_24`;
// process.env[`NC_DB`] = `pg://localhost:5432?u=postgres&p=password&d=abcde`;
// process.env[`NC_TRY`] = 'true';

4
packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts

@ -70,6 +70,10 @@ export default abstract class BaseApiBuilder<T extends Noco>
return this.sqlClient?.knex || this.dbDriver;
}
public get prefix() {
return this.projectBuilder?.prefix;
}
public get apiType(): string {
return this.connectionConfig?.meta?.api?.type;
}

254
packages/nocodb/src/lib/noco/common/handlers/xcMetaDiffSync.ts

@ -14,6 +14,7 @@ export default async function(this: BaseApiBuilder<any> | any) {
},
getProjectId: () => this.getProjectId(),
getDbAlias: () => this.getDbAlias(),
getBuilder: () => this,
xcMeta: this.xcMeta
},
{}
@ -25,9 +26,35 @@ export default async function(this: BaseApiBuilder<any> | any) {
columns: {},
oldMetas: {}
};
const populateViewsParams: XcTablesPopulateParams = {
tableNames: [],
type: 'view',
columns: {},
oldMetas: {}
};
// @ts-ignore
const tableList = (await this.sqlClient.tableList())?.data?.list?.filter(
t => {
if (this?.prefix) {
return t.tn?.startsWith(this?.prefix);
}
return true;
}
);
// @ts-ignore
const tableList = (await this.getSqlClient().tableList())?.data?.list;
const viewList = (await this.sqlClient.viewList())?.data?.list
?.map(v => {
v.type = 'view';
v.tn = v.view_name;
return v;
})
.filter(t => {
if (this?.prefix) {
return t.tn?.startsWith(this?.prefix);
}
return true;
});
// @ts-ignore
const relationList = (await this.getSqlClient().tableList())?.data?.list;
@ -38,7 +65,15 @@ export default async function(this: BaseApiBuilder<any> | any) {
{ condition: { type: 'table' } }
);
const oldViewModels = await this.xcMeta.metaList(
this.getProjectId(),
this.getDbAlias(),
'nc_models',
{ condition: { type: 'view' } }
);
const oldMetasRef = {};
const oldViewMetasRef = {};
// @ts-ignore
const oldMetas = oldModels.map(m => {
const meta = JSON.parse(m.meta);
@ -47,6 +82,15 @@ export default async function(this: BaseApiBuilder<any> | any) {
populateParams.oldMetas[meta.tn] = meta;
oldMetasRef[meta.tn] = meta;
return meta;
}); // @ts-ignore
const oldViewMetas = oldViewModels.map(m => {
const meta = JSON.parse(m.meta);
XcCache.del([this.projectId, this.dbAlias, 'table', meta.tn].join('::'));
XcCache.del([this.projectId, this.dbAlias, 'view', meta.tn].join('::'));
meta.id = m.id;
populateViewsParams.oldMetas[meta.tn] = meta;
oldViewMetasRef[meta.tn] = meta;
return meta;
});
const oldQueryParams = oldModels.map(m => JSON.parse(m.query_params));
@ -497,6 +541,213 @@ export default async function(this: BaseApiBuilder<any> | any) {
});
}
break;
case XcMetaDiffType.VIEW_NEW:
// add table to list
populateViewsParams.tableNames.push({ tn });
break;
case XcMetaDiffType.VIEW_REMOVE:
{
// delete table meta and views
// delete lookup relation etc
// todo: enable
await this.deleteTableNameInACL(tn);
await this.xcMeta.metaDelete(
this.projectId,
this.dbAlias,
'nc_shared_views',
{
model_name: tn
}
);
await this.xcMeta.metaDelete(
this.projectId,
this.dbAlias,
'nc_models',
{
type: 'view'
},
{
_or: [
{
title: { eq: tn }
},
{
parent_model_title: { eq: tn }
}
]
}
);
if (delete this.metas[tn]) delete this.metas[tn];
if (delete this.models[tn]) delete this.models[tn];
}
break;
case XcMetaDiffType.VIEW_COLUMN_ADD:
// update old
populateViewsParams.tableNames.push({ tn });
populateViewsParams.oldMetas[tn] = oldViewMetas.find(
m => m.tn === tn
);
break;
case XcMetaDiffType.VIEW_COLUMN_TYPE_CHANGE:
// update type in old meta
populateViewsParams.oldMetas[tn] = oldViewMetas.find(
m => m.tn === tn
);
populateViewsParams.tableNames.push({
tn,
_tn: populateViewsParams.oldMetas[tn]?._tn
});
break;
case XcMetaDiffType.VIEW_COLUMN_REMOVE:
{
const oldViewMetaIdx = oldViewMetas.findIndex(m => m.tn === tn);
if (oldViewMetaIdx === -1)
throw new Error('Old meta not found : ' + tn);
const oldViewMeta = oldViewMetas[oldViewMetaIdx];
populateViewsParams.oldMetas[tn] = oldViewMeta;
populateViewsParams.tableNames.push({
tn,
_tn: populateViewsParams.oldMetas[tn]?._tn
});
const queryParams = oldQueryParams[oldViewMetaIdx];
const oldColumn = oldViewMeta.columns.find(
c => c.cn === change?.cn
);
const {
// virtualViews,
virtualViewsParamsArr
// @ts-ignore
} = await this.extractSharedAndVirtualViewsParams(tn);
// virtual views param update
for (const qp of [queryParams, ...virtualViewsParamsArr]) {
if (!qp) continue;
// @ts-ignore
const {
filters = {},
sortList = [],
showFields = {},
fieldsOrder = [],
extraViewParams = {}
} = qp;
/* update sort field */
/* const sIndex = (sortList || []).findIndex(
v => v.field === oldColumn._cn
);
if (sIndex > -1) {
sortList.splice(sIndex, 1);
}*/
for (const sort of sortList || []) {
if (
sort?.field === oldColumn.cn ||
sort?.field === oldColumn._cn
) {
sortList.splice(sortList.indexOf(sort), 1);
}
}
/* update show field */
if (oldColumn.cn in showFields || oldColumn._cn in showFields) {
delete showFields[oldColumn.cn];
delete showFields[oldColumn._cn];
}
/* update filters */
// todo: remove only corresponding filter and compare field name
/* if (
filters &&
(JSON.stringify(filters)?.includes(`"${oldColumn.cn}"`) ||
JSON.stringify(filters)?.includes(`"${oldColumn._cn}"`))
) {
filters.splice(0, filters.length);
}*/
for (const filter of filters) {
if (
filter?.field === oldColumn.cn ||
filter?.field === oldColumn._cn
) {
filters.splice(filters.indexOf(filter), 1);
}
}
/* update fieldsOrder */
let index = fieldsOrder.indexOf(oldColumn.cn);
if (index > -1) {
fieldsOrder.splice(index, 1);
}
index = fieldsOrder.indexOf(oldColumn._cn);
if (index > -1) {
fieldsOrder.splice(index, 1);
}
/* update formView params */
// extraViewParams.formParams.fields
if (extraViewParams?.formParams?.fields?.[oldColumn.cn]) {
delete extraViewParams.formParams.fields[oldColumn.cn];
}
if (extraViewParams?.formParams?.fields?.[oldColumn._cn]) {
delete extraViewParams.formParams.fields[oldColumn._cn];
}
}
// Delete lookup columns mapping to current column
// update column name in belongs to
if (oldViewMeta.belongsTo?.length) {
for (const bt of oldViewMeta.belongsTo) {
// filter out lookup columns which maps to current col
oldMetasRef[bt.rtn].v = oldMetasRef[bt.rtn].v?.filter(v => {
if (v.lk && v.lk.ltn === tn && v.lk.lcn === oldColumn.cn) {
relationTableMetas.add(oldMetasRef[bt.rtn]);
return false;
}
return true;
});
}
}
// update column name in has many
if (oldViewMeta.hasMany?.length) {
for (const hm of oldViewMeta.hasMany) {
// filter out lookup columns which maps to current col
oldMetasRef[hm.tn].v = oldMetasRef[hm.tn].v?.filter(v => {
if (v.lk && v.lk.ltn === tn && v.lk.lcn === change.cn) {
relationTableMetas.add(oldMetasRef[hm.tn]);
return false;
}
return true;
});
}
}
// update column name in many to many
if (oldViewMeta.manyToMany?.length) {
for (const mm of oldViewMeta.manyToMany) {
// filter out lookup columns which maps to current col
oldMetasRef[mm.rtn].v = oldMetasRef[mm.rtn].v?.filter(v => {
if (v.lk && v.lk.ltn === tn && v.lk.lcn === change.cn) {
relationTableMetas.add(oldMetasRef[mm.rtn]);
return false;
}
return true;
});
}
}
}
break;
}
}
}
@ -516,6 +767,7 @@ export default async function(this: BaseApiBuilder<any> | any) {
return t === populateParams.tableNames.find(t1 => t1.tn === t.tn);
});
await this.xcTablesPopulate(populateParams);
await this.xcTablesPopulate(populateViewsParams);
return populateParams;
}

172
packages/nocodb/src/lib/noco/meta/handlers/xcMetaDiff.ts

@ -6,6 +6,11 @@ enum XcMetaDiffType {
TABLE_COLUMN_ADD = 'TABLE_COLUMN_ADD',
TABLE_COLUMN_TYPE_CHANGE = 'TABLE_COLUMN_TYPE_CHANGE',
TABLE_COLUMN_REMOVE = 'TABLE_COLUMN_REMOVE',
VIEW_NEW = 'VIEW_NEW',
VIEW_REMOVE = 'VIEW_REMOVE',
VIEW_COLUMN_ADD = 'VIEW_COLUMN_ADD',
VIEW_COLUMN_TYPE_CHANGE = 'VIEW_COLUMN_TYPE_CHANGE',
VIEW_COLUMN_REMOVE = 'VIEW_COLUMN_REMOVE',
TABLE_RELATION_ADD = 'TABLE_RELATION_ADD',
TABLE_RELATION_REMOVE = 'TABLE_RELATION_REMOVE',
TABLE_VIRTUAL_RELATION_ADD = 'TABLE_VIRTUAL_RELATION_ADD',
@ -29,11 +34,19 @@ export default async function(
): Promise<Array<NcMetaDiff>> {
const changes = [];
const builder = this.getBuilder(args);
// @ts-ignore
const sqlClient = this.projectGetSqlClient(args);
// @ts-ignore
const tableList = (await sqlClient.tableList())?.data?.list;
const tableList = (await sqlClient.tableList())?.data?.list?.filter(t => {
if (builder?.prefix) {
return t.tn?.startsWith(builder?.prefix);
}
return true;
});
const colListRef = {};
// @ts-ignore
const oldMetas = (
@ -41,7 +54,9 @@ export default async function(
this.getProjectId(args),
this.getDbAlias(args),
'nc_models',
{ condition: { type: 'table' } }
{
condition: { type: 'table' }
}
)
).map(m => JSON.parse(m.meta));
@ -179,6 +194,159 @@ export default async function(
});
}
// @ts-ignore
const viewList = (await sqlClient.viewList())?.data?.list
?.map(v => {
v.type = 'view';
v.tn = v.view_name;
return v;
})
.filter(t => {
if (builder?.prefix) {
return t.tn?.startsWith(builder?.prefix);
}
return true;
}); // @ts-ignore
const oldViewMetas = (
await this.xcMeta.metaList(
this.getProjectId(args),
this.getDbAlias(args),
'nc_models',
{
condition: { type: 'view' }
}
)
).map(m => JSON.parse(m.meta));
for (const view of viewList) {
const oldViewMetaIdx = oldViewMetas.findIndex(m => m.tn === view.tn);
// new table
if (oldViewMetaIdx === -1) {
changes.push({
tn: view.tn,
detectedChanges: [
{
type: XcMetaDiffType.VIEW_NEW,
msg: `New view`
}
]
});
continue;
}
const oldViewMeta = oldViewMetas[oldViewMetaIdx];
oldViewMetas.splice(oldViewMetaIdx, 1);
const viewProp = {
tn: view.tn,
detectedChanges: []
};
changes.push(viewProp);
// check for column change
colListRef[view.tn] = (
await sqlClient.columnList({ tn: view.tn })
)?.data?.list;
for (const column of colListRef[view.tn]) {
const oldColIdx = oldViewMeta.columns.findIndex(c => c.cn === column.cn);
// new table
if (oldColIdx === -1) {
viewProp.detectedChanges.push({
type: XcMetaDiffType.VIEW_COLUMN_ADD,
msg: `New column(${column.cn})`,
cn: column.cn
});
continue;
}
const oldCol = oldViewMeta.columns[oldColIdx];
oldViewMeta.columns.splice(oldColIdx, 1);
if (oldCol.dt !== column.dt) {
viewProp.detectedChanges.push({
type: XcMetaDiffType.VIEW_COLUMN_TYPE_CHANGE,
msg: `Column type changed(${column.cn})`,
cn: oldCol.cn
});
}
}
for (const { cn } of oldViewMeta.columns) {
viewProp.detectedChanges.push({
type: XcMetaDiffType.VIEW_COLUMN_REMOVE,
msg: `Column removed(${cn})`,
cn
});
}
/* for (const vCol of oldViewMeta.v) {
if (!vCol.mm) continue;
// check related tables & columns
const rTable = tableList.find(t => t.tn === vCol.mm?.rtn);
const m2mTable = tableList.find(t => t.tn === vCol.mm?.vtn);
if (!rTable) {
viewProp.detectedChanges.push({
...vCol,
type: XcMetaDiffType.VIEW_VIRTUAL_M2M_REMOVE,
msg: `Many to many removed(${vCol.mm?.rtn} removed)`
});
continue;
}
if (!m2mTable) {
viewProp.detectedChanges.push({
...vCol,
type: XcMetaDiffType.VIEW_VIRTUAL_M2M_REMOVE,
msg: `Many to many removed(${vCol.mm?.vtn} removed)`
});
continue;
}
// verify columns
const pColumns = (colListRef[vCol.mm.tn] =
colListRef[vCol.mm.tn] ||
(await sqlClient.columnList({ tn: vCol.mm.tn }))?.data?.list);
const cColumns = (colListRef[vCol.mm.rtn] =
colListRef[vCol.mm.rtn] ||
(await sqlClient.columnList({ tn: vCol.mm.rtn }))?.data?.list);
const vColumns = (colListRef[vCol.mm.vtn] =
colListRef[vCol.mm.vtn] ||
(await sqlClient.columnList({ tn: vCol.mm.vtn }))?.data?.list);
if (
pColumns.every(c => c.cn !== vCol.mm.cn) ||
cColumns.every(c => c.cn !== vCol.mm.rcn) ||
vColumns.every(c => c.cn !== vCol.mm.vcn) ||
vColumns.every(c => c.cn !== vCol.mm.vrcn)
) {
viewProp.detectedChanges.push({
...vCol,
type: XcMetaDiffType.VIEW_VIRTUAL_M2M_REMOVE,
msg: `Many to many removed(One of the relation column removed)`
});
continue;
}
}*/
}
for (const { tn } of oldViewMetas) {
changes.push({
tn: tn,
detectedChanges: [
{
type: XcMetaDiffType.VIEW_REMOVE,
msg: `View removed`
}
]
});
}
// extract unique relations
const oldRelations = (
await this.xcMeta.metaList(

Loading…
Cancel
Save