Browse Source

feat: meta sync

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/894/head
Pranav C 3 years ago
parent
commit
5150830725
  1. 144
      packages/nc-gui/components/project/projectMetadata/sync/disableOrEnableTables.vue
  2. 2
      packages/nocodb/src/example/dockerRunMysql.ts
  3. 3
      packages/nocodb/src/lib/noco/NcProjectBuilder.ts
  4. 157
      packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts
  5. 570
      packages/nocodb/src/lib/noco/common/handlers/xcMetaDiffSync.ts
  6. 6
      packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts
  7. 2
      packages/nocodb/src/lib/noco/meta/NcMetaIOImpl.ts
  8. 11
      packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts
  9. 202
      packages/nocodb/src/lib/noco/meta/handlers/xcMetaDiff.ts
  10. 41
      packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts
  11. 13
      packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts
  12. 54
      scripts/metaSync/queries.js

144
packages/nc-gui/components/project/projectMetadata/sync/disableOrEnableTables.vue

@ -27,10 +27,20 @@
small
color="primary"
icon="refresh"
@click="loadModels();loadTableList()"
@click="loadXcDiff()"
>
Reload
</x-btn>
<!-- <x-btn
outlined
tooltip="Reload list"
small
color="primary"
icon="refresh"
@click="loadModels();loadTableList()"
>
Reload
</x-btn>-->
<x-btn
outlined
:loading="updating"
@ -50,9 +60,7 @@
<thead>
<tr>
<th class="grey--text">
Models <span v-show="!isNewOrDeletedModelFound" class="caption ml-1">({{
enableCountText
}})</span>
Models</span>
</th>
<!-- <th>APIs</th>-->
<th class="grey--text">
@ -62,12 +70,12 @@
</tr>
</thead>
<tbody>
<tr v-for="model in comparedModelList" :key="model.title">
<tr v-for="model in diff" :key="model.title">
<!-- v-if="model.alias.toLowerCase().indexOf(filter.toLowerCase()) > -1">-->
<td>
<v-tooltip bottom>
<template #activator="{on}">
<span v-on="on">{{ model.alias }}</span>
<span v-on="on">{{ model.tn }}</span>
</template>
<span class="caption">{{ model.title }}</span>
</v-tooltip>
@ -80,16 +88,72 @@
@change="edited = true"
/>
</td>-->
<td>
<x-icon
small
color="primary"
tooltip="Recreate metadata"
>
mdi-reload
</x-icon>
</td>
<td>
<span
v-if="model.detectedChanges && model.detectedChanges.length"
class="caption error--text"
>{{ model.detectedChanges.map(m => m.msg).join(', ') }}</span>
<!-- <span v-else class="caption grey&#45;&#45;text">Recreate metadata.</span>-->
</td>
</tr>
</tbody>
</v-simple-table>
<!-- </div> <div class="d-flex d-100 justify-center">
<v-simple-table dense style="min-width: 400px">
<thead>
<tr>
<th class="grey&#45;&#45;text">
Models <span v-show="!isNewOrDeletedModelFound" class="caption ml-1">({{
enableCountText
}})</span>
</th>
&lt;!&ndash; <th>APIs</th>&ndash;&gt;
<th class="grey&#45;&#45;text">
Actions
</th>
<th />
</tr>
</thead>
<tbody>
<tr v-for="model in comparedModelList" :key="model.title">
&lt;!&ndash; v-if="model.alias.toLowerCase().indexOf(filter.toLowerCase()) > -1">&ndash;&gt;
<td>
<v-tooltip bottom>
<template #activator="{on}">
<span v-on="on">{{ model.alias }}</span>
</template>
<span class="caption">{{ model.title }}</span>
</v-tooltip>
</td>
&lt;!&ndash; <td>
<v-checkbox
v-model="model.enabled"
dense
:disabled="model.new || model.deleted"
@change="edited = true"
/>
</td>&ndash;&gt;
<td>
<template v-if="model.new">
<!-- <x-icon small color="success success" tooltip="Add and sync meta information"-->
<!-- @click="addTableMeta([model.title])">mdi-plus-circle-outline-->
<!-- </x-icon>-->
&lt;!&ndash; <x-icon small color="success success" tooltip="Add and sync meta information"&ndash;&gt;
&lt;!&ndash; @click="addTableMeta([model.title])">mdi-plus-circle-outline&ndash;&gt;
&lt;!&ndash; </x-icon>&ndash;&gt;
</template>
<template v-else-if="model.deleted">
<!-- <x-icon small v-else-if="model.deleted" color="error error" tooltip="Delete meta information"-->
<!-- @click="deleteTableMeta([model.title])">mdi-delete-outline-->
<!-- </x-icon>-->
&lt;!&ndash; <x-icon small v-else-if="model.deleted" color="error error" tooltip="Delete meta information"&ndash;&gt;
&lt;!&ndash; @click="deleteTableMeta([model.title])">mdi-delete-outline&ndash;&gt;
&lt;!&ndash; </x-icon>&ndash;&gt;
</template>
<x-icon
v-else
@ -105,14 +169,15 @@
<td>
<span
v-if="model.new"
class="caption success--text"
class="caption success&#45;&#45;text"
>New table found in DB. Yet to be synced.</span>
<span v-else-if="model.deleted" class="caption error--text">This table doesn't exist in DB. Yet to be synced.</span>
<!-- <span v-else class="caption grey&#45;&#45;text">Recreate metadata.</span>-->
<span v-else-if="model.deleted" class="caption error&#45;&#45;text">This table doesn't exist in DB. Yet to be synced.</span>
&lt;!&ndash; <span v-else class="caption grey&#45;&#45;text">Recreate metadata.</span>&ndash;&gt;
</td>
</tr>
</tbody>
</v-simple-table>
</div>-->
</div>
</v-card>
</v-col>
@ -155,8 +220,10 @@
</v-tooltip>
<v-spacer />
</div>
<div v-if="isNewOrDeletedModelFound" class="d-flex justify-center">
<x-btn
<!-- <div-->
<!-- v-if="isNewOrDeletedModelFound" -->
<div class="d-flex justify-center">
<!-- <x-btn
x-large
btn.class="mx-auto primary nc-btn-sync-meta-data"
tooltip="Sync metadata"
@ -166,6 +233,18 @@
mdi-database-sync
</v-icon>
Sync Now
</x-btn>-->
<x-btn
x-large
btn.class="mx-auto primary"
tooltip="Sync metadata"
@click="syncMetaDiff"
>
<v-icon color="white" class="mr-2 mt-n1">
mdi-database-sync
</v-icon>
Sync Now
</x-btn>
</div>
</v-col>
@ -193,13 +272,21 @@ export default {
updating: false,
dbsTab: 0,
filter: '',
tables: null
tables: null,
diff: null
}),
async mounted() {
await this.loadXcDiff()
await this.loadModels()
await this.loadTableList()
},
methods: {
async loadXcDiff() {
this.diff = await this.$store.dispatch('sqlMgr/ActSqlOp', [{
dbAlias: this.db.meta.dbAlias,
env: this.$store.getters['project/GtrEnv']
}, 'xcMetaDiff'])
},
async addTableMeta(tables) {
try {
await this.$store.dispatch('sqlMgr/ActSqlOp', [{
@ -243,6 +330,19 @@ export default {
}
},
async syncMetaDiff() {
try {
await this.$store.dispatch('sqlMgr/ActSqlOp', [{
dbAlias: this.db.meta.dbAlias,
env: this.$store.getters['project/GtrEnv']
}, 'xcMetaDiffSync', {}])
this.$toast.success('Table metadata recreated successfully').goAway(3000)
await this.loadXcDiff()
} catch (e) {
this.$toast[e.response?.status === 402 ? 'info' : 'error'](e.message).goAway(3000)
}
},
async recreateTableMeta(table) {
try {
await this.$store.dispatch('sqlMgr/ActSqlOp', [{
@ -307,8 +407,12 @@ export default {
comparedModelList() {
const res = []
const getPriority = (item) => {
if (item.new) { return 2 }
if (item.deleted) { return 1 }
if (item.new) {
return 2
}
if (item.deleted) {
return 1
}
return 0
}
if (this.tables && this.models) {

2
packages/nocodb/src/example/dockerRunMysql.ts

@ -17,7 +17,7 @@ const date = new Date();
process.env[
`NC_DB`
] = `mysql2://localhost:3306?u=root&p=password&d=meta_${date.getFullYear()}_${date.getMonth() +
1}_${date.getDate()}`;
1}_${date.getDate() - 1}`;
// process.env[`NC_DB`] = `pg://localhost:3306?u=root&p=password&d=mar_24`;
// process.env[`NC_DB`] = `pg://localhost:5432?u=postgres&p=password&d=abcde`;
// process.env[`NC_TRY`] = 'true';

3
packages/nocodb/src/lib/noco/NcProjectBuilder.ts

@ -397,6 +397,9 @@ export default class NcProjectBuilder {
await curBuilder.onProcedureCreate(procedure);
}
}
case 'xcMetaDiffSync':
await curBuilder.xcMetaDiffSync();
break;
case 'tableMetaCreate':
XCEeError.throw();
break;

157
packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts

@ -31,6 +31,7 @@ import addErrorOnColumnDeleteInFormula from './helpers/addErrorOnColumnDeleteInF
import ncModelsOrderUpgrader from './jobs/ncModelsOrderUpgrader';
import ncParentModelTitleUpgrader from './jobs/ncParentModelTitleUpgrader';
import ncRemoveDuplicatedRelationRows from './jobs/ncRemoveDuplicatedRelationRows';
import xcMetaDiffSync from './handlers/xcMetaDiffSync';
const log = debug('nc:api:base');
@ -181,7 +182,12 @@ export default abstract class BaseApiBuilder<T extends Noco>
}
public getSqlClient(): any {
return this.sqlClient;
return NcConnectionMgr.getSqlClient({
dbAlias: this.dbAlias,
env: this.config.env,
config: this.config,
projectId: this.projectId
});
}
public abstract onViewCreate(viewName: string): Promise<void>;
@ -208,15 +214,22 @@ export default abstract class BaseApiBuilder<T extends Noco>
public async onTableDelete(
tn: string,
extras?: { ignoreVirtualRelations?: boolean; ignoreViews?: boolean }
extras?: {
ignoreVirtualRelations?: boolean;
ignoreRelations?: boolean;
ignoreViews?: boolean;
}
): Promise<void> {
this.baseLog(`onTableDelete : '%s'`, tn);
XcCache.del([this.projectId, this.dbAlias, 'table', tn].join('::'));
if (!extras?.ignoreRelations)
await this.xcMeta.metaDelete(
this.projectId,
this.dbAlias,
'nc_relations',
null,
{
_and: [
{
_or: [
{
@ -229,8 +242,12 @@ export default abstract class BaseApiBuilder<T extends Noco>
eq: tn
}
}
],
...(extras?.ignoreVirtualRelations ? { type: { eq: 'virtual' } } : {})
]
},
...(extras?.ignoreVirtualRelations
? [{ type: { neq: 'virtual' } }]
: [])
]
}
);
@ -1399,10 +1416,11 @@ export default abstract class BaseApiBuilder<T extends Noco>
return this.xcMeta;
}
public async xcTablesRowDelete(tn: string): Promise<void> {
public async xcTablesRowDelete(tn: string, extras?: any): Promise<void> {
await this.xcMeta.metaDelete(this.projectId, this.dbAlias, 'nc_models', {
title: tn
});
if (!extras?.ignoreViews)
await this.xcMeta.metaDelete(this.projectId, this.dbAlias, 'nc_models', {
parent_model_title: tn
});
@ -2957,17 +2975,47 @@ export default abstract class BaseApiBuilder<T extends Noco>
return this.metas?.[tableName];
}
/* protected async getAndSync(tableName: string): Promise<any[]> {
const tableRelations = (await this.getRelationList()).filter(
rel => rel.tn === tableName
);
const existingTableRelations = await this.xcMeta.metaList(
this.projectId,
this.dbAlias,
'nc_relations',
{
xcCondition: {
type: 'real',
_or: [
{
tn: {
eq: tableName
}
},
{
rtn: {
eq: tableName
}
}
]
}
}
);
ret;
}*/
public async onTableMetaRecreate(tableName: string): Promise<void> {
this.baseLog(`onTableMetaRecreate : '%s'`, tableName);
const oldMeta = this.getMeta(tableName);
const virtualRelations = await this.xcMeta.metaList(
this.projectId,
this.dbAlias,
'nc_relations',
{
xcCondition: {
type: 'virtual',
_or: [
{
tn: {
@ -2983,18 +3031,35 @@ export default abstract class BaseApiBuilder<T extends Noco>
}
}
);
const colListRef = {};
const tableList =
(await this.getSqlClient()?.tableList())?.data?.list || [];
colListRef[tableName] = await this.getColumnList(tableName);
// @ts-ignore
const relations = await this.getRelationList();
for (const vCol of oldMeta.v || []) {
if (vCol.lk) {
}
if (vCol.rl) {
}
}
const tableList = (await this.getSqlClient().tableList()?.data?.list) || [];
for (const rel of virtualRelations) {
const tnColList = await this.getColumnList(rel.tn);
const rtnColList = await this.getColumnList(rel.rtn);
colListRef[rel.tn] =
colListRef[rel.tn] || (await this.getColumnList(rel.tn));
colListRef[rel.rtn] =
colListRef[rel.rtn] || (await this.getColumnList(rel.rtn));
// todo: compare with real relation list
if (
!(
tableList.find(t => t.tn === rel.rtn) &&
tableList.find(t => t.tn === rel.tn) &&
tnColList.find(t => t.cn === rel.cn) &&
rtnColList.find(t => t.cn === rel.rcn)
colListRef[rel.tn].find(t => t.cn === rel.cn) &&
colListRef[rel.rtn].find(t => t.cn === rel.rcn)
)
)
await this.xcMeta.metaDelete(
@ -3004,6 +3069,7 @@ export default abstract class BaseApiBuilder<T extends Noco>
rel.id
);
}
// todo : handle query params
const oldModelRow = await this.xcMeta.metaGet(
this.projectId,
@ -3011,8 +3077,9 @@ export default abstract class BaseApiBuilder<T extends Noco>
'nc_models',
{ title: tableName }
);
await this.onTableDelete(tableName, {
ignoreVirtualRelations: true,
ignoreRelations: true,
ignoreViews: true
} as any);
@ -3028,12 +3095,20 @@ export default abstract class BaseApiBuilder<T extends Noco>
virtualViewsParamsArr
} = await this.extractSharedAndVirtualViewsParams(tableName);
await this.onTableCreate(tableName, { oldMeta: oldMeta });
for (const oldColumn of oldMeta.columns) {
if (colListRef[tableName].find(c => c.cn === oldColumn.cn)) continue;
addErrorOnColumnDeleteInFormula({
virtualColumns: oldMeta.v,
columnName: oldColumn.cn
});
}
await this.onTableCreate(tableName, { oldMeta });
const meta = this.getMeta(tableName);
for (const oldColumn of oldMeta.columns) {
if (meta.columns.find(c => c.c === oldColumn.cn)) continue;
if (meta.columns.find(c => c.cn === oldColumn.cn)) continue;
// virtual views param update
for (const qp of [queryParams, ...virtualViewsParamsArr]) {
@ -3047,35 +3122,46 @@ export default abstract class BaseApiBuilder<T extends Noco>
fieldsOrder = [],
extraViewParams = {}
} = qp;
/* update sort field */
const sIndex = (sortList || []).findIndex(
v => v.field === oldColumn.cno
v => v.field === oldColumn._cn
);
if (sIndex > -1) {
sortList.splice(sIndex, 1);
}
/* update show field */
if (oldColumn.cno in showFields) {
delete showFields[oldColumn.cno];
if (oldColumn.cn in showFields || oldColumn._cn in showFields) {
delete showFields[oldColumn.cn];
delete showFields[oldColumn._cn];
}
/* update filters */
// todo: remove only corresponding filter and compare field name
if (
filters &&
JSON.stringify(filters)?.includes(`"${oldColumn.cno}"`)
(JSON.stringify(filters)?.includes(`"${oldColumn.cn}"`) ||
JSON.stringify(filters)?.includes(`"${oldColumn._cn}"`))
) {
filters.splice(0, filters.length);
}
/* update fieldsOrder */
const index = fieldsOrder.indexOf(oldColumn.cno);
let index = fieldsOrder.indexOf(oldColumn.cn);
if (index > -1) {
fieldsOrder.splice(index, 1);
}
index = fieldsOrder.indexOf(oldColumn._cn);
if (index > -1) {
fieldsOrder.splice(index, 1);
}
/* update formView params */
// extraViewParams.formParams.fields
if (extraViewParams?.formParams?.fields?.[oldColumn.cno]) {
delete extraViewParams.formParams.fields[oldColumn.cno];
if (extraViewParams?.formParams?.fields?.[oldColumn.cn]) {
delete extraViewParams.formParams.fields[oldColumn.cn];
}
if (extraViewParams?.formParams?.fields?.[oldColumn._cn]) {
delete extraViewParams.formParams.fields[oldColumn._cn];
}
}
}
@ -3109,6 +3195,12 @@ export default abstract class BaseApiBuilder<T extends Noco>
)?.max || 0;
return order;
}
public async xcMetaDiffSync(): Promise<void> {
return xcMetaDiffSync.call(this);
}
public abstract xcTablesPopulate(args?: XcTablesPopulateParams): Promise<any>;
}
interface NcBuilderUpgraderCtx {
@ -3134,8 +3226,25 @@ interface NcMetaData {
[key: string]: any;
}
export { IGNORE_TABLES, NcBuilderUpgraderCtx, NcMetaData };
type XcTablesPopulateParams = {
tableNames?: Array<{
tn: string;
_tn?: string;
}>;
type?: 'table' | 'view' | 'function' | 'procedure';
columns?: {
[tn: string]: any;
};
oldMetas?: {
[tn: string]: NcMetaData;
};
};
export {
IGNORE_TABLES,
NcBuilderUpgraderCtx,
NcMetaData,
XcTablesPopulateParams
};
/**
* @copyright Copyright (c) 2021, Xgene Cloud Ltd

570
packages/nocodb/src/lib/noco/common/handlers/xcMetaDiffSync.ts

@ -0,0 +1,570 @@
import BaseApiBuilder, { XcTablesPopulateParams } from '../BaseApiBuilder';
import xcMetaDiff, {
NcMetaDiff as NcMetaDiffType,
XcMetaDiffType
} from '../../meta/handlers/xcMetaDiff';
import XcCache from '../../plugins/adapters/cache/XcCache';
// @ts-ignore
export default async function(this: BaseApiBuilder<any> | any) {
const changes: Array<NcMetaDiffType> = await xcMetaDiff.call(
{
projectGetSqlClient: () => {
return this.getSqlClient();
},
getProjectId: () => this.getProjectId(),
getDbAlias: () => this.getDbAlias(),
xcMeta: this.xcMeta
},
{}
);
const populateParams: XcTablesPopulateParams = {
tableNames: [],
type: 'table',
columns: {},
oldMetas: {}
};
// @ts-ignore
const tableList = (await this.getSqlClient().tableList())?.data?.list;
// @ts-ignore
const relationList = (await this.getSqlClient().tableList())?.data?.list;
const oldModels = await this.xcMeta.metaList(
this.getProjectId(),
this.getDbAlias(),
'nc_models',
{ condition: { type: 'table' } }
);
// @ts-ignore
const oldMetas = oldModels.map(m => {
const meta = JSON.parse(m.meta);
XcCache.del([this.projectId, this.dbAlias, 'table', meta.tn].join('::'));
meta.id = m.id;
return meta;
});
const oldQueryParams = oldModels.map(m => JSON.parse(m.query_params));
const relationTableMetas = new Set();
for (const { tn, detectedChanges } of changes) {
if (!detectedChanges?.length) continue;
for (const change of detectedChanges) {
switch (change.type) {
case XcMetaDiffType.TABLE_NEW:
// add table to list
populateParams.tableNames.push({ tn });
break;
case XcMetaDiffType.TABLE_REMOVE:
{
// delete table meta and views
// delete lookup relation etc
// todo: enable
await this.deleteTableNameInACL(tn);
await this.xcMeta.metaDelete(
this.projectId,
this.dbAlias,
'nc_shared_views',
{
model_name: tn
}
);
await this.xcMeta.metaDelete(
this.projectId,
this.dbAlias,
'nc_models',
{
title: tn
}
);
if (delete this.metas[tn]) delete this.metas[tn];
if (delete this.models[tn]) delete this.models[tn];
}
break;
case XcMetaDiffType.TABLE_COLUMN_ADD:
// update old
populateParams.tableNames.push({ tn });
populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn);
break;
case XcMetaDiffType.TABLE_COLUMN_TYPE_CHANGE:
// update type in old meta
populateParams.oldMetas[tn] = oldMetas.find(m => m.tn === tn);
populateParams.tableNames.push({
tn,
_tn: populateParams.oldMetas[tn]?._tn
});
break;
case XcMetaDiffType.TABLE_COLUMN_REMOVE:
{
const oldMetaIdx = oldMetas.findIndex(m => m.tn === tn);
if (oldMetaIdx === -1)
throw new Error('Old meta not found : ' + tn);
const oldMeta = oldMetas[oldMetaIdx];
populateParams.oldMetas[tn] = oldMeta;
populateParams.tableNames.push({
tn,
_tn: populateParams.oldMetas[tn]?._tn
});
const queryParams = oldQueryParams[oldMetaIdx];
const oldColumn = oldMeta.columns.find(c => c.cn === change?.cn);
const {
// virtualViews,
virtualViewsParamsArr
// @ts-ignore
} = await this.extractSharedAndVirtualViewsParams(tn);
// virtual views param update
for (const qp of [queryParams, ...virtualViewsParamsArr]) {
if (!qp) continue;
// @ts-ignore
const {
filters = {},
sortList = [],
showFields = {},
fieldsOrder = [],
extraViewParams = {}
} = qp;
/* update sort field */
/* const sIndex = (sortList || []).findIndex(
v => v.field === oldColumn._cn
);
if (sIndex > -1) {
sortList.splice(sIndex, 1);
}*/
for (const sort of sortList || []) {
if (
sort?.field === oldColumn.cn ||
sort?.field === oldColumn._cn
) {
sortList.splice(sortList.indexOf(sort), 1);
}
}
/* update show field */
if (oldColumn.cn in showFields || oldColumn._cn in showFields) {
delete showFields[oldColumn.cn];
delete showFields[oldColumn._cn];
}
/* update filters */
// todo: remove only corresponding filter and compare field name
/* if (
filters &&
(JSON.stringify(filters)?.includes(`"${oldColumn.cn}"`) ||
JSON.stringify(filters)?.includes(`"${oldColumn._cn}"`))
) {
filters.splice(0, filters.length);
}*/
for (const filter of filters) {
if (
filter?.field === oldColumn.cn ||
filter?.field === oldColumn._cn
) {
filters.splice(filters.indexOf(filter), 1);
}
}
/* update fieldsOrder */
let index = fieldsOrder.indexOf(oldColumn.cn);
if (index > -1) {
fieldsOrder.splice(index, 1);
}
index = fieldsOrder.indexOf(oldColumn._cn);
if (index > -1) {
fieldsOrder.splice(index, 1);
}
/* update formView params */
// extraViewParams.formParams.fields
if (extraViewParams?.formParams?.fields?.[oldColumn.cn]) {
delete extraViewParams.formParams.fields[oldColumn.cn];
}
if (extraViewParams?.formParams?.fields?.[oldColumn._cn]) {
delete extraViewParams.formParams.fields[oldColumn._cn];
}
}
// Delete lookup columns mapping to current column
// update column name in belongs to
if (oldMeta.belongsTo?.length) {
for (const bt of oldMeta.belongsTo) {
// filter out lookup columns which maps to current col
this.metas[bt.rtn].v = this.metas[bt.rtn].v?.filter(v => {
if (v.lk && v.lk.ltn === tn && v.lk.lcn === oldColumn.cn) {
relationTableMetas.add(this.metas[bt.rtn]);
return false;
}
return true;
});
}
}
// update column name in has many
if (oldMeta.hasMany?.length) {
for (const hm of oldMeta.hasMany) {
// filter out lookup columns which maps to current col
this.metas[hm.tn].v = this.metas[hm.tn].v?.filter(v => {
if (v.lk && v.lk.ltn === tn && v.lk.lcn === change.cn) {
relationTableMetas.add(this.metas[hm.tn]);
return false;
}
return true;
});
}
}
// update column name in many to many
if (oldMeta.manyToMany?.length) {
for (const mm of oldMeta.manyToMany) {
// filter out lookup columns which maps to current col
this.metas[mm.rtn].v = this.metas[mm.rtn].v?.filter(v => {
if (v.lk && v.lk.ltn === tn && v.lk.lcn === change.cn) {
relationTableMetas.add(this.metas[mm.rtn]);
return false;
}
return true;
});
}
}
}
break;
case XcMetaDiffType.TABLE_RELATION_ADD:
{
if (change.tn === tn)
// todo : enable
// ignore duplicate
await this.xcMeta.metaInsert(
this.projectId,
this.dbAlias,
'nc_relations',
{
tn: change.tn,
_tn: this.getTableNameAlias(change.tn),
cn: change.cn,
rtn: change.rtn,
_rtn: this.getTableNameAlias(change.rtn),
rcn: change.rcn,
type: 'real',
db_type: this.connectionConfig?.client
// todo: get these info
/* dr: ,
ur: onUpdate,
fkn*/
}
);
}
break;
case XcMetaDiffType.TABLE_RELATION_REMOVE:
{
// todo: remove from nc_relations
// todo:enable
await this.xcMeta.metaDelete(
this.projectId,
this.dbAlias,
'nc_relations',
{
tn: change.tn,
cn: change.cn,
rtn: change.rtn,
rcn: change.rcn,
type: 'real'
// db_type: this.connectionConfig?.client
}
);
await this.deleteRelationInACL(change.rtn, change.tn);
for (const tn of [change.tn, change.rtn]) {
const {
// @ts-ignore
virtualViews,
virtualViewsParamsArr
} = await this.extractSharedAndVirtualViewsParams(tn);
const oldMeta = oldMetas.find(m => m.tn === tn);
populateParams.oldMetas[tn] = oldMeta;
populateParams.tableNames.push({
tn,
_tn: populateParams.oldMetas[tn]?._tn
});
// extract alias of relation virtual column
const relation = change.tn === tn ? 'bt' : 'hm';
const alias = oldMeta?.v?.find(
v =>
v?.[relation]?.tn === change.tn &&
v?.[relation]?.rtn === change.rtn
)?._cn;
// virtual views param update
for (const qp of virtualViewsParamsArr) {
// @ts-ignore
const {
showFields = {},
fieldsOrder,
extraViewParams = {}
} = qp;
/* update show field */
if (alias in showFields) {
delete showFields[alias];
}
/* update fieldsOrder */
const index = fieldsOrder.indexOf(alias);
if (index > -1) {
fieldsOrder.splice(index, 1);
}
/* update formView params */
if (extraViewParams?.formParams?.fields?.[alias]) {
delete extraViewParams.formParams.fields[alias];
}
}
// todo: enable
await this.updateSharedAndVirtualViewsParams(
virtualViewsParamsArr,
virtualViews
);
}
// todo: bt
const childMeta = oldMetas.find(m => m.tn === change.tn);
Object.assign(childMeta, {
v: childMeta.v.filter(
({ bt, lk }) =>
(!bt || bt.rtn !== change.rtn || bt.tn !== change.tn) &&
!(
lk &&
lk.type === 'bt' &&
lk.rtn === change.rtn &&
lk.tn === change.tn
)
)
});
// todo: hm
const parentMeta = oldMetas.find(m => m.tn === change.rtn);
Object.assign(parentMeta, {
v: parentMeta.v.filter(
({ hm, lk, rl }) =>
(!hm || hm.rtn !== change.rtn || hm.tn !== change.tn) &&
!(
lk &&
lk.type === 'hm' &&
lk.rtn === change.rtn &&
lk.tn === change.tn
) &&
!(
rl &&
rl.type === 'hm' &&
rl.rtn === change.rtn &&
rl.tn === change.tn
)
)
});
}
break;
}
}
}
// todo: optimize
// remove duplicate from list
populateParams.tableNames = populateParams.tableNames?.filter(t => {
return t === populateParams.tableNames.find(t1 => t1.tn === t.tn);
});
await this.xcTablesPopulate(populateParams);
return populateParams;
}
// public async onTableMetaRecreate(tableName: string): Promise<void> {
// this.baseLog(`onTableMetaRecreate : '%s'`, tableName);
// const oldMeta = this.getMeta(tableName);
//
// const virtualRelations = await this.xcMeta.metaList(
// this.projectId,
// this.dbAlias,
// 'nc_relations',
// {
// xcCondition: {
// _or: [
// {
// tn: {
// eq: tableName
// }
// },
// {
// rtn: {
// eq: tableName
// }
// }
// ]
// }
// }
// );
// const colListRef = {};
// const tableList =
// (await this.getSqlClient()?.tableList())?.data?.list || [];
//
// colListRef[tableName] = await this.getColumnList(tableName);
//
// // @ts-ignore
// const relations = await this.getRelationList();
//
// for (const vCol of oldMeta.v || []) {
// if (vCol.lk) {
// }
// if (vCol.rl) {
// }
// }
//
// for (const rel of virtualRelations) {
// colListRef[rel.tn] =
// colListRef[rel.tn] || (await this.getColumnList(rel.tn));
// colListRef[rel.rtn] =
// colListRef[rel.rtn] || (await this.getColumnList(rel.rtn));
//
// // todo: compare with real relation list
// if (
// !(
// tableList.find(t => t.tn === rel.rtn) &&
// tableList.find(t => t.tn === rel.tn) &&
// colListRef[rel.tn].find(t => t.cn === rel.cn) &&
// colListRef[rel.rtn].find(t => t.cn === rel.rcn)
// )
// )
// await this.xcMeta.metaDelete(
// this.projectId,
// this.dbAlias,
// 'nc_relations',
// rel.id
// );
// }
//
// // todo : handle query params
// const oldModelRow = await this.xcMeta.metaGet(
// this.projectId,
// this.dbAlias,
// 'nc_models',
// { title: tableName }
// );
//
// await this.onTableDelete(tableName, {
// ignoreRelations: true,
// ignoreViews: true
// } as any);
//
// let queryParams: any;
// try {
// queryParams = JSON.parse(oldModelRow.query_params);
// } catch (e) {
// queryParams = {};
// }
//
// const {
// virtualViews,
// virtualViewsParamsArr
// } = await this.extractSharedAndVirtualViewsParams(tableName);
//
// for (const oldColumn of oldMeta.columns) {
// if (colListRef[tableName].find(c => c.cn === oldColumn.cn)) continue;
// addErrorOnColumnDeleteInFormula({
// virtualColumns: oldMeta.v,
// columnName: oldColumn.cn
// });
// }
//
// await this.onTableCreate(tableName, { oldMeta });
//
// const meta = this.getMeta(tableName);
//
// for (const oldColumn of oldMeta.columns) {
// if (meta.columns.find(c => c.cn === oldColumn.cn)) continue;
//
// // virtual views param update
// for (const qp of [queryParams, ...virtualViewsParamsArr]) {
// if (!qp) continue;
//
// // @ts-ignore
// const {
// filters = {},
// sortList = [],
// showFields = {},
// fieldsOrder = [],
// extraViewParams = {}
// } = qp;
//
// /* update sort field */
// const sIndex = (sortList || []).findIndex(
// v => v.field === oldColumn._cn
// );
// if (sIndex > -1) {
// sortList.splice(sIndex, 1);
// }
// /* update show field */
// if (oldColumn.cn in showFields || oldColumn._cn in showFields) {
// delete showFields[oldColumn.cn];
// delete showFields[oldColumn._cn];
// }
// /* update filters */
// // todo: remove only corresponding filter and compare field name
// if (
// filters &&
// (JSON.stringify(filters)?.includes(`"${oldColumn.cn}"`) ||
// JSON.stringify(filters)?.includes(`"${oldColumn._cn}"`))
// ) {
// filters.splice(0, filters.length);
// }
//
// /* update fieldsOrder */
// let index = fieldsOrder.indexOf(oldColumn.cn);
// if (index > -1) {
// fieldsOrder.splice(index, 1);
// }
// index = fieldsOrder.indexOf(oldColumn._cn);
// if (index > -1) {
// fieldsOrder.splice(index, 1);
// }
//
// /* update formView params */
// // extraViewParams.formParams.fields
// if (extraViewParams?.formParams?.fields?.[oldColumn.cn]) {
// delete extraViewParams.formParams.fields[oldColumn.cn];
// }
// if (extraViewParams?.formParams?.fields?.[oldColumn._cn]) {
// delete extraViewParams.formParams.fields[oldColumn._cn];
// }
// }
// }
//
// await this.updateSharedAndVirtualViewsParams(
// virtualViewsParamsArr,
// virtualViews
// );
// await this.xcMeta.metaUpdate(
// this.projectId,
// this.dbAlias,
// 'nc_models',
// {
// query_params: JSON.stringify(queryParams)
// },
// { title: tableName, type: 'table' }
// );
// }

6
packages/nocodb/src/lib/noco/gql/GqlApiBuilder.ts

@ -203,7 +203,7 @@ export class GqlApiBuilder extends BaseApiBuilder<Noco> implements XcMetaMgr {
await super.onTableDelete(tn, extras);
this.log(`onTableDelete : '%s' `, tn);
delete this.models[tn];
await this.xcTablesRowDelete(tn);
await this.xcTablesRowDelete(tn, extras);
delete this.resolvers[tn];
delete this.schemas[tn];
@ -1250,9 +1250,9 @@ export class GqlApiBuilder extends BaseApiBuilder<Noco> implements XcMetaMgr {
}
// NOTE: xc-meta
public async xcTablesRowDelete(tn: string): Promise<void> {
public async xcTablesRowDelete(tn: string, extras?: any): Promise<void> {
this.log(`xcTablesRowDelete : Deleting metadata of '%s' table`, tn);
await super.xcTablesRowDelete(tn);
await super.xcTablesRowDelete(tn, extras);
await this.xcMeta.metaDelete(this.projectId, this.dbAlias, 'nc_resolvers', {
title: tn

2
packages/nocodb/src/lib/noco/meta/NcMetaIOImpl.ts

@ -234,6 +234,8 @@ export default class NcMetaIOImpl extends NcMetaIO {
query.select(...args.fields);
}
console.log(query.toQuery());
return query;
}

11
packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts

@ -38,6 +38,7 @@ import { promisify } from 'util';
import NcTemplateParser from '../../templateParser/NcTemplateParser';
import UITypes from '../../sqlUi/UITypes';
import { defaultConnectionConfig } from '../../utils/NcConfigFactory';
import xcMetaDiff from './handlers/xcMetaDiff';
const XC_PLUGIN_DET = 'XC_PLUGIN_DET';
@ -1437,6 +1438,9 @@ export default class NcMetaMgr {
case 'xcVirtualTableDelete':
result = await this.xcVirtualTableDelete(args, req);
break;
case 'xcMetaDiff':
result = await this.xcMetaDiff(args, req);
break;
case 'xcVirtualTableList':
result = await this.xcVirtualTableList(args, req);
break;
@ -1946,6 +1950,7 @@ export default class NcMetaMgr {
case 'functionMetaCreate':
case 'functionMetaDelete':
case 'functionMetaRecreate':
case 'xcMetaDiffSync':
result = { msg: 'success' };
break;
@ -4973,7 +4978,7 @@ export default class NcMetaMgr {
}
for (const [title, aclObj] of Object.entries(viewsObj)) {
for (const role in result[title].disabled) {
for (const role in result[title]?.disabled || []) {
result[title].disabled[role] = Object.values(aclObj).every(
v => v[role]
);
@ -5464,6 +5469,10 @@ export default class NcMetaMgr {
return res;
}
protected async xcMetaDiff(args, req): Promise<any> {
return xcMetaDiff.call(this, { args, req });
}
// @ts-ignore
protected async eeVerify() {
try {

202
packages/nocodb/src/lib/noco/meta/handlers/xcMetaDiff.ts

@ -0,0 +1,202 @@
import NcMetaMgr from '../NcMetaMgr';
enum XcMetaDiffType {
TABLE_NEW = 'TABLE_NEW',
TABLE_REMOVE = 'TABLE_REMOVE',
TABLE_COLUMN_ADD = 'TABLE_COLUMN_ADD',
TABLE_COLUMN_TYPE_CHANGE = 'TABLE_COLUMN_TYPE_CHANGE',
TABLE_COLUMN_REMOVE = 'TABLE_COLUMN_REMOVE',
TABLE_RELATION_ADD = 'TABLE_RELATION_ADD',
TABLE_RELATION_REMOVE = 'TABLE_RELATION_REMOVE',
TABLE_VIRTUAL_RELATION_ADD = 'TABLE_VIRTUAL_RELATION_ADD',
TABLE_VIRTUAL_RELATION_DELETE = 'TABLE_VIRTUAL_RELATION_DELETE'
}
interface NcMetaDiff {
tn: string;
detectedChanges: Array<{
type: XcMetaDiffType;
msg?: string;
[key: string]: any;
}>;
}
// @ts-ignore
export default async function(
this: NcMetaMgr,
{ args }: { args: any; req: any }
): Promise<Array<NcMetaDiff>> {
const changes = [];
// @ts-ignore
const sqlClient = this.projectGetSqlClient(args);
// @ts-ignore
const tableList = (await sqlClient.tableList())?.data?.list;
// @ts-ignore
const oldMetas = (
await this.xcMeta.metaList(
this.getProjectId(args),
this.getDbAlias(args),
'nc_models',
{ condition: { type: 'table' } }
)
).map(m => JSON.parse(m.meta));
for (const table of tableList) {
if (table.tn === 'nc_evolutions') continue;
const oldMetaIdx = oldMetas.findIndex(m => m.tn === table.tn);
// new table
if (oldMetaIdx === -1) {
changes.push({
tn: table.tn,
detectedChanges: [
{
type: XcMetaDiffType.TABLE_NEW,
msg: `New table`
}
]
});
continue;
}
const oldMeta = oldMetas[oldMetaIdx];
oldMetas.splice(oldMetaIdx, 1);
const tableProp = {
tn: table.tn,
detectedChanges: []
};
changes.push(tableProp);
// check for column change
const columnList = (await sqlClient.columnList({ tn: table.tn }))?.data
?.list;
for (const column of columnList) {
const oldColIdx = oldMeta.columns.findIndex(c => c.cn === column.cn);
// new table
if (oldColIdx === -1) {
tableProp.detectedChanges.push({
type: XcMetaDiffType.TABLE_COLUMN_ADD,
msg: `New column(${column.cn})`,
cn: column.cn
});
continue;
}
const oldCol = oldMeta.columns[oldColIdx];
oldMeta.columns.splice(oldColIdx, 1);
if (oldCol.dt !== column.dt) {
tableProp.detectedChanges.push({
type: XcMetaDiffType.TABLE_COLUMN_TYPE_CHANGE,
msg: `Column type changed(${column.cn})`,
cn: oldCol.cn
});
}
}
for (const { cn } of oldMeta.columns) {
tableProp.detectedChanges.push({
type: XcMetaDiffType.TABLE_COLUMN_REMOVE,
msg: `Column removed(${cn})`,
cn
});
}
}
for (const { tn } of oldMetas) {
changes.push({
tn: tn,
detectedChanges: [
{
type: XcMetaDiffType.TABLE_REMOVE,
msg: `Table removed`
}
]
});
}
// @ts-ignore
const relationList = (await sqlClient.relationListAll())?.data?.list;
// todo: handle virtual
const oldRelations = await this.xcMeta.metaList(
this.getProjectId(args),
this.getDbAlias(args),
'nc_relations',
{
condition: {
type: 'real'
}
}
);
// check relations
for (const rel of relationList) {
const oldRelIdx = oldRelations.findIndex(oldRel => {
return (
rel.tn === oldRel.tn &&
rel.rtn === oldRel.rtn &&
rel.cn === oldRel.cn &&
rel.rcn === oldRel.rcn
);
});
// new table
if (oldRelIdx === -1) {
changes
.find(t => t.tn === rel.tn)
?.detectedChanges.push({
type: XcMetaDiffType.TABLE_RELATION_ADD,
tn: rel.tn,
rtn: rel.rtn,
cn: rel.cn,
rcn: rel.rcn,
msg: `New relation added`
});
changes
.find(t => t.tn === rel.rtn)
?.detectedChanges.push({
type: XcMetaDiffType.TABLE_RELATION_ADD,
tn: rel.tn,
rtn: rel.rtn,
cn: rel.cn,
rcn: rel.rcn,
msg: `New relation added`
});
continue;
}
oldRelations.splice(oldRelIdx, 1);
}
for (const oldRel of oldRelations) {
changes
.find(t => t.tn === oldRel.tn)
?.detectedChanges.push({
type: XcMetaDiffType.TABLE_RELATION_REMOVE,
tn: oldRel.tn,
rtn: oldRel.rtn,
cn: oldRel.cn,
rcn: oldRel.rcn,
msg: `Relation removed`
});
changes
.find(t => t.tn === oldRel.rtn)
?.detectedChanges.push({
type: XcMetaDiffType.TABLE_RELATION_REMOVE,
tn: oldRel.tn,
rtn: oldRel.rtn,
cn: oldRel.cn,
rcn: oldRel.rcn,
msg: `Relation removed`
});
}
return changes;
}
export { XcMetaDiffType, NcMetaDiff };

41
packages/nocodb/src/lib/noco/rest/RestApiBuilder.ts

@ -21,7 +21,8 @@ import NcProjectBuilder from '../NcProjectBuilder';
import Noco from '../Noco';
import BaseApiBuilder, {
IGNORE_TABLES,
NcMetaData
NcMetaData,
XcTablesPopulateParams
} from '../common/BaseApiBuilder';
import NcMetaIO from '../meta/NcMetaIO';
@ -340,19 +341,7 @@ export class RestApiBuilder extends BaseApiBuilder<Noco> {
// minRouter.mapRoutes(this.router)
}
public async xcTablesPopulate(args?: {
tableNames?: Array<{
tn: string;
_tn?: string;
}>;
type?: 'table' | 'view' | 'function' | 'procedure';
columns?: {
[tn: string]: any;
};
oldMetas?: {
[tn: string]: NcMetaData;
};
}): Promise<any> {
public async xcTablesPopulate(args?: XcTablesPopulateParams): Promise<any> {
this.log(
`xcTablesPopulate : tables - %o , type - %s`,
args?.tableNames,
@ -543,6 +532,24 @@ export class RestApiBuilder extends BaseApiBuilder<Noco> {
type: table.type || 'table'
}
);
} else if (args?.oldMetas?.[table.tn]?.id) {
this.log(
"xcTablesPopulate : Updating model metadata for '%s' - %s",
table.tn,
table.type
);
await this.xcMeta.metaUpdate(
this.projectId,
this.dbAlias,
'nc_models',
{
title: table.tn,
alias: meta._tn,
meta: JSON.stringify(meta),
type: table.type || 'table'
},
args?.oldMetas?.[table.tn]?.id
);
}
/* create routes for table */
@ -855,8 +862,8 @@ export class RestApiBuilder extends BaseApiBuilder<Noco> {
}
// NOTE: xc-meta
public async xcTablesRowDelete(tn: string): Promise<void> {
await super.xcTablesRowDelete(tn);
public async xcTablesRowDelete(tn: string, extras?: any): Promise<void> {
await super.xcTablesRowDelete(tn, extras);
await this.xcMeta.metaDelete(this.projectId, this.dbAlias, 'nc_routes', {
tn
});
@ -895,7 +902,7 @@ export class RestApiBuilder extends BaseApiBuilder<Noco> {
}
delete this.models[tn];
await this.xcTablesRowDelete(tn);
await this.xcTablesRowDelete(tn, extras);
delete this.routers[tn];
this.swaggerUpdate({

13
packages/nocodb/src/lib/sqlMgr/code/models/xc/BaseModelXcMeta.ts

@ -95,7 +95,7 @@ abstract class BaseModelXcMeta extends BaseRender {
}
public getVitualColumns(): any[] {
return [
const virtualColumns = [
...(this.ctx.hasMany || []).map(hm => ({
hm,
_cn: `${hm._rtn} => ${hm._tn}`
@ -106,11 +106,14 @@ abstract class BaseModelXcMeta extends BaseRender {
}))
];
// const oldVirtualCols = this.ctx?.oldMeta?.v || [];
const oldVirtualCols = this.ctx?.oldMeta?.v || [];
// for (const oldVCol of oldVirtualCols) {
// if
// }
for (const oldVCol of oldVirtualCols) {
console.log(oldVCol);
virtualColumns.push(oldVCol);
}
return virtualColumns;
}
public mapDefaultPrimaryValue(columnsArr: any[]): void {

54
scripts/metaSync/queries.js

@ -0,0 +1,54 @@
const init = `CREATE TABLE \`table_col_delete\` (
\`id\` int NOT NULL AUTO_INCREMENT,
\`title\` varchar(45) DEFAULT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE \`table_to_col_add\` (
\`id\` int NOT NULL AUTO_INCREMENT,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE \`table_to_rel_add_child\` (
\`id\` int NOT NULL AUTO_INCREMENT,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE \`table_to_rel_add_parent\` (
\`id\` int NOT NULL AUTO_INCREMENT,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE \`table_to_rel_remove_child\` (
\`id\` int NOT NULL AUTO_INCREMENT,
\`table_to_rel_remove_parent_id\` int DEFAULT NULL,
PRIMARY KEY (\`id\`),
KEY \`table_to_rel_remove_parent_fk_idx\` (\`table_to_rel_remove_parent_id\`),
CONSTRAINT \`table_to_rel_remove_parent_fk\` FOREIGN KEY (\`table_to_rel_remove_parent_id\`) REFERENCES \`table_to_rel_remove_parent\` (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE \`table_to_rel_remove_parent\` (
\`id\` int NOT NULL AUTO_INCREMENT,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE \`table_to_remove\` (
\`id\` int NOT NULL AUTO_INCREMENT,
\`table_to_removecol\` varchar(45) NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
`
const colDel = `ALTER TABLE \`sakila\`.\`table_col_delete\`
DROP COLUMN \`title\`;`
const relDel =`ALTER TABLE \`sakila\`.\`table_to_rel_remove_child\`
DROP FOREIGN KEY \`table_to_rel_remove_parent_fk\`;
ALTER TABLE \`sakila\`.\`table_to_rel_remove_child\`
DROP COLUMN \`table_to_rel_remove_parent_id\`,
DROP INDEX \`table_to_rel_remove_parent_fk_idx\` ;
;
`
const tableDel =`DROP TABLE \`sakila\`.\`table_to_remove\`;`
Loading…
Cancel
Save