github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: implement upgrader to migrate existing xcdb relations 

Signed-off-by: Pranav C <pranavxc@gmail.com>
test/reset-fail
Pranav C 2 years ago
parent
0d3e517931
commit
d466c2342e
4 changed files with 115 additions and 351 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      packages/nocodb/src/modules/global/init-meta-service.provider.ts
  2. 2
      packages/nocodb/src/version-upgrader/NcUpgrader.ts
  3. 384
      packages/nocodb/src/version-upgrader/ncFilterUpgrader_0104004.ts
  4. 78
      packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts

2
packages/nocodb/src/modules/global/init-meta-service.provider.ts
Unescape View File

@ -26,7 +26,7 @@ export const InitMetaServiceProvider: Provider = {
const config = await NcConfig.createByEnv();
// set version
process.env.NC_VERSION = '0107004';
process.env.NC_VERSION = '0108002';
// init cache
await NocoCache.init();

2
packages/nocodb/src/version-upgrader/NcUpgrader.ts
Unescape View File

@ -14,6 +14,7 @@ import ncProjectEnvUpgrader0011045 from './ncProjectEnvUpgrader0011045';
import ncProjectEnvUpgrader from './ncProjectEnvUpgrader';
import ncHookUpgrader from './ncHookUpgrader';
import ncProjectConfigUpgrader from './ncProjectConfigUpgrader';
import ncXcdbLTARUpgrader from './ncXcdbLTARUpgrader';
import type { MetaService } from '../meta/meta.service';
import type { NcConfig } from '../interface/config';
@ -50,6 +51,7 @@ export default class NcUpgrader {
{ name: '0105003', handler: ncFilterUpgrader_0105003 },
{ name: '0105004', handler: ncHookUpgrader },
{ name: '0107004', handler: ncProjectConfigUpgrader },
{ name: '0108002', handler: ncXcdbLTARUpgrader },
];
if (!(await ctx.ncMeta.knexConnection?.schema?.hasTable?.('nc_store'))) {
return;

384
packages/nocodb/src/version-upgrader/ncFilterUpgrader_0104004.ts
Unescape View File

@ -1,359 +1,43 @@
import { UITypes } from 'nocodb-sdk';
import { OrgUserRoles } from 'nocodb-sdk';
import { NC_APP_SETTINGS } from '../constants';
import Store from '../models/Store';
import { MetaTable } from '../utils/globals';
import Column from '../models/Column';
import Filter from '../models/Filter';
import Project from '../models/Project';
import type { MetaService } from '../meta/meta.service';
import type { NcUpgraderCtx } from './NcUpgrader';
import type { SelectOptionsType } from 'nocodb-sdk';
// as of 0.104.3, almost all filter operators are available to all column types
// while some of them aren't supposed to be shown
// this upgrader is to remove those unsupported filters / migrate to the correct filter
// Change Summary:
// - Text-based columns:
// - remove `>`, `<`, `>=`, `<=`
// - Numeric-based / SingleSelect columns:
// - remove `like`
// - migrate `null`, and `empty` to `blank`
// - Checkbox columns:
// - remove `equal`
// - migrate `empty` and `null` to `notchecked`
// - MultiSelect columns:
// - remove `like`
// - migrate `equal`, `null`, `empty`
// - Attachment columns:
// - remove `>`, `<`, `>=`, `<=`, `equal`
// - migrate `empty`, `null` to `blank`
// - LTAR columns:
// - remove `>`, `<`, `>=`, `<=`
// - migrate `empty`, `null` to `blank`
// - Lookup columns:
// - migrate `empty`, `null` to `blank`
// - Duration columns:
// - remove `like`
// - migrate `empty`, `null` to `blank`
const removeEqualFilters = (filter, ncMeta) => {
const actions = [];
// remove `is equal`, `is not equal`
if (['eq', 'neq'].includes(filter.comparison_op)) {
actions.push(Filter.delete(filter.id, ncMeta));
}
return actions;
};
const removeArithmeticFilters = (filter, ncMeta) => {
const actions = [];
// remove `>`, `<`, `>=`, `<=`
if (['gt', 'lt', 'gte', 'lte'].includes(filter.comparison_op)) {
actions.push(Filter.delete(filter.id, ncMeta));
}
return actions;
};
const removeLikeFilters = (filter, ncMeta) => {
const actions = [];
// remove `is like`, `is not like`
if (['like', 'nlike'].includes(filter.comparison_op)) {
actions.push(Filter.delete(filter.id, ncMeta));
}
return actions;
};
const migrateNullAndEmptyToBlankFilters = (filter, ncMeta) => {
const actions = [];
if (['empty', 'null'].includes(filter.comparison_op)) {
// migrate to blank
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'blank',
},
ncMeta,
),
);
} else if (['notempty', 'notnull'].includes(filter.comparison_op)) {
// migrate to not blank
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'notblank',
},
ncMeta,
),
);
}
return actions;
};
const migrateMultiSelectEq = async (filter, col: Column, ncMeta) => {
// only allow eq / neq
if (!['eq', 'neq'].includes(filter.comparison_op)) return;
// if there is no value -> delete this filter
if (!filter.value) {
return await Filter.delete(filter.id, ncMeta);
}
// options inputted from users
const options = filter.value.split(',');
// retrieve the possible col options
const colOptions = (await col.getColOptions()) as SelectOptionsType;
// only include valid options as the input value becomes dropdown type now
const validOptions = [];
for (const option of options) {
if (colOptions.options.includes(option)) {
validOptions.push(option);
}
}
const newFilterValue = validOptions.join(',');
// if all inputted options are invalid -> delete this filter
if (!newFilterValue) {
return await Filter.delete(filter.id, ncMeta);
}
const actions = [];
if (filter.comparison_op === 'eq') {
// migrate to `contains all of`
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'anyof',
value: newFilterValue,
},
ncMeta,
),
);
} else if (filter.comparison_op === 'neq') {
// migrate to `doesn't contain all of`
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'nanyof',
value: newFilterValue,
},
ncMeta,
),
);
}
return await Promise.all(actions);
};
const migrateToCheckboxFilter = (filter, ncMeta) => {
const actions = [];
const possibleTrueValues = ['true', 'True', '1', 'T', 'Y'];
const possibleFalseValues = ['false', 'False', '0', 'F', 'N'];
if (['empty', 'null'].includes(filter.comparison_op)) {
// migrate to not checked
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'notchecked',
},
ncMeta,
),
);
} else if (['notempty', 'notnull'].includes(filter.comparison_op)) {
// migrate to checked
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'checked',
},
ncMeta,
),
/** Upgrader for upgrading roles */
export default async function ({ ncMeta }: NcUpgraderCtx) {
const users = await ncMeta.metaList2(null, null, MetaTable.USERS);
for (const user of users) {
user.roles = user.roles
.split(',')
.map((r) => {
// update old role names with new roles
if (r === 'user') {
return OrgUserRoles.CREATOR;
} else if (r === 'user-new') {
return OrgUserRoles.VIEWER;
}
return r;
})
.join(',');
await ncMeta.metaUpdate(
null,
null,
MetaTable.USERS,
{ roles: user.roles },
user.id,
);
} else if (filter.comparison_op === 'eq') {
if (possibleTrueValues.includes(filter.value)) {
// migrate to checked
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'checked',
value: '',
},
ncMeta,
),
);
} else if (possibleFalseValues.includes(filter.value)) {
// migrate to notchecked
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'notchecked',
value: '',
},
ncMeta,
),
);
} else {
// invalid value - good to delete
actions.push(Filter.delete(filter.id, ncMeta));
}
} else if (filter.comparison_op === 'neq') {
if (possibleFalseValues.includes(filter.value)) {
// migrate to checked
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'checked',
value: '',
},
ncMeta,
),
);
} else if (possibleTrueValues.includes(filter.value)) {
// migrate to not checked
actions.push(
Filter.update(
filter.id,
{
comparison_op: 'notchecked',
value: '',
},
ncMeta,
),
);
} else {
// invalid value - good to delete
actions.push(Filter.delete(filter.id, ncMeta));
}
}
return actions;
};
async function migrateFilters(ncMeta: MetaService) {
const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP);
for (const filter of filters) {
if (!filter.fk_column_id || filter.is_group) {
continue;
}
const col = await Column.get({ colId: filter.fk_column_id }, ncMeta);
if (
[
UITypes.SingleLineText,
UITypes.LongText,
UITypes.PhoneNumber,
UITypes.Email,
UITypes.URL,
].includes(col.uidt)
) {
await Promise.all(removeArithmeticFilters(filter, ncMeta));
} else if (
[
// numeric fields
UITypes.Duration,
UITypes.Currency,
UITypes.Percent,
UITypes.Number,
UITypes.Decimal,
UITypes.Rating,
UITypes.Rollup,
// select fields
UITypes.SingleSelect,
].includes(col.uidt)
) {
await Promise.all([
...removeLikeFilters(filter, ncMeta),
...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
]);
} else if (col.uidt === UITypes.Checkbox) {
await Promise.all(migrateToCheckboxFilter(filter, ncMeta));
} else if (col.uidt === UITypes.MultiSelect) {
await Promise.all([
...removeLikeFilters(filter, ncMeta),
...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
]);
await migrateMultiSelectEq(filter, col, ncMeta);
} else if (col.uidt === UITypes.Attachment) {
await Promise.all([
...removeArithmeticFilters(filter, ncMeta),
...removeEqualFilters(filter, ncMeta),
...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
]);
} else if (col.uidt === UITypes.LinkToAnotherRecord) {
await Promise.all([
...removeArithmeticFilters(filter, ncMeta),
...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
]);
} else if (col.uidt === UITypes.Lookup) {
await Promise.all([
...removeArithmeticFilters(filter, ncMeta),
...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
]);
} else if (col.uidt === UITypes.Duration) {
await Promise.all([
...removeLikeFilters(filter, ncMeta),
...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
]);
}
}
}
async function updateProjectMeta(ncMeta: MetaService) {
const projectHasEmptyOrFilters: Record<string, boolean> = {};
const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP);
const actions = [];
for (const filter of filters) {
if (
['notempty', 'notnull', 'empty', 'null'].includes(filter.comparison_op)
) {
projectHasEmptyOrFilters[filter.project_id] = true;
}
}
const projects = await ncMeta.metaList2(null, null, MetaTable.PROJECT);
const defaultProjectMeta = {
showNullAndEmptyInFilter: false,
};
for (const project of projects) {
const oldProjectMeta = project.meta;
let newProjectMeta = defaultProjectMeta;
try {
newProjectMeta =
(typeof oldProjectMeta === 'string'
? JSON.parse(oldProjectMeta)
: oldProjectMeta) ?? defaultProjectMeta;
} catch {}
newProjectMeta = {
...newProjectMeta,
showNullAndEmptyInFilter: projectHasEmptyOrFilters[project.id] ?? false,
};
actions.push(
Project.update(
project.id,
{
meta: JSON.stringify(newProjectMeta),
},
ncMeta,
),
// set invite only signup if user have environment variable set
if (process.env.NC_INVITE_ONLY_SIGNUP) {
await Store.saveOrUpdate(
{
value: '{ "invite_only_signup": true }',
key: NC_APP_SETTINGS,
},
ncMeta,
);
}
await Promise.all(actions);
}
export default async function ({ ncMeta }: NcUpgraderCtx) {
// fix the existing filter behaviours or
// migrate `null` or `empty` filters to `blank`
await migrateFilters(ncMeta);
// enrich `showNullAndEmptyInFilter` in project meta
// if there is empty / null filters in existing projects,
// then set `showNullAndEmptyInFilter` to true
// else set to false
await updateProjectMeta(ncMeta);
}

78
packages/nocodb/src/version-upgrader/ncXcdbLTARUpgrader.ts
Unescape View File

@ -0,0 +1,78 @@
import { RelationTypes, UITypes } from 'nocodb-sdk';
import ProjectMgrv2 from '../db/sql-mgr/v2/ProjectMgrv2';
import type SqlMgrv2 from '../db/sql-mgr/v2/SqlMgrv2';
import type { MetaService } from '../meta/meta.service';
import type { LinkToAnotherRecordColumn, Model } from '../models';
import type { NcUpgraderCtx } from './NcUpgrader';
async function upgradeModelRelations({
model,
sqlMgr,
ncMeta,
}: {
ncMeta: MetaService;
model: Model;
sqlMgr: SqlMgrv2;
}) {
// Iterate over each column and upgrade LTAR
for (const column of await model.getColumns()) {
if (column.uidt !== UITypes.LinkToAnotherRecord) {
continue;
}
const colOptions = await column.getColOptions<LinkToAnotherRecordColumn>();
switch (colOptions.type) {
// case RelationTypes.MANY_TO_MANY:
//
// break;
case RelationTypes.HAS_MANY:
{
// delete the foreign key constraint if exists
// create a new index for the column
}
break;
// case RelationTypes.BELONGS_TO:
// break;
}
}
}
// An upgrader for upgrading any existing relation in xcdb
async function upgradeBaseRelations({
ncMeta,
base,
}: {
ncMeta: MetaService;
base: any;
}) {
const sqlMgr = ProjectMgrv2.getSqlMgr({ id: base.project_id }, ncMeta);
// get models for the base
const models = await ncMeta.metaList2(null, base.id, 'models');
// get all columns and filter out relations and upgrade
for (const model of models) {
await upgradeModelRelations({ ncMeta, model, sqlMgr });
}
}
// database to virtual relation and create an index for it
export default async function ({ ncMeta }: NcUpgraderCtx) {
// get all xcdb bases
const bases = await ncMeta.metaList2(null, null, 'bases', {
condition: {
is_meta: 1,
},
orderBy: {},
});
// iterate and upgrade each base
for (const base of bases) {
await upgradeBaseRelations({
ncMeta,
base,
});
}
}
Write Preview
Loading…
Cancel
Save