github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: merge upgraders for created and updated system fields

pull/7373/head
mertmit 10 months ago
parent
b6829c4866
commit
45e0925597
5 changed files with 389 additions and 446 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      packages/nocodb/src/modules/global/init-meta-service.provider.ts
  2. 6
      packages/nocodb/src/version-upgrader/NcUpgrader.ts
  3. 125
      packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedByUpgrader.ts
  4. 386
      packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedSystemFieldsUpgrader.ts
  5. 316
      packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedTimeUpgrader.ts

2
packages/nocodb/src/modules/global/init-meta-service.provider.ts
Unescape View File

@ -27,7 +27,7 @@ export const InitMetaServiceProvider: Provider = {
const config = await NcConfig.createByEnv();
// set version
process.env.NC_VERSION = '0111004';
process.env.NC_VERSION = '0111005';
// init cache
await NocoCache.init();

6
packages/nocodb/src/version-upgrader/NcUpgrader.ts
Unescape View File

@ -16,8 +16,7 @@ import ncHookUpgrader from './ncHookUpgrader';
import ncProjectConfigUpgrader from './ncProjectConfigUpgrader';
import ncXcdbLTARUpgrader from './ncXcdbLTARUpgrader';
import ncXcdbLTARIndexUpgrader from './ncXcdbLTARIndexUpgrader';
import ncXcdbCreatedAndUpdatedTimeUpgrader from './ncXcdbCreatedAndUpdatedTimeUpgrader';
import ncXcdbCreatedAndUpdatedByUpgrader from './ncXcdbCreatedAndUpdatedByUpgrader';
import ncXcdbCreatedAndUpdatedSystemFieldsUpgrader from './ncXcdbCreatedAndUpdatedSystemFieldsUpgrader';
import type { MetaService } from '~/meta/meta.service';
import type { NcConfig } from '~/interface/config';
@ -146,8 +145,7 @@ export default class NcUpgrader {
{ name: '0107004', handler: ncProjectConfigUpgrader },
{ name: '0108002', handler: ncXcdbLTARUpgrader },
{ name: '0111002', handler: ncXcdbLTARIndexUpgrader },
{ name: '0111004', handler: ncXcdbCreatedAndUpdatedTimeUpgrader },
{ name: '0111005', handler: ncXcdbCreatedAndUpdatedByUpgrader },
{ name: '0111005', handler: ncXcdbCreatedAndUpdatedSystemFieldsUpgrader },
];
}
}

125
packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedByUpgrader.ts
Unescape View File

@ -1,125 +0,0 @@
import { UITypes } from 'nocodb-sdk';
import { Logger } from '@nestjs/common';
import type { NcUpgraderCtx } from './NcUpgrader';
import type { MetaService } from '~/meta/meta.service';
import { MetaTable } from '~/utils/globals';
import { Column, Model, Source } from '~/models';
import {
getUniqueColumnAliasName,
getUniqueColumnName,
} from '~/helpers/getUniqueName';
import getColumnPropsFromUIDT from '~/helpers/getColumnPropsFromUIDT';
import ProjectMgrv2 from '~/db/sql-mgr/v2/ProjectMgrv2';
import { Altered } from '~/services/columns.service';
// An upgrader for adding created_by and updated_by columns to all tables as system column
const logger = new Logger('ncXcdbCreatedAndUpdatedByUpgrader');
async function upgradeModels({
ncMeta,
source,
}: {
ncMeta: MetaService;
source: any;
}) {
const models = await Model.list(
{
base_id: source.base_id,
source_id: source.id,
},
ncMeta,
);
await Promise.all(
models.map(async (model: any) => {
if (model.mm) return;
const newColumns = [];
const columns = await model.getColumns(ncMeta);
const oldColumns = columns.map((c) => ({ ...c, cn: c.column_name }));
newColumns.push({
...(await getColumnPropsFromUIDT(
{
uidt: UITypes.CreatedBy,
column_name: getUniqueColumnName(columns, 'created_by'),
title: getUniqueColumnAliasName(columns, 'nc_created_by'),
},
source,
)),
cdf: null,
system: true,
altered: Altered.NEW_COLUMN,
});
newColumns.push({
...(await getColumnPropsFromUIDT(
{
uidt: UITypes.LastModifiedBy,
column_name: getUniqueColumnName(columns, 'updated_by'),
title: getUniqueColumnAliasName(columns, 'nc_updated_by'),
},
source,
)),
cdf: null,
system: true,
altered: Altered.NEW_COLUMN,
});
// update column in db
const tableUpdateBody = {
...model,
tn: model.table_name,
originalColumns: oldColumns,
columns: [...columns, ...newColumns].map((c) => ({
...c,
cn: c.column_name,
})),
};
const sqlMgr = ProjectMgrv2.getSqlMgr({ id: source.base_id }, ncMeta);
await sqlMgr.sqlOpPlus(source, 'tableUpdate', tableUpdateBody);
for (const newColumn of newColumns) {
await Column.insert(
{
...newColumn,
system: 1,
fk_model_id: model.id,
},
ncMeta,
);
}
logger.log(`Upgraded model ${model.name} from source ${source.name}`);
}),
);
}
export default async function ({ ncMeta }: NcUpgraderCtx) {
// get all xcdb sources
const sources = await ncMeta.metaList2(null, null, MetaTable.BASES, {
xcCondition: {
_or: [
{
is_meta: {
eq: 1,
},
},
{
is_local: {
eq: 1,
},
},
],
},
});
// iterate and upgrade each base
for (const source of sources) {
logger.log(`Upgrading source ${source.name}`);
// update the meta props
await upgradeModels({ ncMeta, source: new Source(source) });
}
}

386
packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedSystemFieldsUpgrader.ts
Unescape View File

@ -0,0 +1,386 @@
import { UITypes } from 'nocodb-sdk';
import type { NcUpgraderCtx } from './NcUpgrader';
import type { MetaService } from '~/meta/meta.service';
import type { Base } from '~/models';
import { MetaTable } from '~/utils/globals';
import { Column, Model, Source } from '~/models';
import {
getUniqueColumnAliasName,
getUniqueColumnName,
} from '~/helpers/getUniqueName';
import getColumnPropsFromUIDT from '~/helpers/getColumnPropsFromUIDT';
import ProjectMgrv2 from '~/db/sql-mgr/v2/ProjectMgrv2';
import { Altered } from '~/services/columns.service';
import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
import getColumnUiType from '~/helpers/getColumnUiType';
import RequestQueue from '~/utils/RequestQueue';
// Example Usage:
// An upgrader for upgrading created_at and updated_at columns
// to system column and convert to new uidt CreatedTime and LastModifiedTime
const logger = {
log: (message: string) => {
console.log(
`[ncXcdbCreatedAndUpdatedSystemFieldsUpgrader ${Date.now()}] ` + message,
);
},
};
/* Enable if planning to remove trigger
async function deletePgTrigger({
column,
ncMeta,
model,
}: {
ncMeta: MetaService;
column: Column;
model: Model;
}) {
// delete pg trigger
const triggerFnName = `xc_au_${model.table_name}_${column.column_name}`;
const triggerName = `xc_trigger_${model.table_name}_${column.column_name}`;
await ncMeta.knex.raw(`DROP TRIGGER IF EXISTS ?? ON ??;`, [
triggerName,
model.table_name,
]);
await ncMeta.knex.raw(`DROP FUNCTION IF EXISTS ??()`, [triggerFnName]);
}
*/
async function upgradeModels({
ncMeta,
source,
base,
}: {
ncMeta: MetaService;
source: Source;
base: Base;
}) {
const models = await Model.list(
{
base_id: source.base_id,
source_id: source.id,
},
ncMeta,
);
await Promise.all(
models.map(async (model: any) => {
if (model.mm) return;
logger.log(
`Upgrading model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}})`,
);
const columns = await model.getColumns(ncMeta);
const oldColumns = columns.map((c) => ({ ...c, cn: c.column_name }));
let isCreatedTimeExists = false;
let isLastModifiedTimeExists = false;
let isCreatedByExists = false;
let isLastModifiedByExists = false;
for (const column of columns) {
if (
![
UITypes.DateTime,
UITypes.CreatedTime,
UITypes.LastModifiedTime,
UITypes.CreatedBy,
UITypes.LastModifiedBy,
].includes(column.uidt)
)
continue;
if (column.uidt === UITypes.CreatedBy && column.system) {
isCreatedByExists = true;
continue;
}
if (column.uidt === UITypes.LastModifiedBy && column.system) {
isLastModifiedByExists = true;
continue;
}
if ([UITypes.CreatedBy, UITypes.LastModifiedBy].includes(column.uidt)) {
continue;
}
if (column.uidt === UITypes.CreatedTime && column.system) {
isCreatedTimeExists = true;
continue;
}
if (column.uidt === UITypes.LastModifiedTime && column.system) {
isLastModifiedTimeExists = true;
continue;
}
// if column is created_at or updated_at, update the uidt in meta
if (column.column_name === 'created_at') {
isCreatedTimeExists = true;
await Column.update(
column.id,
{
...column,
uidt: UITypes.CreatedTime,
system: true,
},
ncMeta,
true,
);
/* Enable if planning to remove trigger
if (source.type === 'pg') {
// delete pg trigger if exists
await deletePgTrigger({ column, ncMeta, model });
}*/
}
if (column.column_name === 'updated_at') {
isLastModifiedTimeExists = true;
await Column.update(
column.id,
{
...column,
uidt: UITypes.LastModifiedTime,
system: true,
cdf: '',
au: false,
},
ncMeta,
true,
);
}
}
// get existing columns from database
const sqlClient = await NcConnectionMgrv2.getSqlClient(
source,
ncMeta.knex,
);
const dbColumns =
(
await sqlClient.columnList({
tn: model.table_name,
schema: source.getConfig()?.schema,
})
)?.data?.list?.map((c) => ({ ...c, column_name: c.cn })) || [];
// if no columns found skip since table might not be there
if (!dbColumns.length) {
logger.log(
`Skipping upgrade of model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}}) since columns not found`,
);
return;
}
// create created_at & updated_at and created_by & updated_by columns
const newColumns = [];
const existingDbColumns = [];
if (!isCreatedTimeExists) {
// check column exist and add to meta if found
const columnName = getUniqueColumnName(columns, 'created_at');
const dbColumn = dbColumns.find((c) => c.cn === columnName);
// if column already exist, just update the meta
if (
dbColumn &&
getColumnUiType(source, dbColumn) === UITypes.DateTime
) {
existingDbColumns.push({
...dbColumn,
uidt: UITypes.CreatedTime,
column_name: columnName,
title: getUniqueColumnAliasName(columns, 'CreatedAt'),
system: true,
});
} else {
newColumns.push({
...(await getColumnPropsFromUIDT(
{
uidt: UITypes.CreatedTime,
column_name: getUniqueColumnName(
[...columns, ...dbColumns],
'created_at',
),
title: getUniqueColumnAliasName(columns, 'CreatedAt'),
},
source,
)),
cdf: null,
system: true,
altered: Altered.NEW_COLUMN,
});
}
}
if (!isLastModifiedTimeExists) {
const columnName = getUniqueColumnName(columns, 'created_at');
const dbColumn = dbColumns.find((c) => c.cn === columnName);
// if column already exist, just update the meta
if (
dbColumn &&
getColumnUiType(source, dbColumn) === UITypes.DateTime
) {
existingDbColumns.push({
uidt: UITypes.LastModifiedTime,
...dbColumn,
column_name: columnName,
title: getUniqueColumnAliasName(columns, 'UpdatedAt'),
system: true,
});
} else {
newColumns.push({
...(await getColumnPropsFromUIDT(
{
uidt: UITypes.LastModifiedTime,
column_name: getUniqueColumnName(
[...columns, ...dbColumns],
'updated_at',
),
title: getUniqueColumnAliasName(columns, 'UpdatedAt'),
},
source,
)),
cdf: null,
system: true,
altered: Altered.NEW_COLUMN,
});
}
}
if (!isCreatedByExists) {
newColumns.push({
...(await getColumnPropsFromUIDT(
{
uidt: UITypes.CreatedBy,
column_name: getUniqueColumnName(
[...columns, ...dbColumns],
'created_by',
),
title: getUniqueColumnAliasName(columns, 'nc_created_by'),
},
source,
)),
cdf: null,
system: true,
altered: Altered.NEW_COLUMN,
});
}
if (!isLastModifiedByExists) {
newColumns.push({
...(await getColumnPropsFromUIDT(
{
uidt: UITypes.LastModifiedBy,
column_name: getUniqueColumnName(
[...columns, ...dbColumns],
'updated_by',
),
title: getUniqueColumnAliasName(columns, 'nc_updated_by'),
},
source,
)),
cdf: null,
system: true,
altered: Altered.NEW_COLUMN,
});
}
// alter table and add new columns if any
if (newColumns.length) {
logger.log(
`Altering table '${model.title}'(${model.id}) from base '${base.title}'(${base.id}}) for new columns`,
);
// update column in db
const tableUpdateBody = {
...model,
tn: model.table_name,
originalColumns: oldColumns,
columns: [...columns, ...newColumns].map((c) => ({
...c,
cn: c.column_name,
})),
};
const sqlMgr = ProjectMgrv2.getSqlMgr({ id: source.base_id }, ncMeta);
await sqlMgr.sqlOpPlus(source, 'tableUpdate', tableUpdateBody);
}
for (const newColumn of [...existingDbColumns, ...newColumns]) {
await Column.insert(
{
...newColumn,
system: 1,
fk_model_id: model.id,
},
ncMeta,
);
}
logger.log(
`Upgraded model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}})`,
);
}),
);
}
// database to virtual relation and create an index for it
export default async function ({ ncMeta }: NcUpgraderCtx) {
// get all xcdb sources
const sources = await ncMeta.metaList2(null, null, MetaTable.BASES, {
xcCondition: {
_or: [
{
is_meta: {
eq: 1,
},
},
{
is_local: {
eq: 1,
},
},
],
},
});
const requestQueue = new RequestQueue();
// iterate and upgrade each base
await Promise.all(
sources.map(async (_source, i) => {
const source = new Source(_source);
const base = await source.getProject(ncMeta);
// skip deleted base bases
if (!base || base.deleted) {
logger.log(
`Skipped deleted base source '${source.alias || source.id}' - ${
base.id
}`,
);
return Promise.resolve();
}
// update the meta props
return requestQueue.enqueue(async () => {
logger.log(
`Upgrading base ${base.title}(${base.id},${source.id}) (${i + 1}/${
sources.length
})`,
);
return upgradeModels({ ncMeta, source, base }).then(() => {
logger.log(
`Upgraded base '${base.title}'(${base.id},${source.id}) (${i + 1}/${
sources.length
})`,
);
});
});
}),
);
}

316
packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedTimeUpgrader.ts
Unescape View File

@ -1,316 +0,0 @@
import { UITypes } from 'nocodb-sdk';
import type { NcUpgraderCtx } from './NcUpgrader';
import type { MetaService } from '~/meta/meta.service';
import type { Base } from '~/models';
import { MetaTable } from '~/utils/globals';
import { Column, Model, Source } from '~/models';
import {
getUniqueColumnAliasName,
getUniqueColumnName,
} from '~/helpers/getUniqueName';
import getColumnPropsFromUIDT from '~/helpers/getColumnPropsFromUIDT';
import ProjectMgrv2 from '~/db/sql-mgr/v2/ProjectMgrv2';
import { Altered } from '~/services/columns.service';
import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
import getColumnUiType from '~/helpers/getColumnUiType';
import RequestQueue from '~/utils/RequestQueue';
// Example Usage:
// An upgrader for upgrading created_at and updated_at columns
// to system column and convert to new uidt CreatedTime and LastModifiedTime
const logger = {
log: (message: string) => {
console.log(
`[ncXcdbCreatedAndUpdatedTimeUpgrader ${Date.now()}] ` + message,
);
},
};
/* Enable if planning to remove trigger
async function deletePgTrigger({
column,
ncMeta,
model,
}: {
ncMeta: MetaService;
column: Column;
model: Model;
}) {
// delete pg trigger
const triggerFnName = `xc_au_${model.table_name}_${column.column_name}`;
const triggerName = `xc_trigger_${model.table_name}_${column.column_name}`;
await ncMeta.knex.raw(`DROP TRIGGER IF EXISTS ?? ON ??;`, [
triggerName,
model.table_name,
]);
await ncMeta.knex.raw(`DROP FUNCTION IF EXISTS ??()`, [triggerFnName]);
}
*/
async function upgradeModels({
ncMeta,
source,
base,
}: {
ncMeta: MetaService;
source: Source;
base: Base;
}) {
const models = await Model.list(
{
base_id: source.base_id,
source_id: source.id,
},
ncMeta,
);
await Promise.all(
models.map(async (model: any) => {
if (model.mm) return;
logger.log(
`Upgrading model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}})`,
);
const columns = await model.getColumns(ncMeta);
const oldColumns = columns.map((c) => ({ ...c, cn: c.column_name }));
let isCreatedTimeExists = false;
let isLastModifiedTimeExists = false;
for (const column of columns) {
if (column.uidt !== UITypes.DateTime) continue;
// if column is created_at or updated_at, update the uidt in meta
if (column.column_name === 'created_at') {
isCreatedTimeExists = true;
await Column.update(
column.id,
{
...column,
uidt: UITypes.CreatedTime,
system: true,
},
ncMeta,
true,
);
/* Enable if planning to remove trigger
if (source.type === 'pg') {
// delete pg trigger if exists
await deletePgTrigger({ column, ncMeta, model });
}*/
}
if (column.column_name === 'updated_at') {
isLastModifiedTimeExists = true;
await Column.update(
column.id,
{
...column,
uidt: UITypes.LastModifiedTime,
system: true,
cdf: '',
au: false,
},
ncMeta,
true,
);
}
}
if (!isCreatedTimeExists || !isLastModifiedTimeExists) {
// get existing columns from database
const sqlClient = await NcConnectionMgrv2.getSqlClient(
source,
ncMeta.knex,
);
const dbColumns =
(
await sqlClient.columnList({
tn: model.table_name,
schema: source.getConfig()?.schema,
})
)?.data?.list?.map((c) => ({ ...c, column_name: c.cn })) || [];
// if no columns found skip since table might not be there
if (!dbColumns.length) {
logger.log(
`Skipping upgrade of model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}}) since columns not found`,
);
return;
}
// create created_at and updated_at columns
const newColumns = [];
const existingDbColumns = [];
if (!isCreatedTimeExists) {
// check column exist and add to meta if found
const columnName = getUniqueColumnName(columns, 'created_at');
const dbColumn = dbColumns.find((c) => c.cn === columnName);
// if column already exist, just update the meta
if (
dbColumn &&
getColumnUiType(source, dbColumn) === UITypes.DateTime
) {
existingDbColumns.push({
...dbColumn,
uidt: UITypes.CreatedTime,
column_name: columnName,
title: getUniqueColumnAliasName(columns, 'CreatedAt'),
system: true,
});
} else {
newColumns.push({
...(await getColumnPropsFromUIDT(
{
uidt: UITypes.CreatedTime,
column_name: getUniqueColumnName(
[...columns, ...dbColumns],
'created_at',
),
title: getUniqueColumnAliasName(columns, 'CreatedAt'),
},
source,
)),
cdf: null,
system: true,
altered: Altered.NEW_COLUMN,
});
}
}
if (!isLastModifiedTimeExists) {
const columnName = getUniqueColumnName(columns, 'created_at');
const dbColumn = dbColumns.find((c) => c.cn === columnName);
// if column already exist, just update the meta
if (
dbColumn &&
getColumnUiType(source, dbColumn) === UITypes.DateTime
) {
existingDbColumns.push({
uidt: UITypes.LastModifiedTime,
...dbColumn,
column_name: columnName,
title: getUniqueColumnAliasName(columns, 'UpdatedAt'),
system: true,
});
} else {
newColumns.push({
...(await getColumnPropsFromUIDT(
{
uidt: UITypes.LastModifiedTime,
column_name: getUniqueColumnName(
[...columns, ...dbColumns],
'updated_at',
),
title: getUniqueColumnAliasName(columns, 'UpdatedAt'),
},
source,
)),
cdf: null,
system: true,
altered: Altered.NEW_COLUMN,
});
}
}
// alter table and add new columns if any
if (newColumns.length) {
logger.log(
`Altering table '${model.title}'(${model.id}) from base '${base.title}'(${base.id}}) for new columns`,
);
// update column in db
const tableUpdateBody = {
...model,
tn: model.table_name,
originalColumns: oldColumns,
columns: [...columns, ...newColumns].map((c) => ({
...c,
cn: c.column_name,
})),
};
const sqlMgr = ProjectMgrv2.getSqlMgr({ id: source.base_id }, ncMeta);
await sqlMgr.sqlOpPlus(source, 'tableUpdate', tableUpdateBody);
}
for (const newColumn of [...existingDbColumns, ...newColumns]) {
await Column.insert(
{
...newColumn,
system: 1,
fk_model_id: model.id,
},
ncMeta,
);
}
}
logger.log(
`Upgraded model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}})`,
);
}),
);
}
// database to virtual relation and create an index for it
export default async function ({ ncMeta }: NcUpgraderCtx) {
// get all xcdb sources
const sources = await ncMeta.metaList2(null, null, MetaTable.BASES, {
xcCondition: {
_or: [
{
is_meta: {
eq: 1,
},
},
{
is_local: {
eq: 1,
},
},
],
},
});
const requestQueue = new RequestQueue();
// iterate and upgrade each base
await Promise.all(
sources.map(async (_source, i) => {
const source = new Source(_source);
const base = await source.getProject(ncMeta);
// skip deleted base bases
if (!base || base.deleted) {
logger.log(
`Skipped deleted base source '${source.alias || source.id}' - ${
base.id
}`,
);
return Promise.resolve();
}
// update the meta props
return requestQueue.enqueue(async () => {
logger.log(
`Upgrading base ${base.title}(${base.id},${source.id}) (${i + 1}/${
sources.length
})`,
);
return upgradeModels({ ncMeta, source, base }).then(() => {
logger.log(
`Upgraded base '${base.title}'(${base.id},${source.id}) (${i + 1}/${
sources.length
})`,
);
});
});
}),
);
}
Write Preview
Loading…
Cancel
Save