diff --git a/packages/nocodb/src/modules/global/init-meta-service.provider.ts b/packages/nocodb/src/modules/global/init-meta-service.provider.ts index a027194cce..7d873f0540 100644 --- a/packages/nocodb/src/modules/global/init-meta-service.provider.ts +++ b/packages/nocodb/src/modules/global/init-meta-service.provider.ts @@ -27,7 +27,7 @@ export const InitMetaServiceProvider: Provider = { const config = await NcConfig.createByEnv(); // set version - process.env.NC_VERSION = '0111004'; + process.env.NC_VERSION = '0111005'; // init cache await NocoCache.init(); diff --git a/packages/nocodb/src/version-upgrader/NcUpgrader.ts b/packages/nocodb/src/version-upgrader/NcUpgrader.ts index 88f81ae1fd..7caa5ca98c 100644 --- a/packages/nocodb/src/version-upgrader/NcUpgrader.ts +++ b/packages/nocodb/src/version-upgrader/NcUpgrader.ts @@ -16,8 +16,7 @@ import ncHookUpgrader from './ncHookUpgrader'; import ncProjectConfigUpgrader from './ncProjectConfigUpgrader'; import ncXcdbLTARUpgrader from './ncXcdbLTARUpgrader'; import ncXcdbLTARIndexUpgrader from './ncXcdbLTARIndexUpgrader'; -import ncXcdbCreatedAndUpdatedTimeUpgrader from './ncXcdbCreatedAndUpdatedTimeUpgrader'; -import ncXcdbCreatedAndUpdatedByUpgrader from './ncXcdbCreatedAndUpdatedByUpgrader'; +import ncXcdbCreatedAndUpdatedSystemFieldsUpgrader from './ncXcdbCreatedAndUpdatedSystemFieldsUpgrader'; import type { MetaService } from '~/meta/meta.service'; import type { NcConfig } from '~/interface/config'; @@ -146,8 +145,7 @@ export default class NcUpgrader { { name: '0107004', handler: ncProjectConfigUpgrader }, { name: '0108002', handler: ncXcdbLTARUpgrader }, { name: '0111002', handler: ncXcdbLTARIndexUpgrader }, - { name: '0111004', handler: ncXcdbCreatedAndUpdatedTimeUpgrader }, - { name: '0111005', handler: ncXcdbCreatedAndUpdatedByUpgrader }, + { name: '0111005', handler: ncXcdbCreatedAndUpdatedSystemFieldsUpgrader }, ]; } } diff --git a/packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedByUpgrader.ts b/packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedByUpgrader.ts deleted file mode 100644 index a4407fd1c3..0000000000 --- a/packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedByUpgrader.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { UITypes } from 'nocodb-sdk'; -import { Logger } from '@nestjs/common'; -import type { NcUpgraderCtx } from './NcUpgrader'; -import type { MetaService } from '~/meta/meta.service'; -import { MetaTable } from '~/utils/globals'; -import { Column, Model, Source } from '~/models'; -import { - getUniqueColumnAliasName, - getUniqueColumnName, -} from '~/helpers/getUniqueName'; -import getColumnPropsFromUIDT from '~/helpers/getColumnPropsFromUIDT'; -import ProjectMgrv2 from '~/db/sql-mgr/v2/ProjectMgrv2'; -import { Altered } from '~/services/columns.service'; - -// An upgrader for adding created_by and updated_by columns to all tables as system column - -const logger = new Logger('ncXcdbCreatedAndUpdatedByUpgrader'); - -async function upgradeModels({ - ncMeta, - source, -}: { - ncMeta: MetaService; - source: any; -}) { - const models = await Model.list( - { - base_id: source.base_id, - source_id: source.id, - }, - ncMeta, - ); - - await Promise.all( - models.map(async (model: any) => { - if (model.mm) return; - - const newColumns = []; - - const columns = await model.getColumns(ncMeta); - const oldColumns = columns.map((c) => ({ ...c, cn: c.column_name })); - - newColumns.push({ - ...(await getColumnPropsFromUIDT( - { - uidt: UITypes.CreatedBy, - column_name: getUniqueColumnName(columns, 'created_by'), - title: getUniqueColumnAliasName(columns, 'nc_created_by'), - }, - source, - )), - cdf: null, - system: true, - altered: Altered.NEW_COLUMN, - }); - - newColumns.push({ - ...(await getColumnPropsFromUIDT( - { - uidt: UITypes.LastModifiedBy, - column_name: getUniqueColumnName(columns, 'updated_by'), - title: getUniqueColumnAliasName(columns, 'nc_updated_by'), - }, - source, - )), - cdf: null, - system: true, - altered: Altered.NEW_COLUMN, - }); - - // update column in db - const tableUpdateBody = { - ...model, - tn: model.table_name, - originalColumns: oldColumns, - columns: [...columns, ...newColumns].map((c) => ({ - ...c, - cn: c.column_name, - })), - }; - const sqlMgr = ProjectMgrv2.getSqlMgr({ id: source.base_id }, ncMeta); - await sqlMgr.sqlOpPlus(source, 'tableUpdate', tableUpdateBody); - - for (const newColumn of newColumns) { - await Column.insert( - { - ...newColumn, - system: 1, - fk_model_id: model.id, - }, - ncMeta, - ); - } - - logger.log(`Upgraded model ${model.name} from source ${source.name}`); - }), - ); -} - -export default async function ({ ncMeta }: NcUpgraderCtx) { - // get all xcdb sources - const sources = await ncMeta.metaList2(null, null, MetaTable.BASES, { - xcCondition: { - _or: [ - { - is_meta: { - eq: 1, - }, - }, - { - is_local: { - eq: 1, - }, - }, - ], - }, - }); - - // iterate and upgrade each base - for (const source of sources) { - logger.log(`Upgrading source ${source.name}`); - // update the meta props - await upgradeModels({ ncMeta, source: new Source(source) }); - } -} diff --git a/packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedSystemFieldsUpgrader.ts b/packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedSystemFieldsUpgrader.ts new file mode 100644 index 0000000000..663cb3a0fb --- /dev/null +++ b/packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedSystemFieldsUpgrader.ts @@ -0,0 +1,386 @@ +import { UITypes } from 'nocodb-sdk'; +import type { NcUpgraderCtx } from './NcUpgrader'; +import type { MetaService } from '~/meta/meta.service'; +import type { Base } from '~/models'; +import { MetaTable } from '~/utils/globals'; +import { Column, Model, Source } from '~/models'; +import { + getUniqueColumnAliasName, + getUniqueColumnName, +} from '~/helpers/getUniqueName'; +import getColumnPropsFromUIDT from '~/helpers/getColumnPropsFromUIDT'; +import ProjectMgrv2 from '~/db/sql-mgr/v2/ProjectMgrv2'; +import { Altered } from '~/services/columns.service'; +import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2'; +import getColumnUiType from '~/helpers/getColumnUiType'; +import RequestQueue from '~/utils/RequestQueue'; + +// Example Usage: + +// An upgrader for upgrading created_at and updated_at columns +// to system column and convert to new uidt CreatedTime and LastModifiedTime + +const logger = { + log: (message: string) => { + console.log( + `[ncXcdbCreatedAndUpdatedSystemFieldsUpgrader ${Date.now()}] ` + message, + ); + }, +}; + +/* Enable if planning to remove trigger +async function deletePgTrigger({ + column, + ncMeta, + model, +}: { + ncMeta: MetaService; + column: Column; + model: Model; +}) { + // delete pg trigger + const triggerFnName = `xc_au_${model.table_name}_${column.column_name}`; + const triggerName = `xc_trigger_${model.table_name}_${column.column_name}`; + + await ncMeta.knex.raw(`DROP TRIGGER IF EXISTS ?? ON ??;`, [ + triggerName, + model.table_name, + ]); + await ncMeta.knex.raw(`DROP FUNCTION IF EXISTS ??()`, [triggerFnName]); +} +*/ + +async function upgradeModels({ + ncMeta, + source, + base, +}: { + ncMeta: MetaService; + source: Source; + base: Base; +}) { + const models = await Model.list( + { + base_id: source.base_id, + source_id: source.id, + }, + ncMeta, + ); + + await Promise.all( + models.map(async (model: any) => { + if (model.mm) return; + + logger.log( + `Upgrading model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}})`, + ); + + const columns = await model.getColumns(ncMeta); + const oldColumns = columns.map((c) => ({ ...c, cn: c.column_name })); + let isCreatedTimeExists = false; + let isLastModifiedTimeExists = false; + let isCreatedByExists = false; + let isLastModifiedByExists = false; + for (const column of columns) { + if ( + ![ + UITypes.DateTime, + UITypes.CreatedTime, + UITypes.LastModifiedTime, + UITypes.CreatedBy, + UITypes.LastModifiedBy, + ].includes(column.uidt) + ) + continue; + + if (column.uidt === UITypes.CreatedBy && column.system) { + isCreatedByExists = true; + continue; + } + + if (column.uidt === UITypes.LastModifiedBy && column.system) { + isLastModifiedByExists = true; + continue; + } + + if ([UITypes.CreatedBy, UITypes.LastModifiedBy].includes(column.uidt)) { + continue; + } + + if (column.uidt === UITypes.CreatedTime && column.system) { + isCreatedTimeExists = true; + continue; + } + + if (column.uidt === UITypes.LastModifiedTime && column.system) { + isLastModifiedTimeExists = true; + continue; + } + + // if column is created_at or updated_at, update the uidt in meta + if (column.column_name === 'created_at') { + isCreatedTimeExists = true; + await Column.update( + column.id, + { + ...column, + uidt: UITypes.CreatedTime, + system: true, + }, + ncMeta, + true, + ); + + /* Enable if planning to remove trigger + if (source.type === 'pg') { + // delete pg trigger if exists + await deletePgTrigger({ column, ncMeta, model }); + }*/ + } + if (column.column_name === 'updated_at') { + isLastModifiedTimeExists = true; + await Column.update( + column.id, + { + ...column, + uidt: UITypes.LastModifiedTime, + system: true, + cdf: '', + au: false, + }, + ncMeta, + true, + ); + } + } + + // get existing columns from database + const sqlClient = await NcConnectionMgrv2.getSqlClient( + source, + ncMeta.knex, + ); + + const dbColumns = + ( + await sqlClient.columnList({ + tn: model.table_name, + schema: source.getConfig()?.schema, + }) + )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })) || []; + + // if no columns found skip since table might not be there + if (!dbColumns.length) { + logger.log( + `Skipping upgrade of model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}}) since columns not found`, + ); + return; + } + + // create created_at & updated_at and created_by & updated_by columns + const newColumns = []; + const existingDbColumns = []; + + if (!isCreatedTimeExists) { + // check column exist and add to meta if found + const columnName = getUniqueColumnName(columns, 'created_at'); + const dbColumn = dbColumns.find((c) => c.cn === columnName); + + // if column already exist, just update the meta + if ( + dbColumn && + getColumnUiType(source, dbColumn) === UITypes.DateTime + ) { + existingDbColumns.push({ + ...dbColumn, + uidt: UITypes.CreatedTime, + column_name: columnName, + title: getUniqueColumnAliasName(columns, 'CreatedAt'), + system: true, + }); + } else { + newColumns.push({ + ...(await getColumnPropsFromUIDT( + { + uidt: UITypes.CreatedTime, + column_name: getUniqueColumnName( + [...columns, ...dbColumns], + 'created_at', + ), + title: getUniqueColumnAliasName(columns, 'CreatedAt'), + }, + source, + )), + cdf: null, + system: true, + altered: Altered.NEW_COLUMN, + }); + } + } + + if (!isLastModifiedTimeExists) { + const columnName = getUniqueColumnName(columns, 'created_at'); + const dbColumn = dbColumns.find((c) => c.cn === columnName); + + // if column already exist, just update the meta + if ( + dbColumn && + getColumnUiType(source, dbColumn) === UITypes.DateTime + ) { + existingDbColumns.push({ + uidt: UITypes.LastModifiedTime, + ...dbColumn, + column_name: columnName, + title: getUniqueColumnAliasName(columns, 'UpdatedAt'), + system: true, + }); + } else { + newColumns.push({ + ...(await getColumnPropsFromUIDT( + { + uidt: UITypes.LastModifiedTime, + column_name: getUniqueColumnName( + [...columns, ...dbColumns], + 'updated_at', + ), + title: getUniqueColumnAliasName(columns, 'UpdatedAt'), + }, + source, + )), + cdf: null, + system: true, + altered: Altered.NEW_COLUMN, + }); + } + } + + if (!isCreatedByExists) { + newColumns.push({ + ...(await getColumnPropsFromUIDT( + { + uidt: UITypes.CreatedBy, + column_name: getUniqueColumnName( + [...columns, ...dbColumns], + 'created_by', + ), + title: getUniqueColumnAliasName(columns, 'nc_created_by'), + }, + source, + )), + cdf: null, + system: true, + altered: Altered.NEW_COLUMN, + }); + } + + if (!isLastModifiedByExists) { + newColumns.push({ + ...(await getColumnPropsFromUIDT( + { + uidt: UITypes.LastModifiedBy, + column_name: getUniqueColumnName( + [...columns, ...dbColumns], + 'updated_by', + ), + title: getUniqueColumnAliasName(columns, 'nc_updated_by'), + }, + source, + )), + cdf: null, + system: true, + altered: Altered.NEW_COLUMN, + }); + } + + // alter table and add new columns if any + if (newColumns.length) { + logger.log( + `Altering table '${model.title}'(${model.id}) from base '${base.title}'(${base.id}}) for new columns`, + ); + // update column in db + const tableUpdateBody = { + ...model, + tn: model.table_name, + originalColumns: oldColumns, + columns: [...columns, ...newColumns].map((c) => ({ + ...c, + cn: c.column_name, + })), + }; + const sqlMgr = ProjectMgrv2.getSqlMgr({ id: source.base_id }, ncMeta); + await sqlMgr.sqlOpPlus(source, 'tableUpdate', tableUpdateBody); + } + for (const newColumn of [...existingDbColumns, ...newColumns]) { + await Column.insert( + { + ...newColumn, + system: 1, + fk_model_id: model.id, + }, + ncMeta, + ); + } + + logger.log( + `Upgraded model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}})`, + ); + }), + ); +} + +// database to virtual relation and create an index for it +export default async function ({ ncMeta }: NcUpgraderCtx) { + // get all xcdb sources + const sources = await ncMeta.metaList2(null, null, MetaTable.BASES, { + xcCondition: { + _or: [ + { + is_meta: { + eq: 1, + }, + }, + { + is_local: { + eq: 1, + }, + }, + ], + }, + }); + + const requestQueue = new RequestQueue(); + // iterate and upgrade each base + await Promise.all( + sources.map(async (_source, i) => { + const source = new Source(_source); + + const base = await source.getProject(ncMeta); + + // skip deleted base bases + if (!base || base.deleted) { + logger.log( + `Skipped deleted base source '${source.alias || source.id}' - ${ + base.id + }`, + ); + return Promise.resolve(); + } + + // update the meta props + return requestQueue.enqueue(async () => { + logger.log( + `Upgrading base ${base.title}(${base.id},${source.id}) (${i + 1}/${ + sources.length + })`, + ); + + return upgradeModels({ ncMeta, source, base }).then(() => { + logger.log( + `Upgraded base '${base.title}'(${base.id},${source.id}) (${i + 1}/${ + sources.length + })`, + ); + }); + }); + }), + ); +} diff --git a/packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedTimeUpgrader.ts b/packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedTimeUpgrader.ts deleted file mode 100644 index ff71175d05..0000000000 --- a/packages/nocodb/src/version-upgrader/ncXcdbCreatedAndUpdatedTimeUpgrader.ts +++ /dev/null @@ -1,316 +0,0 @@ -import { UITypes } from 'nocodb-sdk'; -import type { NcUpgraderCtx } from './NcUpgrader'; -import type { MetaService } from '~/meta/meta.service'; -import type { Base } from '~/models'; -import { MetaTable } from '~/utils/globals'; -import { Column, Model, Source } from '~/models'; -import { - getUniqueColumnAliasName, - getUniqueColumnName, -} from '~/helpers/getUniqueName'; -import getColumnPropsFromUIDT from '~/helpers/getColumnPropsFromUIDT'; -import ProjectMgrv2 from '~/db/sql-mgr/v2/ProjectMgrv2'; -import { Altered } from '~/services/columns.service'; -import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2'; -import getColumnUiType from '~/helpers/getColumnUiType'; -import RequestQueue from '~/utils/RequestQueue'; - -// Example Usage: - -// An upgrader for upgrading created_at and updated_at columns -// to system column and convert to new uidt CreatedTime and LastModifiedTime - -const logger = { - log: (message: string) => { - console.log( - `[ncXcdbCreatedAndUpdatedTimeUpgrader ${Date.now()}] ` + message, - ); - }, -}; - -/* Enable if planning to remove trigger -async function deletePgTrigger({ - column, - ncMeta, - model, -}: { - ncMeta: MetaService; - column: Column; - model: Model; -}) { - // delete pg trigger - const triggerFnName = `xc_au_${model.table_name}_${column.column_name}`; - const triggerName = `xc_trigger_${model.table_name}_${column.column_name}`; - - await ncMeta.knex.raw(`DROP TRIGGER IF EXISTS ?? ON ??;`, [ - triggerName, - model.table_name, - ]); - await ncMeta.knex.raw(`DROP FUNCTION IF EXISTS ??()`, [triggerFnName]); -} -*/ - -async function upgradeModels({ - ncMeta, - source, - base, -}: { - ncMeta: MetaService; - source: Source; - base: Base; -}) { - const models = await Model.list( - { - base_id: source.base_id, - source_id: source.id, - }, - ncMeta, - ); - - await Promise.all( - models.map(async (model: any) => { - if (model.mm) return; - - logger.log( - `Upgrading model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}})`, - ); - - const columns = await model.getColumns(ncMeta); - const oldColumns = columns.map((c) => ({ ...c, cn: c.column_name })); - let isCreatedTimeExists = false; - let isLastModifiedTimeExists = false; - for (const column of columns) { - if (column.uidt !== UITypes.DateTime) continue; - - // if column is created_at or updated_at, update the uidt in meta - if (column.column_name === 'created_at') { - isCreatedTimeExists = true; - await Column.update( - column.id, - { - ...column, - uidt: UITypes.CreatedTime, - system: true, - }, - ncMeta, - true, - ); - - /* Enable if planning to remove trigger - if (source.type === 'pg') { - // delete pg trigger if exists - await deletePgTrigger({ column, ncMeta, model }); - }*/ - } - if (column.column_name === 'updated_at') { - isLastModifiedTimeExists = true; - await Column.update( - column.id, - { - ...column, - uidt: UITypes.LastModifiedTime, - system: true, - cdf: '', - au: false, - }, - ncMeta, - true, - ); - } - } - - if (!isCreatedTimeExists || !isLastModifiedTimeExists) { - // get existing columns from database - - const sqlClient = await NcConnectionMgrv2.getSqlClient( - source, - ncMeta.knex, - ); - - const dbColumns = - ( - await sqlClient.columnList({ - tn: model.table_name, - schema: source.getConfig()?.schema, - }) - )?.data?.list?.map((c) => ({ ...c, column_name: c.cn })) || []; - - // if no columns found skip since table might not be there - if (!dbColumns.length) { - logger.log( - `Skipping upgrade of model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}}) since columns not found`, - ); - return; - } - - // create created_at and updated_at columns - const newColumns = []; - const existingDbColumns = []; - - if (!isCreatedTimeExists) { - // check column exist and add to meta if found - const columnName = getUniqueColumnName(columns, 'created_at'); - const dbColumn = dbColumns.find((c) => c.cn === columnName); - - // if column already exist, just update the meta - if ( - dbColumn && - getColumnUiType(source, dbColumn) === UITypes.DateTime - ) { - existingDbColumns.push({ - ...dbColumn, - uidt: UITypes.CreatedTime, - column_name: columnName, - title: getUniqueColumnAliasName(columns, 'CreatedAt'), - system: true, - }); - } else { - newColumns.push({ - ...(await getColumnPropsFromUIDT( - { - uidt: UITypes.CreatedTime, - column_name: getUniqueColumnName( - [...columns, ...dbColumns], - 'created_at', - ), - title: getUniqueColumnAliasName(columns, 'CreatedAt'), - }, - source, - )), - cdf: null, - system: true, - altered: Altered.NEW_COLUMN, - }); - } - } - - if (!isLastModifiedTimeExists) { - const columnName = getUniqueColumnName(columns, 'created_at'); - const dbColumn = dbColumns.find((c) => c.cn === columnName); - - // if column already exist, just update the meta - if ( - dbColumn && - getColumnUiType(source, dbColumn) === UITypes.DateTime - ) { - existingDbColumns.push({ - uidt: UITypes.LastModifiedTime, - ...dbColumn, - column_name: columnName, - title: getUniqueColumnAliasName(columns, 'UpdatedAt'), - system: true, - }); - } else { - newColumns.push({ - ...(await getColumnPropsFromUIDT( - { - uidt: UITypes.LastModifiedTime, - column_name: getUniqueColumnName( - [...columns, ...dbColumns], - 'updated_at', - ), - title: getUniqueColumnAliasName(columns, 'UpdatedAt'), - }, - source, - )), - cdf: null, - system: true, - altered: Altered.NEW_COLUMN, - }); - } - } - - // alter table and add new columns if any - if (newColumns.length) { - logger.log( - `Altering table '${model.title}'(${model.id}) from base '${base.title}'(${base.id}}) for new columns`, - ); - // update column in db - const tableUpdateBody = { - ...model, - tn: model.table_name, - originalColumns: oldColumns, - columns: [...columns, ...newColumns].map((c) => ({ - ...c, - cn: c.column_name, - })), - }; - const sqlMgr = ProjectMgrv2.getSqlMgr({ id: source.base_id }, ncMeta); - await sqlMgr.sqlOpPlus(source, 'tableUpdate', tableUpdateBody); - } - for (const newColumn of [...existingDbColumns, ...newColumns]) { - await Column.insert( - { - ...newColumn, - system: 1, - fk_model_id: model.id, - }, - ncMeta, - ); - } - } - - logger.log( - `Upgraded model '${model.title}'(${model.id}) from base '${base.title}'(${base.id}})`, - ); - }), - ); -} - -// database to virtual relation and create an index for it -export default async function ({ ncMeta }: NcUpgraderCtx) { - // get all xcdb sources - const sources = await ncMeta.metaList2(null, null, MetaTable.BASES, { - xcCondition: { - _or: [ - { - is_meta: { - eq: 1, - }, - }, - { - is_local: { - eq: 1, - }, - }, - ], - }, - }); - - const requestQueue = new RequestQueue(); - // iterate and upgrade each base - await Promise.all( - sources.map(async (_source, i) => { - const source = new Source(_source); - - const base = await source.getProject(ncMeta); - - // skip deleted base bases - if (!base || base.deleted) { - logger.log( - `Skipped deleted base source '${source.alias || source.id}' - ${ - base.id - }`, - ); - return Promise.resolve(); - } - - // update the meta props - return requestQueue.enqueue(async () => { - logger.log( - `Upgrading base ${base.title}(${base.id},${source.id}) (${i + 1}/${ - sources.length - })`, - ); - - return upgradeModels({ ncMeta, source, base }).then(() => { - logger.log( - `Upgraded base '${base.title}'(${base.id},${source.id}) (${i + 1}/${ - sources.length - })`, - ); - }); - }); - }), - ); -}