From 77be7f922603ec995346e81001cd4cc4233b8b16 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 28 Nov 2024 06:05:24 +0000 Subject: [PATCH] fix: add upgrader to merge and recover broken plugins --- .../src/controllers/plugins.controller.ts | 8 +- packages/nocodb/src/helpers/NcPluginMgrv2.ts | 22 ++++- packages/nocodb/src/models/Plugin.ts | 29 +++--- .../nocodb/src/plugins/backblaze/index.ts | 2 +- packages/nocodb/src/plugins/linode/index.ts | 2 +- packages/nocodb/src/plugins/ovhCloud/index.ts | 2 +- packages/nocodb/src/plugins/r2/index.ts | 2 +- packages/nocodb/src/plugins/scaleway/index.ts | 2 +- .../src/plugins/twilioWhatsapp/index.ts | 2 +- packages/nocodb/src/plugins/upcloud/index.ts | 2 +- packages/nocodb/src/plugins/vultr/index.ts | 2 +- .../providers/init-meta-service.provider.ts | 2 +- .../nocodb/src/services/plugins.service.ts | 4 +- .../src/types/nc-plugin/lib/XcPluginConfig.ts | 2 +- .../nocodb/src/version-upgrader/NcUpgrader.ts | 2 + .../0258003_ncDuplicatePluginMerge.ts | 99 +++++++++++++++++++ 16 files changed, 152 insertions(+), 32 deletions(-) create mode 100644 packages/nocodb/src/version-upgrader/upgraders/0258003_ncDuplicatePluginMerge.ts diff --git a/packages/nocodb/src/controllers/plugins.controller.ts b/packages/nocodb/src/controllers/plugins.controller.ts index 45e16b1b02..6b2f633640 100644 --- a/packages/nocodb/src/controllers/plugins.controller.ts +++ b/packages/nocodb/src/controllers/plugins.controller.ts @@ -82,15 +82,15 @@ export class PluginsController { } @Get([ - '/api/v1/db/meta/plugins/:pluginTitle/status', - '/api/v2/meta/plugins/:pluginTitle/status', + '/api/v1/db/meta/plugins/:pluginId/status', + '/api/v2/meta/plugins/:pluginId/status', ]) @Acl('isPluginActive', { scope: 'org', }) - async isPluginActive(@Param('pluginTitle') pluginTitle: string) { + async isPluginActive(@Param('pluginId') pluginId: string) { return await this.pluginsService.isPluginActive({ - pluginTitle: pluginTitle, + pluginId, }); } } diff --git a/packages/nocodb/src/helpers/NcPluginMgrv2.ts b/packages/nocodb/src/helpers/NcPluginMgrv2.ts index 7dee51a1e6..7762974f3c 100644 --- a/packages/nocodb/src/helpers/NcPluginMgrv2.ts +++ b/packages/nocodb/src/helpers/NcPluginMgrv2.ts @@ -67,15 +67,26 @@ class NcPluginMgrv2 { // } public static async init(ncMeta = Noco.ncMeta): Promise { + // extract duplicate plugin ids from default plugins and throw error + const duplicateIds = defaultPlugins + .map((p) => p.id) + .filter((id, index, self) => self.indexOf(id) !== index); + + if (duplicateIds.length) { + throw new Error( + `Duplicate plugin ids found in default plugins: ${duplicateIds.join( + ', ', + )}`, + ); + } + /* Populate rows into nc_plugins table if not present */ for (const plugin of defaultPlugins) { const pluginConfig = await ncMeta.metaGet( RootScopes.ROOT, RootScopes.ROOT, MetaTable.PLUGIN, - { - title: plugin.title, - }, + plugin.id, ); if (!pluginConfig) { @@ -84,6 +95,7 @@ class NcPluginMgrv2 { RootScopes.ROOT, MetaTable.PLUGIN, { + id: plugin.id, title: plugin.title, version: plugin.version, logo: plugin.logo, @@ -121,7 +133,7 @@ class NcPluginMgrv2 { * */ if (process.env.NC_S3_BUCKET_NAME && process.env.NC_S3_REGION) { - const s3Plugin = await Plugin.getPluginByTitle(S3PluginConfig.title); + const s3Plugin = await Plugin.getPlugin(S3PluginConfig.id); const s3CfgData: Record = { bucket: process.env.NC_S3_BUCKET_NAME, region: process.env.NC_S3_REGION, @@ -144,7 +156,7 @@ class NcPluginMgrv2 { process.env.NC_SMTP_HOST && process.env.NC_SMTP_PORT ) { - const smtpPlugin = await Plugin.getPluginByTitle(SMTPPluginConfig.title); + const smtpPlugin = await Plugin.getPlugin(SMTPPluginConfig.title); await Plugin.update(smtpPlugin.id, { active: true, input: JSON.stringify({ diff --git a/packages/nocodb/src/models/Plugin.ts b/packages/nocodb/src/models/Plugin.ts index 37ab8072bb..5ea2120787 100644 --- a/packages/nocodb/src/models/Plugin.ts +++ b/packages/nocodb/src/models/Plugin.ts @@ -89,23 +89,22 @@ export default class Plugin implements PluginType { ); await NocoCache.update(`${CacheScope.PLUGIN}:${pluginId}`, updateObj); - await NocoCache.update(`${CacheScope.PLUGIN}:${plugin.title}`, updateObj); return this.get(pluginId); } - public static async isPluginActive(title: string) { - return !!(await this.getPluginByTitle(title))?.active; + public static async isPluginActive(id: string) { + return !!(await this.getPlugin(id))?.active; } /** - * get plugin by title + * get plugin by id */ - public static async getPluginByTitle(title: string, ncMeta = Noco.ncMeta) { + public static async getPlugin(id: string, ncMeta = Noco.ncMeta) { let plugin = - title && + id && (await NocoCache.get( - `${CacheScope.PLUGIN}:${title}`, + `${CacheScope.PLUGIN}:${id}`, CacheGetType.TYPE_OBJECT, )); if (!plugin) { @@ -113,12 +112,20 @@ export default class Plugin implements PluginType { RootScopes.ROOT, RootScopes.ROOT, MetaTable.PLUGIN, - { - title, - }, + id, ); - await NocoCache.set(`${CacheScope.PLUGIN}:${title}`, plugin); + await NocoCache.set(`${CacheScope.PLUGIN}:${id}`, plugin); } return plugin; } + + // keeping it for backward compatibility, if someone configured google auth via plugin it still relies on this + static async getPluginByTitle(title: string, ncMeta = Noco.ncMeta) { + return await ncMeta.metaGet2( + RootScopes.ROOT, + RootScopes.ROOT, + MetaTable.PLUGIN, + { title }, + ); + } } diff --git a/packages/nocodb/src/plugins/backblaze/index.ts b/packages/nocodb/src/plugins/backblaze/index.ts index df0d0dec66..2e26503fac 100644 --- a/packages/nocodb/src/plugins/backblaze/index.ts +++ b/packages/nocodb/src/plugins/backblaze/index.ts @@ -6,7 +6,7 @@ const config: XcPluginConfig = { builder: BackblazePlugin, id: 'backblaze', title: 'Backblaze', - fallbackTitles: ['Backblaze B2'], + fallbackTitle: 'Backblaze B2', version: '0.0.5', logo: 'plugins/backblaze.jpeg', tags: 'Storage', diff --git a/packages/nocodb/src/plugins/linode/index.ts b/packages/nocodb/src/plugins/linode/index.ts index 127e2d9ece..dedcda2e8d 100644 --- a/packages/nocodb/src/plugins/linode/index.ts +++ b/packages/nocodb/src/plugins/linode/index.ts @@ -5,7 +5,7 @@ import type { XcPluginConfig } from '~/types/nc-plugin'; const config: XcPluginConfig = { builder: LinodeObjectStoragePlugin, id: 'linode', - fallbackTitles: ['Linode Object Storage'], + fallbackTitle: 'Linode Object Storage', title: 'Linode', version: '0.0.4', logo: 'plugins/linode.svg', diff --git a/packages/nocodb/src/plugins/ovhCloud/index.ts b/packages/nocodb/src/plugins/ovhCloud/index.ts index f8af06953c..d4d8e973db 100644 --- a/packages/nocodb/src/plugins/ovhCloud/index.ts +++ b/packages/nocodb/src/plugins/ovhCloud/index.ts @@ -6,7 +6,7 @@ const config: XcPluginConfig = { builder: OvhCloud, id: 'ovh', title: 'Ovh', - fallbackTitles: ['OvhCloud Object Storage'], + fallbackTitle: 'OvhCloud Object Storage', version: '0.0.4', logo: 'plugins/ovhCloud.png', tags: 'Storage', diff --git a/packages/nocodb/src/plugins/r2/index.ts b/packages/nocodb/src/plugins/r2/index.ts index a85b60470e..d95fcc7dba 100644 --- a/packages/nocodb/src/plugins/r2/index.ts +++ b/packages/nocodb/src/plugins/r2/index.ts @@ -6,7 +6,7 @@ const config: XcPluginConfig = { builder: R2Plugin, id: 'cloudflare-r2', title: 'Cloudflare R2', - fallbackTitles: ['Cloudflare R2 Storage'], + fallbackTitle: 'Cloudflare R2 Storage', version: '0.0.3', logo: 'plugins/r2.png', description: diff --git a/packages/nocodb/src/plugins/scaleway/index.ts b/packages/nocodb/src/plugins/scaleway/index.ts index 505f3100e4..b5197d304d 100644 --- a/packages/nocodb/src/plugins/scaleway/index.ts +++ b/packages/nocodb/src/plugins/scaleway/index.ts @@ -6,7 +6,7 @@ const config: XcPluginConfig = { builder: ScalewayObjectStoragePlugin, id: 'scaleway', title: 'Scaleway', - fallbackTitles: ['Scaleway Object Storage'], + fallbackTitle: 'Scaleway Object Storage', version: '0.0.4', logo: 'plugins/scaleway.png', tags: 'Storage', diff --git a/packages/nocodb/src/plugins/twilioWhatsapp/index.ts b/packages/nocodb/src/plugins/twilioWhatsapp/index.ts index c95ff71264..2e17cb340d 100644 --- a/packages/nocodb/src/plugins/twilioWhatsapp/index.ts +++ b/packages/nocodb/src/plugins/twilioWhatsapp/index.ts @@ -4,7 +4,7 @@ import type { XcPluginConfig } from '~/types/nc-plugin'; const config: XcPluginConfig = { builder: TwilioWhatsappPlugin, - id: 'twilio', + id: 'twilio-whatsapp', title: 'Whatsapp Twilio', version: '0.0.1', logo: 'plugins/whatsapp.png', diff --git a/packages/nocodb/src/plugins/upcloud/index.ts b/packages/nocodb/src/plugins/upcloud/index.ts index 6df3e2ae2a..a60c0e6e7d 100644 --- a/packages/nocodb/src/plugins/upcloud/index.ts +++ b/packages/nocodb/src/plugins/upcloud/index.ts @@ -6,7 +6,7 @@ const config: XcPluginConfig = { builder: UpCloudPlugin, id: 'upcloud', title: 'UpCloud', - fallbackTitles: ['UpCloud Object Storage'], + fallbackTitle: 'UpCloud Object Storage', version: '0.0.4', logo: 'plugins/upcloud.png', description: diff --git a/packages/nocodb/src/plugins/vultr/index.ts b/packages/nocodb/src/plugins/vultr/index.ts index 9a8c5dd07d..b70054e5ed 100644 --- a/packages/nocodb/src/plugins/vultr/index.ts +++ b/packages/nocodb/src/plugins/vultr/index.ts @@ -6,7 +6,7 @@ const config: XcPluginConfig = { builder: VultrPlugin, title: 'Vultr', id: 'vultr', - fallbackTitles: ['Vultr Object Storage'], + fallbackTitle: 'Vultr Object Storage', version: '0.0.4', logo: 'plugins/vultr.png', description: diff --git a/packages/nocodb/src/providers/init-meta-service.provider.ts b/packages/nocodb/src/providers/init-meta-service.provider.ts index d5adc91795..a25ee2cfe3 100644 --- a/packages/nocodb/src/providers/init-meta-service.provider.ts +++ b/packages/nocodb/src/providers/init-meta-service.provider.ts @@ -31,7 +31,7 @@ export const InitMetaServiceProvider: FactoryProvider = { const config = await NcConfig.createByEnv(); // set version - process.env.NC_VERSION = '0225002'; + process.env.NC_VERSION = '0258003'; // set migration jobs version process.env.NC_MIGRATION_JOBS_VERSION = '2'; diff --git a/packages/nocodb/src/services/plugins.service.ts b/packages/nocodb/src/services/plugins.service.ts index 9ee7e7d17e..43f69900e1 100644 --- a/packages/nocodb/src/services/plugins.service.ts +++ b/packages/nocodb/src/services/plugins.service.ts @@ -50,8 +50,8 @@ export class PluginsService { return plugin; } - async isPluginActive(param: { pluginTitle: string }) { - return await Plugin.isPluginActive(param.pluginTitle); + async isPluginActive(param: { pluginId: string }) { + return await Plugin.isPluginActive(param.pluginId); } async webhookPluginList() { diff --git a/packages/nocodb/src/types/nc-plugin/lib/XcPluginConfig.ts b/packages/nocodb/src/types/nc-plugin/lib/XcPluginConfig.ts index 23a289c226..6f8d593e7a 100644 --- a/packages/nocodb/src/types/nc-plugin/lib/XcPluginConfig.ts +++ b/packages/nocodb/src/types/nc-plugin/lib/XcPluginConfig.ts @@ -6,7 +6,7 @@ import type XcPluginMigration from './XcPluginMigration'; export default interface XcPluginConfig { id: string; title: string; - fallbackTitles?: string[]; + fallbackTitle?: string; logo?: string; tags?: string; description?: string; diff --git a/packages/nocodb/src/version-upgrader/NcUpgrader.ts b/packages/nocodb/src/version-upgrader/NcUpgrader.ts index 09b2e72276..5a3e7b2087 100644 --- a/packages/nocodb/src/version-upgrader/NcUpgrader.ts +++ b/packages/nocodb/src/version-upgrader/NcUpgrader.ts @@ -12,6 +12,7 @@ import ncXcdbLTARUpgrader from './upgraders/0108002_ncXcdbLTARUpgrader'; import ncXcdbLTARIndexUpgrader from './upgraders/0111002_ncXcdbLTARIndexUpgrader'; import ncXcdbCreatedAndUpdatedSystemFieldsUpgrader from './upgraders/0111005_ncXcdbCreatedAndUpdatedSystemFieldsUpgrader'; import ncDatasourceDecrypt from './upgraders/0225002_ncDatasourceDecrypt'; +import ncDuplicatePluginMerge from './upgraders/0258003_ncDuplicatePluginMerge'; import type { MetaService } from '~/meta/meta.service'; import type { NcConfig } from '~/interface/config'; import { T } from '~/utils'; @@ -150,6 +151,7 @@ export default class NcUpgrader { { name: '0111002', handler: ncXcdbLTARIndexUpgrader }, { name: '0111005', handler: ncXcdbCreatedAndUpdatedSystemFieldsUpgrader }, { name: '0225002', handler: ncDatasourceDecrypt }, + { name: '0258003', handler: ncDuplicatePluginMerge }, ]; } } diff --git a/packages/nocodb/src/version-upgrader/upgraders/0258003_ncDuplicatePluginMerge.ts b/packages/nocodb/src/version-upgrader/upgraders/0258003_ncDuplicatePluginMerge.ts new file mode 100644 index 0000000000..71c1964eb0 --- /dev/null +++ b/packages/nocodb/src/version-upgrader/upgraders/0258003_ncDuplicatePluginMerge.ts @@ -0,0 +1,99 @@ +import type { NcUpgraderCtx } from '~/version-upgrader/NcUpgrader'; +import SlackPluginConfig from '~/plugins/slack'; +import TeamsPluginConfig from '~/plugins/teams'; +import DiscordPluginConfig from '~/plugins/discord'; +import TwilioWhatsappPluginConfig from '~/plugins/twilioWhatsapp'; +import TwilioPluginConfig from '~/plugins/twilio'; +import S3PluginConfig from '~/plugins/s3'; +import MinioPluginConfig from '~/plugins/mino'; +import GcsPluginConfig from '~/plugins/gcs'; +import MattermostPluginConfig from '~/plugins/mattermost'; +import SpacesPluginConfig from '~/plugins/spaces'; +import BackblazePluginConfig from '~/plugins/backblaze'; +import VultrPluginConfig from '~/plugins/vultr'; +import OvhCloudPluginConfig from '~/plugins/ovhCloud'; +import LinodePluginConfig from '~/plugins/linode'; +import UpcloudPluginConfig from '~/plugins/upcloud'; +import SMTPPluginConfig from '~/plugins/smtp'; +import MailerSendConfig from '~/plugins/mailerSend'; +import ScalewayPluginConfig from '~/plugins/scaleway'; +import SESPluginConfig from '~/plugins/ses'; +import R2PluginConfig from '~/plugins/r2'; +import { MetaTable } from '~/cli'; + +const defaultPlugins = [ + SlackPluginConfig, + TeamsPluginConfig, + DiscordPluginConfig, + TwilioWhatsappPluginConfig, + TwilioPluginConfig, + S3PluginConfig, + MinioPluginConfig, + GcsPluginConfig, + MattermostPluginConfig, + SpacesPluginConfig, + BackblazePluginConfig, + VultrPluginConfig, + OvhCloudPluginConfig, + LinodePluginConfig, + UpcloudPluginConfig, + SMTPPluginConfig, + MailerSendConfig, + ScalewayPluginConfig, + SESPluginConfig, + R2PluginConfig, +]; + +// This upgrader helps to merge the duplicate plugins and recover the broken plugins +// and also adds a unique id to the plugin to avoid the duplicate plugins in the future +export default async function ({ ncMeta }: NcUpgraderCtx) { + // get the plugins which are valid and matches the plugin title + // update the plugin with the new id + for (const pluginConfig of defaultPlugins) { + // get the valid plugin + const plugin = await ncMeta + .knex(MetaTable.PLUGIN) + .where('title', pluginConfig.title) + .first(); + + if (plugin) { + // update the plugin with the new id + await ncMeta + .knex(MetaTable.PLUGIN) + .where('id', plugin.id) + .update({ id: pluginConfig.id }); + } + + if (pluginConfig.fallbackTitle) { + // get the plugin with old title + const oldPlugin = await ncMeta + .knex(MetaTable.PLUGIN) + .where('title', pluginConfig.fallbackTitle) + .first(); + + if (plugin) { + // if the old plugin is present then update the new plugin with the old plugin configuration + // and only if new plugin is not configured and active + if (!plugin.active && oldPlugin.active) { + await ncMeta + .knex(MetaTable.PLUGIN) + .update({ + input: oldPlugin.input, + active: true, + }) + .where('id', plugin.id); + } + + // delete the old plugin + await ncMeta.knex(MetaTable.PLUGIN).where('id', oldPlugin.id).delete(); + } else { + // if new plugin is not present then update the old plugin with the new id + // we can skip rest of the props since it will get updated from the existing plugin initialization + await ncMeta + .knex(MetaTable.PLUGIN) + .where('id', oldPlugin.id) + .update({ id: pluginConfig.id }); + } + } + } +}