From ed6817a70e454cb68898461ce937a441828bcb3d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 3 May 2023 12:54:32 +0530 Subject: [PATCH 1/4] feat: service for webhook handler Signed-off-by: Pranav C --- packages/nocodb/src/app.module.ts | 2 + packages/nocodb/src/db/BaseModelSqlv2.ts | 26 ++-- .../src/services/hook-handler.service.spec.ts | 18 +++ .../src/services/hook-handler.service.ts | 143 ++++++++++++++++++ 4 files changed, 179 insertions(+), 10 deletions(-) create mode 100644 packages/nocodb/src/services/hook-handler.service.spec.ts create mode 100644 packages/nocodb/src/services/hook-handler.service.ts diff --git a/packages/nocodb/src/app.module.ts b/packages/nocodb/src/app.module.ts index f44bd3c6ee..9af6d4919e 100644 --- a/packages/nocodb/src/app.module.ts +++ b/packages/nocodb/src/app.module.ts @@ -24,6 +24,7 @@ import type { MiddlewareConsumer, OnApplicationBootstrap, } from '@nestjs/common'; +import { HookHandlerService } from './services/hook-handler.service'; @Module({ imports: [ @@ -43,6 +44,7 @@ import type { LocalStrategy, AuthTokenStrategy, BaseViewStrategy, + HookHandlerService, ], }) export class AppModule implements OnApplicationBootstrap { diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 846a95f9c1..807b451ab8 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -11,24 +11,17 @@ import { UITypes, ViewTypes, } from 'nocodb-sdk'; -import ejs from 'ejs'; import Validator from 'validator'; import { customAlphabet } from 'nanoid'; import DOMPurify from 'isomorphic-dompurify'; import { v4 as uuidv4 } from 'uuid'; import { NcError } from '../helpers/catchError'; import getAst from '../helpers/getAst'; -import NcPluginMgrv2 from '../helpers/NcPluginMgrv2'; -import { - _transformSubmittedFormDataForEmail, - invokeWebhook, -} from '../helpers/webhookHelpers'; + import { Audit, Column, Filter, - FormView, - Hook, Model, Project, Sort, @@ -40,7 +33,8 @@ import { COMPARISON_SUB_OPS, IS_WITHIN_COMPARISON_SUB_OPS, } from '../models/Filter'; -import formSubmissionEmailTemplate from '../utils/common/formSubmissionEmailTemplate'; +import Noco from '../Noco' +import { HANDLE_WEBHOOK } from '../services/hook-handler.service' import formulaQueryBuilderv2 from './formulav2/formulaQueryBuilderv2'; import genRollupSelectv2 from './genRollupSelectv2'; import conditionV2 from './conditionV2'; @@ -2496,6 +2490,18 @@ class BaseModelSqlv2 { } private async handleHooks(hookName, prevData, newData, req): Promise { + + Noco.eventEmitter.emit(HANDLE_WEBHOOK, { + hookName, + prevData, + newData, + user: req?.user, + viewId: this.viewId, + modelId: this.model.id, + tnPath: this.tnPath, + }) +/* + const view = await View.get(this.viewId); // handle form view data submission @@ -2585,7 +2591,7 @@ class BaseModelSqlv2 { } } catch (e) { console.log('hooks :: error', hookName, e); - } + }*/ } // @ts-ignore diff --git a/packages/nocodb/src/services/hook-handler.service.spec.ts b/packages/nocodb/src/services/hook-handler.service.spec.ts new file mode 100644 index 0000000000..9b79fd5af6 --- /dev/null +++ b/packages/nocodb/src/services/hook-handler.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { HookHandlerService } from './hook-handler.service'; + +describe('HookHandlerService', () => { + let service: HookHandlerService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [HookHandlerService], + }).compile(); + + service = module.get(HookHandlerService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/packages/nocodb/src/services/hook-handler.service.ts b/packages/nocodb/src/services/hook-handler.service.ts new file mode 100644 index 0000000000..1b50a6c39d --- /dev/null +++ b/packages/nocodb/src/services/hook-handler.service.ts @@ -0,0 +1,143 @@ +import { Injectable } from '@nestjs/common'; +import { UITypes, ViewTypes } from 'nocodb-sdk'; +import ejs from 'ejs'; +import NcPluginMgrv2 from '../helpers/NcPluginMgrv2'; +import { + _transformSubmittedFormDataForEmail, + invokeWebhook, +} from '../helpers/webhookHelpers'; +import { FormView, Hook, Model, View } from '../models'; +import { IEventEmitter } from '../modules/event-emitter/event-emitter.interface'; +import formSubmissionEmailTemplate from '../utils/common/formSubmissionEmailTemplate'; +import type { OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import type { UserType } from 'nocodb-sdk'; + +export const HANDLE_WEBHOOK = '__nc_handleHooks'; + +@Injectable() +export class HookHandlerService implements OnModuleInit, OnModuleDestroy { + private unsubscribe: () => void; + + constructor(private readonly eventEmitter: IEventEmitter) {} + + private async handleHooks({ + hookName, + prevData, + newData, + user, + viewId, + modelId, + tnPath, + }: { + hookName; + prevData; + newData; + user: UserType; + viewId: string; + modelId: string; + tnPath: string; + }): Promise { + const view = await View.get(viewId); + const model = await Model.get(modelId); + + // handle form view data submission + if ( + (hookName === 'after.insert' || hookName === 'after.bulkInsert') && + view.type === ViewTypes.FORM + ) { + try { + const formView = await view.getView(); + const { columns } = await FormView.getWithInfo(formView.fk_view_id); + const allColumns = await model.getColumns(); + const fieldById = columns.reduce( + (o: Record, f: Record) => ({ + ...o, + [f.fk_column_id]: f, + }), + {}, + ); + let order = 1; + const filteredColumns = allColumns + ?.map((c: Record) => ({ + ...c, + fk_column_id: c.id, + fk_view_id: formView.fk_view_id, + ...(fieldById[c.id] ? fieldById[c.id] : {}), + order: (fieldById[c.id] && fieldById[c.id].order) || order++, + id: fieldById[c.id] && fieldById[c.id].id, + })) + .sort( + (a: Record, b: Record) => + a.order - b.order, + ) + .filter( + (f: Record) => + f.show && + f.uidt !== UITypes.Rollup && + f.uidt !== UITypes.Lookup && + f.uidt !== UITypes.Formula && + f.uidt !== UITypes.QrCode && + f.uidt !== UITypes.Barcode && + f.uidt !== UITypes.SpecificDBType, + ) + .sort( + (a: Record, b: Record) => + a.order - b.order, + ) + .map((c: Record) => ({ + ...c, + required: !!(c.required || 0), + })); + + const emails = Object.entries(JSON.parse(formView?.email) || {}) + .filter((a) => a[1]) + .map((a) => a[0]); + if (emails?.length) { + const transformedData = _transformSubmittedFormDataForEmail( + newData, + formView, + filteredColumns, + ); + (await NcPluginMgrv2.emailAdapter(false))?.mailSend({ + to: emails.join(','), + subject: 'NocoDB Form', + html: ejs.render(formSubmissionEmailTemplate, { + data: transformedData, + tn: tnPath, + _tn: model.title, + }), + }); + } + } catch (e) { + console.log(e); + } + } + + try { + const [event, operation] = hookName.split('.'); + const hooks = await Hook.list({ + fk_model_id: modelId, + event, + operation, + }); + for (const hook of hooks) { + if (hook.active) { + invokeWebhook(hook, model, view, prevData, newData, user); + } + } + } catch (e) { + console.log('hooks :: error', hookName, e); + } + } + + onModuleInit(): any { + this.unsubscribe = this.eventEmitter.on( + HANDLE_WEBHOOK, + this.handleHooks.bind(this), + ); + } + + onModuleDestroy() { + this.unsubscribe?.(); + } +} From d5235a2361312743d3c12bea108ebf51a327f234 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 4 May 2023 00:09:20 +0530 Subject: [PATCH 2/4] feat: use event emitter for webhook(WIP) Signed-off-by: Pranav C --- packages/nocodb/src/Noco.ts | 2 ++ packages/nocodb/src/app.module.ts | 5 +++- .../event-emitter/event-emitter.interface.ts | 6 +++++ .../event-emitter/event-emitter.module.ts | 16 +++++++++++ .../event-emitter/fallback-event-emitter.ts | 27 +++++++++++++++++++ .../event-emitter/nestjs-event-emitter.ts | 23 ++++++++++++++++ 6 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 packages/nocodb/src/modules/event-emitter/event-emitter.interface.ts create mode 100644 packages/nocodb/src/modules/event-emitter/event-emitter.module.ts create mode 100644 packages/nocodb/src/modules/event-emitter/fallback-event-emitter.ts create mode 100644 packages/nocodb/src/modules/event-emitter/nestjs-event-emitter.ts diff --git a/packages/nocodb/src/Noco.ts b/packages/nocodb/src/Noco.ts index a86f260f9a..9c58bf2204 100644 --- a/packages/nocodb/src/Noco.ts +++ b/packages/nocodb/src/Noco.ts @@ -9,6 +9,7 @@ import { NC_LICENSE_KEY } from './constants'; import Store from './models/Store'; import type { Express } from 'express'; import type * as http from 'http'; +import { IEventEmitter } from './modules/event-emitter/event-emitter.interface' export default class Noco { private static _this: Noco; @@ -30,6 +31,7 @@ export default class Noco { } public static config: any; + public static eventEmitter: IEventEmitter; public readonly router: express.Router; public readonly projectRouter: express.Router; public static _ncMeta: any; diff --git a/packages/nocodb/src/app.module.ts b/packages/nocodb/src/app.module.ts index 9af6d4919e..93b61077a1 100644 --- a/packages/nocodb/src/app.module.ts +++ b/packages/nocodb/src/app.module.ts @@ -1,4 +1,4 @@ -import { Module, RequestMethod } from '@nestjs/common'; +import { Inject, Module, RequestMethod } from '@nestjs/common' import { APP_FILTER } from '@nestjs/core'; import { Connection } from './connection/connection'; import { GlobalExceptionFilter } from './filters/global-exception/global-exception.filter'; @@ -7,6 +7,7 @@ import { GlobalMiddleware } from './middlewares/global/global.middleware'; import { GuiMiddleware } from './middlewares/gui/gui.middleware'; import { PublicMiddleware } from './middlewares/public/public.middleware'; import { DatasModule } from './modules/datas/datas.module'; +import { IEventEmitter } from './modules/event-emitter/event-emitter.interface' import { AuthService } from './services/auth.service'; import { UsersModule } from './modules/users/users.module'; import { MetaService } from './meta/meta.service'; @@ -51,6 +52,7 @@ export class AppModule implements OnApplicationBootstrap { constructor( private readonly connection: Connection, private readonly metaService: MetaService, + @Inject('IEventEmitter') private readonly eventEmitter: IEventEmitter, ) {} // Global Middleware @@ -80,6 +82,7 @@ export class AppModule implements OnApplicationBootstrap { // temporary hack Noco._ncMeta = this.metaService; Noco.config = this.connection.config; + Noco.eventEmitter = this.eventEmitter; // init plugin manager await NcPluginMgrv2.init(Noco.ncMeta); diff --git a/packages/nocodb/src/modules/event-emitter/event-emitter.interface.ts b/packages/nocodb/src/modules/event-emitter/event-emitter.interface.ts new file mode 100644 index 0000000000..59084e31f8 --- /dev/null +++ b/packages/nocodb/src/modules/event-emitter/event-emitter.interface.ts @@ -0,0 +1,6 @@ +export interface IEventEmitter { + emit(event: string, arg: any): void; + on(event: string, listener: (arg: any) => void): () => void; + removeListener(event: string, listener: (arg: any) => void): void; + removeAllListeners(event?: string): void; +} diff --git a/packages/nocodb/src/modules/event-emitter/event-emitter.module.ts b/packages/nocodb/src/modules/event-emitter/event-emitter.module.ts new file mode 100644 index 0000000000..a6eab8436d --- /dev/null +++ b/packages/nocodb/src/modules/event-emitter/event-emitter.module.ts @@ -0,0 +1,16 @@ +import { Global, Module } from '@nestjs/common'; +import { FallbackEventEmitter } from './fallback-event-emitter'; + +@Global() +@Module({ + providers: [ + { + provide: 'IEventEmitter', + useFactory: () => { + return new FallbackEventEmitter(); + }, + }, + ], + exports: ['IEventEmitter'], +}) +export class EventEmitterModule {} diff --git a/packages/nocodb/src/modules/event-emitter/fallback-event-emitter.ts b/packages/nocodb/src/modules/event-emitter/fallback-event-emitter.ts new file mode 100644 index 0000000000..4ee3ac5020 --- /dev/null +++ b/packages/nocodb/src/modules/event-emitter/fallback-event-emitter.ts @@ -0,0 +1,27 @@ +import Emittery from 'emittery'; +import { IEventEmitter } from './event-emitter.interface'; + +export class FallbackEventEmitter implements IEventEmitter { + private readonly emitter: Emittery; + + constructor() { + this.emitter = new Emittery(); + } + + emit(event: string, data: any): void { + this.emitter.emit(event, data); + } + + on(event: string, listener: (...args: any[]) => void) { + this.emitter.on(event, listener); + return () => this.emitter.off(event, listener); + } + + removeListener(event: string, listener: (...args: any[]) => void): void { + this.emitter.off(event, listener); + } + + removeAllListeners(event?: string): void { + this.emitter.clearListeners(event); + } +} diff --git a/packages/nocodb/src/modules/event-emitter/nestjs-event-emitter.ts b/packages/nocodb/src/modules/event-emitter/nestjs-event-emitter.ts new file mode 100644 index 0000000000..14e83fe5fa --- /dev/null +++ b/packages/nocodb/src/modules/event-emitter/nestjs-event-emitter.ts @@ -0,0 +1,23 @@ +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { IEventEmitter } from './event-emitter.interface'; + +export class NestjsEventEmitter implements IEventEmitter { + constructor(private readonly eventEmitter: EventEmitter2) {} + + emit(event: string, data: any): void { + this.eventEmitter.emit(event, data); + } + + on(event: string, listener: (...args: any[]) => void) { + this.eventEmitter.on(event, listener); + return () => this.eventEmitter.removeListener(event, listener); + } + + removeListener(event: string, listener: (...args: any[]) => void): void { + this.eventEmitter.removeListener(event, listener); + } + + removeAllListeners(event?: string): void { + this.eventEmitter.removeAllListeners(event); + } +} From b55031fd876cb91ca73a9ee9492852984f998e60 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 5 May 2023 11:33:46 +0530 Subject: [PATCH 3/4] fix: set injection key and add event emitter module Signed-off-by: Pranav C --- packages/nocodb/src/app.module.ts | 8 +++++--- packages/nocodb/src/modules/metas/metas.module.ts | 2 +- packages/nocodb/src/services/hook-handler.service.ts | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/nocodb/src/app.module.ts b/packages/nocodb/src/app.module.ts index 93b61077a1..0cf3915ea9 100644 --- a/packages/nocodb/src/app.module.ts +++ b/packages/nocodb/src/app.module.ts @@ -1,4 +1,4 @@ -import { Inject, Module, RequestMethod } from '@nestjs/common' +import { Inject, Module, RequestMethod } from '@nestjs/common'; import { APP_FILTER } from '@nestjs/core'; import { Connection } from './connection/connection'; import { GlobalExceptionFilter } from './filters/global-exception/global-exception.filter'; @@ -7,13 +7,15 @@ import { GlobalMiddleware } from './middlewares/global/global.middleware'; import { GuiMiddleware } from './middlewares/gui/gui.middleware'; import { PublicMiddleware } from './middlewares/public/public.middleware'; import { DatasModule } from './modules/datas/datas.module'; -import { IEventEmitter } from './modules/event-emitter/event-emitter.interface' +import { IEventEmitter } from './modules/event-emitter/event-emitter.interface'; +import { EventEmitterModule } from './modules/event-emitter/event-emitter.module'; import { AuthService } from './services/auth.service'; import { UsersModule } from './modules/users/users.module'; import { MetaService } from './meta/meta.service'; import Noco from './Noco'; import { TestModule } from './modules/test/test.module'; import { GlobalModule } from './modules/global/global.module'; +import { HookHandlerService } from './services/hook-handler.service'; import { LocalStrategy } from './strategies/local.strategy'; import { AuthTokenStrategy } from './strategies/authtoken.strategy/authtoken.strategy'; import { BaseViewStrategy } from './strategies/base-view.strategy/base-view.strategy'; @@ -25,7 +27,6 @@ import type { MiddlewareConsumer, OnApplicationBootstrap, } from '@nestjs/common'; -import { HookHandlerService } from './services/hook-handler.service'; @Module({ imports: [ @@ -34,6 +35,7 @@ import { HookHandlerService } from './services/hook-handler.service'; ...(process.env['PLAYWRIGHT_TEST'] === 'true' ? [TestModule] : []), MetasModule, DatasModule, + EventEmitterModule, ], controllers: [], providers: [ diff --git a/packages/nocodb/src/modules/metas/metas.module.ts b/packages/nocodb/src/modules/metas/metas.module.ts index 13be9585a7..ea6bbd7a35 100644 --- a/packages/nocodb/src/modules/metas/metas.module.ts +++ b/packages/nocodb/src/modules/metas/metas.module.ts @@ -68,10 +68,10 @@ import { UtilsService } from '../../services/utils.service'; import { ViewColumnsService } from '../../services/view-columns.service'; import { ViewsService } from '../../services/views.service'; import { ApiDocsService } from '../../services/api-docs/api-docs.service'; +import { EventEmitterModule } from '../event-emitter/event-emitter.module' import { GlobalModule } from '../global/global.module'; import { ProjectUsersController } from '../../controllers/project-users.controller'; import { ProjectUsersService } from '../../services/project-users/project-users.service'; -import { DatasModule } from '../datas/datas.module'; @Module({ imports: [ diff --git a/packages/nocodb/src/services/hook-handler.service.ts b/packages/nocodb/src/services/hook-handler.service.ts index 1b50a6c39d..d7dac48aa4 100644 --- a/packages/nocodb/src/services/hook-handler.service.ts +++ b/packages/nocodb/src/services/hook-handler.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common' import { UITypes, ViewTypes } from 'nocodb-sdk'; import ejs from 'ejs'; import NcPluginMgrv2 from '../helpers/NcPluginMgrv2'; @@ -18,7 +18,7 @@ export const HANDLE_WEBHOOK = '__nc_handleHooks'; export class HookHandlerService implements OnModuleInit, OnModuleDestroy { private unsubscribe: () => void; - constructor(private readonly eventEmitter: IEventEmitter) {} + constructor(@Inject('IEventEmitter') private readonly eventEmitter: IEventEmitter) {} private async handleHooks({ hookName, From df6f0501eb171881f1583f8a31f084d5b72e887b Mon Sep 17 00:00:00 2001 From: Raju Udava <86527202+dstala@users.noreply.github.com> Date: Mon, 8 May 2023 16:10:53 +0530 Subject: [PATCH 4/4] test: kludge - add delay for response Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- tests/playwright/tests/db/01-webhook.spec.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/playwright/tests/db/01-webhook.spec.ts b/tests/playwright/tests/db/01-webhook.spec.ts index cf0dce6e4d..b02130544c 100644 --- a/tests/playwright/tests/db/01-webhook.spec.ts +++ b/tests/playwright/tests/db/01-webhook.spec.ts @@ -23,6 +23,9 @@ async function clearServerData({ request }) { async function getWebhookResponses({ request, count = 1 }) { let response; + // kludge- add delay to allow server to process webhook + await new Promise(resolve => setTimeout(resolve, 1000)); + // retry since there can be lag between the time the hook is triggered and the time the server receives the request for (let i = 0; i < 20; i++) { response = await request.get(hookPath + '/count'); @@ -425,6 +428,9 @@ test.describe.serial('Webhook', () => { test('Bulk operations', async ({ request, page }) => { async function verifyBulkOperationTrigger(rsp, type) { + // kludge- add delay to allow server to process webhook + await new Promise(resolve => setTimeout(resolve, 1000)); + for (let i = 0; i < rsp.length; i++) { expect(rsp[i].type).toBe(type); expect(rsp[i].data.table_name).toBe('Test');