Browse Source

Merge pull request #5597 from nocodb/feat/webhook-with-event-emitter

feat: Use event emitter for webook
pull/5615/head
աɨռɢӄաօռɢ 2 years ago committed by GitHub
parent
commit
b10024cf5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      packages/nocodb/src/Noco.ts
  2. 9
      packages/nocodb/src/app.module.ts
  3. 26
      packages/nocodb/src/db/BaseModelSqlv2.ts
  4. 6
      packages/nocodb/src/modules/event-emitter/event-emitter.interface.ts
  5. 16
      packages/nocodb/src/modules/event-emitter/event-emitter.module.ts
  6. 27
      packages/nocodb/src/modules/event-emitter/fallback-event-emitter.ts
  7. 23
      packages/nocodb/src/modules/event-emitter/nestjs-event-emitter.ts
  8. 2
      packages/nocodb/src/modules/metas/metas.module.ts
  9. 18
      packages/nocodb/src/services/hook-handler.service.spec.ts
  10. 143
      packages/nocodb/src/services/hook-handler.service.ts
  11. 6
      tests/playwright/tests/db/01-webhook.spec.ts

2
packages/nocodb/src/Noco.ts

@ -9,6 +9,7 @@ import { NC_LICENSE_KEY } from './constants';
import Store from './models/Store';
import type { Express } from 'express';
import type * as http from 'http';
import { IEventEmitter } from './modules/event-emitter/event-emitter.interface'
export default class Noco {
private static _this: Noco;
@ -30,6 +31,7 @@ export default class Noco {
}
public static config: any;
public static eventEmitter: IEventEmitter;
public readonly router: express.Router;
public readonly projectRouter: express.Router;
public static _ncMeta: any;

9
packages/nocodb/src/app.module.ts

@ -1,4 +1,4 @@
import { Module, RequestMethod } from '@nestjs/common';
import { Inject, Module, RequestMethod } from '@nestjs/common';
import { APP_FILTER } from '@nestjs/core';
import { Connection } from './connection/connection';
import { GlobalExceptionFilter } from './filters/global-exception/global-exception.filter';
@ -7,12 +7,15 @@ import { GlobalMiddleware } from './middlewares/global/global.middleware';
import { GuiMiddleware } from './middlewares/gui/gui.middleware';
import { PublicMiddleware } from './middlewares/public/public.middleware';
import { DatasModule } from './modules/datas/datas.module';
import { IEventEmitter } from './modules/event-emitter/event-emitter.interface';
import { EventEmitterModule } from './modules/event-emitter/event-emitter.module';
import { AuthService } from './services/auth.service';
import { UsersModule } from './modules/users/users.module';
import { MetaService } from './meta/meta.service';
import Noco from './Noco';
import { TestModule } from './modules/test/test.module';
import { GlobalModule } from './modules/global/global.module';
import { HookHandlerService } from './services/hook-handler.service';
import { LocalStrategy } from './strategies/local.strategy';
import { AuthTokenStrategy } from './strategies/authtoken.strategy/authtoken.strategy';
import { BaseViewStrategy } from './strategies/base-view.strategy/base-view.strategy';
@ -32,6 +35,7 @@ import type {
...(process.env['PLAYWRIGHT_TEST'] === 'true' ? [TestModule] : []),
MetasModule,
DatasModule,
EventEmitterModule,
],
controllers: [],
providers: [
@ -43,12 +47,14 @@ import type {
LocalStrategy,
AuthTokenStrategy,
BaseViewStrategy,
HookHandlerService,
],
})
export class AppModule implements OnApplicationBootstrap {
constructor(
private readonly connection: Connection,
private readonly metaService: MetaService,
@Inject('IEventEmitter') private readonly eventEmitter: IEventEmitter,
) {}
// Global Middleware
@ -78,6 +84,7 @@ export class AppModule implements OnApplicationBootstrap {
// temporary hack
Noco._ncMeta = this.metaService;
Noco.config = this.connection.config;
Noco.eventEmitter = this.eventEmitter;
// init plugin manager
await NcPluginMgrv2.init(Noco.ncMeta);

26
packages/nocodb/src/db/BaseModelSqlv2.ts

@ -11,24 +11,17 @@ import {
UITypes,
ViewTypes,
} from 'nocodb-sdk';
import ejs from 'ejs';
import Validator from 'validator';
import { customAlphabet } from 'nanoid';
import DOMPurify from 'isomorphic-dompurify';
import { v4 as uuidv4 } from 'uuid';
import { NcError } from '../helpers/catchError';
import getAst from '../helpers/getAst';
import NcPluginMgrv2 from '../helpers/NcPluginMgrv2';
import {
_transformSubmittedFormDataForEmail,
invokeWebhook,
} from '../helpers/webhookHelpers';
import {
Audit,
Column,
Filter,
FormView,
Hook,
Model,
Project,
Sort,
@ -40,7 +33,8 @@ import {
COMPARISON_SUB_OPS,
IS_WITHIN_COMPARISON_SUB_OPS,
} from '../models/Filter';
import formSubmissionEmailTemplate from '../utils/common/formSubmissionEmailTemplate';
import Noco from '../Noco'
import { HANDLE_WEBHOOK } from '../services/hook-handler.service'
import formulaQueryBuilderv2 from './formulav2/formulaQueryBuilderv2';
import genRollupSelectv2 from './genRollupSelectv2';
import conditionV2 from './conditionV2';
@ -2496,6 +2490,18 @@ class BaseModelSqlv2 {
}
private async handleHooks(hookName, prevData, newData, req): Promise<void> {
Noco.eventEmitter.emit(HANDLE_WEBHOOK, {
hookName,
prevData,
newData,
user: req?.user,
viewId: this.viewId,
modelId: this.model.id,
tnPath: this.tnPath,
})
/*
const view = await View.get(this.viewId);
// handle form view data submission
@ -2585,7 +2591,7 @@ class BaseModelSqlv2 {
}
} catch (e) {
console.log('hooks :: error', hookName, e);
}
}*/
}
// @ts-ignore

6
packages/nocodb/src/modules/event-emitter/event-emitter.interface.ts

@ -0,0 +1,6 @@
export interface IEventEmitter {
emit(event: string, arg: any): void;
on(event: string, listener: (arg: any) => void): () => void;
removeListener(event: string, listener: (arg: any) => void): void;
removeAllListeners(event?: string): void;
}

16
packages/nocodb/src/modules/event-emitter/event-emitter.module.ts

@ -0,0 +1,16 @@
import { Global, Module } from '@nestjs/common';
import { FallbackEventEmitter } from './fallback-event-emitter';
@Global()
@Module({
providers: [
{
provide: 'IEventEmitter',
useFactory: () => {
return new FallbackEventEmitter();
},
},
],
exports: ['IEventEmitter'],
})
export class EventEmitterModule {}

27
packages/nocodb/src/modules/event-emitter/fallback-event-emitter.ts

@ -0,0 +1,27 @@
import Emittery from 'emittery';
import { IEventEmitter } from './event-emitter.interface';
export class FallbackEventEmitter implements IEventEmitter {
private readonly emitter: Emittery;
constructor() {
this.emitter = new Emittery();
}
emit(event: string, data: any): void {
this.emitter.emit(event, data);
}
on(event: string, listener: (...args: any[]) => void) {
this.emitter.on(event, listener);
return () => this.emitter.off(event, listener);
}
removeListener(event: string, listener: (...args: any[]) => void): void {
this.emitter.off(event, listener);
}
removeAllListeners(event?: string): void {
this.emitter.clearListeners(event);
}
}

23
packages/nocodb/src/modules/event-emitter/nestjs-event-emitter.ts

@ -0,0 +1,23 @@
import { EventEmitter2 } from '@nestjs/event-emitter';
import { IEventEmitter } from './event-emitter.interface';
export class NestjsEventEmitter implements IEventEmitter {
constructor(private readonly eventEmitter: EventEmitter2) {}
emit(event: string, data: any): void {
this.eventEmitter.emit(event, data);
}
on(event: string, listener: (...args: any[]) => void) {
this.eventEmitter.on(event, listener);
return () => this.eventEmitter.removeListener(event, listener);
}
removeListener(event: string, listener: (...args: any[]) => void): void {
this.eventEmitter.removeListener(event, listener);
}
removeAllListeners(event?: string): void {
this.eventEmitter.removeAllListeners(event);
}
}

2
packages/nocodb/src/modules/metas/metas.module.ts

@ -68,10 +68,10 @@ import { UtilsService } from '../../services/utils.service';
import { ViewColumnsService } from '../../services/view-columns.service';
import { ViewsService } from '../../services/views.service';
import { ApiDocsService } from '../../services/api-docs/api-docs.service';
import { EventEmitterModule } from '../event-emitter/event-emitter.module'
import { GlobalModule } from '../global/global.module';
import { ProjectUsersController } from '../../controllers/project-users.controller';
import { ProjectUsersService } from '../../services/project-users/project-users.service';
import { DatasModule } from '../datas/datas.module';
@Module({
imports: [

18
packages/nocodb/src/services/hook-handler.service.spec.ts

@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { HookHandlerService } from './hook-handler.service';
describe('HookHandlerService', () => {
let service: HookHandlerService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [HookHandlerService],
}).compile();
service = module.get<HookHandlerService>(HookHandlerService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

143
packages/nocodb/src/services/hook-handler.service.ts

@ -0,0 +1,143 @@
import { Inject, Injectable } from '@nestjs/common'
import { UITypes, ViewTypes } from 'nocodb-sdk';
import ejs from 'ejs';
import NcPluginMgrv2 from '../helpers/NcPluginMgrv2';
import {
_transformSubmittedFormDataForEmail,
invokeWebhook,
} from '../helpers/webhookHelpers';
import { FormView, Hook, Model, View } from '../models';
import { IEventEmitter } from '../modules/event-emitter/event-emitter.interface';
import formSubmissionEmailTemplate from '../utils/common/formSubmissionEmailTemplate';
import type { OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import type { UserType } from 'nocodb-sdk';
export const HANDLE_WEBHOOK = '__nc_handleHooks';
@Injectable()
export class HookHandlerService implements OnModuleInit, OnModuleDestroy {
private unsubscribe: () => void;
constructor(@Inject('IEventEmitter') private readonly eventEmitter: IEventEmitter) {}
private async handleHooks({
hookName,
prevData,
newData,
user,
viewId,
modelId,
tnPath,
}: {
hookName;
prevData;
newData;
user: UserType;
viewId: string;
modelId: string;
tnPath: string;
}): Promise<void> {
const view = await View.get(viewId);
const model = await Model.get(modelId);
// handle form view data submission
if (
(hookName === 'after.insert' || hookName === 'after.bulkInsert') &&
view.type === ViewTypes.FORM
) {
try {
const formView = await view.getView<FormView>();
const { columns } = await FormView.getWithInfo(formView.fk_view_id);
const allColumns = await model.getColumns();
const fieldById = columns.reduce(
(o: Record<string, any>, f: Record<string, any>) => ({
...o,
[f.fk_column_id]: f,
}),
{},
);
let order = 1;
const filteredColumns = allColumns
?.map((c: Record<string, any>) => ({
...c,
fk_column_id: c.id,
fk_view_id: formView.fk_view_id,
...(fieldById[c.id] ? fieldById[c.id] : {}),
order: (fieldById[c.id] && fieldById[c.id].order) || order++,
id: fieldById[c.id] && fieldById[c.id].id,
}))
.sort(
(a: Record<string, any>, b: Record<string, any>) =>
a.order - b.order,
)
.filter(
(f: Record<string, any>) =>
f.show &&
f.uidt !== UITypes.Rollup &&
f.uidt !== UITypes.Lookup &&
f.uidt !== UITypes.Formula &&
f.uidt !== UITypes.QrCode &&
f.uidt !== UITypes.Barcode &&
f.uidt !== UITypes.SpecificDBType,
)
.sort(
(a: Record<string, any>, b: Record<string, any>) =>
a.order - b.order,
)
.map((c: Record<string, any>) => ({
...c,
required: !!(c.required || 0),
}));
const emails = Object.entries(JSON.parse(formView?.email) || {})
.filter((a) => a[1])
.map((a) => a[0]);
if (emails?.length) {
const transformedData = _transformSubmittedFormDataForEmail(
newData,
formView,
filteredColumns,
);
(await NcPluginMgrv2.emailAdapter(false))?.mailSend({
to: emails.join(','),
subject: 'NocoDB Form',
html: ejs.render(formSubmissionEmailTemplate, {
data: transformedData,
tn: tnPath,
_tn: model.title,
}),
});
}
} catch (e) {
console.log(e);
}
}
try {
const [event, operation] = hookName.split('.');
const hooks = await Hook.list({
fk_model_id: modelId,
event,
operation,
});
for (const hook of hooks) {
if (hook.active) {
invokeWebhook(hook, model, view, prevData, newData, user);
}
}
} catch (e) {
console.log('hooks :: error', hookName, e);
}
}
onModuleInit(): any {
this.unsubscribe = this.eventEmitter.on(
HANDLE_WEBHOOK,
this.handleHooks.bind(this),
);
}
onModuleDestroy() {
this.unsubscribe?.();
}
}

6
tests/playwright/tests/db/01-webhook.spec.ts

@ -23,6 +23,9 @@ async function clearServerData({ request }) {
async function getWebhookResponses({ request, count = 1 }) {
let response;
// kludge- add delay to allow server to process webhook
await new Promise(resolve => setTimeout(resolve, 1000));
// retry since there can be lag between the time the hook is triggered and the time the server receives the request
for (let i = 0; i < 20; i++) {
response = await request.get(hookPath + '/count');
@ -425,6 +428,9 @@ test.describe.serial('Webhook', () => {
test('Bulk operations', async ({ request, page }) => {
async function verifyBulkOperationTrigger(rsp, type) {
// kludge- add delay to allow server to process webhook
await new Promise(resolve => setTimeout(resolve, 1000));
for (let i = 0; i < rsp.length; i++) {
expect(rsp[i].type).toBe(type);
expect(rsp[i].data.table_name).toBe('Test');

Loading…
Cancel
Save