From be39067ec135925b459f021ee0d7677bcadb3c36 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Tue, 10 Sep 2024 09:45:53 +0000 Subject: [PATCH] refactor: event related modules --- packages/nocodb/src/Noco.ts | 2 +- .../src/db/sql-client/lib/KnexClient.ts | 2 +- packages/nocodb/src/db/sql-mgr/SqlMgr.ts | 2 +- .../nocodb/src/gateways/socket.gateway.ts | 2 +- packages/nocodb/src/helpers/apiMetrics.ts | 2 +- .../nocodb/src/helpers/initAdminFromEnv.ts | 2 +- .../providers/init-meta-service.provider.ts | 2 +- .../services/app-hooks-listener.service.ts | 2 +- .../nocodb/src/services/telemetry.service.ts | 4 +- .../src/services/users/users.service.ts | 2 +- .../nocodb/src/utils/TeleBatchProcessor.ts | 50 ++++ packages/nocodb/src/utils/feedbackForm.ts | 13 + packages/nocodb/src/utils/index.ts | 2 + packages/nocodb/src/utils/tele.ts | 280 ++++++++++++++++++ .../nocodb/src/version-upgrader/NcUpgrader.ts | 2 +- 15 files changed, 357 insertions(+), 12 deletions(-) create mode 100644 packages/nocodb/src/utils/TeleBatchProcessor.ts create mode 100644 packages/nocodb/src/utils/feedbackForm.ts create mode 100644 packages/nocodb/src/utils/tele.ts diff --git a/packages/nocodb/src/Noco.ts b/packages/nocodb/src/Noco.ts index d6163cd335..53cc0e5102 100644 --- a/packages/nocodb/src/Noco.ts +++ b/packages/nocodb/src/Noco.ts @@ -2,7 +2,7 @@ import path from 'path'; import { NestFactory } from '@nestjs/core'; import clear from 'clear'; import * as express from 'express'; -import { T } from 'nc-help'; +import { T } from '~/utils'; import { v4 as uuidv4 } from 'uuid'; import dotenv from 'dotenv'; import { IoAdapter } from '@nestjs/platform-socket.io'; diff --git a/packages/nocodb/src/db/sql-client/lib/KnexClient.ts b/packages/nocodb/src/db/sql-client/lib/KnexClient.ts index 726cb4e5be..941d88da9c 100644 --- a/packages/nocodb/src/db/sql-client/lib/KnexClient.ts +++ b/packages/nocodb/src/db/sql-client/lib/KnexClient.ts @@ -3,7 +3,7 @@ import fs from 'fs'; import { promisify } from 'util'; import path from 'path'; import { knex } from 'knex'; -import { T } from 'nc-help'; +import { T } from '~/utils'; import findIndex from 'lodash/findIndex'; import find from 'lodash/find'; import jsonfile from 'jsonfile'; diff --git a/packages/nocodb/src/db/sql-mgr/SqlMgr.ts b/packages/nocodb/src/db/sql-mgr/SqlMgr.ts index f3c3a5b6d3..1dcf5df509 100644 --- a/packages/nocodb/src/db/sql-mgr/SqlMgr.ts +++ b/packages/nocodb/src/db/sql-mgr/SqlMgr.ts @@ -6,7 +6,7 @@ import fsExtra from 'fs-extra'; import importFresh from 'import-fresh'; import inflection from 'inflection'; import slash from 'slash'; -import { T } from 'nc-help'; +import { T } from '~/utils'; import { customAlphabet } from 'nanoid'; import type MssqlClient from '~/db/sql-client/lib/mssql/MssqlClient'; import type MysqlClient from '~/db/sql-client/lib/mysql/MysqlClient'; diff --git a/packages/nocodb/src/gateways/socket.gateway.ts b/packages/nocodb/src/gateways/socket.gateway.ts index e6eb13ffed..65f7e4ca0d 100644 --- a/packages/nocodb/src/gateways/socket.gateway.ts +++ b/packages/nocodb/src/gateways/socket.gateway.ts @@ -1,7 +1,7 @@ import crypto from 'crypto'; import { Inject, Injectable } from '@nestjs/common'; import { HttpAdapterHost } from '@nestjs/core'; -import { T } from 'nc-help'; +import { T } from '~/utils'; import { Server } from 'socket.io'; import { AuthGuard } from '@nestjs/passport'; import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host'; diff --git a/packages/nocodb/src/helpers/apiMetrics.ts b/packages/nocodb/src/helpers/apiMetrics.ts index 10dbd3b737..c0ea094610 100644 --- a/packages/nocodb/src/helpers/apiMetrics.ts +++ b/packages/nocodb/src/helpers/apiMetrics.ts @@ -1,4 +1,4 @@ -import { T } from 'nc-help'; +import { T } from '~/utils'; import type { Request } from 'express'; const countMap = {}; diff --git a/packages/nocodb/src/helpers/initAdminFromEnv.ts b/packages/nocodb/src/helpers/initAdminFromEnv.ts index b45113a78a..49f8e7f84f 100644 --- a/packages/nocodb/src/helpers/initAdminFromEnv.ts +++ b/packages/nocodb/src/helpers/initAdminFromEnv.ts @@ -3,7 +3,7 @@ import { v4 as uuidv4 } from 'uuid'; import bcrypt from 'bcryptjs'; import { validatePassword } from 'nocodb-sdk'; import boxen from 'boxen'; -import { T } from 'nc-help'; +import { T } from '~/utils'; import isEmail from 'validator/lib/isEmail'; import NocoCache from '~/cache/NocoCache'; import Noco from '~/Noco'; diff --git a/packages/nocodb/src/providers/init-meta-service.provider.ts b/packages/nocodb/src/providers/init-meta-service.provider.ts index 19f60d1ce8..ed736a5043 100644 --- a/packages/nocodb/src/providers/init-meta-service.provider.ts +++ b/packages/nocodb/src/providers/init-meta-service.provider.ts @@ -1,4 +1,4 @@ -import { T } from 'nc-help'; +import { T } from '~/utils'; import type { FactoryProvider } from '@nestjs/common'; import type { IEventEmitter } from '~/modules/event-emitter/event-emitter.interface'; import { populatePluginsForCloud } from '~/utils/cloud/populateCloudPlugins'; diff --git a/packages/nocodb/src/services/app-hooks-listener.service.ts b/packages/nocodb/src/services/app-hooks-listener.service.ts index 8752a0235b..1aec1a2327 100644 --- a/packages/nocodb/src/services/app-hooks-listener.service.ts +++ b/packages/nocodb/src/services/app-hooks-listener.service.ts @@ -4,7 +4,7 @@ import { AuditOperationSubTypes, AuditOperationTypes, } from 'nocodb-sdk'; -import { T } from 'nc-help'; +import { T } from '~/utils'; import type { AuditType } from 'nocodb-sdk'; import type { OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import type { diff --git a/packages/nocodb/src/services/telemetry.service.ts b/packages/nocodb/src/services/telemetry.service.ts index cecf359473..d4716890a2 100644 --- a/packages/nocodb/src/services/telemetry.service.ts +++ b/packages/nocodb/src/services/telemetry.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; -import { packageInfo } from 'nc-help'; -import { T } from 'nc-help'; +import { packageInfo } from '~/utils'; +import { T } from '~/utils'; @Injectable() export class TelemetryService { diff --git a/packages/nocodb/src/services/users/users.service.ts b/packages/nocodb/src/services/users/users.service.ts index b7fb40b695..22306427f9 100644 --- a/packages/nocodb/src/services/users/users.service.ts +++ b/packages/nocodb/src/services/users/users.service.ts @@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common'; import { AppEvents, OrgUserRoles, validatePassword } from 'nocodb-sdk'; import { v4 as uuidv4 } from 'uuid'; import isEmail from 'validator/lib/isEmail'; -import { T } from 'nc-help'; +import { T } from '~/utils'; import * as ejs from 'ejs'; import bcrypt from 'bcryptjs'; import type { diff --git a/packages/nocodb/src/utils/TeleBatchProcessor.ts b/packages/nocodb/src/utils/TeleBatchProcessor.ts new file mode 100644 index 0000000000..1b20a5ec5f --- /dev/null +++ b/packages/nocodb/src/utils/TeleBatchProcessor.ts @@ -0,0 +1,50 @@ +const axios = require('axios'); + +// This class is used to batch process telemetry data +class TeleBatchProcessor { + private batch: any[]; + private flushAt: number; + private flushInterval: number; + private timeoutRef: any; + + constructor({ flushAt = 100, flushInterval = 10000 } = {}) { + this.batch = []; + this.flushAt = flushAt; + this.flushInterval = flushInterval; + } + + capture(item) { + this.batch.push(item); + + if (this.batch.length >= this.flushAt) { + this.flushBackground(); + } + + if (this.flushInterval) { + this.timeoutRef = setTimeout(() => { + this.flushBackground(); + }, this.flushInterval); + } + } + + flushBackground() { + if (this.timeoutRef) { + clearTimeout(this.timeoutRef); + this.timeoutRef = null; + } + this.flush().catch(() => { + // do nothing + }); + } + + async flush() { + const batch = this.batch.splice(0, this.batch.length); + if (!batch.length) { + return; + } + + await axios.post('https://nocodb.com/api/v1/telemetry', batch); + } +} + +export default TeleBatchProcessor; diff --git a/packages/nocodb/src/utils/feedbackForm.ts b/packages/nocodb/src/utils/feedbackForm.ts new file mode 100644 index 0000000000..e2daa1fb1d --- /dev/null +++ b/packages/nocodb/src/utils/feedbackForm.ts @@ -0,0 +1,13 @@ +import axios from 'axios'; + +export const getFeedBackForm = async () => { + try { + const response = await axios + .get('https://nocodb.com/api/v1/feedback_form', { + timeout: 5000 + }); + return response.data; + } catch (e) { + return { error: e.message }; + } +}; diff --git a/packages/nocodb/src/utils/index.ts b/packages/nocodb/src/utils/index.ts index 834c0286b1..4de678014f 100644 --- a/packages/nocodb/src/utils/index.ts +++ b/packages/nocodb/src/utils/index.ts @@ -3,5 +3,7 @@ export * from './sanitiseUserObj'; export * from './emailUtils'; export * from './circularReplacer'; export * from './nocoExecute'; +export * from './Tele'; +export * from './packageVersion'; export const isEE = false; diff --git a/packages/nocodb/src/utils/tele.ts b/packages/nocodb/src/utils/tele.ts new file mode 100644 index 0000000000..d651e7710a --- /dev/null +++ b/packages/nocodb/src/utils/tele.ts @@ -0,0 +1,280 @@ +import os from 'os'; +import Emittery from 'emittery'; +import { machineIdSync } from 'node-machine-id'; +import axios from 'axios'; +import isDocker from 'is-docker'; +import { packageVersion } from '~/utils/packageVersion'; +import TeleBatchProcessor from '~/utils/TeleBatchProcessor'; + +const isDisabled = !!process.env.NC_DISABLE_TELE; +const cache = !!process.env.NC_REDIS_URL; +const executable = !!process.env.NC_BINARY_BUILD; +const litestream = !!( + process.env.LITESTREAM_S3_BUCKET && + process.env.LITESTREAM_S3_SECRET_ACCESS_KEY && + process.env.LITESTREAM_S3_ACCESS_KEY_ID +); + +const sendEvt = () => { + try { + const upTime = Math.round(process.uptime() / 3600); + Tele.emit('evt', { + evt_type: 'alive', + count: global.NC_COUNT, + upTime, + cache, + litestream, + executable, + }); + } catch {} +}; +setInterval(sendEvt, 8 * 60 * 60 * 1000); + +export class Tele { + static emitter; + static machineId; + static config: Record; + static client: TeleBatchProcessor; + + static emit(event, data) { + try { + this._init(); + Tele.emitter.emit(event, data); + } catch (e) {} + } + + static init(config: Record) { + Tele.config = config; + Tele._init(); + } + + static page(args: Record) { + this.emit('page', args); + } + + static event(args: Record) { + this.emit('ph_event', args); + } + + static _init() { + try { + if (!Tele.emitter) { + Tele.emitter = new Emittery(); + Tele.machineId = machineIdSync(); + + let package_id = ''; + let xc_version = ''; + xc_version = process.env.NC_SERVER_UUID; + package_id = packageVersion; + + const teleData: Record = { + package_id, + os_type: os.type(), + os_platform: os.platform(), + os_release: os.release(), + node_version: process.version, + docker: isDocker(), + xc_version: xc_version, + env: process.env.NODE_ENV || 'production', + oneClick: !!process.env.NC_ONE_CLICK, + }; + teleData.machine_id = `${machineIdSync()},,`; + Tele.emitter.on('evt_app_started', async (msg) => { + try { + await waitForMachineId(teleData); + if (isDisabled) return; + + if (msg && msg.count !== undefined) { + global.NC_COUNT = msg.count; + } + + await axios.post('https://telemetry.nocodb.com/v1/telemetry', { + ...teleData, + evt_type: 'started', + payload: { + count: global.NC_COUNT, + }, + }); + } catch (e) { + } finally { + sendEvt(); + } + }); + + Tele.emitter.on('evt', async (payload) => { + try { + const instanceMeta = (await Tele.getInstanceMeta()) || {}; + + await waitForMachineId(teleData); + if (payload.check) { + teleData.machine_id = `${machineIdSync()},,`; + } + if ( + isDisabled && + !( + payload.evt_type && + payload.evt_type.startsWith('a:sync-request:') + ) + ) + return; + + if (payload.evt_type === 'project:invite') { + global.NC_COUNT = payload.count || global.NC_COUNT; + } + if (payload.evt_type === 'user:first_signup') { + global.NC_COUNT = +global.NC_COUNT || 1; + } + + await axios.post('https://telemetry.nocodb.com/v1/telemetry', { + ...teleData, + evt_type: payload.evt_type, + payload: { ...instanceMeta, ...(payload || {}) }, + }); + } catch {} + }); + + Tele.emitter.on('evt_api_created', async (data) => { + try { + await waitForMachineId(teleData); + const stats = { + ...teleData, + table_count: data.tablesCount || 0, + relation_count: data.relationsCount || 0, + view_count: data.viewsCount || 0, + api_count: data.apiCount || 0, + function_count: data.functionsCount || 0, + procedure_count: data.proceduresCount || 0, + mysql: data.dbType === 'mysql2' ? 1 : 0, + pg: data.dbType === 'pg' ? 1 : 0, + mssql: data.dbType === 'mssql' ? 1 : 0, + sqlite3: data.dbType === 'sqlite3' ? 1 : 0, + oracledb: data.dbType === 'oracledb' ? 1 : 0, + rest: data.type === 'rest' ? 1 : 0, + graphql: data.type === 'graphql' ? 1 : 0, + grpc: data.type === 'grpc' ? 1 : 0, + time_taken: data.timeTaken, + }; + if (isDisabled) return; + await axios.post( + 'https://telemetry.nocodb.com/v1/telemetry/apis_created', + stats, + ); + } catch (e) {} + }); + + Tele.emitter.on('evt_subscribe', async (email) => { + try { + if (isDisabled) return; + await axios.post( + 'https://telemetry.nocodb.com/v1/newsletter/sdhjh34u3yuy34bj343jhj4iwolaAdsdj3434uiut4nn', + { + email, + }, + ); + } catch (e) {} + }); + + Tele.emitter.on('page', async (args) => { + try { + if (isDisabled) return; + const instanceMeta = await Tele.getInstanceMeta(); + + await this.client.capture({ + distinctId: args.id || `${this.machineId}:public`, + event: '$pageview', + properties: { + ...teleData, + ...instanceMeta, + $current_url: args.path, + }, + }); + } catch {} + }); + Tele.emitter.on('ph_event', async (payload: Record) => { + try { + if ( + isDisabled && + !( + payload.evt_type && + payload.evt_type.startsWith('a:sync-request:') + ) + ) + return; + const instanceMeta = await this.getInstanceMeta(); + let id = payload.id; + + if (!id) { + if (payload.event && payload.event.startsWith('a:api:')) { + id = this.machineId; + } else { + id = `${this.machineId}:public`; + } + } + await this.client.capture({ + // userId: id, + distinctId: id, + event: payload.event, + properties: { + ...teleData, + ...instanceMeta, + ...(payload.data || {}), + }, + }); + } catch {} + }); + } + } catch (e) {} + + try { + if (!this.client) { + this.client = new TeleBatchProcessor(); + } + } catch {} + } + + static async getInstanceMeta() { + try { + return ( + (Tele.config && + Tele.config.instance && + (await Tele.config.instance())) || + {} + ); + } catch { + return {}; + } + } + + static get id() { + return this.machineId || machineIdSync(); + } +} + +export { Tele as T }; + +async function waitForMachineId(teleData) { + let i = 5; + while (i-- && !teleData.machine_id) { + await new Promise((resolve) => setTimeout(() => resolve(null), 500)); + } +} + +// this is to keep the server alive +if (process.env.NC_PUBLIC_URL) { + setInterval(() => { + axios({ + method: 'get', + url: process.env.NC_PUBLIC_URL, + }) + .then(() => {}) + .catch(() => {}); + }, 2 * 60 * 60 * 1000); +} + +if (process.env.NC_ONE_CLICK) { + try { + Tele.emit('evt', { + evt_type: 'ONE_CLICK', + }); + } catch {} +} diff --git a/packages/nocodb/src/version-upgrader/NcUpgrader.ts b/packages/nocodb/src/version-upgrader/NcUpgrader.ts index 510fa88bfc..34e462eda7 100644 --- a/packages/nocodb/src/version-upgrader/NcUpgrader.ts +++ b/packages/nocodb/src/version-upgrader/NcUpgrader.ts @@ -1,5 +1,5 @@ import debug from 'debug'; -import { T } from 'nc-help'; +import { T } from '~/utils'; import boxen from 'boxen'; import ncAttachmentUpgrader from './ncAttachmentUpgrader'; import ncAttachmentUpgrader_0104002 from './ncAttachmentUpgrader_0104002';