Browse Source

refactor: event related modules

pull/9459/head
Pranav C 3 months ago
parent
commit
be39067ec1
  1. 2
      packages/nocodb/src/Noco.ts
  2. 2
      packages/nocodb/src/db/sql-client/lib/KnexClient.ts
  3. 2
      packages/nocodb/src/db/sql-mgr/SqlMgr.ts
  4. 2
      packages/nocodb/src/gateways/socket.gateway.ts
  5. 2
      packages/nocodb/src/helpers/apiMetrics.ts
  6. 2
      packages/nocodb/src/helpers/initAdminFromEnv.ts
  7. 2
      packages/nocodb/src/providers/init-meta-service.provider.ts
  8. 2
      packages/nocodb/src/services/app-hooks-listener.service.ts
  9. 4
      packages/nocodb/src/services/telemetry.service.ts
  10. 2
      packages/nocodb/src/services/users/users.service.ts
  11. 50
      packages/nocodb/src/utils/TeleBatchProcessor.ts
  12. 13
      packages/nocodb/src/utils/feedbackForm.ts
  13. 2
      packages/nocodb/src/utils/index.ts
  14. 280
      packages/nocodb/src/utils/tele.ts
  15. 2
      packages/nocodb/src/version-upgrader/NcUpgrader.ts

2
packages/nocodb/src/Noco.ts

@ -2,7 +2,7 @@ import path from 'path';
import { NestFactory } from '@nestjs/core';
import clear from 'clear';
import * as express from 'express';
import { T } from 'nc-help';
import { T } from '~/utils';
import { v4 as uuidv4 } from 'uuid';
import dotenv from 'dotenv';
import { IoAdapter } from '@nestjs/platform-socket.io';

2
packages/nocodb/src/db/sql-client/lib/KnexClient.ts

@ -3,7 +3,7 @@ import fs from 'fs';
import { promisify } from 'util';
import path from 'path';
import { knex } from 'knex';
import { T } from 'nc-help';
import { T } from '~/utils';
import findIndex from 'lodash/findIndex';
import find from 'lodash/find';
import jsonfile from 'jsonfile';

2
packages/nocodb/src/db/sql-mgr/SqlMgr.ts

@ -6,7 +6,7 @@ import fsExtra from 'fs-extra';
import importFresh from 'import-fresh';
import inflection from 'inflection';
import slash from 'slash';
import { T } from 'nc-help';
import { T } from '~/utils';
import { customAlphabet } from 'nanoid';
import type MssqlClient from '~/db/sql-client/lib/mssql/MssqlClient';
import type MysqlClient from '~/db/sql-client/lib/mysql/MysqlClient';

2
packages/nocodb/src/gateways/socket.gateway.ts

@ -1,7 +1,7 @@
import crypto from 'crypto';
import { Inject, Injectable } from '@nestjs/common';
import { HttpAdapterHost } from '@nestjs/core';
import { T } from 'nc-help';
import { T } from '~/utils';
import { Server } from 'socket.io';
import { AuthGuard } from '@nestjs/passport';
import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host';

2
packages/nocodb/src/helpers/apiMetrics.ts

@ -1,4 +1,4 @@
import { T } from 'nc-help';
import { T } from '~/utils';
import type { Request } from 'express';
const countMap = {};

2
packages/nocodb/src/helpers/initAdminFromEnv.ts

@ -3,7 +3,7 @@ import { v4 as uuidv4 } from 'uuid';
import bcrypt from 'bcryptjs';
import { validatePassword } from 'nocodb-sdk';
import boxen from 'boxen';
import { T } from 'nc-help';
import { T } from '~/utils';
import isEmail from 'validator/lib/isEmail';
import NocoCache from '~/cache/NocoCache';
import Noco from '~/Noco';

2
packages/nocodb/src/providers/init-meta-service.provider.ts

@ -1,4 +1,4 @@
import { T } from 'nc-help';
import { T } from '~/utils';
import type { FactoryProvider } from '@nestjs/common';
import type { IEventEmitter } from '~/modules/event-emitter/event-emitter.interface';
import { populatePluginsForCloud } from '~/utils/cloud/populateCloudPlugins';

2
packages/nocodb/src/services/app-hooks-listener.service.ts

@ -4,7 +4,7 @@ import {
AuditOperationSubTypes,
AuditOperationTypes,
} from 'nocodb-sdk';
import { T } from 'nc-help';
import { T } from '~/utils';
import type { AuditType } from 'nocodb-sdk';
import type { OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import type {

4
packages/nocodb/src/services/telemetry.service.ts

@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common';
import { packageInfo } from 'nc-help';
import { T } from 'nc-help';
import { packageInfo } from '~/utils';
import { T } from '~/utils';
@Injectable()
export class TelemetryService {

2
packages/nocodb/src/services/users/users.service.ts

@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common';
import { AppEvents, OrgUserRoles, validatePassword } from 'nocodb-sdk';
import { v4 as uuidv4 } from 'uuid';
import isEmail from 'validator/lib/isEmail';
import { T } from 'nc-help';
import { T } from '~/utils';
import * as ejs from 'ejs';
import bcrypt from 'bcryptjs';
import type {

50
packages/nocodb/src/utils/TeleBatchProcessor.ts

@ -0,0 +1,50 @@
const axios = require('axios');
// This class is used to batch process telemetry data
class TeleBatchProcessor {
private batch: any[];
private flushAt: number;
private flushInterval: number;
private timeoutRef: any;
constructor({ flushAt = 100, flushInterval = 10000 } = {}) {
this.batch = [];
this.flushAt = flushAt;
this.flushInterval = flushInterval;
}
capture(item) {
this.batch.push(item);
if (this.batch.length >= this.flushAt) {
this.flushBackground();
}
if (this.flushInterval) {
this.timeoutRef = setTimeout(() => {
this.flushBackground();
}, this.flushInterval);
}
}
flushBackground() {
if (this.timeoutRef) {
clearTimeout(this.timeoutRef);
this.timeoutRef = null;
}
this.flush().catch(() => {
// do nothing
});
}
async flush() {
const batch = this.batch.splice(0, this.batch.length);
if (!batch.length) {
return;
}
await axios.post('https://nocodb.com/api/v1/telemetry', batch);
}
}
export default TeleBatchProcessor;

13
packages/nocodb/src/utils/feedbackForm.ts

@ -0,0 +1,13 @@
import axios from 'axios';
export const getFeedBackForm = async () => {
try {
const response = await axios
.get('https://nocodb.com/api/v1/feedback_form', {
timeout: 5000
});
return response.data;
} catch (e) {
return { error: e.message };
}
};

2
packages/nocodb/src/utils/index.ts

@ -3,5 +3,7 @@ export * from './sanitiseUserObj';
export * from './emailUtils';
export * from './circularReplacer';
export * from './nocoExecute';
export * from './Tele';
export * from './packageVersion';
export const isEE = false;

280
packages/nocodb/src/utils/tele.ts

@ -0,0 +1,280 @@
import os from 'os';
import Emittery from 'emittery';
import { machineIdSync } from 'node-machine-id';
import axios from 'axios';
import isDocker from 'is-docker';
import { packageVersion } from '~/utils/packageVersion';
import TeleBatchProcessor from '~/utils/TeleBatchProcessor';
const isDisabled = !!process.env.NC_DISABLE_TELE;
const cache = !!process.env.NC_REDIS_URL;
const executable = !!process.env.NC_BINARY_BUILD;
const litestream = !!(
process.env.LITESTREAM_S3_BUCKET &&
process.env.LITESTREAM_S3_SECRET_ACCESS_KEY &&
process.env.LITESTREAM_S3_ACCESS_KEY_ID
);
const sendEvt = () => {
try {
const upTime = Math.round(process.uptime() / 3600);
Tele.emit('evt', {
evt_type: 'alive',
count: global.NC_COUNT,
upTime,
cache,
litestream,
executable,
});
} catch {}
};
setInterval(sendEvt, 8 * 60 * 60 * 1000);
export class Tele {
static emitter;
static machineId;
static config: Record<string, any>;
static client: TeleBatchProcessor;
static emit(event, data) {
try {
this._init();
Tele.emitter.emit(event, data);
} catch (e) {}
}
static init(config: Record<string, any>) {
Tele.config = config;
Tele._init();
}
static page(args: Record<string, any>) {
this.emit('page', args);
}
static event(args: Record<string, any>) {
this.emit('ph_event', args);
}
static _init() {
try {
if (!Tele.emitter) {
Tele.emitter = new Emittery();
Tele.machineId = machineIdSync();
let package_id = '';
let xc_version = '';
xc_version = process.env.NC_SERVER_UUID;
package_id = packageVersion;
const teleData: Record<string, any> = {
package_id,
os_type: os.type(),
os_platform: os.platform(),
os_release: os.release(),
node_version: process.version,
docker: isDocker(),
xc_version: xc_version,
env: process.env.NODE_ENV || 'production',
oneClick: !!process.env.NC_ONE_CLICK,
};
teleData.machine_id = `${machineIdSync()},,`;
Tele.emitter.on('evt_app_started', async (msg) => {
try {
await waitForMachineId(teleData);
if (isDisabled) return;
if (msg && msg.count !== undefined) {
global.NC_COUNT = msg.count;
}
await axios.post('https://telemetry.nocodb.com/v1/telemetry', {
...teleData,
evt_type: 'started',
payload: {
count: global.NC_COUNT,
},
});
} catch (e) {
} finally {
sendEvt();
}
});
Tele.emitter.on('evt', async (payload) => {
try {
const instanceMeta = (await Tele.getInstanceMeta()) || {};
await waitForMachineId(teleData);
if (payload.check) {
teleData.machine_id = `${machineIdSync()},,`;
}
if (
isDisabled &&
!(
payload.evt_type &&
payload.evt_type.startsWith('a:sync-request:')
)
)
return;
if (payload.evt_type === 'project:invite') {
global.NC_COUNT = payload.count || global.NC_COUNT;
}
if (payload.evt_type === 'user:first_signup') {
global.NC_COUNT = +global.NC_COUNT || 1;
}
await axios.post('https://telemetry.nocodb.com/v1/telemetry', {
...teleData,
evt_type: payload.evt_type,
payload: { ...instanceMeta, ...(payload || {}) },
});
} catch {}
});
Tele.emitter.on('evt_api_created', async (data) => {
try {
await waitForMachineId(teleData);
const stats = {
...teleData,
table_count: data.tablesCount || 0,
relation_count: data.relationsCount || 0,
view_count: data.viewsCount || 0,
api_count: data.apiCount || 0,
function_count: data.functionsCount || 0,
procedure_count: data.proceduresCount || 0,
mysql: data.dbType === 'mysql2' ? 1 : 0,
pg: data.dbType === 'pg' ? 1 : 0,
mssql: data.dbType === 'mssql' ? 1 : 0,
sqlite3: data.dbType === 'sqlite3' ? 1 : 0,
oracledb: data.dbType === 'oracledb' ? 1 : 0,
rest: data.type === 'rest' ? 1 : 0,
graphql: data.type === 'graphql' ? 1 : 0,
grpc: data.type === 'grpc' ? 1 : 0,
time_taken: data.timeTaken,
};
if (isDisabled) return;
await axios.post(
'https://telemetry.nocodb.com/v1/telemetry/apis_created',
stats,
);
} catch (e) {}
});
Tele.emitter.on('evt_subscribe', async (email) => {
try {
if (isDisabled) return;
await axios.post(
'https://telemetry.nocodb.com/v1/newsletter/sdhjh34u3yuy34bj343jhj4iwolaAdsdj3434uiut4nn',
{
email,
},
);
} catch (e) {}
});
Tele.emitter.on('page', async (args) => {
try {
if (isDisabled) return;
const instanceMeta = await Tele.getInstanceMeta();
await this.client.capture({
distinctId: args.id || `${this.machineId}:public`,
event: '$pageview',
properties: {
...teleData,
...instanceMeta,
$current_url: args.path,
},
});
} catch {}
});
Tele.emitter.on('ph_event', async (payload: Record<string, any>) => {
try {
if (
isDisabled &&
!(
payload.evt_type &&
payload.evt_type.startsWith('a:sync-request:')
)
)
return;
const instanceMeta = await this.getInstanceMeta();
let id = payload.id;
if (!id) {
if (payload.event && payload.event.startsWith('a:api:')) {
id = this.machineId;
} else {
id = `${this.machineId}:public`;
}
}
await this.client.capture({
// userId: id,
distinctId: id,
event: payload.event,
properties: {
...teleData,
...instanceMeta,
...(payload.data || {}),
},
});
} catch {}
});
}
} catch (e) {}
try {
if (!this.client) {
this.client = new TeleBatchProcessor();
}
} catch {}
}
static async getInstanceMeta() {
try {
return (
(Tele.config &&
Tele.config.instance &&
(await Tele.config.instance())) ||
{}
);
} catch {
return {};
}
}
static get id() {
return this.machineId || machineIdSync();
}
}
export { Tele as T };
async function waitForMachineId(teleData) {
let i = 5;
while (i-- && !teleData.machine_id) {
await new Promise((resolve) => setTimeout(() => resolve(null), 500));
}
}
// this is to keep the server alive
if (process.env.NC_PUBLIC_URL) {
setInterval(() => {
axios({
method: 'get',
url: process.env.NC_PUBLIC_URL,
})
.then(() => {})
.catch(() => {});
}, 2 * 60 * 60 * 1000);
}
if (process.env.NC_ONE_CLICK) {
try {
Tele.emit('evt', {
evt_type: 'ONE_CLICK',
});
} catch {}
}

2
packages/nocodb/src/version-upgrader/NcUpgrader.ts

@ -1,5 +1,5 @@
import debug from 'debug';
import { T } from 'nc-help';
import { T } from '~/utils';
import boxen from 'boxen';
import ncAttachmentUpgrader from './ncAttachmentUpgrader';
import ncAttachmentUpgrader_0104002 from './ncAttachmentUpgrader_0104002';

Loading…
Cancel
Save