Merge pull request #9459 from nocodb/nc-refactor/tel

refactor: Event related modules

packages/nocodb/src/Noco.ts

@@ -2,7 +2,6 @@ import path from 'path';
 import { NestFactory } from '@nestjs/core';
 import clear from 'clear';
 import * as express from 'express';
-import { T } from 'nc-help';
 import { v4 as uuidv4 } from 'uuid';
 import dotenv from 'dotenv';
 import { IoAdapter } from '@nestjs/platform-socket.io';
@@ -15,7 +14,7 @@ import type { Express } from 'express';
 import type http from 'http';
 import { MetaTable, RootScopes } from '~/utils/globals';
 import { AppModule } from '~/app.module';
-import { isEE } from '~/utils';
+import { isEE, T } from '~/utils';
 
 dotenv.config();
 

packages/nocodb/src/db/CustomKnex.ts

@@ -657,7 +657,12 @@ const parseCondition = (obj, columnAliases, qb, pKey?) => {
         });
         break;
       default:
-        if (val && typeof val === 'object' && !(val instanceof Date) && !Array.isArray(val)) {
+        if (
+          val &&
+          typeof val === 'object' &&
+          !(val instanceof Date) &&
+          !Array.isArray(val)
+        ) {
           qb = parseCondition(val, columnAliases, qb, key);
         } else {
           const fieldName = columnAliases[pKey] || pKey;

packages/nocodb/src/db/aggregation.ts

@@ -156,13 +156,15 @@ export default async function applyAggregation({
       break;
 
     case UITypes.Formula:
-      const formula = await column.getColOptions<FormulaColumn>(context);
+      {
-      if (formula.error) {
+        const formula = await column.getColOptions<FormulaColumn>(context);
-        aggregation = CommonAggregations.None;
+        if (formula.error) {
-      } else {
+          aggregation = CommonAggregations.None;
-        column_name_query = (
+        } else {
-          await baseModelSqlv2.getSelectQueryBuilderForFormula(column)
+          column_name_query = (
-        ).builder;
+            await baseModelSqlv2.getSelectQueryBuilderForFormula(column)
+          ).builder;
+        }
       }
       break;
 

packages/nocodb/src/db/sql-client/lib/KnexClient.ts

@@ -3,7 +3,6 @@ import fs from 'fs';
 import { promisify } from 'util';
 import path from 'path';
 import { knex } from 'knex';
-import { T } from 'nc-help';
 import findIndex from 'lodash/findIndex';
 import find from 'lodash/find';
 import jsonfile from 'jsonfile';
@@ -14,6 +13,7 @@ import Debug from '../../util/Debug';
 import * as dataHelp from './data.helper';
 import SqlClient from './SqlClient';
 import type { Knex } from 'knex';
+import { T } from '~/utils';
 
 const evt = new Emit();
 

packages/nocodb/src/db/sql-mgr/SqlMgr.ts

@@ -6,13 +6,13 @@ import fsExtra from 'fs-extra';
 import importFresh from 'import-fresh';
 import inflection from 'inflection';
 import slash from 'slash';
-import { T } from 'nc-help';
 import { customAlphabet } from 'nanoid';
 import type MssqlClient from '~/db/sql-client/lib/mssql/MssqlClient';
 import type MysqlClient from '~/db/sql-client/lib/mysql/MysqlClient';
 import type OracleClient from '~/db/sql-client/lib/oracle/OracleClient';
 import type PGClient from '~/db/sql-client/lib/pg/PgClient';
 import type SqliteClient from '~/db/sql-client/lib/sqlite/SqliteClient';
+import { T } from '~/utils';
 import Result from '~/db/util/Result';
 import Debug from '~/db/util/Debug';
 import KnexMigrator from '~/db/sql-migrator/lib/KnexMigrator';

packages/nocodb/src/gateways/socket.gateway.ts

@@ -1,13 +1,13 @@
 import crypto from 'crypto';
 import { Inject, Injectable } from '@nestjs/common';
 import { HttpAdapterHost } from '@nestjs/core';
-import { T } from 'nc-help';
 import { Server } from 'socket.io';
 import { AuthGuard } from '@nestjs/passport';
 import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host';
 import { WebSocketGateway, WebSocketServer } from '@nestjs/websockets';
 import type { OnModuleInit } from '@nestjs/common';
 import type { Socket } from 'socket.io';
+import { T } from '~/utils';
 import { JwtStrategy } from '~/strategies/jwt.strategy';
 import { TelemetryService } from '~/services/telemetry.service';
 

packages/nocodb/src/helpers/apiMetrics.ts

@@ -1,5 +1,5 @@
-import { T } from 'nc-help';
 import type { Request } from 'express';
+import { T } from '~/utils';
 
 const countMap = {};
 

packages/nocodb/src/helpers/initAdminFromEnv.ts

@@ -3,8 +3,8 @@ import { v4 as uuidv4 } from 'uuid';
 import bcrypt from 'bcryptjs';
 import { validatePassword } from 'nocodb-sdk';
 import boxen from 'boxen';
-import { T } from 'nc-help';
 import isEmail from 'validator/lib/isEmail';
+import { T } from '~/utils';
 import NocoCache from '~/cache/NocoCache';
 import Noco from '~/Noco';
 import { BaseUser, User } from '~/models';

packages/nocodb/src/providers/init-meta-service.provider.ts

@@ -1,6 +1,6 @@
-import { T } from 'nc-help';
 import type { FactoryProvider } from '@nestjs/common';
 import type { IEventEmitter } from '~/modules/event-emitter/event-emitter.interface';
+import { T } from '~/utils';
 import { populatePluginsForCloud } from '~/utils/cloud/populateCloudPlugins';
 import { MetaService } from '~/meta/meta.service';
 import Noco from '~/Noco';

packages/nocodb/src/services/app-hooks-listener.service.ts

@@ -4,7 +4,6 @@ import {
   AuditOperationSubTypes,
   AuditOperationTypes,
 } from 'nocodb-sdk';
-import { T } from 'nc-help';
 import type { AuditType } from 'nocodb-sdk';
 import type { OnModuleDestroy, OnModuleInit } from '@nestjs/common';
 import type {
@@ -21,6 +20,7 @@ import type {
   UserSigninEvent,
   UserSignupEvent,
 } from '~/services/app-hooks/interfaces';
+import { T } from '~/utils';
 import { Audit, User } from '~/models';
 import { TelemetryService } from '~/services/telemetry.service';
 import { AppHooksService } from '~/services/app-hooks/app-hooks.service';

packages/nocodb/src/services/telemetry.service.ts

@@ -1,6 +1,5 @@
 import { Injectable } from '@nestjs/common';
-import { packageInfo } from 'nc-help';
+import { packageInfo, T } from '~/utils';
-import { T } from 'nc-help';
 
 @Injectable()
 export class TelemetryService {

packages/nocodb/src/services/users/users.service.ts

@@ -3,7 +3,6 @@ import { Injectable } from '@nestjs/common';
 import { AppEvents, OrgUserRoles, validatePassword } from 'nocodb-sdk';
 import { v4 as uuidv4 } from 'uuid';
 import isEmail from 'validator/lib/isEmail';
-import { T } from 'nc-help';
 import * as ejs from 'ejs';
 import bcrypt from 'bcryptjs';
 import type {
@@ -14,6 +13,7 @@ import type {
   UserType,
 } from 'nocodb-sdk';
 import type { NcRequest } from '~/interface/config';
+import { T } from '~/utils';
 import { genJwt, setTokenCookie } from '~/services/users/helpers';
 import { NC_APP_SETTINGS } from '~/constants';
 import { AppHooksService } from '~/services/app-hooks/app-hooks.service';

packages/nocodb/src/utils/TeleBatchProcessor.ts

@@ -0,0 +1,50 @@
+import axios from 'axios';
+
+// This class is used to batch process telemetry data
+class TeleBatchProcessor {
+  private batch: any[];
+  private flushAt: number;
+  private flushInterval: number;
+  private timeoutRef: any;
+
+  constructor({ flushAt = 100, flushInterval = 10000 } = {}) {
+    this.batch = [];
+    this.flushAt = flushAt;
+    this.flushInterval = flushInterval;
+  }
+
+  capture(item: Record<string, any>) {
+    this.batch.push(item);
+
+    if (this.batch.length >= this.flushAt) {
+      this.flushBackground();
+    }
+
+    if (this.flushInterval) {
+      this.timeoutRef = setTimeout(() => {
+        this.flushBackground();
+      }, this.flushInterval);
+    }
+  }
+
+  flushBackground() {
+    if (this.timeoutRef) {
+      clearTimeout(this.timeoutRef);
+      this.timeoutRef = null;
+    }
+    this.flush().catch(() => {
+      // do nothing
+    });
+  }
+
+  async flush() {
+    const batch = this.batch.splice(0, this.batch.length);
+    if (!batch.length) {
+      return;
+    }
+
+    await axios.post('https://nocodb.com/api/v1/telemetry', batch);
+  }
+}
+
+export default TeleBatchProcessor;

packages/nocodb/src/utils/feedbackForm.ts

@@ -0,0 +1,15 @@
+import axios from 'axios';
+
+export const getFeedBackForm = async () => {
+  try {
+    const response = await axios.get(
+      'https://nocodb.com/api/v1/feedback_form',
+      {
+        timeout: 5000,
+      },
+    );
+    return response.data;
+  } catch (e) {
+    return { error: e.message };
+  }
+};

packages/nocodb/src/utils/index.ts

@@ -3,5 +3,7 @@ export * from './sanitiseUserObj';
 export * from './emailUtils';
 export * from './circularReplacer';
 export * from './nocoExecute';
+export { Tele as T } from './tele';
+export * from './packageVersion';
 
 export const isEE = false;

packages/nocodb/src/utils/tele.ts

@@ -0,0 +1,280 @@
+import os from 'os';
+import Emittery from 'emittery';
+import { machineIdSync } from 'node-machine-id';
+import axios from 'axios';
+import isDocker from 'is-docker';
+import { packageVersion } from '~/utils/packageVersion';
+import TeleBatchProcessor from '~/utils/TeleBatchProcessor';
+
+const isDisabled = !!process.env.NC_DISABLE_TELE;
+const cache = !!process.env.NC_REDIS_URL;
+const executable = !!process.env.NC_BINARY_BUILD;
+const litestream = !!(
+  process.env.LITESTREAM_S3_BUCKET &&
+  process.env.LITESTREAM_S3_SECRET_ACCESS_KEY &&
+  process.env.LITESTREAM_S3_ACCESS_KEY_ID
+);
+
+const sendEvt = () => {
+  try {
+    const upTime = Math.round(process.uptime() / 3600);
+    Tele.emit('evt', {
+      evt_type: 'alive',
+      count: global.NC_COUNT,
+      upTime,
+      cache,
+      litestream,
+      executable,
+    });
+  } catch {}
+};
+setInterval(sendEvt, 8 * 60 * 60 * 1000);
+
+class Tele {
+  static emitter;
+  static machineId;
+  static config: Record<string, any>;
+  static client: TeleBatchProcessor;
+
+  static emit(event, data) {
+    try {
+      this._init();
+      Tele.emitter.emit(event, data);
+    } catch (e) {}
+  }
+
+  static init(config: Record<string, any>) {
+    Tele.config = config;
+    Tele._init();
+  }
+
+  static page(args: Record<string, any>) {
+    this.emit('page', args);
+  }
+
+  static event(args: Record<string, any>) {
+    this.emit('ph_event', args);
+  }
+
+  static _init() {
+    try {
+      if (!Tele.emitter) {
+        Tele.emitter = new Emittery();
+        Tele.machineId = machineIdSync();
+
+        let package_id = '';
+        let xc_version = '';
+        xc_version = process.env.NC_SERVER_UUID;
+        package_id = packageVersion;
+
+        const teleData: Record<string, any> = {
+          package_id,
+          os_type: os.type(),
+          os_platform: os.platform(),
+          os_release: os.release(),
+          node_version: process.version,
+          docker: isDocker(),
+          xc_version: xc_version,
+          env: process.env.NODE_ENV || 'production',
+          oneClick: !!process.env.NC_ONE_CLICK,
+        };
+        teleData.machine_id = `${machineIdSync()},,`;
+        Tele.emitter.on('evt_app_started', async (msg) => {
+          try {
+            await waitForMachineId(teleData);
+            if (isDisabled) return;
+
+            if (msg && msg.count !== undefined) {
+              global.NC_COUNT = msg.count;
+            }
+
+            await axios.post('https://telemetry.nocodb.com/api/v1/telemetry', {
+              ...teleData,
+              evt_type: 'started',
+              payload: {
+                count: global.NC_COUNT,
+              },
+            });
+          } catch (e) {
+          } finally {
+            sendEvt();
+          }
+        });
+
+        Tele.emitter.on('evt', async (payload) => {
+          try {
+            const instanceMeta = (await Tele.getInstanceMeta()) || {};
+
+            await waitForMachineId(teleData);
+            if (payload.check) {
+              teleData.machine_id = `${machineIdSync()},,`;
+            }
+            if (
+              isDisabled &&
+              !(
+                payload.evt_type &&
+                payload.evt_type.startsWith('a:sync-request:')
+              )
+            )
+              return;
+
+            if (payload.evt_type === 'project:invite') {
+              global.NC_COUNT = payload.count || global.NC_COUNT;
+            }
+            if (payload.evt_type === 'user:first_signup') {
+              global.NC_COUNT = +global.NC_COUNT || 1;
+            }
+
+            await axios.post('https://telemetry.nocodb.com/api/v1/telemetry', {
+              ...teleData,
+              evt_type: payload.evt_type,
+              payload: { ...instanceMeta, ...(payload || {}) },
+            });
+          } catch {}
+        });
+
+        Tele.emitter.on('evt_api_created', async (data) => {
+          try {
+            await waitForMachineId(teleData);
+            const stats = {
+              ...teleData,
+              table_count: data.tablesCount || 0,
+              relation_count: data.relationsCount || 0,
+              view_count: data.viewsCount || 0,
+              api_count: data.apiCount || 0,
+              function_count: data.functionsCount || 0,
+              procedure_count: data.proceduresCount || 0,
+              mysql: data.dbType === 'mysql2' ? 1 : 0,
+              pg: data.dbType === 'pg' ? 1 : 0,
+              mssql: data.dbType === 'mssql' ? 1 : 0,
+              sqlite3: data.dbType === 'sqlite3' ? 1 : 0,
+              oracledb: data.dbType === 'oracledb' ? 1 : 0,
+              rest: data.type === 'rest' ? 1 : 0,
+              graphql: data.type === 'graphql' ? 1 : 0,
+              grpc: data.type === 'grpc' ? 1 : 0,
+              time_taken: data.timeTaken,
+            };
+            if (isDisabled) return;
+            await axios.post(
+              'https://telemetry.nocodb.com/api/v1/telemetry/apis_created',
+              stats,
+            );
+          } catch (e) {}
+        });
+
+        Tele.emitter.on('evt_subscribe', async (email) => {
+          try {
+            if (isDisabled) return;
+            await axios.post(
+              'https://telemetry.nocodb.com/api/v1/newsletter/sdhjh34u3yuy34bj343jhj4iwolaAdsdj3434uiut4nn',
+              {
+                email,
+              },
+            );
+          } catch (e) {}
+        });
+
+        Tele.emitter.on('page', async (args) => {
+          try {
+            if (isDisabled) return;
+            const instanceMeta = await Tele.getInstanceMeta();
+
+            await this.client.capture({
+              distinctId: args.id || `${this.machineId}:public`,
+              event: '$pageview',
+              properties: {
+                ...teleData,
+                ...instanceMeta,
+                $current_url: args.path,
+              },
+            });
+          } catch {}
+        });
+        Tele.emitter.on('ph_event', async (payload: Record<string, any>) => {
+          try {
+            if (
+              isDisabled &&
+              !(
+                payload.evt_type &&
+                payload.evt_type.startsWith('a:sync-request:')
+              )
+            )
+              return;
+            const instanceMeta = await this.getInstanceMeta();
+            let id = payload.id;
+
+            if (!id) {
+              if (payload.event && payload.event.startsWith('a:api:')) {
+                id = this.machineId;
+              } else {
+                id = `${this.machineId}:public`;
+              }
+            }
+            await this.client.capture({
+              // userId: id,
+              distinctId: id,
+              event: payload.event,
+              properties: {
+                ...teleData,
+                ...instanceMeta,
+                ...(payload.data || {}),
+              },
+            });
+          } catch {}
+        });
+      }
+    } catch (e) {}
+
+    try {
+      if (!this.client) {
+        this.client = new TeleBatchProcessor();
+      }
+    } catch {}
+  }
+
+  static async getInstanceMeta() {
+    try {
+      return (
+        (Tele.config &&
+          Tele.config.instance &&
+          (await Tele.config.instance())) ||
+        {}
+      );
+    } catch {
+      return {};
+    }
+  }
+
+  static get id() {
+    return this.machineId || machineIdSync();
+  }
+}
+
+async function waitForMachineId(teleData) {
+  let i = 5;
+  while (i-- && !teleData.machine_id) {
+    await new Promise((resolve) => setTimeout(() => resolve(null), 500));
+  }
+}
+
+// this is to keep the server alive
+if (process.env.NC_PUBLIC_URL) {
+  setInterval(() => {
+    axios({
+      method: 'get',
+      url: process.env.NC_PUBLIC_URL,
+    })
+      .then(() => {})
+      .catch(() => {});
+  }, 2 * 60 * 60 * 1000);
+}
+
+if (process.env.NC_ONE_CLICK) {
+  try {
+    Tele.emit('evt', {
+      evt_type: 'ONE_CLICK',
+    });
+  } catch {}
+}
+
+export { Tele };

packages/nocodb/src/version-upgrader/NcUpgrader.ts

@@ -1,5 +1,4 @@
 import debug from 'debug';
-import { T } from 'nc-help';
 import boxen from 'boxen';
 import ncAttachmentUpgrader from './ncAttachmentUpgrader';
 import ncAttachmentUpgrader_0104002 from './ncAttachmentUpgrader_0104002';
@@ -14,6 +13,7 @@ import ncXcdbLTARIndexUpgrader from './ncXcdbLTARIndexUpgrader';
 import ncXcdbCreatedAndUpdatedSystemFieldsUpgrader from './ncXcdbCreatedAndUpdatedSystemFieldsUpgrader';
 import type { MetaService } from '~/meta/meta.service';
 import type { NcConfig } from '~/interface/config';
+import { T } from '~/utils';
 import { MetaTable, RootScopes } from '~/utils/globals';
 
 const log = debug('nc:version-upgrader');