diff --git a/packages/nocodb/src/Noco.ts b/packages/nocodb/src/Noco.ts index d6163cd335..20761ac0cb 100644 --- a/packages/nocodb/src/Noco.ts +++ b/packages/nocodb/src/Noco.ts @@ -2,7 +2,6 @@ import path from 'path'; import { NestFactory } from '@nestjs/core'; import clear from 'clear'; import * as express from 'express'; -import { T } from 'nc-help'; import { v4 as uuidv4 } from 'uuid'; import dotenv from 'dotenv'; import { IoAdapter } from '@nestjs/platform-socket.io'; @@ -15,7 +14,7 @@ import type { Express } from 'express'; import type http from 'http'; import { MetaTable, RootScopes } from '~/utils/globals'; import { AppModule } from '~/app.module'; -import { isEE } from '~/utils'; +import { isEE, T } from '~/utils'; dotenv.config(); diff --git a/packages/nocodb/src/db/CustomKnex.ts b/packages/nocodb/src/db/CustomKnex.ts index 30421068e9..c4075fd06e 100644 --- a/packages/nocodb/src/db/CustomKnex.ts +++ b/packages/nocodb/src/db/CustomKnex.ts @@ -657,7 +657,12 @@ const parseCondition = (obj, columnAliases, qb, pKey?) => { }); break; default: - if (val && typeof val === 'object' && !(val instanceof Date) && !Array.isArray(val)) { + if ( + val && + typeof val === 'object' && + !(val instanceof Date) && + !Array.isArray(val) + ) { qb = parseCondition(val, columnAliases, qb, key); } else { const fieldName = columnAliases[pKey] || pKey; diff --git a/packages/nocodb/src/db/aggregation.ts b/packages/nocodb/src/db/aggregation.ts index e094e62d09..9c83f4e783 100644 --- a/packages/nocodb/src/db/aggregation.ts +++ b/packages/nocodb/src/db/aggregation.ts @@ -156,13 +156,15 @@ export default async function applyAggregation({ break; case UITypes.Formula: - const formula = await column.getColOptions(context); - if (formula.error) { - aggregation = CommonAggregations.None; - } else { - column_name_query = ( - await baseModelSqlv2.getSelectQueryBuilderForFormula(column) - ).builder; + { + const formula = await column.getColOptions(context); + if (formula.error) { + aggregation = CommonAggregations.None; + } else { + column_name_query = ( + await baseModelSqlv2.getSelectQueryBuilderForFormula(column) + ).builder; + } } break; diff --git a/packages/nocodb/src/db/sql-client/lib/KnexClient.ts b/packages/nocodb/src/db/sql-client/lib/KnexClient.ts index 726cb4e5be..3c84644cce 100644 --- a/packages/nocodb/src/db/sql-client/lib/KnexClient.ts +++ b/packages/nocodb/src/db/sql-client/lib/KnexClient.ts @@ -3,7 +3,6 @@ import fs from 'fs'; import { promisify } from 'util'; import path from 'path'; import { knex } from 'knex'; -import { T } from 'nc-help'; import findIndex from 'lodash/findIndex'; import find from 'lodash/find'; import jsonfile from 'jsonfile'; @@ -14,6 +13,7 @@ import Debug from '../../util/Debug'; import * as dataHelp from './data.helper'; import SqlClient from './SqlClient'; import type { Knex } from 'knex'; +import { T } from '~/utils'; const evt = new Emit(); diff --git a/packages/nocodb/src/db/sql-mgr/SqlMgr.ts b/packages/nocodb/src/db/sql-mgr/SqlMgr.ts index f3c3a5b6d3..0a25235f98 100644 --- a/packages/nocodb/src/db/sql-mgr/SqlMgr.ts +++ b/packages/nocodb/src/db/sql-mgr/SqlMgr.ts @@ -6,13 +6,13 @@ import fsExtra from 'fs-extra'; import importFresh from 'import-fresh'; import inflection from 'inflection'; import slash from 'slash'; -import { T } from 'nc-help'; import { customAlphabet } from 'nanoid'; import type MssqlClient from '~/db/sql-client/lib/mssql/MssqlClient'; import type MysqlClient from '~/db/sql-client/lib/mysql/MysqlClient'; import type OracleClient from '~/db/sql-client/lib/oracle/OracleClient'; import type PGClient from '~/db/sql-client/lib/pg/PgClient'; import type SqliteClient from '~/db/sql-client/lib/sqlite/SqliteClient'; +import { T } from '~/utils'; import Result from '~/db/util/Result'; import Debug from '~/db/util/Debug'; import KnexMigrator from '~/db/sql-migrator/lib/KnexMigrator'; diff --git a/packages/nocodb/src/gateways/socket.gateway.ts b/packages/nocodb/src/gateways/socket.gateway.ts index e6eb13ffed..4562c677a4 100644 --- a/packages/nocodb/src/gateways/socket.gateway.ts +++ b/packages/nocodb/src/gateways/socket.gateway.ts @@ -1,13 +1,13 @@ import crypto from 'crypto'; import { Inject, Injectable } from '@nestjs/common'; import { HttpAdapterHost } from '@nestjs/core'; -import { T } from 'nc-help'; import { Server } from 'socket.io'; import { AuthGuard } from '@nestjs/passport'; import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host'; import { WebSocketGateway, WebSocketServer } from '@nestjs/websockets'; import type { OnModuleInit } from '@nestjs/common'; import type { Socket } from 'socket.io'; +import { T } from '~/utils'; import { JwtStrategy } from '~/strategies/jwt.strategy'; import { TelemetryService } from '~/services/telemetry.service'; diff --git a/packages/nocodb/src/helpers/apiMetrics.ts b/packages/nocodb/src/helpers/apiMetrics.ts index 10dbd3b737..4476e7655a 100644 --- a/packages/nocodb/src/helpers/apiMetrics.ts +++ b/packages/nocodb/src/helpers/apiMetrics.ts @@ -1,5 +1,5 @@ -import { T } from 'nc-help'; import type { Request } from 'express'; +import { T } from '~/utils'; const countMap = {}; diff --git a/packages/nocodb/src/helpers/initAdminFromEnv.ts b/packages/nocodb/src/helpers/initAdminFromEnv.ts index b45113a78a..d6ec9ce019 100644 --- a/packages/nocodb/src/helpers/initAdminFromEnv.ts +++ b/packages/nocodb/src/helpers/initAdminFromEnv.ts @@ -3,8 +3,8 @@ import { v4 as uuidv4 } from 'uuid'; import bcrypt from 'bcryptjs'; import { validatePassword } from 'nocodb-sdk'; import boxen from 'boxen'; -import { T } from 'nc-help'; import isEmail from 'validator/lib/isEmail'; +import { T } from '~/utils'; import NocoCache from '~/cache/NocoCache'; import Noco from '~/Noco'; import { BaseUser, User } from '~/models'; diff --git a/packages/nocodb/src/providers/init-meta-service.provider.ts b/packages/nocodb/src/providers/init-meta-service.provider.ts index 19f60d1ce8..241bd75529 100644 --- a/packages/nocodb/src/providers/init-meta-service.provider.ts +++ b/packages/nocodb/src/providers/init-meta-service.provider.ts @@ -1,6 +1,6 @@ -import { T } from 'nc-help'; import type { FactoryProvider } from '@nestjs/common'; import type { IEventEmitter } from '~/modules/event-emitter/event-emitter.interface'; +import { T } from '~/utils'; import { populatePluginsForCloud } from '~/utils/cloud/populateCloudPlugins'; import { MetaService } from '~/meta/meta.service'; import Noco from '~/Noco'; diff --git a/packages/nocodb/src/services/app-hooks-listener.service.ts b/packages/nocodb/src/services/app-hooks-listener.service.ts index 8752a0235b..c1a665ec38 100644 --- a/packages/nocodb/src/services/app-hooks-listener.service.ts +++ b/packages/nocodb/src/services/app-hooks-listener.service.ts @@ -4,7 +4,6 @@ import { AuditOperationSubTypes, AuditOperationTypes, } from 'nocodb-sdk'; -import { T } from 'nc-help'; import type { AuditType } from 'nocodb-sdk'; import type { OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import type { @@ -21,6 +20,7 @@ import type { UserSigninEvent, UserSignupEvent, } from '~/services/app-hooks/interfaces'; +import { T } from '~/utils'; import { Audit, User } from '~/models'; import { TelemetryService } from '~/services/telemetry.service'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; diff --git a/packages/nocodb/src/services/telemetry.service.ts b/packages/nocodb/src/services/telemetry.service.ts index cecf359473..df4e56769a 100644 --- a/packages/nocodb/src/services/telemetry.service.ts +++ b/packages/nocodb/src/services/telemetry.service.ts @@ -1,6 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { packageInfo } from 'nc-help'; -import { T } from 'nc-help'; +import { packageInfo, T } from '~/utils'; @Injectable() export class TelemetryService { diff --git a/packages/nocodb/src/services/users/users.service.ts b/packages/nocodb/src/services/users/users.service.ts index b7fb40b695..3f002388e2 100644 --- a/packages/nocodb/src/services/users/users.service.ts +++ b/packages/nocodb/src/services/users/users.service.ts @@ -3,7 +3,6 @@ import { Injectable } from '@nestjs/common'; import { AppEvents, OrgUserRoles, validatePassword } from 'nocodb-sdk'; import { v4 as uuidv4 } from 'uuid'; import isEmail from 'validator/lib/isEmail'; -import { T } from 'nc-help'; import * as ejs from 'ejs'; import bcrypt from 'bcryptjs'; import type { @@ -14,6 +13,7 @@ import type { UserType, } from 'nocodb-sdk'; import type { NcRequest } from '~/interface/config'; +import { T } from '~/utils'; import { genJwt, setTokenCookie } from '~/services/users/helpers'; import { NC_APP_SETTINGS } from '~/constants'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; diff --git a/packages/nocodb/src/utils/TeleBatchProcessor.ts b/packages/nocodb/src/utils/TeleBatchProcessor.ts new file mode 100644 index 0000000000..3bd66169f1 --- /dev/null +++ b/packages/nocodb/src/utils/TeleBatchProcessor.ts @@ -0,0 +1,50 @@ +import axios from 'axios'; + +// This class is used to batch process telemetry data +class TeleBatchProcessor { + private batch: any[]; + private flushAt: number; + private flushInterval: number; + private timeoutRef: any; + + constructor({ flushAt = 100, flushInterval = 10000 } = {}) { + this.batch = []; + this.flushAt = flushAt; + this.flushInterval = flushInterval; + } + + capture(item: Record) { + this.batch.push(item); + + if (this.batch.length >= this.flushAt) { + this.flushBackground(); + } + + if (this.flushInterval) { + this.timeoutRef = setTimeout(() => { + this.flushBackground(); + }, this.flushInterval); + } + } + + flushBackground() { + if (this.timeoutRef) { + clearTimeout(this.timeoutRef); + this.timeoutRef = null; + } + this.flush().catch(() => { + // do nothing + }); + } + + async flush() { + const batch = this.batch.splice(0, this.batch.length); + if (!batch.length) { + return; + } + + await axios.post('https://nocodb.com/api/v1/telemetry', batch); + } +} + +export default TeleBatchProcessor; diff --git a/packages/nocodb/src/utils/feedbackForm.ts b/packages/nocodb/src/utils/feedbackForm.ts new file mode 100644 index 0000000000..53cc8b9074 --- /dev/null +++ b/packages/nocodb/src/utils/feedbackForm.ts @@ -0,0 +1,15 @@ +import axios from 'axios'; + +export const getFeedBackForm = async () => { + try { + const response = await axios.get( + 'https://nocodb.com/api/v1/feedback_form', + { + timeout: 5000, + }, + ); + return response.data; + } catch (e) { + return { error: e.message }; + } +}; diff --git a/packages/nocodb/src/utils/index.ts b/packages/nocodb/src/utils/index.ts index 834c0286b1..c155cb9168 100644 --- a/packages/nocodb/src/utils/index.ts +++ b/packages/nocodb/src/utils/index.ts @@ -3,5 +3,7 @@ export * from './sanitiseUserObj'; export * from './emailUtils'; export * from './circularReplacer'; export * from './nocoExecute'; +export { Tele as T } from './tele'; +export * from './packageVersion'; export const isEE = false; diff --git a/packages/nocodb/src/utils/tele.ts b/packages/nocodb/src/utils/tele.ts new file mode 100644 index 0000000000..c2fb1969e5 --- /dev/null +++ b/packages/nocodb/src/utils/tele.ts @@ -0,0 +1,280 @@ +import os from 'os'; +import Emittery from 'emittery'; +import { machineIdSync } from 'node-machine-id'; +import axios from 'axios'; +import isDocker from 'is-docker'; +import { packageVersion } from '~/utils/packageVersion'; +import TeleBatchProcessor from '~/utils/TeleBatchProcessor'; + +const isDisabled = !!process.env.NC_DISABLE_TELE; +const cache = !!process.env.NC_REDIS_URL; +const executable = !!process.env.NC_BINARY_BUILD; +const litestream = !!( + process.env.LITESTREAM_S3_BUCKET && + process.env.LITESTREAM_S3_SECRET_ACCESS_KEY && + process.env.LITESTREAM_S3_ACCESS_KEY_ID +); + +const sendEvt = () => { + try { + const upTime = Math.round(process.uptime() / 3600); + Tele.emit('evt', { + evt_type: 'alive', + count: global.NC_COUNT, + upTime, + cache, + litestream, + executable, + }); + } catch {} +}; +setInterval(sendEvt, 8 * 60 * 60 * 1000); + +class Tele { + static emitter; + static machineId; + static config: Record; + static client: TeleBatchProcessor; + + static emit(event, data) { + try { + this._init(); + Tele.emitter.emit(event, data); + } catch (e) {} + } + + static init(config: Record) { + Tele.config = config; + Tele._init(); + } + + static page(args: Record) { + this.emit('page', args); + } + + static event(args: Record) { + this.emit('ph_event', args); + } + + static _init() { + try { + if (!Tele.emitter) { + Tele.emitter = new Emittery(); + Tele.machineId = machineIdSync(); + + let package_id = ''; + let xc_version = ''; + xc_version = process.env.NC_SERVER_UUID; + package_id = packageVersion; + + const teleData: Record = { + package_id, + os_type: os.type(), + os_platform: os.platform(), + os_release: os.release(), + node_version: process.version, + docker: isDocker(), + xc_version: xc_version, + env: process.env.NODE_ENV || 'production', + oneClick: !!process.env.NC_ONE_CLICK, + }; + teleData.machine_id = `${machineIdSync()},,`; + Tele.emitter.on('evt_app_started', async (msg) => { + try { + await waitForMachineId(teleData); + if (isDisabled) return; + + if (msg && msg.count !== undefined) { + global.NC_COUNT = msg.count; + } + + await axios.post('https://telemetry.nocodb.com/api/v1/telemetry', { + ...teleData, + evt_type: 'started', + payload: { + count: global.NC_COUNT, + }, + }); + } catch (e) { + } finally { + sendEvt(); + } + }); + + Tele.emitter.on('evt', async (payload) => { + try { + const instanceMeta = (await Tele.getInstanceMeta()) || {}; + + await waitForMachineId(teleData); + if (payload.check) { + teleData.machine_id = `${machineIdSync()},,`; + } + if ( + isDisabled && + !( + payload.evt_type && + payload.evt_type.startsWith('a:sync-request:') + ) + ) + return; + + if (payload.evt_type === 'project:invite') { + global.NC_COUNT = payload.count || global.NC_COUNT; + } + if (payload.evt_type === 'user:first_signup') { + global.NC_COUNT = +global.NC_COUNT || 1; + } + + await axios.post('https://telemetry.nocodb.com/api/v1/telemetry', { + ...teleData, + evt_type: payload.evt_type, + payload: { ...instanceMeta, ...(payload || {}) }, + }); + } catch {} + }); + + Tele.emitter.on('evt_api_created', async (data) => { + try { + await waitForMachineId(teleData); + const stats = { + ...teleData, + table_count: data.tablesCount || 0, + relation_count: data.relationsCount || 0, + view_count: data.viewsCount || 0, + api_count: data.apiCount || 0, + function_count: data.functionsCount || 0, + procedure_count: data.proceduresCount || 0, + mysql: data.dbType === 'mysql2' ? 1 : 0, + pg: data.dbType === 'pg' ? 1 : 0, + mssql: data.dbType === 'mssql' ? 1 : 0, + sqlite3: data.dbType === 'sqlite3' ? 1 : 0, + oracledb: data.dbType === 'oracledb' ? 1 : 0, + rest: data.type === 'rest' ? 1 : 0, + graphql: data.type === 'graphql' ? 1 : 0, + grpc: data.type === 'grpc' ? 1 : 0, + time_taken: data.timeTaken, + }; + if (isDisabled) return; + await axios.post( + 'https://telemetry.nocodb.com/api/v1/telemetry/apis_created', + stats, + ); + } catch (e) {} + }); + + Tele.emitter.on('evt_subscribe', async (email) => { + try { + if (isDisabled) return; + await axios.post( + 'https://telemetry.nocodb.com/api/v1/newsletter/sdhjh34u3yuy34bj343jhj4iwolaAdsdj3434uiut4nn', + { + email, + }, + ); + } catch (e) {} + }); + + Tele.emitter.on('page', async (args) => { + try { + if (isDisabled) return; + const instanceMeta = await Tele.getInstanceMeta(); + + await this.client.capture({ + distinctId: args.id || `${this.machineId}:public`, + event: '$pageview', + properties: { + ...teleData, + ...instanceMeta, + $current_url: args.path, + }, + }); + } catch {} + }); + Tele.emitter.on('ph_event', async (payload: Record) => { + try { + if ( + isDisabled && + !( + payload.evt_type && + payload.evt_type.startsWith('a:sync-request:') + ) + ) + return; + const instanceMeta = await this.getInstanceMeta(); + let id = payload.id; + + if (!id) { + if (payload.event && payload.event.startsWith('a:api:')) { + id = this.machineId; + } else { + id = `${this.machineId}:public`; + } + } + await this.client.capture({ + // userId: id, + distinctId: id, + event: payload.event, + properties: { + ...teleData, + ...instanceMeta, + ...(payload.data || {}), + }, + }); + } catch {} + }); + } + } catch (e) {} + + try { + if (!this.client) { + this.client = new TeleBatchProcessor(); + } + } catch {} + } + + static async getInstanceMeta() { + try { + return ( + (Tele.config && + Tele.config.instance && + (await Tele.config.instance())) || + {} + ); + } catch { + return {}; + } + } + + static get id() { + return this.machineId || machineIdSync(); + } +} + +async function waitForMachineId(teleData) { + let i = 5; + while (i-- && !teleData.machine_id) { + await new Promise((resolve) => setTimeout(() => resolve(null), 500)); + } +} + +// this is to keep the server alive +if (process.env.NC_PUBLIC_URL) { + setInterval(() => { + axios({ + method: 'get', + url: process.env.NC_PUBLIC_URL, + }) + .then(() => {}) + .catch(() => {}); + }, 2 * 60 * 60 * 1000); +} + +if (process.env.NC_ONE_CLICK) { + try { + Tele.emit('evt', { + evt_type: 'ONE_CLICK', + }); + } catch {} +} + +export { Tele }; diff --git a/packages/nocodb/src/version-upgrader/NcUpgrader.ts b/packages/nocodb/src/version-upgrader/NcUpgrader.ts index 510fa88bfc..c78a00ed47 100644 --- a/packages/nocodb/src/version-upgrader/NcUpgrader.ts +++ b/packages/nocodb/src/version-upgrader/NcUpgrader.ts @@ -1,5 +1,4 @@ import debug from 'debug'; -import { T } from 'nc-help'; import boxen from 'boxen'; import ncAttachmentUpgrader from './ncAttachmentUpgrader'; import ncAttachmentUpgrader_0104002 from './ncAttachmentUpgrader_0104002'; @@ -14,6 +13,7 @@ import ncXcdbLTARIndexUpgrader from './ncXcdbLTARIndexUpgrader'; import ncXcdbCreatedAndUpdatedSystemFieldsUpgrader from './ncXcdbCreatedAndUpdatedSystemFieldsUpgrader'; import type { MetaService } from '~/meta/meta.service'; import type { NcConfig } from '~/interface/config'; +import { T } from '~/utils'; import { MetaTable, RootScopes } from '~/utils/globals'; const log = debug('nc:version-upgrader');