diff --git a/packages/nc-gui/components/dashboard/TreeView.vue b/packages/nc-gui/components/dashboard/TreeView.vue index 2f5f97ee4b..5940b634c5 100644 --- a/packages/nc-gui/components/dashboard/TreeView.vue +++ b/packages/nc-gui/components/dashboard/TreeView.vue @@ -399,8 +399,8 @@ const duplicateTable = async (table: TableType) => { const { close } = useDialog(resolveComponent('DlgTableDuplicate'), { 'modelValue': isOpen, 'table': table, - 'onOk': async (jobData: { name: string; id: string }) => { - $jobs.subscribe({ name: jobData.name, id: jobData.id }, undefined, async (status: string, data?: any) => { + 'onOk': async (jobData: { id: string }) => { + $jobs.subscribe({ id: jobData.id }, undefined, async (status: string, data?: any) => { if (status === JobStatus.COMPLETED) { await loadTables() const newTable = tables.value.find((el) => el.id === data?.result?.id) diff --git a/packages/nc-gui/nuxt-shim.d.ts b/packages/nc-gui/nuxt-shim.d.ts index dec8a6e113..4dc4ac605a 100644 --- a/packages/nc-gui/nuxt-shim.d.ts +++ b/packages/nc-gui/nuxt-shim.d.ts @@ -18,7 +18,6 @@ declare module '#app/nuxt' { job: | { id: string - name: string } | any, subscribedCb?: () => void, diff --git a/packages/nc-gui/pages/index/index/index.vue b/packages/nc-gui/pages/index/index/index.vue index bc22e036b3..486786b72b 100644 --- a/packages/nc-gui/pages/index/index/index.vue +++ b/packages/nc-gui/pages/index/index/index.vue @@ -90,10 +90,10 @@ const duplicateProject = (project: ProjectType) => { const { close } = useDialog(resolveComponent('DlgProjectDuplicate'), { 'modelValue': isOpen, 'project': project, - 'onOk': async (jobData: { name: string; id: string }) => { + 'onOk': async (jobData: { id: string }) => { await loadProjects() - $jobs.subscribe({ name: jobData.name, id: jobData.id }, undefined, async (status: string) => { + $jobs.subscribe({ id: jobData.id }, undefined, async (status: string) => { if (status === JobStatus.COMPLETED) { await loadProjects() } else if (status === JobStatus.FAILED) { diff --git a/packages/nc-gui/plugins/jobs.ts b/packages/nc-gui/plugins/jobs.ts index aa6b2cddb7..cc12c3aa86 100644 --- a/packages/nc-gui/plugins/jobs.ts +++ b/packages/nc-gui/plugins/jobs.ts @@ -29,22 +29,22 @@ export default defineNuxtPlugin(async (nuxtApp) => { await init(nuxtApp.$state.token.value) } - const send = (name: string, data: any) => { + const send = (evt: string, data: any) => { if (socket) { const _id = messageIndex++ - socket.emit(name, { _id, data }) + socket.emit(evt, { _id, data }) return _id } } const jobs = { subscribe( - job: { id: string; name: string } | any, + job: { id: string } | any, subscribedCb?: () => void, statusCb?: (status: JobStatus, data?: any) => void, logCb?: (data: { message: string }) => void, ) { - const logFn = (data: { id: string; name: string; data: { message: string } }) => { + const logFn = (data: { id: string; data: { message: string } }) => { if (data.id === job.id) { if (logCb) logCb(data.data) } @@ -61,11 +61,10 @@ export default defineNuxtPlugin(async (nuxtApp) => { const _id = send('subscribe', job) - const subscribeFn = (data: { _id: number; name: string; id: string }) => { + const subscribeFn = (data: { _id: number; id: string }) => { if (data._id === _id) { - if (data.id !== job.id || data.name !== job.name) { + if (data.id !== job.id) { job.id = data.id - job.name = data.name } if (subscribedCb) subscribedCb() socket?.on('log', logFn) @@ -75,10 +74,10 @@ export default defineNuxtPlugin(async (nuxtApp) => { } socket?.on('subscribed', subscribeFn) }, - getStatus(name: string, id: string): Promise { + getStatus(id: string): Promise { return new Promise((resolve) => { if (socket) { - const _id = send('status', { name, id }) + const _id = send('status', { id }) const tempFn = (data: any) => { if (data._id === _id) { resolve(data.status) diff --git a/packages/nocodb/src/app.module.ts b/packages/nocodb/src/app.module.ts index f5a4e8f1b7..9bc91efdc0 100644 --- a/packages/nocodb/src/app.module.ts +++ b/packages/nocodb/src/app.module.ts @@ -1,6 +1,5 @@ import { Module, RequestMethod } from '@nestjs/common'; import { APP_FILTER } from '@nestjs/core'; -import { BullModule } from '@nestjs/bull'; import { EventEmitterModule as NestJsEventEmitter } from '@nestjs/event-emitter'; import { GlobalExceptionFilter } from './filters/global-exception/global-exception.filter'; import { GlobalMiddleware } from './middlewares/global/global.middleware'; @@ -17,7 +16,6 @@ import { AuthTokenStrategy } from './strategies/authtoken.strategy/authtoken.str import { BaseViewStrategy } from './strategies/base-view.strategy/base-view.strategy'; import { MetasModule } from './modules/metas/metas.module'; import { JobsModule } from './modules/jobs/jobs.module'; -import { AppInitService } from './services/app-init.service'; import type { MiddlewareConsumer } from '@nestjs/common'; @Module({ @@ -30,13 +28,6 @@ import type { MiddlewareConsumer } from '@nestjs/common'; EventEmitterModule, JobsModule, NestJsEventEmitter.forRoot(), - ...(process.env['NC_REDIS_URL'] - ? [ - BullModule.forRoot({ - redis: process.env.NC_REDIS_URL, - }), - ] - : []), ], controllers: [], providers: [ @@ -49,7 +40,6 @@ import type { MiddlewareConsumer } from '@nestjs/common'; AuthTokenStrategy, BaseViewStrategy, HookHandlerService, - AppInitService, ], }) export class AppModule { diff --git a/packages/nocodb/src/cache/RedisCacheMgr.ts b/packages/nocodb/src/cache/RedisCacheMgr.ts index 04f2aca34d..e0dafae4a1 100644 --- a/packages/nocodb/src/cache/RedisCacheMgr.ts +++ b/packages/nocodb/src/cache/RedisCacheMgr.ts @@ -12,8 +12,12 @@ export default class RedisCacheMgr extends CacheMgr { constructor(config: any) { super(); this.client = new Redis(config); - // flush the existing db with selected key (Default: 0) - this.client.flushdb(); + + // avoid flushing db in worker container + if (process.env.NC_WORKER_CONTAINER !== 'true') { + // flush the existing db with selected key (Default: 0) + this.client.flushdb(); + } // TODO(cache): fetch orgs once it's implemented const orgs = 'noco'; diff --git a/packages/nocodb/src/connection/connection.spec.ts b/packages/nocodb/src/connection/connection.spec.ts deleted file mode 100644 index 0c14346a55..0000000000 --- a/packages/nocodb/src/connection/connection.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { Connection } from './knex'; -import type { TestingModule } from '@nestjs/testing'; - -describe('Knex', () => { - let provider: Connection; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [Connection], - }).compile(); - - provider = module.get(Connection); - }); - - it('should be defined', () => { - expect(provider).toBeDefined(); - }); -}); diff --git a/packages/nocodb/src/connection/connection.ts b/packages/nocodb/src/connection/connection.ts deleted file mode 100644 index cc12bf4f8d..0000000000 --- a/packages/nocodb/src/connection/connection.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Global, Injectable, Scope } from '@nestjs/common'; - -import { XKnex } from '../db/CustomKnex'; -import NcConfigFactory from '../utils/NcConfigFactory'; -import type * as knex from 'knex'; - -@Injectable({ - scope: Scope.DEFAULT, -}) -export class Connection { - public static knex: knex.Knex; - public static _config: any; - - get knexInstance(): knex.Knex { - return Connection.knex; - } - - get config(): knex.Knex { - return Connection._config; - } - - // init metadb connection - static async init(): Promise { - Connection._config = await NcConfigFactory.make(); - if (!Connection.knex) { - Connection.knex = XKnex({ - ...this._config.meta.db, - useNullAsDefault: true, - }); - } - } - - // init metadb connection - async init(): Promise { - return await Connection.init(); - } -} diff --git a/packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts b/packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts index f8ad0dde20..5f1c6bb291 100644 --- a/packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts +++ b/packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts @@ -10,7 +10,7 @@ import Debug from '../../util/Debug'; import Emit from '../../util/emit'; import Result from '../../util/Result'; import * as fileHelp from '../../util/file.help'; -import NcConfigFactory from '../../../utils/NcConfigFactory'; +import { getToolDir, NcConfig } from '../../../utils/nc-config'; import SqlMigrator from './SqlMigrator'; const evt = new Emit(); @@ -39,7 +39,7 @@ export default class KnexMigrator extends SqlMigrator { this.project_id = projectObj?.project_id; this.project = projectObj?.config; this.metaDb = projectObj?.metaDb; - this.toolDir = NcConfigFactory.getToolDir(); + this.toolDir = getToolDir(); } emit(data, _args?) { @@ -312,8 +312,12 @@ export default class KnexMigrator extends SqlMigrator { if (exists) { await this._readProjectJson(projJsonFilePath); this.emit('Migrator for project initalised successfully'); - } else if (NcConfigFactory.hasDbUrl()) { - this.project = await NcConfigFactory.make(); + } else if ( + Object.keys(process.env).some((envKey) => + envKey.startsWith('NC_DB_URL'), + ) + ) { + this.project = await NcConfig.createByEnv(); } else { args.type = args.type || 'sqlite'; diff --git a/packages/nocodb/src/index.ts b/packages/nocodb/src/index.ts index 439a258b32..ca9b3f35ad 100644 --- a/packages/nocodb/src/index.ts +++ b/packages/nocodb/src/index.ts @@ -1,6 +1,5 @@ import Noco from './Noco'; -import NcConfigFactory from './utils/NcConfigFactory'; export default Noco; -export { Noco, NcConfigFactory }; +export { Noco }; diff --git a/packages/nocodb/src/init.ts b/packages/nocodb/src/init.ts index c333fe2da7..cb3a874bf7 100644 --- a/packages/nocodb/src/init.ts +++ b/packages/nocodb/src/init.ts @@ -1,12 +1,12 @@ -import { Connection } from './connection/connection'; import { MetaService } from './meta/meta.service'; +import { NcConfig } from './utils/nc-config'; import Noco from './Noco'; // run upgrader import NcUpgrader from './version-upgrader/NcUpgrader'; export default async () => { - await Connection.init(); - Noco._ncMeta = new MetaService(new Connection()); + const config = await NcConfig.createByEnv(); + Noco._ncMeta = new MetaService(config); await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta }); }; diff --git a/packages/nocodb/src/main.ts b/packages/nocodb/src/main.ts index 4f1d5a3d69..12d87347b8 100644 --- a/packages/nocodb/src/main.ts +++ b/packages/nocodb/src/main.ts @@ -5,12 +5,22 @@ import { AppModule } from './app.module'; async function bootstrap() { const app = await NestFactory.create(AppModule); - app.use(express.json({ limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' })); - app.use( - cors({ - exposedHeaders: 'xc-db-response', - }), - ); - await app.listen(8080); + if (process.env.NC_WORKER_CONTAINER !== 'true') { + app.use( + express.json({ limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' }), + ); + app.use( + cors({ + exposedHeaders: 'xc-db-response', + }), + ); + await app.listen(8080); + } else { + if (!process.env.NC_REDIS_URL) { + throw new Error('NC_REDIS_URL is required'); + } + process.env.NC_DISABLE_TELE = 'true'; + await app.init(); + } } bootstrap(); diff --git a/packages/nocodb/src/meta/meta.service.ts b/packages/nocodb/src/meta/meta.service.ts index 3d7b09fe5c..5d30444980 100644 --- a/packages/nocodb/src/meta/meta.service.ts +++ b/packages/nocodb/src/meta/meta.service.ts @@ -1,22 +1,15 @@ -import { - Global, - Inject, - Injectable, - OnApplicationBootstrap, - OnModuleInit, - Optional, -} from '@nestjs/common'; +import { Injectable, Optional } from '@nestjs/common'; import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; import timezone from 'dayjs/plugin/timezone'; import { customAlphabet } from 'nanoid'; import CryptoJS from 'crypto-js'; -import { Connection } from '../connection/connection'; -import Noco from '../Noco'; -import NocoCache from '../cache/NocoCache'; +import { XKnex } from '../db/CustomKnex'; +import { NcConfig } from '../utils/nc-config'; import XcMigrationSourcev2 from './migrations/XcMigrationSourcev2'; import XcMigrationSource from './migrations/XcMigrationSource'; import type { Knex } from 'knex'; +import type * as knex from 'knex'; dayjs.extend(utc); dayjs.extend(timezone); @@ -192,18 +185,38 @@ const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14); @Injectable() export class MetaService { - constructor(private metaConnection: Connection, @Optional() trx = null) { + private _knex: knex.Knex; + private _config: any; + + constructor(config: NcConfig, @Optional() trx = null) { + this._config = config; + this._knex = XKnex({ + ...this._config.meta.db, + useNullAsDefault: true, + }); this.trx = trx; } + get knexInstance(): knex.Knex { + return this._knex; + } + + get config(): NcConfig { + return this._config; + } + public get connection() { - return this.trx ?? this.metaConnection.knexInstance; + return this.trx ?? this.knexInstance; } get knexConnection() { return this.connection; } + public get knex(): any { + return this.knexConnection; + } + public async metaGet( project_id: string, dbAlias: string, @@ -758,7 +771,7 @@ export class MetaService { }); // todo: tobe done - return new MetaService(this.metaConnection, trx); + return new MetaService(this.config, trx); } async metaReset( @@ -1027,10 +1040,6 @@ export class MetaService { .delete(); } - public get knex(): any { - return this.knexConnection; - } - private getNanoId() { return nanoid(); } diff --git a/packages/nocodb/src/modules/datas/datas.module.ts b/packages/nocodb/src/modules/datas/datas.module.ts index 6c9f6f713b..d08854bf89 100644 --- a/packages/nocodb/src/modules/datas/datas.module.ts +++ b/packages/nocodb/src/modules/datas/datas.module.ts @@ -27,14 +27,18 @@ import { PublicDatasService } from '../../services/public-datas.service'; }), ], controllers: [ - DatasController, - BulkDataAliasController, - DataAliasController, - DataAliasNestedController, - DataAliasExportController, - OldDatasController, - PublicDatasController, - PublicDatasExportController, + ...(process.env.NC_WORKER_CONTAINER !== 'true' + ? [ + DatasController, + BulkDataAliasController, + DataAliasController, + DataAliasNestedController, + DataAliasExportController, + OldDatasController, + PublicDatasController, + PublicDatasExportController, + ] + : []), ], providers: [ DatasService, diff --git a/packages/nocodb/src/modules/global/global.module.ts b/packages/nocodb/src/modules/global/global.module.ts index b7aa2d6005..819eece5f7 100644 --- a/packages/nocodb/src/modules/global/global.module.ts +++ b/packages/nocodb/src/modules/global/global.module.ts @@ -1,25 +1,18 @@ import { Global, Module } from '@nestjs/common'; import { ExtractJwt } from 'passport-jwt'; -import { - AppInitService, - appInitServiceProvider, -} from '../../services/app-init.service'; import { SocketGateway } from '../../gateways/socket.gateway'; -import { Connection } from '../../connection/connection'; import { GlobalGuard } from '../../guards/global/global.guard'; import { MetaService } from '../../meta/meta.service'; -import Noco from '../../Noco'; import { JwtStrategy } from '../../strategies/jwt.strategy'; import { UsersService } from '../../services/users/users.service'; +import Noco from '../../Noco'; +import { InitMetaServiceProvider } from './init-meta-service.provider'; import type { Provider } from '@nestjs/common'; export const JwtStrategyProvider: Provider = { provide: JwtStrategy, - useFactory: async ( - usersService: UsersService, - appInitService: AppInitService, - ) => { - const config = appInitService.appConfig; + useFactory: async (usersService: UsersService, metaService: MetaService) => { + const config = metaService.config; await Noco.initJwt(); @@ -34,29 +27,25 @@ export const JwtStrategyProvider: Provider = { return new JwtStrategy(options, usersService); }, - inject: [UsersService, AppInitService], + inject: [UsersService, MetaService], }; @Global() @Module({ imports: [], providers: [ - appInitServiceProvider, - Connection, - MetaService, + InitMetaServiceProvider, UsersService, JwtStrategyProvider, GlobalGuard, - SocketGateway, + ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [SocketGateway] : []), ], exports: [ - AppInitService, - Connection, MetaService, JwtStrategyProvider, UsersService, GlobalGuard, - SocketGateway, + ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [SocketGateway] : []), ], }) export class GlobalModule {} diff --git a/packages/nocodb/src/modules/global/init-meta-service.provider.ts b/packages/nocodb/src/modules/global/init-meta-service.provider.ts new file mode 100644 index 0000000000..a86646b302 --- /dev/null +++ b/packages/nocodb/src/modules/global/init-meta-service.provider.ts @@ -0,0 +1,65 @@ +import { T } from 'nc-help'; +import { MetaService } from '../../meta/meta.service'; +import Noco from '../../Noco'; +import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2'; +import NcUpgrader from '../../version-upgrader/NcUpgrader'; +import NocoCache from '../../cache/NocoCache'; +import getInstance from '../../utils/getInstance'; +import initAdminFromEnv from '../../helpers/initAdminFromEnv'; +import { User } from '../../models'; +import { NcConfig, prepareEnv } from '../../utils/nc-config'; +import type { Provider } from '@nestjs/common'; +import type { IEventEmitter } from '../event-emitter/event-emitter.interface'; + +export const InitMetaServiceProvider: Provider = { + // initialize app, + // 1. init cache + // 2. init db connection and create if not exist + // 3. init meta and set to Noco + // 4. init jwt + // 5. init plugin manager + // 6. run upgrader + useFactory: async (eventEmitter: IEventEmitter) => { + // NC_DATABASE_URL_FILE, DATABASE_URL_FILE, DATABASE_URL, NC_DATABASE_URL to NC_DB + await prepareEnv(); + + const config = await NcConfig.createByEnv(); + + // set version + process.env.NC_VERSION = '0107004'; + + // init cache + await NocoCache.init(); + + // init meta service + const metaService = new MetaService(config); + await metaService.init(); + + // provide meta and config to Noco + Noco._ncMeta = metaService; + Noco.config = config; + Noco.eventEmitter = eventEmitter; + + // init jwt secret + await Noco.initJwt(); + + // load super admin user from env if env is set + await initAdminFromEnv(metaService); + + // init plugin manager + await NcPluginMgrv2.init(Noco.ncMeta); + await Noco.loadEEState(); + + // run upgrader + await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta }); + + T.init({ + instance: getInstance, + }); + T.emit('evt_app_started', await User.count()); + + return metaService; + }, + provide: MetaService, + inject: ['IEventEmitter'], +}; diff --git a/packages/nocodb/src/modules/jobs/fallback-queue.service.ts b/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts similarity index 81% rename from packages/nocodb/src/modules/jobs/fallback-queue.service.ts rename to packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts index 3250f90506..0fc60d68b1 100644 --- a/packages/nocodb/src/modules/jobs/fallback-queue.service.ts +++ b/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts @@ -1,12 +1,12 @@ import { Injectable } from '@nestjs/common'; import PQueue from 'p-queue'; import Emittery from 'emittery'; -import { JobStatus, JobTypes } from '../../interface/Jobs'; -import { DuplicateProcessor } from './export-import/duplicate.processor'; +import { JobStatus, JobTypes } from '../../../interface/Jobs'; +import { DuplicateProcessor } from '../jobs/export-import/duplicate.processor'; +import { AtImportProcessor } from '../jobs/at-import/at-import.processor'; import { JobsEventService } from './jobs-event.service'; -import { AtImportProcessor } from './at-import/at-import.processor'; -interface Job { +export interface Job { id: string; name: string; status: string; @@ -27,16 +27,12 @@ export class QueueService { private readonly atImportProcessor: AtImportProcessor, ) { this.emitter.on(JobStatus.ACTIVE, (data: { job: Job }) => { - const job = this.queueMemory.find( - (job) => job.id === data.job.id && job.name === data.job.name, - ); + const job = this.queueMemory.find((job) => job.id === data.job.id); job.status = JobStatus.ACTIVE; this.jobsEventService.onActive.apply(this.jobsEventService, [job as any]); }); this.emitter.on(JobStatus.COMPLETED, (data: { job: Job; result: any }) => { - const job = this.queueMemory.find( - (job) => job.id === data.job.id && job.name === data.job.name, - ); + const job = this.queueMemory.find((job) => job.id === data.job.id); job.status = JobStatus.COMPLETED; this.jobsEventService.onCompleted.apply(this.jobsEventService, [ job, @@ -46,9 +42,7 @@ export class QueueService { this.removeJob(job); }); this.emitter.on(JobStatus.FAILED, (data: { job: Job; error: Error }) => { - const job = this.queueMemory.find( - (job) => job.id === data.job.id && job.name === data.job.name, - ); + const job = this.queueMemory.find((job) => job.id === data.job.id); job.status = JobStatus.FAILED; this.jobsEventService.onFailed.apply(this.jobsEventService, [ job, @@ -126,9 +120,7 @@ export class QueueService { // remove job from memory private removeJob(job: Job) { - const fIndex = this.queueMemory.findIndex( - (q) => q.id === job.id && q.name === job.name, - ); + const fIndex = this.queueMemory.findIndex((q) => q.id === job.id); if (fIndex) { this.queueMemory.splice(fIndex, 1); } diff --git a/packages/nocodb/src/modules/jobs/jobs-event.service.ts b/packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts similarity index 71% rename from packages/nocodb/src/modules/jobs/jobs-event.service.ts rename to packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts index 7e72857204..87b8c6aaae 100644 --- a/packages/nocodb/src/modules/jobs/jobs-event.service.ts +++ b/packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts @@ -7,7 +7,7 @@ import { import { Job } from 'bull'; import boxen from 'boxen'; import { EventEmitter2 } from '@nestjs/event-emitter'; -import { JobEvents, JOBS_QUEUE, JobStatus } from '../../interface/Jobs'; +import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs'; @Processor(JOBS_QUEUE) export class JobsEventService { @@ -16,7 +16,6 @@ export class JobsEventService { @OnQueueActive() onActive(job: Job) { this.eventEmitter.emit(JobEvents.STATUS, { - name: job.name, id: job.id.toString(), status: JobStatus.ACTIVE, }); @@ -26,7 +25,7 @@ export class JobsEventService { onFailed(job: Job, error: Error) { console.error( boxen( - `---- !! JOB FAILED !! ----\nname: ${job.name}\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, + `---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, { padding: 1, borderStyle: 'double', @@ -36,7 +35,6 @@ export class JobsEventService { ); this.eventEmitter.emit(JobEvents.STATUS, { - name: job.name, id: job.id.toString(), status: JobStatus.FAILED, data: { @@ -50,7 +48,6 @@ export class JobsEventService { @OnQueueCompleted() onCompleted(job: Job, data: any) { this.eventEmitter.emit(JobEvents.STATUS, { - name: job.name, id: job.id.toString(), status: JobStatus.COMPLETED, data: { @@ -58,12 +55,4 @@ export class JobsEventService { }, }); } - - sendLog(job: Job, data: { message: string }) { - this.eventEmitter.emit(JobEvents.LOG, { - name: job.name, - id: job.id.toString(), - data, - }); - } } diff --git a/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts b/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts new file mode 100644 index 0000000000..b957da4b56 --- /dev/null +++ b/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts @@ -0,0 +1,51 @@ +import { Injectable } from '@nestjs/common'; +import { JobStatus } from '../../../interface/Jobs'; +import { QueueService } from './fallback-queue.service'; + +@Injectable() +export class JobsService { + constructor(private readonly fallbackQueueService: QueueService) {} + + async add(name: string, data: any) { + return this.fallbackQueueService.add(name, data); + } + + async jobStatus(jobId: string) { + return await ( + await this.fallbackQueueService.getJob(jobId) + ).status; + } + + async jobList() { + return await this.fallbackQueueService.getJobs([ + JobStatus.ACTIVE, + JobStatus.WAITING, + JobStatus.DELAYED, + JobStatus.PAUSED, + ]); + } + + async getJobWithData(data: any) { + const jobs = await this.fallbackQueueService.getJobs([ + // 'completed', + JobStatus.WAITING, + JobStatus.ACTIVE, + JobStatus.DELAYED, + // 'failed', + JobStatus.PAUSED, + ]); + + const job = jobs.find((j) => { + for (const key in data) { + if (j.data[key]) { + if (j.data[key] !== data[key]) return false; + } else { + return false; + } + } + return true; + }); + + return job; + } +} diff --git a/packages/nocodb/src/modules/jobs/jobs.gateway.ts b/packages/nocodb/src/modules/jobs/jobs.gateway.ts index 119462e69f..9e3d08aa68 100644 --- a/packages/nocodb/src/modules/jobs/jobs.gateway.ts +++ b/packages/nocodb/src/modules/jobs/jobs.gateway.ts @@ -9,10 +9,10 @@ import { Server, Socket } from 'socket.io'; import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host'; import { AuthGuard } from '@nestjs/passport'; import { OnEvent } from '@nestjs/event-emitter'; +import { Inject } from '@nestjs/common'; import { JobEvents } from '../../interface/Jobs'; -import { JobsService } from './jobs.service'; -import type { JobStatus } from '../../interface/Jobs'; import type { OnModuleInit } from '@nestjs/common'; +import type { JobStatus } from '../../interface/Jobs'; @WebSocketGateway({ cors: { @@ -23,7 +23,7 @@ import type { OnModuleInit } from '@nestjs/common'; namespace: 'jobs', }) export class JobsGateway implements OnModuleInit { - constructor(private readonly jobsService: JobsService) {} + constructor(@Inject('JobsService') private readonly jobsService) {} @WebSocketServer() server: Server; @@ -43,34 +43,28 @@ export class JobsGateway implements OnModuleInit { @SubscribeMessage('subscribe') async subscribe( @MessageBody() - body: { _id: number; data: { id: string; name: string } | any }, + body: { _id: number; data: { id: string } | any }, @ConnectedSocket() client: Socket, ): Promise { const { _id, data } = body; - if ( - Object.keys(data).every((k) => ['name', 'id'].includes(k)) && - data?.name && - data?.id - ) { - const rooms = (await this.jobsService.jobList(data.name)).map( - (j) => `${j.name}-${j.id}`, + if (Object.keys(data).every((k) => ['id'].includes(k)) && data?.id) { + const rooms = (await this.jobsService.jobList()).map( + (j) => `jobs-${j.id}`, ); - const room = rooms.find((r) => r === `${data.name}-${data.id}`); + const room = rooms.find((r) => r === `jobs-${data.id}`); if (room) { - client.join(`${data.name}-${data.id}`); + client.join(`jobs-${data.id}`); client.emit('subscribed', { _id, - name: data.name, id: data.id, }); } } else { const job = await this.jobsService.getJobWithData(data); if (job) { - client.join(`${job.name}-${job.id}`); + client.join(`jobs-${job.id}`); client.emit('subscribed', { _id, - name: job.name, id: job.id, }); } @@ -79,42 +73,30 @@ export class JobsGateway implements OnModuleInit { @SubscribeMessage('status') async status( - @MessageBody() body: { _id: number; data: { id: string; name: string } }, + @MessageBody() body: { _id: number; data: { id: string } }, @ConnectedSocket() client: Socket, ): Promise { const { _id, data } = body; client.emit('status', { _id, id: data.id, - name: data.name, status: await this.jobsService.jobStatus(data.id), }); } @OnEvent(JobEvents.STATUS) - async sendJobStatus(data: { - name: string; - id: string; - status: JobStatus; - data?: any; - }): Promise { - this.server.to(`${data.name}-${data.id}`).emit('status', { + sendJobStatus(data: { id: string; status: JobStatus; data?: any }): void { + this.server.to(`jobs-${data.id}`).emit('status', { id: data.id, - name: data.name, status: data.status, data: data.data, }); } @OnEvent(JobEvents.LOG) - async sendJobLog(data: { - name: string; - id: string; - data: { message: string }; - }): Promise { - this.server.to(`${data.name}-${data.id}`).emit('log', { + sendJobLog(data: { id: string; data: { message: string } }): void { + this.server.to(`jobs-${data.id}`).emit('log', { id: data.id, - name: data.name, data: data.data, }); } diff --git a/packages/nocodb/src/modules/jobs/jobs.module.ts b/packages/nocodb/src/modules/jobs/jobs.module.ts index f211cc5d46..fc7d7a59fb 100644 --- a/packages/nocodb/src/modules/jobs/jobs.module.ts +++ b/packages/nocodb/src/modules/jobs/jobs.module.ts @@ -4,35 +4,59 @@ import { GlobalModule } from '../global/global.module'; import { DatasModule } from '../datas/datas.module'; import { MetasModule } from '../metas/metas.module'; import { JOBS_QUEUE } from '../../interface/Jobs'; -import { JobsService } from './jobs.service'; -import { ExportService } from './export-import/export.service'; -import { ImportService } from './export-import/import.service'; -import { DuplicateController } from './export-import/duplicate.controller'; -import { DuplicateProcessor } from './export-import/duplicate.processor'; +import { ExportService } from './jobs/export-import/export.service'; +import { ImportService } from './jobs/export-import/import.service'; +import { AtImportController } from './jobs/at-import/at-import.controller'; +import { AtImportProcessor } from './jobs/at-import/at-import.processor'; +import { DuplicateController } from './jobs/export-import/duplicate.controller'; +import { DuplicateProcessor } from './jobs/export-import/duplicate.processor'; +import { JobsLogService } from './jobs/jobs-log.service'; import { JobsGateway } from './jobs.gateway'; -import { QueueService } from './fallback-queue.service'; -import { JobsEventService } from './jobs-event.service'; -import { AtImportController } from './at-import/at-import.controller'; -import { AtImportProcessor } from './at-import/at-import.processor'; + +// Redis +import { JobsService } from './redis/jobs.service'; +import { JobsRedisService } from './redis/jobs-redis.service'; +import { JobsEventService } from './redis/jobs-event.service'; + +// Fallback +import { JobsService as FallbackJobsService } from './fallback/jobs.service'; +import { QueueService as FallbackQueueService } from './fallback/fallback-queue.service'; +import { JobsEventService as FallbackJobsEventService } from './fallback/jobs-event.service'; @Module({ imports: [ GlobalModule, DatasModule, MetasModule, - BullModule.registerQueue({ - name: JOBS_QUEUE, - }), + ...(process.env.NC_REDIS_URL + ? [ + BullModule.forRoot({ + url: process.env.NC_REDIS_URL, + }), + BullModule.registerQueue({ + name: JOBS_QUEUE, + }), + ] + : []), + ], + controllers: [ + ...(process.env.NC_WORKER_CONTAINER !== 'true' + ? [DuplicateController, AtImportController] + : []), ], - controllers: [DuplicateController, AtImportController], providers: [ - QueueService, - JobsGateway, - JobsService, - JobsEventService, - DuplicateProcessor, + ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [JobsGateway] : []), + ...(process.env.NC_REDIS_URL + ? [JobsRedisService, JobsEventService] + : [FallbackQueueService, FallbackJobsEventService]), + { + provide: 'JobsService', + useClass: process.env.NC_REDIS_URL ? JobsService : FallbackJobsService, + }, + JobsLogService, ExportService, ImportService, + DuplicateProcessor, AtImportProcessor, ], }) diff --git a/packages/nocodb/src/modules/jobs/jobs.service.ts b/packages/nocodb/src/modules/jobs/jobs.service.ts deleted file mode 100644 index 4a4154bf8c..0000000000 --- a/packages/nocodb/src/modules/jobs/jobs.service.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { InjectQueue } from '@nestjs/bull'; -import { Injectable } from '@nestjs/common'; -import { Queue } from 'bull'; -import { JOBS_QUEUE, JobStatus } from '../../interface/Jobs'; -import { QueueService } from './fallback-queue.service'; - -@Injectable() -export class JobsService { - public activeQueue; - constructor( - @InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue, - private readonly fallbackQueueService: QueueService, - ) { - this.activeQueue = this.fallbackQueueService; - /* process.env.NC_REDIS_URL - ? this.jobsQueue - : this.fallbackQueueService; - */ - } - - async jobStatus(jobId: string) { - return await (await this.activeQueue.getJob(jobId)).getState(); - } - - async jobList(jobType: string) { - return ( - await this.activeQueue.getJobs([ - JobStatus.ACTIVE, - JobStatus.WAITING, - JobStatus.DELAYED, - JobStatus.PAUSED, - ]) - ).filter((j) => j.name === jobType); - } - - async getJobWithData(data: any) { - const jobs = await this.activeQueue.getJobs([ - // 'completed', - JobStatus.WAITING, - JobStatus.ACTIVE, - JobStatus.DELAYED, - // 'failed', - JobStatus.PAUSED, - ]); - - const job = jobs.find((j) => { - for (const key in data) { - if (j.data[key]) { - if (j.data[key] !== data[key]) return false; - } else { - return false; - } - } - return true; - }); - - return job; - } -} diff --git a/packages/nocodb/src/modules/jobs/at-import/at-import.controller.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts similarity index 59% rename from packages/nocodb/src/modules/jobs/at-import/at-import.controller.ts rename to packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts index f27b7bfca8..d1d2a54857 100644 --- a/packages/nocodb/src/modules/jobs/at-import/at-import.controller.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts @@ -1,30 +1,36 @@ -import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common'; -import { GlobalGuard } from '../../../guards/global/global.guard'; -import { ExtractProjectIdMiddleware } from '../../../middlewares/extract-project-id/extract-project-id.middleware'; -import { SyncSource } from '../../../models'; -import { NcError } from '../../../helpers/catchError'; -import { JobsService } from '../jobs.service'; -import { JobTypes } from '../../../interface/Jobs'; +import { + Controller, + HttpCode, + Inject, + Post, + Request, + UseGuards, +} from '@nestjs/common'; +import { GlobalGuard } from '../../../../guards/global/global.guard'; +import { ExtractProjectIdMiddleware } from '../../../../middlewares/extract-project-id/extract-project-id.middleware'; +import { SyncSource } from '../../../../models'; +import { NcError } from '../../../../helpers/catchError'; +import { JobTypes } from '../../../../interface/Jobs'; @Controller() @UseGuards(ExtractProjectIdMiddleware, GlobalGuard) export class AtImportController { - constructor(private readonly jobsService: JobsService) {} + constructor(@Inject('JobsService') private readonly jobsService) {} @Post('/api/v1/db/meta/import/airtable') @HttpCode(200) async importAirtable(@Request() req) { - const job = await this.jobsService.activeQueue.add(JobTypes.AtImport, { + const job = await this.jobsService.add(JobTypes.AtImport, { ...req.body, }); - return { id: job.id, name: job.name }; + return { id: job.id }; } @Post('/api/v1/db/meta/syncs/:syncId/trigger') @HttpCode(200) async triggerSync(@Request() req) { - const jobs = await this.jobsService.jobList(JobTypes.AtImport); + const jobs = await this.jobsService.jobList(); const fnd = jobs.find((j) => j.data.syncId === req.params.syncId); if (fnd) { @@ -44,7 +50,7 @@ export class AtImportController { baseURL = `http://localhost:${process.env.PORT || 8080}`; } - const job = await this.jobsService.activeQueue.add(JobTypes.AtImport, { + const job = await this.jobsService.add(JobTypes.AtImport, { syncId: req.params.syncId, ...(syncSource?.details || {}), projectId: syncSource.project_id, @@ -54,7 +60,7 @@ export class AtImportController { user: user, }); - return { id: job.id, name: job.name }; + return { id: job.id }; } @Post('/api/v1/db/meta/syncs/:syncId/abort') diff --git a/packages/nocodb/src/modules/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts similarity index 98% rename from packages/nocodb/src/modules/jobs/at-import/at-import.processor.ts rename to packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index 32ab68fd4d..bc9a247dad 100644 --- a/packages/nocodb/src/modules/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -9,23 +9,23 @@ import utc from 'dayjs/plugin/utc'; import tinycolor from 'tinycolor2'; import { Process, Processor } from '@nestjs/bull'; import { Job } from 'bull'; -import extractRolesObj from '../../../utils/extractRolesObj'; -import { AttachmentsService } from '../../../services/attachments.service'; -import { ColumnsService } from '../../../services/columns.service'; -import { BulkDataAliasService } from '../../../services/bulk-data-alias.service'; -import { FiltersService } from '../../../services/filters.service'; -import { FormColumnsService } from '../../../services/form-columns.service'; -import { GalleriesService } from '../../../services/galleries.service'; -import { GridsService } from '../../../services/grids.service'; -import { ProjectUsersService } from '../../../services/project-users/project-users.service'; -import { ProjectsService } from '../../../services/projects.service'; -import { SortsService } from '../../../services/sorts.service'; -import { TablesService } from '../../../services/tables.service'; -import { ViewColumnsService } from '../../../services/view-columns.service'; -import { ViewsService } from '../../../services/views.service'; -import { FormsService } from '../../../services/forms.service'; -import { JobsEventService } from '../jobs-event.service'; -import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs'; +import extractRolesObj from '../../../../utils/extractRolesObj'; +import { AttachmentsService } from '../../../../services/attachments.service'; +import { ColumnsService } from '../../../../services/columns.service'; +import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service'; +import { FiltersService } from '../../../../services/filters.service'; +import { FormColumnsService } from '../../../../services/form-columns.service'; +import { GalleriesService } from '../../../../services/galleries.service'; +import { GridsService } from '../../../../services/grids.service'; +import { ProjectUsersService } from '../../../../services/project-users/project-users.service'; +import { ProjectsService } from '../../../../services/projects.service'; +import { SortsService } from '../../../../services/sorts.service'; +import { TablesService } from '../../../../services/tables.service'; +import { ViewColumnsService } from '../../../../services/view-columns.service'; +import { ViewsService } from '../../../../services/views.service'; +import { FormsService } from '../../../../services/forms.service'; +import { JOBS_QUEUE, JobTypes } from '../../../../interface/Jobs'; +import { JobsLogService } from '../jobs-log.service'; import FetchAT from './helpers/fetchAT'; import { importData, importLTARData } from './helpers/readAndProcessData'; import EntityMap from './helpers/EntityMap'; @@ -99,7 +99,7 @@ export class AtImportProcessor { private readonly viewColumnsService: ViewColumnsService, private readonly sortsService: SortsService, private readonly bulkDataAliasService: BulkDataAliasService, - private readonly jobsEventService: JobsEventService, + private readonly jobsLogService: JobsLogService, ) {} @Process(JobTypes.AtImport) @@ -135,11 +135,11 @@ export class AtImportProcessor { }; const logBasic = (log) => { - this.jobsEventService.sendLog(job, { message: log }); + this.jobsLogService.sendLog(job, { message: log }); }; const logDetailed = (log) => { - if (debugMode) this.jobsEventService.sendLog(job, { message: log }); + if (debugMode) this.jobsLogService.sendLog(job, { message: log }); }; const perfStats = []; diff --git a/packages/nocodb/src/modules/jobs/at-import/helpers/EntityMap.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/EntityMap.ts similarity index 100% rename from packages/nocodb/src/modules/jobs/at-import/helpers/EntityMap.ts rename to packages/nocodb/src/modules/jobs/jobs/at-import/helpers/EntityMap.ts diff --git a/packages/nocodb/src/modules/jobs/at-import/helpers/fetchAT.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts similarity index 100% rename from packages/nocodb/src/modules/jobs/at-import/helpers/fetchAT.ts rename to packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts diff --git a/packages/nocodb/src/modules/jobs/at-import/helpers/readAndProcessData.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts similarity index 98% rename from packages/nocodb/src/modules/jobs/at-import/helpers/readAndProcessData.ts rename to packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts index 7de16fad7e..2bd3d627c6 100644 --- a/packages/nocodb/src/modules/jobs/at-import/helpers/readAndProcessData.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts @@ -1,8 +1,8 @@ /* eslint-disable no-async-promise-executor */ import { RelationTypes, UITypes } from 'nocodb-sdk'; import EntityMap from './EntityMap'; -import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service'; -import type { TablesService } from '../../../../services/tables.service'; +import type { BulkDataAliasService } from '../../../../../services/bulk-data-alias.service'; +import type { TablesService } from '../../../../../services/tables.service'; // @ts-ignore import type { AirtableBase } from 'airtable/lib/airtable_base'; import type { TableType } from 'nocodb-sdk'; diff --git a/packages/nocodb/src/modules/jobs/at-import/helpers/syncMap.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/helpers/syncMap.ts similarity index 100% rename from packages/nocodb/src/modules/jobs/at-import/helpers/syncMap.ts rename to packages/nocodb/src/modules/jobs/jobs/at-import/helpers/syncMap.ts diff --git a/packages/nocodb/src/modules/jobs/export-import/duplicate.controller.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts similarity index 71% rename from packages/nocodb/src/modules/jobs/export-import/duplicate.controller.ts rename to packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts index ac1bf3a6f1..71a38ce90c 100644 --- a/packages/nocodb/src/modules/jobs/export-import/duplicate.controller.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts @@ -2,28 +2,28 @@ import { Body, Controller, HttpCode, + Inject, Param, Post, Request, UseGuards, } from '@nestjs/common'; import { ProjectStatus } from 'nocodb-sdk'; -import { GlobalGuard } from '../../../guards/global/global.guard'; +import { GlobalGuard } from '../../../../guards/global/global.guard'; import { Acl, ExtractProjectIdMiddleware, -} from '../../../middlewares/extract-project-id/extract-project-id.middleware'; -import { ProjectsService } from '../../../services/projects.service'; -import { Base, Model, Project } from '../../../models'; -import { generateUniqueName } from '../../../helpers/exportImportHelpers'; -import { JobsService } from '../jobs.service'; -import { JobTypes } from '../../../interface/Jobs'; +} from '../../../../middlewares/extract-project-id/extract-project-id.middleware'; +import { ProjectsService } from '../../../../services/projects.service'; +import { Base, Model, Project } from '../../../../models'; +import { generateUniqueName } from '../../../../helpers/exportImportHelpers'; +import { JobTypes } from '../../../../interface/Jobs'; @Controller() @UseGuards(ExtractProjectIdMiddleware, GlobalGuard) export class DuplicateController { constructor( - private readonly jobsService: JobsService, + @Inject('JobsService') private readonly jobsService, private readonly projectsService: ProjectsService, ) {} @@ -67,7 +67,7 @@ export class DuplicateController { user: { id: req.user.id }, }); - const job = await this.jobsService.activeQueue.add(JobTypes.DuplicateBase, { + const job = await this.jobsService.add(JobTypes.DuplicateBase, { projectId: project.id, baseId: base.id, dupProjectId: dupProject.id, @@ -78,7 +78,7 @@ export class DuplicateController { }, }); - return { id: job.id, name: job.name }; + return { id: job.id }; } @Post('/api/v1/db/meta/duplicate/:projectId/table/:modelId') @@ -116,21 +116,18 @@ export class DuplicateController { models.map((p) => p.title), ); - const job = await this.jobsService.activeQueue.add( - JobTypes.DuplicateModel, - { - projectId: project.id, - baseId: base.id, - modelId: model.id, - title: uniqueTitle, - options, - req: { - user: req.user, - clientIp: req.clientIp, - }, + const job = await this.jobsService.add(JobTypes.DuplicateModel, { + projectId: project.id, + baseId: base.id, + modelId: model.id, + title: uniqueTitle, + options, + req: { + user: req.user, + clientIp: req.clientIp, }, - ); + }); - return { id: job.id, name: job.name }; + return { id: job.id }; } } diff --git a/packages/nocodb/src/modules/jobs/export-import/duplicate.processor.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts similarity index 96% rename from packages/nocodb/src/modules/jobs/export-import/duplicate.processor.ts rename to packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts index 05742a13af..dc471803b8 100644 --- a/packages/nocodb/src/modules/jobs/export-import/duplicate.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts @@ -4,12 +4,12 @@ import { Job } from 'bull'; import papaparse from 'papaparse'; import { UITypes } from 'nocodb-sdk'; import { Logger } from '@nestjs/common'; -import { Base, Column, Model, Project } from '../../../models'; -import { ProjectsService } from '../../../services/projects.service'; -import { findWithIdentifier } from '../../../helpers/exportImportHelpers'; -import { BulkDataAliasService } from '../../../services/bulk-data-alias.service'; -import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs'; -import { elapsedTime, initTime } from '../helpers'; +import { Base, Column, Model, Project } from '../../../../models'; +import { ProjectsService } from '../../../../services/projects.service'; +import { findWithIdentifier } from '../../../../helpers/exportImportHelpers'; +import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service'; +import { JOBS_QUEUE, JobTypes } from '../../../../interface/Jobs'; +import { elapsedTime, initTime } from '../../helpers'; import { ExportService } from './export.service'; import { ImportService } from './import.service'; diff --git a/packages/nocodb/src/modules/jobs/export-import/export.service.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts similarity index 96% rename from packages/nocodb/src/modules/jobs/export-import/export.service.ts rename to packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts index 4a516d1d14..f72ec18d13 100644 --- a/packages/nocodb/src/modules/jobs/export-import/export.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts @@ -2,19 +2,19 @@ import { Readable } from 'stream'; import { UITypes, ViewTypes } from 'nocodb-sdk'; import { unparse } from 'papaparse'; import { Injectable, Logger } from '@nestjs/common'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; -import { getViewAndModelByAliasOrId } from '../../../modules/datas/helpers'; +import NcConnectionMgrv2 from '../../../../utils/common/NcConnectionMgrv2'; +import { getViewAndModelByAliasOrId } from '../../../datas/helpers'; import { clearPrefix, generateBaseIdMap, -} from '../../../helpers/exportImportHelpers'; -import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2'; -import { NcError } from '../../../helpers/catchError'; -import { Base, Hook, Model, Project } from '../../../models'; -import { DatasService } from '../../../services/datas.service'; -import { elapsedTime, initTime } from '../helpers'; -import type { BaseModelSqlv2 } from '../../../db/BaseModelSqlv2'; -import type { View } from '../../../models'; +} from '../../../../helpers/exportImportHelpers'; +import NcPluginMgrv2 from '../../../../helpers/NcPluginMgrv2'; +import { NcError } from '../../../../helpers/catchError'; +import { Base, Hook, Model, Project } from '../../../../models'; +import { DatasService } from '../../../../services/datas.service'; +import { elapsedTime, initTime } from '../../helpers'; +import type { BaseModelSqlv2 } from '../../../../db/BaseModelSqlv2'; +import type { View } from '../../../../models'; @Injectable() export class ExportService { diff --git a/packages/nocodb/src/modules/jobs/export-import/import.service.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts similarity index 97% rename from packages/nocodb/src/modules/jobs/export-import/import.service.ts rename to packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts index 035044df32..08c75b8abe 100644 --- a/packages/nocodb/src/modules/jobs/export-import/import.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts @@ -9,28 +9,28 @@ import { reverseGet, withoutId, withoutNull, -} from '../../../helpers/exportImportHelpers'; -import { NcError } from '../../../helpers/catchError'; -import { Base, Column, Model, Project } from '../../../models'; -import { TablesService } from '../../../services/tables.service'; -import { ColumnsService } from '../../../services/columns.service'; -import { FiltersService } from '../../../services/filters.service'; -import { SortsService } from '../../../services/sorts.service'; -import { ViewColumnsService } from '../../../services/view-columns.service'; -import { GridColumnsService } from '../../../services/grid-columns.service'; -import { FormColumnsService } from '../../../services/form-columns.service'; -import { GridsService } from '../../../services/grids.service'; -import { FormsService } from '../../../services/forms.service'; -import { GalleriesService } from '../../../services/galleries.service'; -import { KanbansService } from '../../../services/kanbans.service'; -import { HooksService } from '../../../services/hooks.service'; -import { ViewsService } from '../../../services/views.service'; -import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2'; -import { BulkDataAliasService } from '../../../services/bulk-data-alias.service'; -import { elapsedTime, initTime } from '../helpers'; +} from '../../../../helpers/exportImportHelpers'; +import { NcError } from '../../../../helpers/catchError'; +import { Base, Column, Model, Project } from '../../../../models'; +import { TablesService } from '../../../../services/tables.service'; +import { ColumnsService } from '../../../../services/columns.service'; +import { FiltersService } from '../../../../services/filters.service'; +import { SortsService } from '../../../../services/sorts.service'; +import { ViewColumnsService } from '../../../../services/view-columns.service'; +import { GridColumnsService } from '../../../../services/grid-columns.service'; +import { FormColumnsService } from '../../../../services/form-columns.service'; +import { GridsService } from '../../../../services/grids.service'; +import { FormsService } from '../../../../services/forms.service'; +import { GalleriesService } from '../../../../services/galleries.service'; +import { KanbansService } from '../../../../services/kanbans.service'; +import { HooksService } from '../../../../services/hooks.service'; +import { ViewsService } from '../../../../services/views.service'; +import NcPluginMgrv2 from '../../../../helpers/NcPluginMgrv2'; +import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service'; +import { elapsedTime, initTime } from '../../helpers'; import type { Readable } from 'stream'; import type { ViewCreateReqType } from 'nocodb-sdk'; -import type { LinkToAnotherRecordColumn, User, View } from '../../../models'; +import type { LinkToAnotherRecordColumn, User, View } from '../../../../models'; @Injectable() export class ImportService { diff --git a/packages/nocodb/src/modules/jobs/jobs/jobs-log.service.ts b/packages/nocodb/src/modules/jobs/jobs/jobs-log.service.ts new file mode 100644 index 0000000000..e6066d975a --- /dev/null +++ b/packages/nocodb/src/modules/jobs/jobs/jobs-log.service.ts @@ -0,0 +1,16 @@ +import { Injectable } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { JobEvents } from '../../../interface/Jobs'; +import type { Job } from 'bull'; + +@Injectable() +export class JobsLogService { + constructor(private eventEmitter: EventEmitter2) {} + + sendLog(job: Job, data: { message: string }) { + this.eventEmitter.emit(JobEvents.LOG, { + id: job.id.toString(), + data, + }); + } +} diff --git a/packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts b/packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts new file mode 100644 index 0000000000..be94733c69 --- /dev/null +++ b/packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts @@ -0,0 +1,107 @@ +import { + OnQueueActive, + OnQueueCompleted, + OnQueueFailed, + Processor, +} from '@nestjs/bull'; +import { Job } from 'bull'; +import boxen from 'boxen'; +import { EventEmitter2, OnEvent } from '@nestjs/event-emitter'; +import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs'; +import { JobsRedisService } from './jobs-redis.service'; + +@Processor(JOBS_QUEUE) +export class JobsEventService { + constructor( + private jobsRedisService: JobsRedisService, + private eventEmitter: EventEmitter2, + ) {} + + @OnQueueActive() + onActive(job: Job) { + if (process.env.NC_WORKER_CONTAINER === 'true') { + this.jobsRedisService.publish(`jobs-${job.id.toString()}`, { + cmd: JobEvents.STATUS, + id: job.id.toString(), + status: JobStatus.ACTIVE, + }); + } else { + this.eventEmitter.emit(JobEvents.STATUS, { + id: job.id.toString(), + status: JobStatus.ACTIVE, + }); + } + } + + @OnQueueFailed() + onFailed(job: Job, error: Error) { + console.error( + boxen( + `---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, + { + padding: 1, + borderStyle: 'double', + borderColor: 'yellow', + }, + ), + ); + + if (process.env.NC_WORKER_CONTAINER === 'true') { + this.jobsRedisService.publish(`jobs-${job.id.toString()}`, { + cmd: JobEvents.STATUS, + id: job.id.toString(), + status: JobStatus.FAILED, + data: { + error: { + message: error?.message, + }, + }, + }); + } else { + this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`); + this.eventEmitter.emit(JobEvents.STATUS, { + id: job.id.toString(), + status: JobStatus.FAILED, + data: { + error: { + message: error?.message, + }, + }, + }); + } + } + + @OnQueueCompleted() + onCompleted(job: Job, data: any) { + if (process.env.NC_WORKER_CONTAINER === 'true') { + this.jobsRedisService.publish(`jobs-${job.id.toString()}`, { + cmd: JobEvents.STATUS, + id: job.id.toString(), + status: JobStatus.COMPLETED, + data: { + result: data, + }, + }); + } else { + this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`); + this.eventEmitter.emit(JobEvents.STATUS, { + id: job.id.toString(), + status: JobStatus.COMPLETED, + data: { + result: data, + }, + }); + } + } + + @OnEvent(JobEvents.LOG) + onLog(data: { id: string; data: { message: string } }) { + if (process.env.NC_WORKER_CONTAINER === 'true') { + this.jobsRedisService.publish(`jobs-${data.id}`, { + cmd: JobEvents.LOG, + id: data.id, + data: data.data, + }); + } + } +} diff --git a/packages/nocodb/src/modules/jobs/redis/jobs-redis.service.ts b/packages/nocodb/src/modules/jobs/redis/jobs-redis.service.ts new file mode 100644 index 0000000000..7a2f9ec757 --- /dev/null +++ b/packages/nocodb/src/modules/jobs/redis/jobs-redis.service.ts @@ -0,0 +1,53 @@ +import { Injectable } from '@nestjs/common'; +import Redis from 'ioredis'; + +@Injectable() +export class JobsRedisService { + private redisClient: Redis; + private redisSubscriber: Redis; + private unsubscribeCallbacks: { [key: string]: () => void } = {}; + + constructor() { + if (process.env.NC_WORKER_CONTAINER === 'true') { + this.redisClient = new Redis(process.env.NC_REDIS_URL); + return; + } + this.redisSubscriber = new Redis(process.env.NC_REDIS_URL); + } + + publish(channel: string, message: string | any) { + if (typeof message === 'string') { + this.redisClient.publish(channel, message); + } else { + try { + this.redisClient.publish(channel, JSON.stringify(message)); + } catch (e) { + console.error(e); + } + } + } + + subscribe(channel: string, callback: (message: any) => void) { + this.redisSubscriber.subscribe(channel); + + const onMessage = (_channel, message) => { + try { + message = JSON.parse(message); + } catch (e) {} + callback(message); + }; + + this.redisSubscriber.on('message', onMessage); + this.unsubscribeCallbacks[channel] = () => { + this.redisSubscriber.unsubscribe(channel); + this.redisSubscriber.off('message', onMessage); + }; + } + + unsubscribe(channel: string) { + if (this.unsubscribeCallbacks[channel]) { + this.unsubscribeCallbacks[channel](); + delete this.unsubscribeCallbacks[channel]; + } + } +} diff --git a/packages/nocodb/src/modules/jobs/redis/jobs.service.ts b/packages/nocodb/src/modules/jobs/redis/jobs.service.ts new file mode 100644 index 0000000000..7e3962142c --- /dev/null +++ b/packages/nocodb/src/modules/jobs/redis/jobs.service.ts @@ -0,0 +1,98 @@ +import { InjectQueue } from '@nestjs/bull'; +import { Injectable } from '@nestjs/common'; +import { Queue } from 'bull'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs'; +import { JobsRedisService } from './jobs-redis.service'; +import type { OnModuleInit } from '@nestjs/common'; + +@Injectable() +export class JobsService implements OnModuleInit { + constructor( + @InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue, + private jobsRedisService: JobsRedisService, + private eventEmitter: EventEmitter2, + ) {} + + // pause primary instance queue + async onModuleInit() { + if (process.env.NC_WORKER_CONTAINER !== 'true') { + await this.jobsQueue.pause(true); + } + } + + async add(name: string, data: any) { + // resume primary instance queue if there is no worker + const workerCount = (await this.jobsQueue.getWorkers()).length; + const localWorkerPaused = await this.jobsQueue.isPaused(true); + + // if there is no worker and primary instance queue is paused, resume it + // if there is any worker and primary instance queue is not paused, pause it + if (workerCount < 1 && localWorkerPaused) { + await this.jobsQueue.resume(true); + } else if (workerCount > 0 && !localWorkerPaused) { + await this.jobsQueue.pause(true); + } + + const job = await this.jobsQueue.add(name, data); + + // subscribe to job events + this.jobsRedisService.subscribe(`jobs-${job.id.toString()}`, (data) => { + const cmd = data.cmd; + delete data.cmd; + switch (cmd) { + case JobEvents.STATUS: + this.eventEmitter.emit(JobEvents.STATUS, data); + if ([JobStatus.COMPLETED, JobStatus.FAILED].includes(data.status)) { + this.jobsRedisService.unsubscribe(`jobs-${data.id.toString()}`); + } + break; + case JobEvents.LOG: + this.eventEmitter.emit(JobEvents.LOG, data); + break; + } + }); + + return job; + } + + async jobStatus(jobId: string) { + const job = await this.jobsQueue.getJob(jobId); + if (job) { + return await job.getState(); + } + } + + async jobList() { + return await this.jobsQueue.getJobs([ + JobStatus.ACTIVE, + JobStatus.WAITING, + JobStatus.DELAYED, + JobStatus.PAUSED, + ]); + } + + async getJobWithData(data: any) { + const jobs = await this.jobsQueue.getJobs([ + // 'completed', + JobStatus.WAITING, + JobStatus.ACTIVE, + JobStatus.DELAYED, + // 'failed', + JobStatus.PAUSED, + ]); + + const job = jobs.find((j) => { + for (const key in data) { + if (j.data[key]) { + if (j.data[key] !== data[key]) return false; + } else { + return false; + } + } + return true; + }); + + return job; + } +} diff --git a/packages/nocodb/src/modules/metas/metas.module.ts b/packages/nocodb/src/modules/metas/metas.module.ts index 0aa6a9ae76..d42e860d7e 100644 --- a/packages/nocodb/src/modules/metas/metas.module.ts +++ b/packages/nocodb/src/modules/metas/metas.module.ts @@ -67,7 +67,6 @@ import { UtilsService } from '../../services/utils.service'; import { ViewColumnsService } from '../../services/view-columns.service'; import { ViewsService } from '../../services/views.service'; import { ApiDocsService } from '../../services/api-docs/api-docs.service'; -import { EventEmitterModule } from '../event-emitter/event-emitter.module'; import { GlobalModule } from '../global/global.module'; import { ProjectUsersController } from '../../controllers/project-users.controller'; import { ProjectUsersService } from '../../services/project-users/project-users.service'; @@ -83,38 +82,42 @@ import { ProjectUsersService } from '../../services/project-users/project-users. GlobalModule, ], controllers: [ - ApiDocsController, - ApiTokensController, - AttachmentsController, - AuditsController, - BasesController, - CachesController, - ColumnsController, - FiltersController, - FormColumnsController, - FormsController, - GalleriesController, - GridColumnsController, - GridsController, - HooksController, - KanbansController, - MapsController, - MetaDiffsController, - ModelVisibilitiesController, - OrgLcenseController, - OrgTokensController, - OrgUsersController, - PluginsController, - ProjectUsersController, - ProjectsController, - PublicMetasController, - ViewsController, - ViewColumnsController, - UtilsController, - TablesController, - SyncController, - SortsController, - SharedBasesController, + ...(process.env.NC_WORKER_CONTAINER !== 'true' + ? [ + ApiDocsController, + ApiTokensController, + AttachmentsController, + AuditsController, + BasesController, + CachesController, + ColumnsController, + FiltersController, + FormColumnsController, + FormsController, + GalleriesController, + GridColumnsController, + GridsController, + HooksController, + KanbansController, + MapsController, + MetaDiffsController, + ModelVisibilitiesController, + OrgLcenseController, + OrgTokensController, + OrgUsersController, + PluginsController, + ProjectUsersController, + ProjectsController, + PublicMetasController, + ViewsController, + ViewColumnsController, + UtilsController, + TablesController, + SyncController, + SortsController, + SharedBasesController, + ] + : []), ], providers: [ ApiDocsService, diff --git a/packages/nocodb/src/modules/test/test.module.ts b/packages/nocodb/src/modules/test/test.module.ts index d86fece015..aeddd77174 100644 --- a/packages/nocodb/src/modules/test/test.module.ts +++ b/packages/nocodb/src/modules/test/test.module.ts @@ -2,6 +2,8 @@ import { Module } from '@nestjs/common'; import { TestController } from '../../controllers/test/test.controller'; @Module({ - controllers: [TestController], + controllers: [ + ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [TestController] : []), + ], }) export class TestModule {} diff --git a/packages/nocodb/src/modules/users/users.module.ts b/packages/nocodb/src/modules/users/users.module.ts index d4297b3c5e..bb1da81911 100644 --- a/packages/nocodb/src/modules/users/users.module.ts +++ b/packages/nocodb/src/modules/users/users.module.ts @@ -10,7 +10,9 @@ import { UsersController } from '../../controllers/users/users.controller'; @Module({ imports: [GlobalModule, PassportModule], - controllers: [UsersController], + controllers: [ + ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [UsersController] : []), + ], providers: [UsersService, GoogleStrategyProvider], exports: [UsersService], }) diff --git a/packages/nocodb/src/plugins/storage/Local.ts b/packages/nocodb/src/plugins/storage/Local.ts index 26740839b9..f0dbebf2a4 100644 --- a/packages/nocodb/src/plugins/storage/Local.ts +++ b/packages/nocodb/src/plugins/storage/Local.ts @@ -3,7 +3,7 @@ import path from 'path'; import { promisify } from 'util'; import mkdirp from 'mkdirp'; import axios from 'axios'; -import NcConfigFactory from '../../utils/NcConfigFactory'; +import { getToolDir } from '../../utils/nc-config'; import type { IStorageAdapterV2, XcFile } from 'nc-plugin'; import type { Readable } from 'stream'; @@ -11,7 +11,7 @@ export default class Local implements IStorageAdapterV2 { constructor() {} public async fileCreate(key: string, file: XcFile): Promise { - const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/')); + const destPath = path.join(getToolDir(), ...key.split('/')); try { await mkdirp(path.dirname(destPath)); const data = await promisify(fs.readFile)(file.path); @@ -24,7 +24,7 @@ export default class Local implements IStorageAdapterV2 { } async fileCreateByUrl(key: string, url: string): Promise { - const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/')); + const destPath = path.join(getToolDir(), ...key.split('/')); return new Promise((resolve, reject) => { axios .get(url, { @@ -71,10 +71,7 @@ export default class Local implements IStorageAdapterV2 { stream: Readable, ): Promise { return new Promise((resolve, reject) => { - const destPath = path.join( - NcConfigFactory.getToolDir(), - ...key.split('/'), - ); + const destPath = path.join(getToolDir(), ...key.split('/')); try { mkdirp(path.dirname(destPath)).then(() => { const writableStream = fs.createWriteStream(destPath); @@ -89,12 +86,12 @@ export default class Local implements IStorageAdapterV2 { } public async fileReadByStream(key: string): Promise { - const srcPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/')); + const srcPath = path.join(getToolDir(), ...key.split('/')); return fs.createReadStream(srcPath, { encoding: 'utf8' }); } public async getDirectoryList(key: string): Promise { - const destDir = path.join(NcConfigFactory.getToolDir(), ...key.split('/')); + const destDir = path.join(getToolDir(), ...key.split('/')); return fs.promises.readdir(destDir); } @@ -106,7 +103,7 @@ export default class Local implements IStorageAdapterV2 { public async fileRead(filePath: string): Promise { try { const fileData = await fs.promises.readFile( - path.join(NcConfigFactory.getToolDir(), ...filePath.split('/')), + path.join(getToolDir(), ...filePath.split('/')), ); return fileData; } catch (e) { diff --git a/packages/nocodb/src/services/app-init.service.spec.ts b/packages/nocodb/src/services/app-init.service.spec.ts deleted file mode 100644 index ac84d33dfb..0000000000 --- a/packages/nocodb/src/services/app-init.service.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { AppInitService } from './app-init.service'; -import type { TestingModule } from '@nestjs/testing'; - -describe('AppInitService', () => { - let service: AppInitService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [AppInitService], - }).compile(); - - service = module.get(AppInitService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/packages/nocodb/src/services/app-init.service.ts b/packages/nocodb/src/services/app-init.service.ts deleted file mode 100644 index e1050d480c..0000000000 --- a/packages/nocodb/src/services/app-init.service.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { T } from 'nc-help'; -import NocoCache from '../cache/NocoCache'; -import { Connection } from '../connection/connection'; -import initAdminFromEnv from '../helpers/initAdminFromEnv'; -import NcPluginMgrv2 from '../helpers/NcPluginMgrv2'; -import { MetaService } from '../meta/meta.service'; -import { User } from '../models'; -import Noco from '../Noco'; -import getInstance from '../utils/getInstance'; -import NcConfigFactory from '../utils/NcConfigFactory'; -import NcUpgrader from '../version-upgrader/NcUpgrader'; -import type { IEventEmitter } from '../modules/event-emitter/event-emitter.interface'; -import type { Provider } from '@nestjs/common'; - -export class AppInitService { - private readonly config: any; - - constructor(config) { - this.config = config; - } - - get appConfig(): any { - return this.config; - } -} - -export const appInitServiceProvider: Provider = { - provide: AppInitService, - // initialize app, - // 1. init cache - // 2. init db connection and create if not exist - // 3. init meta and set to Noco - // 4. init jwt - // 5. init plugin manager - // 6. run upgrader - useFactory: async ( - connection: Connection, - metaService: MetaService, - eventEmitter: IEventEmitter, - ) => { - process.env.NC_VERSION = '0107004'; - - await NocoCache.init(); - - await connection.init(); - - await NcConfigFactory.metaDbCreateIfNotExist(connection.config); - - await metaService.init(); - - // todo: remove - // temporary hack - Noco._ncMeta = metaService; - Noco.config = connection.config; - Noco.eventEmitter = eventEmitter; - - // init jwt secret - await Noco.initJwt(); - - // load super admin user from env if env is set - await initAdminFromEnv(metaService); - - // init plugin manager - await NcPluginMgrv2.init(Noco.ncMeta); - await Noco.loadEEState(); - - // run upgrader - await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta }); - - T.init({ - instance: getInstance, - }); - T.emit('evt_app_started', await User.count()); - - // todo: move app config to app-init service - return new AppInitService(connection.config); - }, - inject: [Connection, MetaService, 'IEventEmitter'], -}; diff --git a/packages/nocodb/src/services/auth.service.ts b/packages/nocodb/src/services/auth.service.ts index 3715bf1edc..287cb16391 100644 --- a/packages/nocodb/src/services/auth.service.ts +++ b/packages/nocodb/src/services/auth.service.ts @@ -5,18 +5,13 @@ import * as bcrypt from 'bcryptjs'; import { v4 as uuidv4 } from 'uuid'; import Noco from '../Noco'; -import { Connection } from '../connection/connection'; import { genJwt } from './users/helpers'; import { UsersService } from './users/users.service'; import type { CreateUserDto } from '../controllers/auth.controller'; @Injectable() export class AuthService { - constructor( - private usersService: UsersService, - // private jwtService: JwtService, - private connection: Connection, - ) {} + constructor(private usersService: UsersService) {} async validateUser(email: string, pass: string): Promise { const user = await this.usersService.findOne(email); diff --git a/packages/nocodb/src/services/projects.service.ts b/packages/nocodb/src/services/projects.service.ts index 54294fb5ba..eb93634a4f 100644 --- a/packages/nocodb/src/services/projects.service.ts +++ b/packages/nocodb/src/services/projects.service.ts @@ -11,7 +11,7 @@ import syncMigration from '../helpers/syncMigration'; import { Project, ProjectUser } from '../models'; import Noco from '../Noco'; import extractRolesObj from '../utils/extractRolesObj'; -import NcConfigFactory from '../utils/NcConfigFactory'; +import { getToolDir } from '../utils/nc-config'; import type { ProjectUpdateReqType } from 'nocodb-sdk'; import type { ProjectReqType } from 'nocodb-sdk'; @@ -96,7 +96,7 @@ export class ProjectsService { // if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each project // each file will be named as nc_.db const fs = require('fs'); - const toolDir = NcConfigFactory.getToolDir(); + const toolDir = getToolDir(); const nanoidv2 = customAlphabet( '1234567890abcdefghijklmnopqrstuvwxyz', 14, diff --git a/packages/nocodb/src/services/utils.service.ts b/packages/nocodb/src/services/utils.service.ts index 2624b8e9a1..574141000d 100644 --- a/packages/nocodb/src/services/utils.service.ts +++ b/packages/nocodb/src/services/utils.service.ts @@ -9,7 +9,7 @@ import { Project, User } from '../models'; import Noco from '../Noco'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import { MetaTable } from '../utils/globals'; -import NcConfigFactory from '../utils/NcConfigFactory'; +import { jdbcToXcConfig } from '../utils/nc-config/helpers'; import { packageVersion } from '../utils/packageVersion'; const versionCache = { @@ -186,7 +186,7 @@ export class UtilsService { }) { const { url } = param.body; try { - const connectionConfig = NcConfigFactory.extractXcUrlFromJdbc(url, true); + const connectionConfig = jdbcToXcConfig(url); return connectionConfig; } catch (error) { return NcError.internalServerError( diff --git a/packages/nocodb/src/utils/NcConfigFactory.ts b/packages/nocodb/src/utils/NcConfigFactory.ts deleted file mode 100644 index 5a9222d998..0000000000 --- a/packages/nocodb/src/utils/NcConfigFactory.ts +++ /dev/null @@ -1,755 +0,0 @@ -import fs from 'fs'; -import { URL } from 'url'; -import { promisify } from 'util'; -import * as path from 'path'; -import parseDbUrl from 'parse-database-url'; -import { SqlClientFactory } from '../db/sql-client/lib/SqlClientFactory'; -// import SqlClientFactory from '../db/sql-client/lib/SqlClientFactory'; -// import type { -// AuthConfig, -// DbConfig, -// MailerConfig, -// NcConfig, -// } from '../../interface/config'; - -// const { -// uniqueNamesGenerator, -// starWars, -// adjectives, -// animals, -// } = require('unique-names-generator'); - -type NcConfig = any; -type DbConfig = any; - -const driverClientMapping = { - mysql: 'mysql2', - mariadb: 'mysql2', - postgres: 'pg', - postgresql: 'pg', - sqlite: 'sqlite3', - mssql: 'mssql', -}; - -const defaultClientPortMapping = { - mysql: 3306, - mysql2: 3306, - postgres: 5432, - pg: 5432, - mssql: 1433, -}; - -const defaultConnectionConfig: any = { - // https://github.com/knex/knex/issues/97 - // timezone: process.env.NC_TIMEZONE || 'UTC', - dateStrings: true, -}; - -// default knex options -const defaultConnectionOptions = { - pool: { - min: 0, - max: 10, - }, -}; - -const knownQueryParams = [ - { - parameter: 'database', - aliases: ['d', 'db'], - }, - { - parameter: 'password', - aliases: ['p'], - }, - { - parameter: 'user', - aliases: ['u'], - }, - { - parameter: 'title', - aliases: ['t'], - }, - { - parameter: 'keyFilePath', - aliases: [], - }, - { - parameter: 'certFilePath', - aliases: [], - }, - { - parameter: 'caFilePath', - aliases: [], - }, - { - parameter: 'ssl', - aliases: [], - }, - { - parameter: 'options', - aliases: ['opt', 'opts'], - }, -]; - -export default class NcConfigFactory { - public static async make(): Promise { - await this.jdbcToXcUrl(); - - const ncConfig = new NcConfigFactory(); - - ncConfig.auth = { - jwt: { - secret: process.env.NC_AUTH_JWT_SECRET, - }, - }; - - ncConfig.port = +(process?.env?.PORT ?? 8080); - ncConfig.env = '_noco'; // process.env?.NODE_ENV || 'dev'; - ncConfig.workingEnv = '_noco'; // process.env?.NODE_ENV || 'dev'; - // ncConfig.toolDir = this.getToolDir(); - ncConfig.projectType = - ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest'; - - if (ncConfig.meta?.db?.connection?.filename) { - ncConfig.meta.db.connection.filename = path.join( - this.getToolDir(), - ncConfig.meta.db.connection.filename, - ); - } - - if (process.env.NC_DB) { - ncConfig.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB); - } else if (process.env.NC_DB_JSON) { - ncConfig.meta.db = JSON.parse(process.env.NC_DB_JSON); - } else if (process.env.NC_DB_JSON_FILE) { - const filePath = process.env.NC_DB_JSON_FILE; - - if (!(await promisify(fs.exists)(filePath))) { - throw new Error(`NC_DB_JSON_FILE not found: ${filePath}`); - } - - const fileContent = await promisify(fs.readFile)(filePath, { - encoding: 'utf8', - }); - ncConfig.meta.db = JSON.parse(fileContent); - } - - if (process.env.NC_TRY) { - ncConfig.try = true; - ncConfig.meta.db = { - client: 'sqlite3', - connection: ':memory:', - pool: { - min: 1, - max: 1, - // disposeTimeout: 360000*1000, - idleTimeoutMillis: 360000 * 1000, - }, - } as any; - } - - if (process.env.NC_PUBLIC_URL) { - ncConfig.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL; - // ncConfig.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL; - ncConfig.publicUrl = process.env.NC_PUBLIC_URL; - } - - if (process.env.NC_DASHBOARD_URL) { - ncConfig.dashboardPath = process.env.NC_DASHBOARD_URL; - } - - return ncConfig; - } - - public static getToolDir() { - return process.env.NC_TOOL_DIR || process.cwd(); - } - - public static hasDbUrl(): boolean { - return Object.keys(process.env).some((envKey) => - envKey.startsWith('NC_DB_URL'), - ); - } - - public static makeFromUrls(urls: string[]): NcConfig { - const config = new NcConfigFactory(); - - // config.envs[process.env.NODE_ENV || 'dev'].db = []; - config.envs['_noco'].db = []; - for (const [i, url] of Object.entries(urls)) { - // config.envs[process.env.NODE_ENV || 'dev'].db.push(this.urlToDbConfig(url, i)); - config.envs['_noco'].db.push(this.urlToDbConfig(url, i)); - } - - return config; - } - - public static urlToDbConfig( - urlString: string, - key = '', - config?: NcConfigFactory, - type?: string, - ): DbConfig { - const url = new URL(urlString); - - let dbConfig: DbConfig; - - if (url.protocol.startsWith('sqlite3')) { - dbConfig = { - client: 'sqlite3', - connection: { - client: 'sqlite3', - connection: { - filename: - url.searchParams.get('d') || url.searchParams.get('database'), - }, - database: - url.searchParams.get('d') || url.searchParams.get('database'), - }, - } as any; - } else { - const parsedQuery = {}; - for (const [key, value] of url.searchParams.entries()) { - const fnd = knownQueryParams.find( - (param) => param.parameter === key || param.aliases.includes(key), - ); - if (fnd) { - parsedQuery[fnd.parameter] = value; - } else { - parsedQuery[key] = value; - } - } - - dbConfig = { - client: url.protocol.replace(':', ''), - connection: { - ...defaultConnectionConfig, - ...parsedQuery, - host: url.hostname, - port: +url.port, - }, - // pool: { - // min: 1, - // max: 1 - // }, - acquireConnectionTimeout: 600000, - } as any; - - if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { - dbConfig.connection.ssl = true; - } - - if ( - url.searchParams.get('keyFilePath') && - url.searchParams.get('certFilePath') && - url.searchParams.get('caFilePath') - ) { - dbConfig.connection.ssl = { - keyFilePath: url.searchParams.get('keyFilePath'), - certFilePath: url.searchParams.get('certFilePath'), - caFilePath: url.searchParams.get('caFilePath'), - }; - } - } - - if (config && !config.title) { - config.title = - url.searchParams.get('t') || - url.searchParams.get('title') || - this.generateRandomTitle(); - } - - Object.assign(dbConfig, { - meta: { - tn: 'nc_evolutions', - allSchemas: - !!url.searchParams.get('allSchemas') || - !(url.searchParams.get('d') || url.searchParams.get('database')), - api: { - prefix: url.searchParams.get('apiPrefix') || '', - swagger: true, - type: - type || - ((url.searchParams.get('api') || - url.searchParams.get('a')) as any) || - 'rest', - }, - dbAlias: url.searchParams.get('dbAlias') || `db${key}`, - metaTables: 'db', - migrations: { - disabled: false, - name: 'nc_evolutions', - }, - }, - }); - - return dbConfig; - } - - private static generateRandomTitle(): string { - return ''; /*uniqueNamesGenerator({ - dictionaries: [[starWars], [adjectives, animals]][ - Math.floor(Math.random() * 2) - ], - }) - .toLowerCase() - .replace(/[ -]/g, '_');*/ - } - - static async metaUrlToDbConfig(urlString) { - const url = new URL(urlString); - - let dbConfig; - - if (url.protocol.startsWith('sqlite3')) { - const db = url.searchParams.get('d') || url.searchParams.get('database'); - dbConfig = { - client: 'sqlite3', - connection: { - filename: db, - }, - ...(db === ':memory:' - ? { - pool: { - min: 1, - max: 1, - // disposeTimeout: 360000*1000, - idleTimeoutMillis: 360000 * 1000, - }, - } - : {}), - }; - } else { - const parsedQuery = {}; - for (const [key, value] of url.searchParams.entries()) { - const fnd = knownQueryParams.find( - (param) => param.parameter === key || param.aliases.includes(key), - ); - if (fnd) { - parsedQuery[fnd.parameter] = value; - } else { - parsedQuery[key] = value; - } - } - - dbConfig = { - client: url.protocol.replace(':', ''), - connection: { - ...defaultConnectionConfig, - ...parsedQuery, - host: url.hostname, - port: +url.port, - }, - acquireConnectionTimeout: 600000, - ...(url.searchParams.has('search_path') - ? { - searchPath: url.searchParams.get('search_path').split(','), - } - : {}), - }; - if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { - dbConfig.connection.ssl = true; - } - } - url.searchParams.forEach((_value, key) => { - let value: any = _value; - if (value === 'true') { - value = true; - } else if (value === 'false') { - value = false; - } else if (/^\d+$/.test(value)) { - value = +value; - } - // todo: implement config read from JSON file or JSON env val read - if ( - ![ - 'password', - 'p', - 'database', - 'd', - 'user', - 'u', - 'search_path', - ].includes(key) - ) { - key.split('.').reduce((obj, k, i, arr) => { - return (obj[k] = i === arr.length - 1 ? value : obj[k] || {}); - }, dbConfig); - } - }); - - if ( - dbConfig?.connection?.ssl && - typeof dbConfig?.connection?.ssl === 'object' - ) { - if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) { - dbConfig.connection.ssl.ca = ( - await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath) - ).toString(); - delete dbConfig.connection.ssl.caFilePath; - } - if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) { - dbConfig.connection.ssl.key = ( - await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath) - ).toString(); - delete dbConfig.connection.ssl.keyFilePath; - } - if ( - dbConfig.connection.ssl.certFilePath && - !dbConfig.connection.ssl.cert - ) { - dbConfig.connection.ssl.cert = ( - await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath) - ).toString(); - delete dbConfig.connection.ssl.certFilePath; - } - } - - return dbConfig; - } - - public static async makeProjectConfigFromUrl( - url, - type?: string, - ): Promise { - const config = new NcConfigFactory(); - const dbConfig = this.urlToDbConfig(url, '', config, type); - // config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig); - config.envs['_noco'].db.push(dbConfig); - - if (process.env.NC_AUTH_ADMIN_SECRET) { - config.auth = { - masterKey: { - secret: process.env.NC_AUTH_ADMIN_SECRET, - }, - }; - } else if (process.env.NC_NO_AUTH) { - config.auth = { - disabled: true, - }; - // } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) { - } else if (config?.envs?.['_noco']?.db?.[0]) { - config.auth = { - jwt: { - // dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias, - dbAlias: - process.env.NC_AUTH_JWT_DB_ALIAS || - config.envs['_noco'].db[0].meta.dbAlias, - secret: process.env.NC_AUTH_JWT_SECRET, - }, - }; - } - - if (process.env.NC_DB) { - config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB); - } - - if (process.env.NC_TRY) { - config.try = true; - config.meta.db = { - client: 'sqlite3', - connection: ':memory:', - pool: { - min: 1, - max: 1, - // disposeTimeout: 360000*1000, - idleTimeoutMillis: 360000 * 1000, - }, - } as any; - } - - if (process.env.NC_MAILER) { - config.mailer = { - from: process.env.NC_MAILER_FROM, - options: { - host: process.env.NC_MAILER_HOST, - port: parseInt(process.env.NC_MAILER_PORT, 10), - secure: process.env.NC_MAILER_SECURE === 'true', - auth: { - user: process.env.NC_MAILER_USER, - pass: process.env.NC_MAILER_PASS, - }, - }, - }; - } - - if (process.env.NC_PUBLIC_URL) { - // config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL; - config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL; - config.publicUrl = process.env.NC_PUBLIC_URL; - } - - config.port = +(process?.env?.PORT ?? 8080); - // config.env = process.env?.NODE_ENV || 'dev'; - // config.workingEnv = process.env?.NODE_ENV || 'dev'; - config.env = '_noco'; - config.workingEnv = '_noco'; - config.toolDir = this.getToolDir(); - config.projectType = - type || - config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type || - 'rest'; - - return config; - } - - public static async makeProjectConfigFromConnection( - dbConnectionConfig: any, - type?: string, - ): Promise { - const config = new NcConfigFactory(); - let dbConfig = dbConnectionConfig; - - if (dbConfig.client === 'sqlite3') { - dbConfig = { - client: 'sqlite3', - connection: { - ...dbConnectionConfig, - database: dbConnectionConfig.connection.filename, - }, - }; - } - - // todo: - const key = ''; - Object.assign(dbConfig, { - meta: { - tn: 'nc_evolutions', - api: { - prefix: '', - swagger: true, - type: type || 'rest', - }, - dbAlias: `db${key}`, - metaTables: 'db', - migrations: { - disabled: false, - name: 'nc_evolutions', - }, - }, - }); - - // config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig); - config.envs['_noco'].db.push(dbConfig); - - if (process.env.NC_AUTH_ADMIN_SECRET) { - config.auth = { - masterKey: { - secret: process.env.NC_AUTH_ADMIN_SECRET, - }, - }; - } else if (process.env.NC_NO_AUTH) { - config.auth = { - disabled: true, - }; - // } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) { - } else if (config?.envs?.['_noco']?.db?.[0]) { - config.auth = { - jwt: { - // dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias, - dbAlias: - process.env.NC_AUTH_JWT_DB_ALIAS || - config.envs['_noco'].db[0].meta.dbAlias, - secret: process.env.NC_AUTH_JWT_SECRET, - }, - }; - } - - if (process.env.NC_DB) { - config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB); - } - - if (process.env.NC_TRY) { - config.try = true; - config.meta.db = { - client: 'sqlite3', - connection: ':memory:', - pool: { - min: 1, - max: 1, - // disposeTimeout: 360000*1000, - idleTimeoutMillis: 360000 * 1000, - }, - } as any; - } - - if (process.env.NC_PUBLIC_URL) { - // config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL; - config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL; - config.publicUrl = process.env.NC_PUBLIC_URL; - } - - config.port = +(process?.env?.PORT ?? 8080); - // config.env = process.env?.NODE_ENV || 'dev'; - // config.workingEnv = process.env?.NODE_ENV || 'dev'; - config.env = '_noco'; - config.workingEnv = '_noco'; - config.toolDir = process.env.NC_TOOL_DIR || process.cwd(); - config.projectType = - type || - config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type || - 'rest'; - - return config; - } - - public static async metaDbCreateIfNotExist(args: NcConfig) { - if (args.meta?.db?.client === 'sqlite3') { - const metaSqlClient = await SqlClientFactory.create({ - ...args.meta.db, - connection: args.meta.db, - }); - await metaSqlClient.createDatabaseIfNotExists({ - database: args.meta.db?.connection?.filename, - }); - } else { - const metaSqlClient = await SqlClientFactory.create(args.meta.db); - await metaSqlClient.createDatabaseIfNotExists(args.meta.db?.connection); - await metaSqlClient.knex.destroy(); - } - - /* const dbPath = path.join(args.toolDir, 'xc.db') - const exists = fs.existsSync(dbPath); - if (!exists) { - const fd = fs.openSync(dbPath, "w"); - fs.closeSync(fd); - } - */ - } - - public version = '0.6'; - public port: number; - public auth?: any; - public env: 'production' | 'dev' | 'test' | string; - public workingEnv: string; - public toolDir: string; - public envs: { - [p: string]: { db: DbConfig[]; api?: any; publicUrl?: string }; - }; - // public projectType: "rest" | "graphql" | "grpc"; - public queriesFolder: string | string[] = ''; - public seedsFolder: string | string[]; - public title: string; - public publicUrl: string; - public projectType; - public meta = { - db: { - client: 'sqlite3', - connection: { - filename: 'noco.db', - }, - }, - }; - public mailer: any; - public try = false; - - public dashboardPath = '/dashboard'; - - constructor() { - this.envs = { _noco: { db: [] } }; - } - - public static async jdbcToXcUrl() { - if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) { - const database_url = await promisify(fs.readFile)( - process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE, - 'utf-8', - ); - process.env.NC_DB = this.extractXcUrlFromJdbc(database_url); - } else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) { - process.env.NC_DB = this.extractXcUrlFromJdbc( - process.env.NC_DATABASE_URL || process.env.DATABASE_URL, - ); - } - } - - public static extractXcUrlFromJdbc(url: string, rtConfig = false) { - // drop the jdbc prefix - if (url.startsWith('jdbc:')) { - url = url.substring(5); - } - - const config = parseDbUrl(url); - - const parsedConfig: { - driver?: string; - host?: string; - port?: string; - database?: string; - user?: string; - password?: string; - ssl?: string; - } = {}; - for (const [key, value] of Object.entries(config)) { - const fnd = knownQueryParams.find( - (param) => param.parameter === key || param.aliases.includes(key), - ); - if (fnd) { - parsedConfig[fnd.parameter] = value; - } else { - parsedConfig[key] = value; - } - } - - if (!parsedConfig?.port) - parsedConfig.port = - defaultClientPortMapping[ - driverClientMapping[parsedConfig.driver] || parsedConfig.driver - ]; - - if (rtConfig) { - const { driver, ...connectionConfig } = parsedConfig; - - const client = driverClientMapping[driver] || driver; - - const avoidSSL = [ - 'localhost', - '127.0.0.1', - 'host.docker.internal', - '172.17.0.1', - ]; - - if ( - client === 'pg' && - !connectionConfig?.ssl && - !avoidSSL.includes(connectionConfig.host) - ) { - connectionConfig.ssl = 'true'; - } - - return { - client: client, - connection: { - ...connectionConfig, - }, - } as any; - } - - const { driver, host, port, database, user, password, ...extra } = - parsedConfig; - - const extraParams = []; - - for (const [key, value] of Object.entries(extra)) { - extraParams.push(`${key}=${value}`); - } - - const res = `${driverClientMapping[driver] || driver}://${host}${ - port ? `:${port}` : '' - }?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${ - database ? `d=${database}&` : '' - }${extraParams.join('&')}`; - - return res; - } - - // public static initOneClickDeployment() { - // if (process.env.NC_ONE_CLICK) { - // const url = NcConfigFactory.extractXcUrlFromJdbc(process.env.DATABASE_URL); - // process.env.NC_DB = url; - // } - // } -} - -export { defaultConnectionConfig, defaultConnectionOptions }; diff --git a/packages/nocodb/src/utils/common/NcConnectionMgr.ts b/packages/nocodb/src/utils/common/NcConnectionMgr.ts index 22f1101ffe..fe8baba635 100644 --- a/packages/nocodb/src/utils/common/NcConnectionMgr.ts +++ b/packages/nocodb/src/utils/common/NcConnectionMgr.ts @@ -2,7 +2,7 @@ import fs from 'fs'; import { promisify } from 'util'; import SqlClientFactory from '../../db/sql-client/lib/SqlClientFactory'; import { XKnex } from '../../db/CustomKnex'; -import { defaultConnectionConfig } from '../NcConfigFactory'; +import { defaultConnectionConfig } from '../nc-config'; // import type { NcConfig } from '../../../interface/config'; import type { Knex } from 'knex'; // import type NcMetaIO from '../../meta/NcMetaIO'; diff --git a/packages/nocodb/src/utils/common/NcConnectionMgrv2.ts b/packages/nocodb/src/utils/common/NcConnectionMgrv2.ts index 7abd807587..592d457a41 100644 --- a/packages/nocodb/src/utils/common/NcConnectionMgrv2.ts +++ b/packages/nocodb/src/utils/common/NcConnectionMgrv2.ts @@ -3,7 +3,7 @@ import { XKnex } from '../../db/CustomKnex'; import { defaultConnectionConfig, defaultConnectionOptions, -} from '../NcConfigFactory'; +} from '../nc-config'; import Noco from '../../Noco'; import type Base from '../../models/Base'; diff --git a/packages/nocodb/src/utils/nc-config/NcConfig.ts b/packages/nocodb/src/utils/nc-config/NcConfig.ts new file mode 100644 index 0000000000..b8d3654551 --- /dev/null +++ b/packages/nocodb/src/utils/nc-config/NcConfig.ts @@ -0,0 +1,182 @@ +import * as path from 'path'; +import fs from 'fs'; +import { promisify } from 'util'; +import { SqlClientFactory } from '../../db/sql-client/lib/SqlClientFactory'; +import { getToolDir, metaUrlToDbConfig } from './helpers'; +import { DriverClient } from './interfaces'; +import type { DbConfig } from './interfaces'; + +export class NcConfig { + version: string; + meta: { + db: DbConfig; + } = { + db: { + client: DriverClient.SQLITE, + connection: { + filename: 'noco.db', + }, + }, + }; + auth: { + jwt: { + secret: string; + options?: any; + }; + }; + + // if this is true, port is not exposed + worker: boolean; + + toolDir: string; + + // exposed instance port + port: number; + + // if this is true, use sqlite3 :memory: as meta db + try: boolean; + + // optional + publicUrl?: string; + dashboardPath?: string; + + // TODO what is this? + envs: any; + + queriesFolder: string; + env: string; + workingEnv: string; + projectType: string; + + private constructor() {} + + public static async create(param: { + meta: { + metaUrl?: string; + metaJson?: string; + metaJsonFile?: string; + }; + secret?: string; + port?: string | number; + tryMode?: boolean; + worker?: boolean; + dashboardPath?: string; + publicUrl?: string; + }): Promise { + const { meta, secret, port, worker, tryMode, publicUrl, dashboardPath } = + param; + + const ncConfig = new NcConfig(); + + ncConfig.auth = { + jwt: { + secret: secret, + }, + }; + + ncConfig.port = +(port ?? 8080); + ncConfig.toolDir = getToolDir(); + ncConfig.worker = worker ?? false; + + ncConfig.env = '_noco'; + ncConfig.workingEnv = '_noco'; + + ncConfig.projectType = + ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest'; + + if (ncConfig.meta?.db?.connection?.filename) { + ncConfig.meta.db.connection.filename = path.join( + ncConfig.toolDir, + ncConfig.meta.db.connection.filename, + ); + } + + if (tryMode) { + ncConfig.try = true; + ncConfig.meta.db = { + client: DriverClient.SQLITE, + connection: ':memory:' as any, + pool: { + min: 1, + max: 1, + // disposeTimeout: 360000*1000, + idleTimeoutMillis: 360000 * 1000, + }, + }; + } else { + if (meta?.metaUrl) { + ncConfig.meta.db = await metaUrlToDbConfig(meta.metaUrl); + } else if (meta?.metaJson) { + ncConfig.meta.db = JSON.parse(meta.metaJson); + } else if (meta?.metaJsonFile) { + if (!(await promisify(fs.exists)(meta.metaJsonFile))) { + throw new Error(`NC_DB_JSON_FILE not found: ${meta.metaJsonFile}`); + } + const fileContent = await promisify(fs.readFile)(meta.metaJsonFile, { + encoding: 'utf8', + }); + ncConfig.meta.db = JSON.parse(fileContent); + } + } + + if (publicUrl) { + ncConfig.envs['_noco'].publicUrl = publicUrl; + ncConfig.publicUrl = publicUrl; + } + + if (dashboardPath) { + ncConfig.dashboardPath = dashboardPath; + } + + try { + // make sure meta db exists + await ncConfig.metaDbCreateIfNotExist(); + } catch (e) { + throw new Error(e); + } + + return ncConfig; + } + + public static async createByEnv(): Promise { + return NcConfig.create({ + meta: { + metaUrl: process.env.NC_DB, + metaJson: process.env.NC_DB_JSON, + metaJsonFile: process.env.NC_DB_JSON_FILE, + }, + secret: process.env.NC_AUTH_JWT_SECRET, + port: process.env.NC_PORT, + tryMode: !!process.env.NC_TRY, + worker: !!process.env.NC_WORKER, + dashboardPath: process.env.NC_DASHBOARD_PATH, + publicUrl: process.env.NC_PUBLIC_URL, + }); + } + + private async metaDbCreateIfNotExist() { + if (this.meta?.db?.client === 'sqlite3') { + const metaSqlClient = await SqlClientFactory.create({ + ...this.meta.db, + connection: this.meta.db, + }); + if (this.meta.db?.connection?.filename) { + await metaSqlClient.createDatabaseIfNotExists({ + database: this.meta.db?.connection?.filename, + }); + } else { + throw new Error('Configuration missing meta db connection'); + } + } else { + const metaSqlClient = await SqlClientFactory.create(this.meta.db); + if (this.meta.db?.connection?.database) { + await metaSqlClient.createDatabaseIfNotExists( + (this.meta.db as any).connection, + ); + await metaSqlClient.knex.destroy(); + } else { + throw new Error('Configuration missing meta db connection'); + } + } + } +} diff --git a/packages/nocodb/src/utils/nc-config/constants.ts b/packages/nocodb/src/utils/nc-config/constants.ts new file mode 100644 index 0000000000..e77a68748d --- /dev/null +++ b/packages/nocodb/src/utils/nc-config/constants.ts @@ -0,0 +1,84 @@ +export const driverClientMapping = { + mysql: 'mysql2', + mariadb: 'mysql2', + postgres: 'pg', + postgresql: 'pg', + sqlite: 'sqlite3', + mssql: 'mssql', +}; + +export const defaultClientPortMapping = { + mysql: 3306, + mysql2: 3306, + postgres: 5432, + pg: 5432, + mssql: 1433, +}; + +export const defaultConnectionConfig: any = { + // https://github.com/knex/knex/issues/97 + // timezone: process.env.NC_TIMEZONE || 'UTC', + dateStrings: true, +}; + +// default knex options +export const defaultConnectionOptions = { + pool: { + min: 0, + max: 10, + }, +}; + +export const avoidSSL = [ + 'localhost', + '127.0.0.1', + 'host.docker.internal', + '172.17.0.1', +]; + +export const knownQueryParams = [ + { + parameter: 'database', + aliases: ['d', 'db'], + }, + { + parameter: 'password', + aliases: ['p'], + }, + { + parameter: 'user', + aliases: ['u'], + }, + { + parameter: 'title', + aliases: ['t'], + }, + { + parameter: 'keyFilePath', + aliases: [], + }, + { + parameter: 'certFilePath', + aliases: [], + }, + { + parameter: 'caFilePath', + aliases: [], + }, + { + parameter: 'ssl', + aliases: [], + }, + { + parameter: 'options', + aliases: ['opt', 'opts'], + }, +]; + +export enum DriverClient { + MYSQL = 'mysql2', + MSSQL = 'mssql', + PG = 'pg', + SQLITE = 'sqlite3', + SNOWFLAKE = 'snowflake', +} diff --git a/packages/nocodb/src/utils/nc-config/helpers.ts b/packages/nocodb/src/utils/nc-config/helpers.ts new file mode 100644 index 0000000000..a8bf1c84cf --- /dev/null +++ b/packages/nocodb/src/utils/nc-config/helpers.ts @@ -0,0 +1,324 @@ +import fs from 'fs'; +import { URL } from 'url'; +import { promisify } from 'util'; +import parseDbUrl from 'parse-database-url'; +import { + avoidSSL, + defaultClientPortMapping, + defaultConnectionConfig, + defaultConnectionOptions, + driverClientMapping, + knownQueryParams, +} from './constants'; +import { DriverClient } from './interfaces'; +import type { Connection, DbConfig } from './interfaces'; + +export async function prepareEnv() { + if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) { + const database_url = await promisify(fs.readFile)( + process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE, + 'utf-8', + ); + process.env.NC_DB = jdbcToXcUrl(database_url); + } else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) { + process.env.NC_DB = jdbcToXcUrl( + process.env.NC_DATABASE_URL || process.env.DATABASE_URL, + ); + } +} + +export function getToolDir() { + return process.env.NC_TOOL_DIR || process.cwd(); +} + +export function jdbcToXcConfig(url: string): DbConfig { + // drop the jdbc prefix + url.replace(/^jdbc:/, ''); + + const config = parseDbUrl(url); + + const parsedConfig: Connection = {}; + + for (const [key, value] of Object.entries(config)) { + const fnd = knownQueryParams.find( + (param) => param.parameter === key || param.aliases.includes(key), + ); + if (fnd) { + parsedConfig[fnd.parameter] = value; + } else { + parsedConfig[key] = value; + } + } + + if (!parsedConfig?.port) { + parsedConfig.port = + defaultClientPortMapping[ + driverClientMapping[parsedConfig.driver] || parsedConfig.driver + ]; + } + + const { driver, ...connectionConfig } = parsedConfig; + + const client = driverClientMapping[driver] || driver; + + if ( + client === 'pg' && + !connectionConfig?.ssl && + !avoidSSL.includes(connectionConfig.host) + ) { + connectionConfig.ssl = true; + } + + return { + client: client, + connection: { + ...connectionConfig, + }, + } as DbConfig; +} + +export function jdbcToXcUrl(url: string): string { + // drop the jdbc prefix + url.replace(/^jdbc:/, ''); + + const config = parseDbUrl(url); + + const parsedConfig: Connection = {}; + + for (const [key, value] of Object.entries(config)) { + const fnd = knownQueryParams.find( + (param) => param.parameter === key || param.aliases.includes(key), + ); + if (fnd) { + parsedConfig[fnd.parameter] = value; + } else { + parsedConfig[key] = value; + } + } + + if (!parsedConfig?.port) { + parsedConfig.port = + defaultClientPortMapping[ + driverClientMapping[parsedConfig.driver] || parsedConfig.driver + ]; + } + + const { driver, host, port, database, user, password, ...extra } = + parsedConfig; + + const extraParams = []; + + for (const [key, value] of Object.entries(extra)) { + extraParams.push(`${key}=${value}`); + } + + const res = `${driverClientMapping[driver] || driver}://${host}${ + port ? `:${port}` : '' + }?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${ + database ? `d=${database}&` : '' + }${extraParams.join('&')}`; + + return res; +} + +export function xcUrlToDbConfig( + urlString: string, + key = '', + type?: string, +): DbConfig { + const url = new URL(urlString); + + let dbConfig: DbConfig; + + if (url.protocol.startsWith('sqlite3')) { + dbConfig = { + client: 'sqlite3', + connection: { + client: 'sqlite3', + connection: { + filename: + url.searchParams.get('d') || url.searchParams.get('database'), + }, + database: url.searchParams.get('d') || url.searchParams.get('database'), + }, + } as any; + } else { + const parsedQuery = {}; + for (const [key, value] of url.searchParams.entries()) { + const fnd = knownQueryParams.find( + (param) => param.parameter === key || param.aliases.includes(key), + ); + if (fnd) { + parsedQuery[fnd.parameter] = value; + } else { + parsedQuery[key] = value; + } + } + + dbConfig = { + client: url.protocol.replace(':', '') as DriverClient, + connection: { + ...parsedQuery, + host: url.hostname, + port: +url.port, + }, + acquireConnectionTimeout: 600000, + }; + + if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { + dbConfig.connection.ssl = true; + } + + if ( + url.searchParams.get('keyFilePath') && + url.searchParams.get('certFilePath') && + url.searchParams.get('caFilePath') + ) { + dbConfig.connection.ssl = { + keyFilePath: url.searchParams.get('keyFilePath'), + certFilePath: url.searchParams.get('certFilePath'), + caFilePath: url.searchParams.get('caFilePath'), + }; + } + } + + /* TODO check if this is needed + if (config && !config.title) { + config.title = + url.searchParams.get('t') || + url.searchParams.get('title') || + this.generateRandomTitle(); + } + */ + + Object.assign(dbConfig, { + meta: { + tn: 'nc_evolutions', + allSchemas: + !!url.searchParams.get('allSchemas') || + !(url.searchParams.get('d') || url.searchParams.get('database')), + api: { + prefix: url.searchParams.get('apiPrefix') || '', + swagger: true, + type: + type || + ((url.searchParams.get('api') || url.searchParams.get('a')) as any) || + 'rest', + }, + dbAlias: url.searchParams.get('dbAlias') || `db${key}`, + metaTables: 'db', + migrations: { + disabled: false, + name: 'nc_evolutions', + }, + }, + }); + + return dbConfig; +} + +export async function metaUrlToDbConfig(urlString): Promise { + const url = new URL(urlString); + + let dbConfig: DbConfig; + + if (url.protocol.startsWith('sqlite3')) { + const db = url.searchParams.get('d') || url.searchParams.get('database'); + dbConfig = { + client: DriverClient.SQLITE, + connection: { + filename: db, + }, + ...(db === ':memory:' + ? { + pool: { + min: 1, + max: 1, + // disposeTimeout: 360000*1000, + idleTimeoutMillis: 360000 * 1000, + }, + } + : {}), + }; + } else { + const parsedQuery = {}; + for (const [key, value] of url.searchParams.entries()) { + const fnd = knownQueryParams.find( + (param) => param.parameter === key || param.aliases.includes(key), + ); + if (fnd) { + parsedQuery[fnd.parameter] = value; + } else { + parsedQuery[key] = value; + } + } + + dbConfig = { + client: url.protocol.replace(':', '') as DriverClient, + connection: { + ...defaultConnectionConfig, + ...parsedQuery, + host: url.hostname, + port: +url.port, + }, + acquireConnectionTimeout: 600000, + ...defaultConnectionOptions, + ...(url.searchParams.has('search_path') + ? { + searchPath: url.searchParams.get('search_path').split(','), + } + : {}), + }; + + if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { + dbConfig.connection.ssl = true; + } + } + + url.searchParams.forEach((_value, key) => { + let value: any = _value; + if (value === 'true') { + value = true; + } else if (value === 'false') { + value = false; + } else if (/^\d+$/.test(value)) { + value = +value; + } + // todo: implement config read from JSON file or JSON env val read + if ( + !['password', 'p', 'database', 'd', 'user', 'u', 'search_path'].includes( + key, + ) + ) { + key.split('.').reduce((obj, k, i, arr) => { + return (obj[k] = i === arr.length - 1 ? value : obj[k] || {}); + }, dbConfig); + } + }); + + if ( + dbConfig?.connection?.ssl && + typeof dbConfig?.connection?.ssl === 'object' + ) { + if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) { + dbConfig.connection.ssl.ca = ( + await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath) + ).toString(); + delete dbConfig.connection.ssl.caFilePath; + } + if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) { + dbConfig.connection.ssl.key = ( + await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath) + ).toString(); + delete dbConfig.connection.ssl.keyFilePath; + } + if (dbConfig.connection.ssl.certFilePath && !dbConfig.connection.ssl.cert) { + dbConfig.connection.ssl.cert = ( + await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath) + ).toString(); + delete dbConfig.connection.ssl.certFilePath; + } + } + + return dbConfig; +} diff --git a/packages/nocodb/src/utils/nc-config/index.ts b/packages/nocodb/src/utils/nc-config/index.ts new file mode 100644 index 0000000000..e70aeceb43 --- /dev/null +++ b/packages/nocodb/src/utils/nc-config/index.ts @@ -0,0 +1,4 @@ +export * from './helpers'; +export * from './interfaces'; +export * from './constants'; +export * from './NcConfig'; diff --git a/packages/nocodb/src/utils/nc-config/interfaces.ts b/packages/nocodb/src/utils/nc-config/interfaces.ts new file mode 100644 index 0000000000..afd296d045 --- /dev/null +++ b/packages/nocodb/src/utils/nc-config/interfaces.ts @@ -0,0 +1,39 @@ +import { DriverClient } from './constants'; + +interface Connection { + driver?: DriverClient; + host?: string; + port?: number; + database?: string; + user?: string; + password?: string; + ssl?: + | boolean + | { + ca?: string; + cert?: string; + key?: string; + caFilePath?: string; + certFilePath?: string; + keyFilePath?: string; + }; + filename?: string; +} + +interface DbConfig { + client: DriverClient; + connection: Connection; + acquireConnectionTimeout?: number; + useNullAsDefault?: boolean; + pool?: { + min?: number; + max?: number; + idleTimeoutMillis?: number; + }; + migrations?: { + directory?: string; + tableName?: string; + }; +} + +export { DriverClient, Connection, DbConfig }; diff --git a/packages/nocodb/tests/unit/TestDbMngr.ts b/packages/nocodb/tests/unit/TestDbMngr.ts index bb11933c0c..aa4f2dd870 100644 --- a/packages/nocodb/tests/unit/TestDbMngr.ts +++ b/packages/nocodb/tests/unit/TestDbMngr.ts @@ -2,9 +2,9 @@ import fs from 'fs'; import process from 'process'; import { knex } from 'knex'; import SqlMgrv2 from '../../src/db/sql-mgr/v2/SqlMgrv2'; +import { jdbcToXcUrl, xcUrlToDbConfig } from '../../src/utils/nc-config'; import type { Knex } from 'knex'; import type { DbConfig } from '../../src/interface/config'; -import NcConfigFactory from '../../src/utils/NcConfigFactory' export default class TestDbMngr { public static readonly dbName = 'test_meta'; @@ -75,7 +75,7 @@ export default class TestDbMngr { private static async isDbConfigured() { const { user, password, host, port, client } = TestDbMngr.connection; - const config = NcConfigFactory.urlToDbConfig( + const config = xcUrlToDbConfig( `${client}://${user}:${password}@${host}:${port}`, ); config.connection = { @@ -84,7 +84,7 @@ export default class TestDbMngr { host, port, }; - const result = await TestDbMngr.testConnection(config); + const result = await TestDbMngr.testConnection(config as any); return result.code !== -1; } static async connectDb() { @@ -95,9 +95,9 @@ export default class TestDbMngr { ] = `${client}://${user}:${password}@${host}:${port}/${TestDbMngr.dbName}`; } - TestDbMngr.dbConfig = NcConfigFactory.urlToDbConfig( - NcConfigFactory.extractXcUrlFromJdbc(process.env[`DATABASE_URL`]), - ); + TestDbMngr.dbConfig = xcUrlToDbConfig( + jdbcToXcUrl(process.env[`DATABASE_URL`]), + ) as any; this.dbConfig.meta = { tn: 'nc_evolutions', dbAlias: 'db', diff --git a/tests/playwright/pages/Dashboard/TreeView.ts b/tests/playwright/pages/Dashboard/TreeView.ts index ce17279547..ad0a5ba676 100644 --- a/tests/playwright/pages/Dashboard/TreeView.ts +++ b/tests/playwright/pages/Dashboard/TreeView.ts @@ -216,7 +216,6 @@ export class TreeViewPage extends BasePage { uiAction: () => this.rootPage.getByRole('button', { name: 'Confirm' }).click(), httpMethodsToMatch: ['POST'], requestUrlPathToMatch: `/api/v1/db/meta/duplicate/`, - responseJsonMatcher: json => json.name === 'duplicate-model', }); await this.get().locator(`[data-testid="tree-view-table-${title} copy"]`).waitFor(); }