Merge pull request #5711 from nocodb/feat/worker

feat: worker container

packages/nc-gui/components/dashboard/TreeView.vue

@@ -399,8 +399,8 @@ const duplicateTable = async (table: TableType) => {
   const { close } = useDialog(resolveComponent('DlgTableDuplicate'), {
     'modelValue': isOpen,
     'table': table,
-    'onOk': async (jobData: { name: string; id: string }) => {
+    'onOk': async (jobData: { id: string }) => {
-      $jobs.subscribe({ name: jobData.name, id: jobData.id }, undefined, async (status: string, data?: any) => {
+      $jobs.subscribe({ id: jobData.id }, undefined, async (status: string, data?: any) => {
         if (status === JobStatus.COMPLETED) {
           await loadTables()
           const newTable = tables.value.find((el) => el.id === data?.result?.id)

packages/nc-gui/nuxt-shim.d.ts

@@ -18,7 +18,6 @@ declare module '#app/nuxt' {
         job:
           | {
               id: string
-              name: string
             }
           | any,
         subscribedCb?: () => void,

packages/nc-gui/pages/index/index/index.vue

@@ -90,10 +90,10 @@ const duplicateProject = (project: ProjectType) => {
   const { close } = useDialog(resolveComponent('DlgProjectDuplicate'), {
     'modelValue': isOpen,
     'project': project,
-    'onOk': async (jobData: { name: string; id: string }) => {
+    'onOk': async (jobData: { id: string }) => {
       await loadProjects()
 
-      $jobs.subscribe({ name: jobData.name, id: jobData.id }, undefined, async (status: string) => {
+      $jobs.subscribe({ id: jobData.id }, undefined, async (status: string) => {
         if (status === JobStatus.COMPLETED) {
           await loadProjects()
         } else if (status === JobStatus.FAILED) {

packages/nc-gui/plugins/jobs.ts

@@ -29,22 +29,22 @@ export default defineNuxtPlugin(async (nuxtApp) => {
     await init(nuxtApp.$state.token.value)
   }
 
-  const send = (name: string, data: any) => {
+  const send = (evt: string, data: any) => {
     if (socket) {
       const _id = messageIndex++
-      socket.emit(name, { _id, data })
+      socket.emit(evt, { _id, data })
       return _id
     }
   }
 
   const jobs = {
     subscribe(
-      job: { id: string; name: string } | any,
+      job: { id: string } | any,
       subscribedCb?: () => void,
       statusCb?: (status: JobStatus, data?: any) => void,
       logCb?: (data: { message: string }) => void,
     ) {
-      const logFn = (data: { id: string; name: string; data: { message: string } }) => {
+      const logFn = (data: { id: string; data: { message: string } }) => {
         if (data.id === job.id) {
           if (logCb) logCb(data.data)
         }
@@ -61,11 +61,10 @@ export default defineNuxtPlugin(async (nuxtApp) => {
 
       const _id = send('subscribe', job)
 
-      const subscribeFn = (data: { _id: number; name: string; id: string }) => {
+      const subscribeFn = (data: { _id: number; id: string }) => {
         if (data._id === _id) {
-          if (data.id !== job.id || data.name !== job.name) {
+          if (data.id !== job.id) {
             job.id = data.id
-            job.name = data.name
           }
           if (subscribedCb) subscribedCb()
           socket?.on('log', logFn)
@@ -75,10 +74,10 @@ export default defineNuxtPlugin(async (nuxtApp) => {
       }
       socket?.on('subscribed', subscribeFn)
     },
-    getStatus(name: string, id: string): Promise<string> {
+    getStatus(id: string): Promise<string> {
       return new Promise((resolve) => {
         if (socket) {
-          const _id = send('status', { name, id })
+          const _id = send('status', { id })
           const tempFn = (data: any) => {
             if (data._id === _id) {
               resolve(data.status)

packages/nocodb/src/app.module.ts

@@ -1,6 +1,5 @@
 import { Module, RequestMethod } from '@nestjs/common';
 import { APP_FILTER } from '@nestjs/core';
-import { BullModule } from '@nestjs/bull';
 import { EventEmitterModule as NestJsEventEmitter } from '@nestjs/event-emitter';
 import { GlobalExceptionFilter } from './filters/global-exception/global-exception.filter';
 import { GlobalMiddleware } from './middlewares/global/global.middleware';
@@ -17,7 +16,6 @@ import { AuthTokenStrategy } from './strategies/authtoken.strategy/authtoken.str
 import { BaseViewStrategy } from './strategies/base-view.strategy/base-view.strategy';
 import { MetasModule } from './modules/metas/metas.module';
 import { JobsModule } from './modules/jobs/jobs.module';
-import { AppInitService } from './services/app-init.service';
 import type { MiddlewareConsumer } from '@nestjs/common';
 
 @Module({
@@ -30,13 +28,6 @@ import type { MiddlewareConsumer } from '@nestjs/common';
     EventEmitterModule,
     JobsModule,
     NestJsEventEmitter.forRoot(),
-    ...(process.env['NC_REDIS_URL']
-      ? [
-          BullModule.forRoot({
-            redis: process.env.NC_REDIS_URL,
-          }),
-        ]
-      : []),
   ],
   controllers: [],
   providers: [
@@ -49,7 +40,6 @@ import type { MiddlewareConsumer } from '@nestjs/common';
     AuthTokenStrategy,
     BaseViewStrategy,
     HookHandlerService,
-    AppInitService,
   ],
 })
 export class AppModule {

packages/nocodb/src/cache/RedisCacheMgr.ts

@@ -12,8 +12,12 @@ export default class RedisCacheMgr extends CacheMgr {
   constructor(config: any) {
     super();
     this.client = new Redis(config);
-    // flush the existing db with selected key (Default: 0)
+
-    this.client.flushdb();
+    // avoid flushing db in worker container
+    if (process.env.NC_WORKER_CONTAINER !== 'true') {
+      // flush the existing db with selected key (Default: 0)
+      this.client.flushdb();
+    }
 
     // TODO(cache): fetch orgs once it's implemented
     const orgs = 'noco';

packages/nocodb/src/connection/connection.spec.ts

@@ -1,19 +0,0 @@
-import { Test } from '@nestjs/testing';
-import { Connection } from './knex';
-import type { TestingModule } from '@nestjs/testing';
-
-describe('Knex', () => {
-  let provider: Connection;
-
-  beforeEach(async () => {
-    const module: TestingModule = await Test.createTestingModule({
-      providers: [Connection],
-    }).compile();
-
-    provider = module.get<Connection>(Connection);
-  });
-
-  it('should be defined', () => {
-    expect(provider).toBeDefined();
-  });
-});

packages/nocodb/src/connection/connection.ts

@@ -1,37 +0,0 @@
-import { Global, Injectable, Scope } from '@nestjs/common';
-
-import { XKnex } from '../db/CustomKnex';
-import NcConfigFactory from '../utils/NcConfigFactory';
-import type * as knex from 'knex';
-
-@Injectable({
-  scope: Scope.DEFAULT,
-})
-export class Connection {
-  public static knex: knex.Knex;
-  public static _config: any;
-
-  get knexInstance(): knex.Knex {
-    return Connection.knex;
-  }
-
-  get config(): knex.Knex {
-    return Connection._config;
-  }
-
-  // init metadb connection
-  static async init(): Promise<void> {
-    Connection._config = await NcConfigFactory.make();
-    if (!Connection.knex) {
-      Connection.knex = XKnex({
-        ...this._config.meta.db,
-        useNullAsDefault: true,
-      });
-    }
-  }
-
-  // init metadb connection
-  async init(): Promise<void> {
-    return await Connection.init();
-  }
-}

packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts

@@ -10,7 +10,7 @@ import Debug from '../../util/Debug';
 import Emit from '../../util/emit';
 import Result from '../../util/Result';
 import * as fileHelp from '../../util/file.help';
-import NcConfigFactory from '../../../utils/NcConfigFactory';
+import { getToolDir, NcConfig } from '../../../utils/nc-config';
 import SqlMigrator from './SqlMigrator';
 
 const evt = new Emit();
@@ -39,7 +39,7 @@ export default class KnexMigrator extends SqlMigrator {
     this.project_id = projectObj?.project_id;
     this.project = projectObj?.config;
     this.metaDb = projectObj?.metaDb;
-    this.toolDir = NcConfigFactory.getToolDir();
+    this.toolDir = getToolDir();
   }
 
   emit(data, _args?) {
@@ -312,8 +312,12 @@ export default class KnexMigrator extends SqlMigrator {
       if (exists) {
         await this._readProjectJson(projJsonFilePath);
         this.emit('Migrator for project initalised successfully');
-      } else if (NcConfigFactory.hasDbUrl()) {
+      } else if (
-        this.project = await NcConfigFactory.make();
+        Object.keys(process.env).some((envKey) =>
+          envKey.startsWith('NC_DB_URL'),
+        )
+      ) {
+        this.project = await NcConfig.createByEnv();
       } else {
         args.type = args.type || 'sqlite';
 

packages/nocodb/src/index.ts

@@ -1,6 +1,5 @@
 import Noco from './Noco';
-import NcConfigFactory from './utils/NcConfigFactory';
 
 export default Noco;
 
-export { Noco, NcConfigFactory };
+export { Noco };

packages/nocodb/src/init.ts

@@ -1,12 +1,12 @@
-import { Connection } from './connection/connection';
 import { MetaService } from './meta/meta.service';
+import { NcConfig } from './utils/nc-config';
 import Noco from './Noco';
 
 // run upgrader
 import NcUpgrader from './version-upgrader/NcUpgrader';
 
 export default async () => {
-  await Connection.init();
+  const config = await NcConfig.createByEnv();
-  Noco._ncMeta = new MetaService(new Connection());
+  Noco._ncMeta = new MetaService(config);
   await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
 };

packages/nocodb/src/main.ts

@@ -5,12 +5,22 @@ import { AppModule } from './app.module';
 
 async function bootstrap() {
   const app = await NestFactory.create(AppModule);
-  app.use(express.json({ limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' }));
+  if (process.env.NC_WORKER_CONTAINER !== 'true') {
-  app.use(
+    app.use(
-    cors({
+      express.json({ limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' }),
-      exposedHeaders: 'xc-db-response',
+    );
-    }),
+    app.use(
-  );
+      cors({
-  await app.listen(8080);
+        exposedHeaders: 'xc-db-response',
+      }),
+    );
+    await app.listen(8080);
+  } else {
+    if (!process.env.NC_REDIS_URL) {
+      throw new Error('NC_REDIS_URL is required');
+    }
+    process.env.NC_DISABLE_TELE = 'true';
+    await app.init();
+  }
 }
 bootstrap();

packages/nocodb/src/meta/meta.service.ts

@@ -1,22 +1,15 @@
-import {
+import { Injectable, Optional } from '@nestjs/common';
-  Global,
-  Inject,
-  Injectable,
-  OnApplicationBootstrap,
-  OnModuleInit,
-  Optional,
-} from '@nestjs/common';
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc';
 import timezone from 'dayjs/plugin/timezone';
 import { customAlphabet } from 'nanoid';
 import CryptoJS from 'crypto-js';
-import { Connection } from '../connection/connection';
+import { XKnex } from '../db/CustomKnex';
-import Noco from '../Noco';
+import { NcConfig } from '../utils/nc-config';
-import NocoCache from '../cache/NocoCache';
 import XcMigrationSourcev2 from './migrations/XcMigrationSourcev2';
 import XcMigrationSource from './migrations/XcMigrationSource';
 import type { Knex } from 'knex';
+import type * as knex from 'knex';
 
 dayjs.extend(utc);
 dayjs.extend(timezone);
@@ -192,18 +185,38 @@ const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14);
 
 @Injectable()
 export class MetaService {
-  constructor(private metaConnection: Connection, @Optional() trx = null) {
+  private _knex: knex.Knex;
+  private _config: any;
+
+  constructor(config: NcConfig, @Optional() trx = null) {
+    this._config = config;
+    this._knex = XKnex({
+      ...this._config.meta.db,
+      useNullAsDefault: true,
+    });
     this.trx = trx;
   }
 
+  get knexInstance(): knex.Knex {
+    return this._knex;
+  }
+
+  get config(): NcConfig {
+    return this._config;
+  }
+
   public get connection() {
-    return this.trx ?? this.metaConnection.knexInstance;
+    return this.trx ?? this.knexInstance;
   }
 
   get knexConnection() {
     return this.connection;
   }
 
+  public get knex(): any {
+    return this.knexConnection;
+  }
+
   public async metaGet(
     project_id: string,
     dbAlias: string,
@@ -758,7 +771,7 @@ export class MetaService {
     });
 
     // todo: tobe done
-    return new MetaService(this.metaConnection, trx);
+    return new MetaService(this.config, trx);
   }
 
   async metaReset(
@@ -1027,10 +1040,6 @@ export class MetaService {
       .delete();
   }
 
-  public get knex(): any {
-    return this.knexConnection;
-  }
-
   private getNanoId() {
     return nanoid();
   }

packages/nocodb/src/modules/datas/datas.module.ts

@@ -27,14 +27,18 @@ import { PublicDatasService } from '../../services/public-datas.service';
     }),
   ],
   controllers: [
-    DatasController,
+    ...(process.env.NC_WORKER_CONTAINER !== 'true'
-    BulkDataAliasController,
+      ? [
-    DataAliasController,
+          DatasController,
-    DataAliasNestedController,
+          BulkDataAliasController,
-    DataAliasExportController,
+          DataAliasController,
-    OldDatasController,
+          DataAliasNestedController,
-    PublicDatasController,
+          DataAliasExportController,
-    PublicDatasExportController,
+          OldDatasController,
+          PublicDatasController,
+          PublicDatasExportController,
+        ]
+      : []),
   ],
   providers: [
     DatasService,

packages/nocodb/src/modules/global/global.module.ts

@@ -1,25 +1,18 @@
 import { Global, Module } from '@nestjs/common';
 import { ExtractJwt } from 'passport-jwt';
-import {
-  AppInitService,
-  appInitServiceProvider,
-} from '../../services/app-init.service';
 import { SocketGateway } from '../../gateways/socket.gateway';
-import { Connection } from '../../connection/connection';
 import { GlobalGuard } from '../../guards/global/global.guard';
 import { MetaService } from '../../meta/meta.service';
-import Noco from '../../Noco';
 import { JwtStrategy } from '../../strategies/jwt.strategy';
 import { UsersService } from '../../services/users/users.service';
+import Noco from '../../Noco';
+import { InitMetaServiceProvider } from './init-meta-service.provider';
 import type { Provider } from '@nestjs/common';
 
 export const JwtStrategyProvider: Provider = {
   provide: JwtStrategy,
-  useFactory: async (
+  useFactory: async (usersService: UsersService, metaService: MetaService) => {
-    usersService: UsersService,
+    const config = metaService.config;
-    appInitService: AppInitService,
-  ) => {
-    const config = appInitService.appConfig;
 
     await Noco.initJwt();
 
@@ -34,29 +27,25 @@ export const JwtStrategyProvider: Provider = {
 
     return new JwtStrategy(options, usersService);
   },
-  inject: [UsersService, AppInitService],
+  inject: [UsersService, MetaService],
 };
 
 @Global()
 @Module({
   imports: [],
   providers: [
-    appInitServiceProvider,
+    InitMetaServiceProvider,
-    Connection,
-    MetaService,
     UsersService,
     JwtStrategyProvider,
     GlobalGuard,
-    SocketGateway,
+    ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [SocketGateway] : []),
   ],
   exports: [
-    AppInitService,
-    Connection,
     MetaService,
     JwtStrategyProvider,
     UsersService,
     GlobalGuard,
-    SocketGateway,
+    ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [SocketGateway] : []),
   ],
 })
 export class GlobalModule {}

packages/nocodb/src/modules/global/init-meta-service.provider.ts

@@ -0,0 +1,65 @@
+import { T } from 'nc-help';
+import { MetaService } from '../../meta/meta.service';
+import Noco from '../../Noco';
+import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2';
+import NcUpgrader from '../../version-upgrader/NcUpgrader';
+import NocoCache from '../../cache/NocoCache';
+import getInstance from '../../utils/getInstance';
+import initAdminFromEnv from '../../helpers/initAdminFromEnv';
+import { User } from '../../models';
+import { NcConfig, prepareEnv } from '../../utils/nc-config';
+import type { Provider } from '@nestjs/common';
+import type { IEventEmitter } from '../event-emitter/event-emitter.interface';
+
+export const InitMetaServiceProvider: Provider = {
+  // initialize app,
+  // 1. init cache
+  // 2. init db connection and create if not exist
+  // 3. init meta and set to Noco
+  // 4. init jwt
+  // 5. init plugin manager
+  // 6. run upgrader
+  useFactory: async (eventEmitter: IEventEmitter) => {
+    // NC_DATABASE_URL_FILE, DATABASE_URL_FILE, DATABASE_URL, NC_DATABASE_URL to NC_DB
+    await prepareEnv();
+
+    const config = await NcConfig.createByEnv();
+
+    // set version
+    process.env.NC_VERSION = '0107004';
+
+    // init cache
+    await NocoCache.init();
+
+    // init meta service
+    const metaService = new MetaService(config);
+    await metaService.init();
+
+    // provide meta and config to Noco
+    Noco._ncMeta = metaService;
+    Noco.config = config;
+    Noco.eventEmitter = eventEmitter;
+
+    // init jwt secret
+    await Noco.initJwt();
+
+    // load super admin user from env if env is set
+    await initAdminFromEnv(metaService);
+
+    // init plugin manager
+    await NcPluginMgrv2.init(Noco.ncMeta);
+    await Noco.loadEEState();
+
+    // run upgrader
+    await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
+
+    T.init({
+      instance: getInstance,
+    });
+    T.emit('evt_app_started', await User.count());
+
+    return metaService;
+  },
+  provide: MetaService,
+  inject: ['IEventEmitter'],
+};

packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts

@@ -1,12 +1,12 @@
 import { Injectable } from '@nestjs/common';
 import PQueue from 'p-queue';
 import Emittery from 'emittery';
-import { JobStatus, JobTypes } from '../../interface/Jobs';
+import { JobStatus, JobTypes } from '../../../interface/Jobs';
-import { DuplicateProcessor } from './export-import/duplicate.processor';
+import { DuplicateProcessor } from '../jobs/export-import/duplicate.processor';
+import { AtImportProcessor } from '../jobs/at-import/at-import.processor';
 import { JobsEventService } from './jobs-event.service';
-import { AtImportProcessor } from './at-import/at-import.processor';
 
-interface Job {
+export interface Job {
   id: string;
   name: string;
   status: string;
@@ -27,16 +27,12 @@ export class QueueService {
     private readonly atImportProcessor: AtImportProcessor,
   ) {
     this.emitter.on(JobStatus.ACTIVE, (data: { job: Job }) => {
-      const job = this.queueMemory.find(
+      const job = this.queueMemory.find((job) => job.id === data.job.id);
-        (job) => job.id === data.job.id && job.name === data.job.name,
-      );
       job.status = JobStatus.ACTIVE;
       this.jobsEventService.onActive.apply(this.jobsEventService, [job as any]);
     });
     this.emitter.on(JobStatus.COMPLETED, (data: { job: Job; result: any }) => {
-      const job = this.queueMemory.find(
+      const job = this.queueMemory.find((job) => job.id === data.job.id);
-        (job) => job.id === data.job.id && job.name === data.job.name,
-      );
       job.status = JobStatus.COMPLETED;
       this.jobsEventService.onCompleted.apply(this.jobsEventService, [
         job,
@@ -46,9 +42,7 @@ export class QueueService {
       this.removeJob(job);
     });
     this.emitter.on(JobStatus.FAILED, (data: { job: Job; error: Error }) => {
-      const job = this.queueMemory.find(
+      const job = this.queueMemory.find((job) => job.id === data.job.id);
-        (job) => job.id === data.job.id && job.name === data.job.name,
-      );
       job.status = JobStatus.FAILED;
       this.jobsEventService.onFailed.apply(this.jobsEventService, [
         job,
@@ -126,9 +120,7 @@ export class QueueService {
 
   // remove job from memory
   private removeJob(job: Job) {
-    const fIndex = this.queueMemory.findIndex(
+    const fIndex = this.queueMemory.findIndex((q) => q.id === job.id);
-      (q) => q.id === job.id && q.name === job.name,
-    );
     if (fIndex) {
       this.queueMemory.splice(fIndex, 1);
     }

packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts

@@ -7,7 +7,7 @@ import {
 import { Job } from 'bull';
 import boxen from 'boxen';
 import { EventEmitter2 } from '@nestjs/event-emitter';
-import { JobEvents, JOBS_QUEUE, JobStatus } from '../../interface/Jobs';
+import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
 
 @Processor(JOBS_QUEUE)
 export class JobsEventService {
@@ -16,7 +16,6 @@ export class JobsEventService {
   @OnQueueActive()
   onActive(job: Job) {
     this.eventEmitter.emit(JobEvents.STATUS, {
-      name: job.name,
       id: job.id.toString(),
       status: JobStatus.ACTIVE,
     });
@@ -26,7 +25,7 @@ export class JobsEventService {
   onFailed(job: Job, error: Error) {
     console.error(
       boxen(
-        `---- !! JOB FAILED !! ----\nname: ${job.name}\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
+        `---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
         {
           padding: 1,
           borderStyle: 'double',
@@ -36,7 +35,6 @@ export class JobsEventService {
     );
 
     this.eventEmitter.emit(JobEvents.STATUS, {
-      name: job.name,
       id: job.id.toString(),
       status: JobStatus.FAILED,
       data: {
@@ -50,7 +48,6 @@ export class JobsEventService {
   @OnQueueCompleted()
   onCompleted(job: Job, data: any) {
     this.eventEmitter.emit(JobEvents.STATUS, {
-      name: job.name,
       id: job.id.toString(),
       status: JobStatus.COMPLETED,
       data: {
@@ -58,12 +55,4 @@ export class JobsEventService {
       },
     });
   }
-
-  sendLog(job: Job, data: { message: string }) {
-    this.eventEmitter.emit(JobEvents.LOG, {
-      name: job.name,
-      id: job.id.toString(),
-      data,
-    });
-  }
 }

packages/nocodb/src/modules/jobs/fallback/jobs.service.ts

@@ -0,0 +1,51 @@
+import { Injectable } from '@nestjs/common';
+import { JobStatus } from '../../../interface/Jobs';
+import { QueueService } from './fallback-queue.service';
+
+@Injectable()
+export class JobsService {
+  constructor(private readonly fallbackQueueService: QueueService) {}
+
+  async add(name: string, data: any) {
+    return this.fallbackQueueService.add(name, data);
+  }
+
+  async jobStatus(jobId: string) {
+    return await (
+      await this.fallbackQueueService.getJob(jobId)
+    ).status;
+  }
+
+  async jobList() {
+    return await this.fallbackQueueService.getJobs([
+      JobStatus.ACTIVE,
+      JobStatus.WAITING,
+      JobStatus.DELAYED,
+      JobStatus.PAUSED,
+    ]);
+  }
+
+  async getJobWithData(data: any) {
+    const jobs = await this.fallbackQueueService.getJobs([
+      // 'completed',
+      JobStatus.WAITING,
+      JobStatus.ACTIVE,
+      JobStatus.DELAYED,
+      // 'failed',
+      JobStatus.PAUSED,
+    ]);
+
+    const job = jobs.find((j) => {
+      for (const key in data) {
+        if (j.data[key]) {
+          if (j.data[key] !== data[key]) return false;
+        } else {
+          return false;
+        }
+      }
+      return true;
+    });
+
+    return job;
+  }
+}

packages/nocodb/src/modules/jobs/jobs.gateway.ts

@@ -9,10 +9,10 @@ import { Server, Socket } from 'socket.io';
 import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host';
 import { AuthGuard } from '@nestjs/passport';
 import { OnEvent } from '@nestjs/event-emitter';
+import { Inject } from '@nestjs/common';
 import { JobEvents } from '../../interface/Jobs';
-import { JobsService } from './jobs.service';
-import type { JobStatus } from '../../interface/Jobs';
 import type { OnModuleInit } from '@nestjs/common';
+import type { JobStatus } from '../../interface/Jobs';
 
 @WebSocketGateway({
   cors: {
@@ -23,7 +23,7 @@ import type { OnModuleInit } from '@nestjs/common';
   namespace: 'jobs',
 })
 export class JobsGateway implements OnModuleInit {
-  constructor(private readonly jobsService: JobsService) {}
+  constructor(@Inject('JobsService') private readonly jobsService) {}
 
   @WebSocketServer()
   server: Server;
@@ -43,34 +43,28 @@ export class JobsGateway implements OnModuleInit {
   @SubscribeMessage('subscribe')
   async subscribe(
     @MessageBody()
-    body: { _id: number; data: { id: string; name: string } | any },
+    body: { _id: number; data: { id: string } | any },
     @ConnectedSocket() client: Socket,
   ): Promise<void> {
     const { _id, data } = body;
-    if (
+    if (Object.keys(data).every((k) => ['id'].includes(k)) && data?.id) {
-      Object.keys(data).every((k) => ['name', 'id'].includes(k)) &&
+      const rooms = (await this.jobsService.jobList()).map(
-      data?.name &&
+        (j) => `jobs-${j.id}`,
-      data?.id
-    ) {
-      const rooms = (await this.jobsService.jobList(data.name)).map(
-        (j) => `${j.name}-${j.id}`,
       );
-      const room = rooms.find((r) => r === `${data.name}-${data.id}`);
+      const room = rooms.find((r) => r === `jobs-${data.id}`);
       if (room) {
-        client.join(`${data.name}-${data.id}`);
+        client.join(`jobs-${data.id}`);
         client.emit('subscribed', {
           _id,
-          name: data.name,
           id: data.id,
         });
       }
     } else {
       const job = await this.jobsService.getJobWithData(data);
       if (job) {
-        client.join(`${job.name}-${job.id}`);
+        client.join(`jobs-${job.id}`);
         client.emit('subscribed', {
           _id,
-          name: job.name,
           id: job.id,
         });
       }
@@ -79,42 +73,30 @@ export class JobsGateway implements OnModuleInit {
 
   @SubscribeMessage('status')
   async status(
-    @MessageBody() body: { _id: number; data: { id: string; name: string } },
+    @MessageBody() body: { _id: number; data: { id: string } },
     @ConnectedSocket() client: Socket,
   ): Promise<void> {
     const { _id, data } = body;
     client.emit('status', {
       _id,
       id: data.id,
-      name: data.name,
       status: await this.jobsService.jobStatus(data.id),
     });
   }
 
   @OnEvent(JobEvents.STATUS)
-  async sendJobStatus(data: {
+  sendJobStatus(data: { id: string; status: JobStatus; data?: any }): void {
-    name: string;
+    this.server.to(`jobs-${data.id}`).emit('status', {
-    id: string;
-    status: JobStatus;
-    data?: any;
-  }): Promise<void> {
-    this.server.to(`${data.name}-${data.id}`).emit('status', {
       id: data.id,
-      name: data.name,
       status: data.status,
       data: data.data,
     });
   }
 
   @OnEvent(JobEvents.LOG)
-  async sendJobLog(data: {
+  sendJobLog(data: { id: string; data: { message: string } }): void {
-    name: string;
+    this.server.to(`jobs-${data.id}`).emit('log', {
-    id: string;
-    data: { message: string };
-  }): Promise<void> {
-    this.server.to(`${data.name}-${data.id}`).emit('log', {
       id: data.id,
-      name: data.name,
       data: data.data,
     });
   }

packages/nocodb/src/modules/jobs/jobs.module.ts

@@ -4,35 +4,59 @@ import { GlobalModule } from '../global/global.module';
 import { DatasModule } from '../datas/datas.module';
 import { MetasModule } from '../metas/metas.module';
 import { JOBS_QUEUE } from '../../interface/Jobs';
-import { JobsService } from './jobs.service';
+import { ExportService } from './jobs/export-import/export.service';
-import { ExportService } from './export-import/export.service';
+import { ImportService } from './jobs/export-import/import.service';
-import { ImportService } from './export-import/import.service';
+import { AtImportController } from './jobs/at-import/at-import.controller';
-import { DuplicateController } from './export-import/duplicate.controller';
+import { AtImportProcessor } from './jobs/at-import/at-import.processor';
-import { DuplicateProcessor } from './export-import/duplicate.processor';
+import { DuplicateController } from './jobs/export-import/duplicate.controller';
+import { DuplicateProcessor } from './jobs/export-import/duplicate.processor';
+import { JobsLogService } from './jobs/jobs-log.service';
 import { JobsGateway } from './jobs.gateway';
-import { QueueService } from './fallback-queue.service';
+
-import { JobsEventService } from './jobs-event.service';
+// Redis
-import { AtImportController } from './at-import/at-import.controller';
+import { JobsService } from './redis/jobs.service';
-import { AtImportProcessor } from './at-import/at-import.processor';
+import { JobsRedisService } from './redis/jobs-redis.service';
+import { JobsEventService } from './redis/jobs-event.service';
+
+// Fallback
+import { JobsService as FallbackJobsService } from './fallback/jobs.service';
+import { QueueService as FallbackQueueService } from './fallback/fallback-queue.service';
+import { JobsEventService as FallbackJobsEventService } from './fallback/jobs-event.service';
 
 @Module({
   imports: [
     GlobalModule,
     DatasModule,
     MetasModule,
-    BullModule.registerQueue({
+    ...(process.env.NC_REDIS_URL
-      name: JOBS_QUEUE,
+      ? [
-    }),
+          BullModule.forRoot({
+            url: process.env.NC_REDIS_URL,
+          }),
+          BullModule.registerQueue({
+            name: JOBS_QUEUE,
+          }),
+        ]
+      : []),
+  ],
+  controllers: [
+    ...(process.env.NC_WORKER_CONTAINER !== 'true'
+      ? [DuplicateController, AtImportController]
+      : []),
   ],
-  controllers: [DuplicateController, AtImportController],
   providers: [
-    QueueService,
+    ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [JobsGateway] : []),
-    JobsGateway,
+    ...(process.env.NC_REDIS_URL
-    JobsService,
+      ? [JobsRedisService, JobsEventService]
-    JobsEventService,
+      : [FallbackQueueService, FallbackJobsEventService]),
-    DuplicateProcessor,
+    {
+      provide: 'JobsService',
+      useClass: process.env.NC_REDIS_URL ? JobsService : FallbackJobsService,
+    },
+    JobsLogService,
     ExportService,
     ImportService,
+    DuplicateProcessor,
     AtImportProcessor,
   ],
 })

packages/nocodb/src/modules/jobs/jobs.service.ts

@@ -1,59 +0,0 @@
-import { InjectQueue } from '@nestjs/bull';
-import { Injectable } from '@nestjs/common';
-import { Queue } from 'bull';
-import { JOBS_QUEUE, JobStatus } from '../../interface/Jobs';
-import { QueueService } from './fallback-queue.service';
-
-@Injectable()
-export class JobsService {
-  public activeQueue;
-  constructor(
-    @InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue,
-    private readonly fallbackQueueService: QueueService,
-  ) {
-    this.activeQueue = this.fallbackQueueService;
-    /* process.env.NC_REDIS_URL
-        ? this.jobsQueue
-        : this.fallbackQueueService;
-    */
-  }
-
-  async jobStatus(jobId: string) {
-    return await (await this.activeQueue.getJob(jobId)).getState();
-  }
-
-  async jobList(jobType: string) {
-    return (
-      await this.activeQueue.getJobs([
-        JobStatus.ACTIVE,
-        JobStatus.WAITING,
-        JobStatus.DELAYED,
-        JobStatus.PAUSED,
-      ])
-    ).filter((j) => j.name === jobType);
-  }
-
-  async getJobWithData(data: any) {
-    const jobs = await this.activeQueue.getJobs([
-      // 'completed',
-      JobStatus.WAITING,
-      JobStatus.ACTIVE,
-      JobStatus.DELAYED,
-      // 'failed',
-      JobStatus.PAUSED,
-    ]);
-
-    const job = jobs.find((j) => {
-      for (const key in data) {
-        if (j.data[key]) {
-          if (j.data[key] !== data[key]) return false;
-        } else {
-          return false;
-        }
-      }
-      return true;
-    });
-
-    return job;
-  }
-}

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts

@@ -1,30 +1,36 @@
-import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common';
+import {
-import { GlobalGuard } from '../../../guards/global/global.guard';
+  Controller,
-import { ExtractProjectIdMiddleware } from '../../../middlewares/extract-project-id/extract-project-id.middleware';
+  HttpCode,
-import { SyncSource } from '../../../models';
+  Inject,
-import { NcError } from '../../../helpers/catchError';
+  Post,
-import { JobsService } from '../jobs.service';
+  Request,
-import { JobTypes } from '../../../interface/Jobs';
+  UseGuards,
+} from '@nestjs/common';
+import { GlobalGuard } from '../../../../guards/global/global.guard';
+import { ExtractProjectIdMiddleware } from '../../../../middlewares/extract-project-id/extract-project-id.middleware';
+import { SyncSource } from '../../../../models';
+import { NcError } from '../../../../helpers/catchError';
+import { JobTypes } from '../../../../interface/Jobs';
 
 @Controller()
 @UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
 export class AtImportController {
-  constructor(private readonly jobsService: JobsService) {}
+  constructor(@Inject('JobsService') private readonly jobsService) {}
 
   @Post('/api/v1/db/meta/import/airtable')
   @HttpCode(200)
   async importAirtable(@Request() req) {
-    const job = await this.jobsService.activeQueue.add(JobTypes.AtImport, {
+    const job = await this.jobsService.add(JobTypes.AtImport, {
       ...req.body,
     });
 
-    return { id: job.id, name: job.name };
+    return { id: job.id };
   }
 
   @Post('/api/v1/db/meta/syncs/:syncId/trigger')
   @HttpCode(200)
   async triggerSync(@Request() req) {
-    const jobs = await this.jobsService.jobList(JobTypes.AtImport);
+    const jobs = await this.jobsService.jobList();
     const fnd = jobs.find((j) => j.data.syncId === req.params.syncId);
 
     if (fnd) {
@@ -44,7 +50,7 @@ export class AtImportController {
       baseURL = `http://localhost:${process.env.PORT || 8080}`;
     }
 
-    const job = await this.jobsService.activeQueue.add(JobTypes.AtImport, {
+    const job = await this.jobsService.add(JobTypes.AtImport, {
       syncId: req.params.syncId,
       ...(syncSource?.details || {}),
       projectId: syncSource.project_id,
@@ -54,7 +60,7 @@ export class AtImportController {
       user: user,
     });
 
-    return { id: job.id, name: job.name };
+    return { id: job.id };
   }
 
   @Post('/api/v1/db/meta/syncs/:syncId/abort')

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@@ -9,23 +9,23 @@ import utc from 'dayjs/plugin/utc';
 import tinycolor from 'tinycolor2';
 import { Process, Processor } from '@nestjs/bull';
 import { Job } from 'bull';
-import extractRolesObj from '../../../utils/extractRolesObj';
+import extractRolesObj from '../../../../utils/extractRolesObj';
-import { AttachmentsService } from '../../../services/attachments.service';
+import { AttachmentsService } from '../../../../services/attachments.service';
-import { ColumnsService } from '../../../services/columns.service';
+import { ColumnsService } from '../../../../services/columns.service';
-import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
+import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
-import { FiltersService } from '../../../services/filters.service';
+import { FiltersService } from '../../../../services/filters.service';
-import { FormColumnsService } from '../../../services/form-columns.service';
+import { FormColumnsService } from '../../../../services/form-columns.service';
-import { GalleriesService } from '../../../services/galleries.service';
+import { GalleriesService } from '../../../../services/galleries.service';
-import { GridsService } from '../../../services/grids.service';
+import { GridsService } from '../../../../services/grids.service';
-import { ProjectUsersService } from '../../../services/project-users/project-users.service';
+import { ProjectUsersService } from '../../../../services/project-users/project-users.service';
-import { ProjectsService } from '../../../services/projects.service';
+import { ProjectsService } from '../../../../services/projects.service';
-import { SortsService } from '../../../services/sorts.service';
+import { SortsService } from '../../../../services/sorts.service';
-import { TablesService } from '../../../services/tables.service';
+import { TablesService } from '../../../../services/tables.service';
-import { ViewColumnsService } from '../../../services/view-columns.service';
+import { ViewColumnsService } from '../../../../services/view-columns.service';
-import { ViewsService } from '../../../services/views.service';
+import { ViewsService } from '../../../../services/views.service';
-import { FormsService } from '../../../services/forms.service';
+import { FormsService } from '../../../../services/forms.service';
-import { JobsEventService } from '../jobs-event.service';
+import { JOBS_QUEUE, JobTypes } from '../../../../interface/Jobs';
-import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs';
+import { JobsLogService } from '../jobs-log.service';
 import FetchAT from './helpers/fetchAT';
 import { importData, importLTARData } from './helpers/readAndProcessData';
 import EntityMap from './helpers/EntityMap';
@@ -99,7 +99,7 @@ export class AtImportProcessor {
     private readonly viewColumnsService: ViewColumnsService,
     private readonly sortsService: SortsService,
     private readonly bulkDataAliasService: BulkDataAliasService,
-    private readonly jobsEventService: JobsEventService,
+    private readonly jobsLogService: JobsLogService,
   ) {}
 
   @Process(JobTypes.AtImport)
@@ -135,11 +135,11 @@ export class AtImportProcessor {
     };
 
     const logBasic = (log) => {
-      this.jobsEventService.sendLog(job, { message: log });
+      this.jobsLogService.sendLog(job, { message: log });
     };
 
     const logDetailed = (log) => {
-      if (debugMode) this.jobsEventService.sendLog(job, { message: log });
+      if (debugMode) this.jobsLogService.sendLog(job, { message: log });
     };
 
     const perfStats = [];

packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts

@@ -1,8 +1,8 @@
 /* eslint-disable no-async-promise-executor */
 import { RelationTypes, UITypes } from 'nocodb-sdk';
 import EntityMap from './EntityMap';
-import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
+import type { BulkDataAliasService } from '../../../../../services/bulk-data-alias.service';
-import type { TablesService } from '../../../../services/tables.service';
+import type { TablesService } from '../../../../../services/tables.service';
 // @ts-ignore
 import type { AirtableBase } from 'airtable/lib/airtable_base';
 import type { TableType } from 'nocodb-sdk';

packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts

@@ -2,28 +2,28 @@ import {
   Body,
   Controller,
   HttpCode,
+  Inject,
   Param,
   Post,
   Request,
   UseGuards,
 } from '@nestjs/common';
 import { ProjectStatus } from 'nocodb-sdk';
-import { GlobalGuard } from '../../../guards/global/global.guard';
+import { GlobalGuard } from '../../../../guards/global/global.guard';
 import {
   Acl,
   ExtractProjectIdMiddleware,
-} from '../../../middlewares/extract-project-id/extract-project-id.middleware';
+} from '../../../../middlewares/extract-project-id/extract-project-id.middleware';
-import { ProjectsService } from '../../../services/projects.service';
+import { ProjectsService } from '../../../../services/projects.service';
-import { Base, Model, Project } from '../../../models';
+import { Base, Model, Project } from '../../../../models';
-import { generateUniqueName } from '../../../helpers/exportImportHelpers';
+import { generateUniqueName } from '../../../../helpers/exportImportHelpers';
-import { JobsService } from '../jobs.service';
+import { JobTypes } from '../../../../interface/Jobs';
-import { JobTypes } from '../../../interface/Jobs';
 
 @Controller()
 @UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
 export class DuplicateController {
   constructor(
-    private readonly jobsService: JobsService,
+    @Inject('JobsService') private readonly jobsService,
     private readonly projectsService: ProjectsService,
   ) {}
 
@@ -67,7 +67,7 @@ export class DuplicateController {
       user: { id: req.user.id },
     });
 
-    const job = await this.jobsService.activeQueue.add(JobTypes.DuplicateBase, {
+    const job = await this.jobsService.add(JobTypes.DuplicateBase, {
       projectId: project.id,
       baseId: base.id,
       dupProjectId: dupProject.id,
@@ -78,7 +78,7 @@ export class DuplicateController {
       },
     });
 
-    return { id: job.id, name: job.name };
+    return { id: job.id };
   }
 
   @Post('/api/v1/db/meta/duplicate/:projectId/table/:modelId')
@@ -116,21 +116,18 @@ export class DuplicateController {
       models.map((p) => p.title),
     );
 
-    const job = await this.jobsService.activeQueue.add(
+    const job = await this.jobsService.add(JobTypes.DuplicateModel, {
-      JobTypes.DuplicateModel,
+      projectId: project.id,
-      {
+      baseId: base.id,
-        projectId: project.id,
+      modelId: model.id,
-        baseId: base.id,
+      title: uniqueTitle,
-        modelId: model.id,
+      options,
-        title: uniqueTitle,
+      req: {
-        options,
+        user: req.user,
-        req: {
+        clientIp: req.clientIp,
-          user: req.user,
-          clientIp: req.clientIp,
-        },
       },
-    );
+    });
 
-    return { id: job.id, name: job.name };
+    return { id: job.id };
   }
 }

packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@@ -4,12 +4,12 @@ import { Job } from 'bull';
 import papaparse from 'papaparse';
 import { UITypes } from 'nocodb-sdk';
 import { Logger } from '@nestjs/common';
-import { Base, Column, Model, Project } from '../../../models';
+import { Base, Column, Model, Project } from '../../../../models';
-import { ProjectsService } from '../../../services/projects.service';
+import { ProjectsService } from '../../../../services/projects.service';
-import { findWithIdentifier } from '../../../helpers/exportImportHelpers';
+import { findWithIdentifier } from '../../../../helpers/exportImportHelpers';
-import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
+import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
-import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs';
+import { JOBS_QUEUE, JobTypes } from '../../../../interface/Jobs';
-import { elapsedTime, initTime } from '../helpers';
+import { elapsedTime, initTime } from '../../helpers';
 import { ExportService } from './export.service';
 import { ImportService } from './import.service';
 

packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@@ -2,19 +2,19 @@ import { Readable } from 'stream';
 import { UITypes, ViewTypes } from 'nocodb-sdk';
 import { unparse } from 'papaparse';
 import { Injectable, Logger } from '@nestjs/common';
-import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
+import NcConnectionMgrv2 from '../../../../utils/common/NcConnectionMgrv2';
-import { getViewAndModelByAliasOrId } from '../../../modules/datas/helpers';
+import { getViewAndModelByAliasOrId } from '../../../datas/helpers';
 import {
   clearPrefix,
   generateBaseIdMap,
-} from '../../../helpers/exportImportHelpers';
+} from '../../../../helpers/exportImportHelpers';
-import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2';
+import NcPluginMgrv2 from '../../../../helpers/NcPluginMgrv2';
-import { NcError } from '../../../helpers/catchError';
+import { NcError } from '../../../../helpers/catchError';
-import { Base, Hook, Model, Project } from '../../../models';
+import { Base, Hook, Model, Project } from '../../../../models';
-import { DatasService } from '../../../services/datas.service';
+import { DatasService } from '../../../../services/datas.service';
-import { elapsedTime, initTime } from '../helpers';
+import { elapsedTime, initTime } from '../../helpers';
-import type { BaseModelSqlv2 } from '../../../db/BaseModelSqlv2';
+import type { BaseModelSqlv2 } from '../../../../db/BaseModelSqlv2';
-import type { View } from '../../../models';
+import type { View } from '../../../../models';
 
 @Injectable()
 export class ExportService {

packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@@ -9,28 +9,28 @@ import {
   reverseGet,
   withoutId,
   withoutNull,
-} from '../../../helpers/exportImportHelpers';
+} from '../../../../helpers/exportImportHelpers';
-import { NcError } from '../../../helpers/catchError';
+import { NcError } from '../../../../helpers/catchError';
-import { Base, Column, Model, Project } from '../../../models';
+import { Base, Column, Model, Project } from '../../../../models';
-import { TablesService } from '../../../services/tables.service';
+import { TablesService } from '../../../../services/tables.service';
-import { ColumnsService } from '../../../services/columns.service';
+import { ColumnsService } from '../../../../services/columns.service';
-import { FiltersService } from '../../../services/filters.service';
+import { FiltersService } from '../../../../services/filters.service';
-import { SortsService } from '../../../services/sorts.service';
+import { SortsService } from '../../../../services/sorts.service';
-import { ViewColumnsService } from '../../../services/view-columns.service';
+import { ViewColumnsService } from '../../../../services/view-columns.service';
-import { GridColumnsService } from '../../../services/grid-columns.service';
+import { GridColumnsService } from '../../../../services/grid-columns.service';
-import { FormColumnsService } from '../../../services/form-columns.service';
+import { FormColumnsService } from '../../../../services/form-columns.service';
-import { GridsService } from '../../../services/grids.service';
+import { GridsService } from '../../../../services/grids.service';
-import { FormsService } from '../../../services/forms.service';
+import { FormsService } from '../../../../services/forms.service';
-import { GalleriesService } from '../../../services/galleries.service';
+import { GalleriesService } from '../../../../services/galleries.service';
-import { KanbansService } from '../../../services/kanbans.service';
+import { KanbansService } from '../../../../services/kanbans.service';
-import { HooksService } from '../../../services/hooks.service';
+import { HooksService } from '../../../../services/hooks.service';
-import { ViewsService } from '../../../services/views.service';
+import { ViewsService } from '../../../../services/views.service';
-import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2';
+import NcPluginMgrv2 from '../../../../helpers/NcPluginMgrv2';
-import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
+import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
-import { elapsedTime, initTime } from '../helpers';
+import { elapsedTime, initTime } from '../../helpers';
 import type { Readable } from 'stream';
 import type { ViewCreateReqType } from 'nocodb-sdk';
-import type { LinkToAnotherRecordColumn, User, View } from '../../../models';
+import type { LinkToAnotherRecordColumn, User, View } from '../../../../models';
 
 @Injectable()
 export class ImportService {

packages/nocodb/src/modules/jobs/jobs/jobs-log.service.ts

@@ -0,0 +1,16 @@
+import { Injectable } from '@nestjs/common';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { JobEvents } from '../../../interface/Jobs';
+import type { Job } from 'bull';
+
+@Injectable()
+export class JobsLogService {
+  constructor(private eventEmitter: EventEmitter2) {}
+
+  sendLog(job: Job, data: { message: string }) {
+    this.eventEmitter.emit(JobEvents.LOG, {
+      id: job.id.toString(),
+      data,
+    });
+  }
+}

packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts

@@ -0,0 +1,107 @@
+import {
+  OnQueueActive,
+  OnQueueCompleted,
+  OnQueueFailed,
+  Processor,
+} from '@nestjs/bull';
+import { Job } from 'bull';
+import boxen from 'boxen';
+import { EventEmitter2, OnEvent } from '@nestjs/event-emitter';
+import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
+import { JobsRedisService } from './jobs-redis.service';
+
+@Processor(JOBS_QUEUE)
+export class JobsEventService {
+  constructor(
+    private jobsRedisService: JobsRedisService,
+    private eventEmitter: EventEmitter2,
+  ) {}
+
+  @OnQueueActive()
+  onActive(job: Job) {
+    if (process.env.NC_WORKER_CONTAINER === 'true') {
+      this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
+        cmd: JobEvents.STATUS,
+        id: job.id.toString(),
+        status: JobStatus.ACTIVE,
+      });
+    } else {
+      this.eventEmitter.emit(JobEvents.STATUS, {
+        id: job.id.toString(),
+        status: JobStatus.ACTIVE,
+      });
+    }
+  }
+
+  @OnQueueFailed()
+  onFailed(job: Job, error: Error) {
+    console.error(
+      boxen(
+        `---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
+        {
+          padding: 1,
+          borderStyle: 'double',
+          borderColor: 'yellow',
+        },
+      ),
+    );
+
+    if (process.env.NC_WORKER_CONTAINER === 'true') {
+      this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
+        cmd: JobEvents.STATUS,
+        id: job.id.toString(),
+        status: JobStatus.FAILED,
+        data: {
+          error: {
+            message: error?.message,
+          },
+        },
+      });
+    } else {
+      this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`);
+      this.eventEmitter.emit(JobEvents.STATUS, {
+        id: job.id.toString(),
+        status: JobStatus.FAILED,
+        data: {
+          error: {
+            message: error?.message,
+          },
+        },
+      });
+    }
+  }
+
+  @OnQueueCompleted()
+  onCompleted(job: Job, data: any) {
+    if (process.env.NC_WORKER_CONTAINER === 'true') {
+      this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
+        cmd: JobEvents.STATUS,
+        id: job.id.toString(),
+        status: JobStatus.COMPLETED,
+        data: {
+          result: data,
+        },
+      });
+    } else {
+      this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`);
+      this.eventEmitter.emit(JobEvents.STATUS, {
+        id: job.id.toString(),
+        status: JobStatus.COMPLETED,
+        data: {
+          result: data,
+        },
+      });
+    }
+  }
+
+  @OnEvent(JobEvents.LOG)
+  onLog(data: { id: string; data: { message: string } }) {
+    if (process.env.NC_WORKER_CONTAINER === 'true') {
+      this.jobsRedisService.publish(`jobs-${data.id}`, {
+        cmd: JobEvents.LOG,
+        id: data.id,
+        data: data.data,
+      });
+    }
+  }
+}

packages/nocodb/src/modules/jobs/redis/jobs-redis.service.ts

@@ -0,0 +1,53 @@
+import { Injectable } from '@nestjs/common';
+import Redis from 'ioredis';
+
+@Injectable()
+export class JobsRedisService {
+  private redisClient: Redis;
+  private redisSubscriber: Redis;
+  private unsubscribeCallbacks: { [key: string]: () => void } = {};
+
+  constructor() {
+    if (process.env.NC_WORKER_CONTAINER === 'true') {
+      this.redisClient = new Redis(process.env.NC_REDIS_URL);
+      return;
+    }
+    this.redisSubscriber = new Redis(process.env.NC_REDIS_URL);
+  }
+
+  publish(channel: string, message: string | any) {
+    if (typeof message === 'string') {
+      this.redisClient.publish(channel, message);
+    } else {
+      try {
+        this.redisClient.publish(channel, JSON.stringify(message));
+      } catch (e) {
+        console.error(e);
+      }
+    }
+  }
+
+  subscribe(channel: string, callback: (message: any) => void) {
+    this.redisSubscriber.subscribe(channel);
+
+    const onMessage = (_channel, message) => {
+      try {
+        message = JSON.parse(message);
+      } catch (e) {}
+      callback(message);
+    };
+
+    this.redisSubscriber.on('message', onMessage);
+    this.unsubscribeCallbacks[channel] = () => {
+      this.redisSubscriber.unsubscribe(channel);
+      this.redisSubscriber.off('message', onMessage);
+    };
+  }
+
+  unsubscribe(channel: string) {
+    if (this.unsubscribeCallbacks[channel]) {
+      this.unsubscribeCallbacks[channel]();
+      delete this.unsubscribeCallbacks[channel];
+    }
+  }
+}

packages/nocodb/src/modules/jobs/redis/jobs.service.ts

@@ -0,0 +1,98 @@
+import { InjectQueue } from '@nestjs/bull';
+import { Injectable } from '@nestjs/common';
+import { Queue } from 'bull';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
+import { JobsRedisService } from './jobs-redis.service';
+import type { OnModuleInit } from '@nestjs/common';
+
+@Injectable()
+export class JobsService implements OnModuleInit {
+  constructor(
+    @InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue,
+    private jobsRedisService: JobsRedisService,
+    private eventEmitter: EventEmitter2,
+  ) {}
+
+  // pause primary instance queue
+  async onModuleInit() {
+    if (process.env.NC_WORKER_CONTAINER !== 'true') {
+      await this.jobsQueue.pause(true);
+    }
+  }
+
+  async add(name: string, data: any) {
+    // resume primary instance queue if there is no worker
+    const workerCount = (await this.jobsQueue.getWorkers()).length;
+    const localWorkerPaused = await this.jobsQueue.isPaused(true);
+
+    // if there is no worker and primary instance queue is paused, resume it
+    // if there is any worker and primary instance queue is not paused, pause it
+    if (workerCount < 1 && localWorkerPaused) {
+      await this.jobsQueue.resume(true);
+    } else if (workerCount > 0 && !localWorkerPaused) {
+      await this.jobsQueue.pause(true);
+    }
+
+    const job = await this.jobsQueue.add(name, data);
+
+    // subscribe to job events
+    this.jobsRedisService.subscribe(`jobs-${job.id.toString()}`, (data) => {
+      const cmd = data.cmd;
+      delete data.cmd;
+      switch (cmd) {
+        case JobEvents.STATUS:
+          this.eventEmitter.emit(JobEvents.STATUS, data);
+          if ([JobStatus.COMPLETED, JobStatus.FAILED].includes(data.status)) {
+            this.jobsRedisService.unsubscribe(`jobs-${data.id.toString()}`);
+          }
+          break;
+        case JobEvents.LOG:
+          this.eventEmitter.emit(JobEvents.LOG, data);
+          break;
+      }
+    });
+
+    return job;
+  }
+
+  async jobStatus(jobId: string) {
+    const job = await this.jobsQueue.getJob(jobId);
+    if (job) {
+      return await job.getState();
+    }
+  }
+
+  async jobList() {
+    return await this.jobsQueue.getJobs([
+      JobStatus.ACTIVE,
+      JobStatus.WAITING,
+      JobStatus.DELAYED,
+      JobStatus.PAUSED,
+    ]);
+  }
+
+  async getJobWithData(data: any) {
+    const jobs = await this.jobsQueue.getJobs([
+      // 'completed',
+      JobStatus.WAITING,
+      JobStatus.ACTIVE,
+      JobStatus.DELAYED,
+      // 'failed',
+      JobStatus.PAUSED,
+    ]);
+
+    const job = jobs.find((j) => {
+      for (const key in data) {
+        if (j.data[key]) {
+          if (j.data[key] !== data[key]) return false;
+        } else {
+          return false;
+        }
+      }
+      return true;
+    });
+
+    return job;
+  }
+}

packages/nocodb/src/modules/metas/metas.module.ts

@@ -67,7 +67,6 @@ import { UtilsService } from '../../services/utils.service';
 import { ViewColumnsService } from '../../services/view-columns.service';
 import { ViewsService } from '../../services/views.service';
 import { ApiDocsService } from '../../services/api-docs/api-docs.service';
-import { EventEmitterModule } from '../event-emitter/event-emitter.module';
 import { GlobalModule } from '../global/global.module';
 import { ProjectUsersController } from '../../controllers/project-users.controller';
 import { ProjectUsersService } from '../../services/project-users/project-users.service';
@@ -83,38 +82,42 @@ import { ProjectUsersService } from '../../services/project-users/project-users.
     GlobalModule,
   ],
   controllers: [
-    ApiDocsController,
+    ...(process.env.NC_WORKER_CONTAINER !== 'true'
-    ApiTokensController,
+      ? [
-    AttachmentsController,
+          ApiDocsController,
-    AuditsController,
+          ApiTokensController,
-    BasesController,
+          AttachmentsController,
-    CachesController,
+          AuditsController,
-    ColumnsController,
+          BasesController,
-    FiltersController,
+          CachesController,
-    FormColumnsController,
+          ColumnsController,
-    FormsController,
+          FiltersController,
-    GalleriesController,
+          FormColumnsController,
-    GridColumnsController,
+          FormsController,
-    GridsController,
+          GalleriesController,
-    HooksController,
+          GridColumnsController,
-    KanbansController,
+          GridsController,
-    MapsController,
+          HooksController,
-    MetaDiffsController,
+          KanbansController,
-    ModelVisibilitiesController,
+          MapsController,
-    OrgLcenseController,
+          MetaDiffsController,
-    OrgTokensController,
+          ModelVisibilitiesController,
-    OrgUsersController,
+          OrgLcenseController,
-    PluginsController,
+          OrgTokensController,
-    ProjectUsersController,
+          OrgUsersController,
-    ProjectsController,
+          PluginsController,
-    PublicMetasController,
+          ProjectUsersController,
-    ViewsController,
+          ProjectsController,
-    ViewColumnsController,
+          PublicMetasController,
-    UtilsController,
+          ViewsController,
-    TablesController,
+          ViewColumnsController,
-    SyncController,
+          UtilsController,
-    SortsController,
+          TablesController,
-    SharedBasesController,
+          SyncController,
+          SortsController,
+          SharedBasesController,
+        ]
+      : []),
   ],
   providers: [
     ApiDocsService,

packages/nocodb/src/modules/test/test.module.ts

@@ -2,6 +2,8 @@ import { Module } from '@nestjs/common';
 import { TestController } from '../../controllers/test/test.controller';
 
 @Module({
-  controllers: [TestController],
+  controllers: [
+    ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [TestController] : []),
+  ],
 })
 export class TestModule {}

packages/nocodb/src/modules/users/users.module.ts

@@ -10,7 +10,9 @@ import { UsersController } from '../../controllers/users/users.controller';
 
 @Module({
   imports: [GlobalModule, PassportModule],
-  controllers: [UsersController],
+  controllers: [
+    ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [UsersController] : []),
+  ],
   providers: [UsersService, GoogleStrategyProvider],
   exports: [UsersService],
 })

packages/nocodb/src/plugins/storage/Local.ts

@@ -3,7 +3,7 @@ import path from 'path';
 import { promisify } from 'util';
 import mkdirp from 'mkdirp';
 import axios from 'axios';
-import NcConfigFactory from '../../utils/NcConfigFactory';
+import { getToolDir } from '../../utils/nc-config';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 
@@ -11,7 +11,7 @@ export default class Local implements IStorageAdapterV2 {
   constructor() {}
 
   public async fileCreate(key: string, file: XcFile): Promise<any> {
-    const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
+    const destPath = path.join(getToolDir(), ...key.split('/'));
     try {
       await mkdirp(path.dirname(destPath));
       const data = await promisify(fs.readFile)(file.path);
@@ -24,7 +24,7 @@ export default class Local implements IStorageAdapterV2 {
   }
 
   async fileCreateByUrl(key: string, url: string): Promise<any> {
-    const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
+    const destPath = path.join(getToolDir(), ...key.split('/'));
     return new Promise((resolve, reject) => {
       axios
         .get(url, {
@@ -71,10 +71,7 @@ export default class Local implements IStorageAdapterV2 {
     stream: Readable,
   ): Promise<void> {
     return new Promise((resolve, reject) => {
-      const destPath = path.join(
+      const destPath = path.join(getToolDir(), ...key.split('/'));
-        NcConfigFactory.getToolDir(),
-        ...key.split('/'),
-      );
       try {
         mkdirp(path.dirname(destPath)).then(() => {
           const writableStream = fs.createWriteStream(destPath);
@@ -89,12 +86,12 @@ export default class Local implements IStorageAdapterV2 {
   }
 
   public async fileReadByStream(key: string): Promise<Readable> {
-    const srcPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
+    const srcPath = path.join(getToolDir(), ...key.split('/'));
     return fs.createReadStream(srcPath, { encoding: 'utf8' });
   }
 
   public async getDirectoryList(key: string): Promise<string[]> {
-    const destDir = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
+    const destDir = path.join(getToolDir(), ...key.split('/'));
     return fs.promises.readdir(destDir);
   }
 
@@ -106,7 +103,7 @@ export default class Local implements IStorageAdapterV2 {
   public async fileRead(filePath: string): Promise<any> {
     try {
       const fileData = await fs.promises.readFile(
-        path.join(NcConfigFactory.getToolDir(), ...filePath.split('/')),
+        path.join(getToolDir(), ...filePath.split('/')),
       );
       return fileData;
     } catch (e) {

packages/nocodb/src/services/app-init.service.spec.ts

@@ -1,19 +0,0 @@
-import { Test } from '@nestjs/testing';
-import { AppInitService } from './app-init.service';
-import type { TestingModule } from '@nestjs/testing';
-
-describe('AppInitService', () => {
-  let service: AppInitService;
-
-  beforeEach(async () => {
-    const module: TestingModule = await Test.createTestingModule({
-      providers: [AppInitService],
-    }).compile();
-
-    service = module.get<AppInitService>(AppInitService);
-  });
-
-  it('should be defined', () => {
-    expect(service).toBeDefined();
-  });
-});

packages/nocodb/src/services/app-init.service.ts

@@ -1,79 +0,0 @@
-import { T } from 'nc-help';
-import NocoCache from '../cache/NocoCache';
-import { Connection } from '../connection/connection';
-import initAdminFromEnv from '../helpers/initAdminFromEnv';
-import NcPluginMgrv2 from '../helpers/NcPluginMgrv2';
-import { MetaService } from '../meta/meta.service';
-import { User } from '../models';
-import Noco from '../Noco';
-import getInstance from '../utils/getInstance';
-import NcConfigFactory from '../utils/NcConfigFactory';
-import NcUpgrader from '../version-upgrader/NcUpgrader';
-import type { IEventEmitter } from '../modules/event-emitter/event-emitter.interface';
-import type { Provider } from '@nestjs/common';
-
-export class AppInitService {
-  private readonly config: any;
-
-  constructor(config) {
-    this.config = config;
-  }
-
-  get appConfig(): any {
-    return this.config;
-  }
-}
-
-export const appInitServiceProvider: Provider = {
-  provide: AppInitService,
-  // initialize app,
-  // 1. init cache
-  // 2. init db connection and create if not exist
-  // 3. init meta and set to Noco
-  // 4. init jwt
-  // 5. init plugin manager
-  // 6. run upgrader
-  useFactory: async (
-    connection: Connection,
-    metaService: MetaService,
-    eventEmitter: IEventEmitter,
-  ) => {
-    process.env.NC_VERSION = '0107004';
-
-    await NocoCache.init();
-
-    await connection.init();
-
-    await NcConfigFactory.metaDbCreateIfNotExist(connection.config);
-
-    await metaService.init();
-
-    // todo: remove
-    // temporary hack
-    Noco._ncMeta = metaService;
-    Noco.config = connection.config;
-    Noco.eventEmitter = eventEmitter;
-
-    // init jwt secret
-    await Noco.initJwt();
-
-    // load super admin user from env if env is set
-    await initAdminFromEnv(metaService);
-
-    // init plugin manager
-    await NcPluginMgrv2.init(Noco.ncMeta);
-    await Noco.loadEEState();
-
-    // run upgrader
-    await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
-
-    T.init({
-      instance: getInstance,
-    });
-    T.emit('evt_app_started', await User.count());
-
-    // todo: move app config to app-init service
-    return new AppInitService(connection.config);
-  },
-  inject: [Connection, MetaService, 'IEventEmitter'],
-};

packages/nocodb/src/services/auth.service.ts

@@ -5,18 +5,13 @@ import * as bcrypt from 'bcryptjs';
 
 import { v4 as uuidv4 } from 'uuid';
 import Noco from '../Noco';
-import { Connection } from '../connection/connection';
 import { genJwt } from './users/helpers';
 import { UsersService } from './users/users.service';
 import type { CreateUserDto } from '../controllers/auth.controller';
 
 @Injectable()
 export class AuthService {
-  constructor(
+  constructor(private usersService: UsersService) {}
-    private usersService: UsersService,
-    // private jwtService: JwtService,
-    private connection: Connection,
-  ) {}
 
   async validateUser(email: string, pass: string): Promise<any> {
     const user = await this.usersService.findOne(email);

packages/nocodb/src/services/projects.service.ts

@@ -11,7 +11,7 @@ import syncMigration from '../helpers/syncMigration';
 import { Project, ProjectUser } from '../models';
 import Noco from '../Noco';
 import extractRolesObj from '../utils/extractRolesObj';
-import NcConfigFactory from '../utils/NcConfigFactory';
+import { getToolDir } from '../utils/nc-config';
 import type { ProjectUpdateReqType } from 'nocodb-sdk';
 import type { ProjectReqType } from 'nocodb-sdk';
 
@@ -96,7 +96,7 @@ export class ProjectsService {
         // if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each project
         // each file will be named as nc_<random_id>.db
         const fs = require('fs');
-        const toolDir = NcConfigFactory.getToolDir();
+        const toolDir = getToolDir();
         const nanoidv2 = customAlphabet(
           '1234567890abcdefghijklmnopqrstuvwxyz',
           14,

packages/nocodb/src/services/utils.service.ts

@@ -9,7 +9,7 @@ import { Project, User } from '../models';
 import Noco from '../Noco';
 import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2';
 import { MetaTable } from '../utils/globals';
-import NcConfigFactory from '../utils/NcConfigFactory';
+import { jdbcToXcConfig } from '../utils/nc-config/helpers';
 import { packageVersion } from '../utils/packageVersion';
 
 const versionCache = {
@@ -186,7 +186,7 @@ export class UtilsService {
   }) {
     const { url } = param.body;
     try {
-      const connectionConfig = NcConfigFactory.extractXcUrlFromJdbc(url, true);
+      const connectionConfig = jdbcToXcConfig(url);
       return connectionConfig;
     } catch (error) {
       return NcError.internalServerError(

packages/nocodb/src/utils/NcConfigFactory.ts

@@ -1,755 +0,0 @@
-import fs from 'fs';
-import { URL } from 'url';
-import { promisify } from 'util';
-import * as path from 'path';
-import parseDbUrl from 'parse-database-url';
-import { SqlClientFactory } from '../db/sql-client/lib/SqlClientFactory';
-// import SqlClientFactory from '../db/sql-client/lib/SqlClientFactory';
-// import type {
-//   AuthConfig,
-//   DbConfig,
-//   MailerConfig,
-//   NcConfig,
-// } from '../../interface/config';
-
-// const {
-//   uniqueNamesGenerator,
-//   starWars,
-//   adjectives,
-//   animals,
-// } = require('unique-names-generator');
-
-type NcConfig = any;
-type DbConfig = any;
-
-const driverClientMapping = {
-  mysql: 'mysql2',
-  mariadb: 'mysql2',
-  postgres: 'pg',
-  postgresql: 'pg',
-  sqlite: 'sqlite3',
-  mssql: 'mssql',
-};
-
-const defaultClientPortMapping = {
-  mysql: 3306,
-  mysql2: 3306,
-  postgres: 5432,
-  pg: 5432,
-  mssql: 1433,
-};
-
-const defaultConnectionConfig: any = {
-  // https://github.com/knex/knex/issues/97
-  // timezone: process.env.NC_TIMEZONE || 'UTC',
-  dateStrings: true,
-};
-
-// default knex options
-const defaultConnectionOptions = {
-  pool: {
-    min: 0,
-    max: 10,
-  },
-};
-
-const knownQueryParams = [
-  {
-    parameter: 'database',
-    aliases: ['d', 'db'],
-  },
-  {
-    parameter: 'password',
-    aliases: ['p'],
-  },
-  {
-    parameter: 'user',
-    aliases: ['u'],
-  },
-  {
-    parameter: 'title',
-    aliases: ['t'],
-  },
-  {
-    parameter: 'keyFilePath',
-    aliases: [],
-  },
-  {
-    parameter: 'certFilePath',
-    aliases: [],
-  },
-  {
-    parameter: 'caFilePath',
-    aliases: [],
-  },
-  {
-    parameter: 'ssl',
-    aliases: [],
-  },
-  {
-    parameter: 'options',
-    aliases: ['opt', 'opts'],
-  },
-];
-
-export default class NcConfigFactory {
-  public static async make(): Promise<any> {
-    await this.jdbcToXcUrl();
-
-    const ncConfig = new NcConfigFactory();
-
-    ncConfig.auth = {
-      jwt: {
-        secret: process.env.NC_AUTH_JWT_SECRET,
-      },
-    };
-
-    ncConfig.port = +(process?.env?.PORT ?? 8080);
-    ncConfig.env = '_noco'; // process.env?.NODE_ENV || 'dev';
-    ncConfig.workingEnv = '_noco'; // process.env?.NODE_ENV || 'dev';
-    // ncConfig.toolDir = this.getToolDir();
-    ncConfig.projectType =
-      ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest';
-
-    if (ncConfig.meta?.db?.connection?.filename) {
-      ncConfig.meta.db.connection.filename = path.join(
-        this.getToolDir(),
-        ncConfig.meta.db.connection.filename,
-      );
-    }
-
-    if (process.env.NC_DB) {
-      ncConfig.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
-    } else if (process.env.NC_DB_JSON) {
-      ncConfig.meta.db = JSON.parse(process.env.NC_DB_JSON);
-    } else if (process.env.NC_DB_JSON_FILE) {
-      const filePath = process.env.NC_DB_JSON_FILE;
-
-      if (!(await promisify(fs.exists)(filePath))) {
-        throw new Error(`NC_DB_JSON_FILE not found: ${filePath}`);
-      }
-
-      const fileContent = await promisify(fs.readFile)(filePath, {
-        encoding: 'utf8',
-      });
-      ncConfig.meta.db = JSON.parse(fileContent);
-    }
-
-    if (process.env.NC_TRY) {
-      ncConfig.try = true;
-      ncConfig.meta.db = {
-        client: 'sqlite3',
-        connection: ':memory:',
-        pool: {
-          min: 1,
-          max: 1,
-          // disposeTimeout: 360000*1000,
-          idleTimeoutMillis: 360000 * 1000,
-        },
-      } as any;
-    }
-
-    if (process.env.NC_PUBLIC_URL) {
-      ncConfig.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
-      // ncConfig.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
-      ncConfig.publicUrl = process.env.NC_PUBLIC_URL;
-    }
-
-    if (process.env.NC_DASHBOARD_URL) {
-      ncConfig.dashboardPath = process.env.NC_DASHBOARD_URL;
-    }
-
-    return ncConfig;
-  }
-
-  public static getToolDir() {
-    return process.env.NC_TOOL_DIR || process.cwd();
-  }
-
-  public static hasDbUrl(): boolean {
-    return Object.keys(process.env).some((envKey) =>
-      envKey.startsWith('NC_DB_URL'),
-    );
-  }
-
-  public static makeFromUrls(urls: string[]): NcConfig {
-    const config = new NcConfigFactory();
-
-    // config.envs[process.env.NODE_ENV || 'dev'].db = [];
-    config.envs['_noco'].db = [];
-    for (const [i, url] of Object.entries(urls)) {
-      // config.envs[process.env.NODE_ENV || 'dev'].db.push(this.urlToDbConfig(url, i));
-      config.envs['_noco'].db.push(this.urlToDbConfig(url, i));
-    }
-
-    return config;
-  }
-
-  public static urlToDbConfig(
-    urlString: string,
-    key = '',
-    config?: NcConfigFactory,
-    type?: string,
-  ): DbConfig {
-    const url = new URL(urlString);
-
-    let dbConfig: DbConfig;
-
-    if (url.protocol.startsWith('sqlite3')) {
-      dbConfig = {
-        client: 'sqlite3',
-        connection: {
-          client: 'sqlite3',
-          connection: {
-            filename:
-              url.searchParams.get('d') || url.searchParams.get('database'),
-          },
-          database:
-            url.searchParams.get('d') || url.searchParams.get('database'),
-        },
-      } as any;
-    } else {
-      const parsedQuery = {};
-      for (const [key, value] of url.searchParams.entries()) {
-        const fnd = knownQueryParams.find(
-          (param) => param.parameter === key || param.aliases.includes(key),
-        );
-        if (fnd) {
-          parsedQuery[fnd.parameter] = value;
-        } else {
-          parsedQuery[key] = value;
-        }
-      }
-
-      dbConfig = {
-        client: url.protocol.replace(':', ''),
-        connection: {
-          ...defaultConnectionConfig,
-          ...parsedQuery,
-          host: url.hostname,
-          port: +url.port,
-        },
-        // pool: {
-        //   min: 1,
-        //   max: 1
-        // },
-        acquireConnectionTimeout: 600000,
-      } as any;
-
-      if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
-        dbConfig.connection.ssl = true;
-      }
-
-      if (
-        url.searchParams.get('keyFilePath') &&
-        url.searchParams.get('certFilePath') &&
-        url.searchParams.get('caFilePath')
-      ) {
-        dbConfig.connection.ssl = {
-          keyFilePath: url.searchParams.get('keyFilePath'),
-          certFilePath: url.searchParams.get('certFilePath'),
-          caFilePath: url.searchParams.get('caFilePath'),
-        };
-      }
-    }
-
-    if (config && !config.title) {
-      config.title =
-        url.searchParams.get('t') ||
-        url.searchParams.get('title') ||
-        this.generateRandomTitle();
-    }
-
-    Object.assign(dbConfig, {
-      meta: {
-        tn: 'nc_evolutions',
-        allSchemas:
-          !!url.searchParams.get('allSchemas') ||
-          !(url.searchParams.get('d') || url.searchParams.get('database')),
-        api: {
-          prefix: url.searchParams.get('apiPrefix') || '',
-          swagger: true,
-          type:
-            type ||
-            ((url.searchParams.get('api') ||
-              url.searchParams.get('a')) as any) ||
-            'rest',
-        },
-        dbAlias: url.searchParams.get('dbAlias') || `db${key}`,
-        metaTables: 'db',
-        migrations: {
-          disabled: false,
-          name: 'nc_evolutions',
-        },
-      },
-    });
-
-    return dbConfig;
-  }
-
-  private static generateRandomTitle(): string {
-    return ''; /*uniqueNamesGenerator({
-      dictionaries: [[starWars], [adjectives, animals]][
-        Math.floor(Math.random() * 2)
-      ],
-    })
-      .toLowerCase()
-      .replace(/[ -]/g, '_');*/
-  }
-
-  static async metaUrlToDbConfig(urlString) {
-    const url = new URL(urlString);
-
-    let dbConfig;
-
-    if (url.protocol.startsWith('sqlite3')) {
-      const db = url.searchParams.get('d') || url.searchParams.get('database');
-      dbConfig = {
-        client: 'sqlite3',
-        connection: {
-          filename: db,
-        },
-        ...(db === ':memory:'
-          ? {
-              pool: {
-                min: 1,
-                max: 1,
-                // disposeTimeout: 360000*1000,
-                idleTimeoutMillis: 360000 * 1000,
-              },
-            }
-          : {}),
-      };
-    } else {
-      const parsedQuery = {};
-      for (const [key, value] of url.searchParams.entries()) {
-        const fnd = knownQueryParams.find(
-          (param) => param.parameter === key || param.aliases.includes(key),
-        );
-        if (fnd) {
-          parsedQuery[fnd.parameter] = value;
-        } else {
-          parsedQuery[key] = value;
-        }
-      }
-
-      dbConfig = {
-        client: url.protocol.replace(':', ''),
-        connection: {
-          ...defaultConnectionConfig,
-          ...parsedQuery,
-          host: url.hostname,
-          port: +url.port,
-        },
-        acquireConnectionTimeout: 600000,
-        ...(url.searchParams.has('search_path')
-          ? {
-              searchPath: url.searchParams.get('search_path').split(','),
-            }
-          : {}),
-      };
-      if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
-        dbConfig.connection.ssl = true;
-      }
-    }
-    url.searchParams.forEach((_value, key) => {
-      let value: any = _value;
-      if (value === 'true') {
-        value = true;
-      } else if (value === 'false') {
-        value = false;
-      } else if (/^\d+$/.test(value)) {
-        value = +value;
-      }
-      // todo: implement config read from JSON file or JSON env val read
-      if (
-        ![
-          'password',
-          'p',
-          'database',
-          'd',
-          'user',
-          'u',
-          'search_path',
-        ].includes(key)
-      ) {
-        key.split('.').reduce((obj, k, i, arr) => {
-          return (obj[k] = i === arr.length - 1 ? value : obj[k] || {});
-        }, dbConfig);
-      }
-    });
-
-    if (
-      dbConfig?.connection?.ssl &&
-      typeof dbConfig?.connection?.ssl === 'object'
-    ) {
-      if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) {
-        dbConfig.connection.ssl.ca = (
-          await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath)
-        ).toString();
-        delete dbConfig.connection.ssl.caFilePath;
-      }
-      if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) {
-        dbConfig.connection.ssl.key = (
-          await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath)
-        ).toString();
-        delete dbConfig.connection.ssl.keyFilePath;
-      }
-      if (
-        dbConfig.connection.ssl.certFilePath &&
-        !dbConfig.connection.ssl.cert
-      ) {
-        dbConfig.connection.ssl.cert = (
-          await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath)
-        ).toString();
-        delete dbConfig.connection.ssl.certFilePath;
-      }
-    }
-
-    return dbConfig;
-  }
-
-  public static async makeProjectConfigFromUrl(
-    url,
-    type?: string,
-  ): Promise<NcConfig> {
-    const config = new NcConfigFactory();
-    const dbConfig = this.urlToDbConfig(url, '', config, type);
-    // config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
-    config.envs['_noco'].db.push(dbConfig);
-
-    if (process.env.NC_AUTH_ADMIN_SECRET) {
-      config.auth = {
-        masterKey: {
-          secret: process.env.NC_AUTH_ADMIN_SECRET,
-        },
-      };
-    } else if (process.env.NC_NO_AUTH) {
-      config.auth = {
-        disabled: true,
-      };
-      // } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
-    } else if (config?.envs?.['_noco']?.db?.[0]) {
-      config.auth = {
-        jwt: {
-          // dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
-          dbAlias:
-            process.env.NC_AUTH_JWT_DB_ALIAS ||
-            config.envs['_noco'].db[0].meta.dbAlias,
-          secret: process.env.NC_AUTH_JWT_SECRET,
-        },
-      };
-    }
-
-    if (process.env.NC_DB) {
-      config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
-    }
-
-    if (process.env.NC_TRY) {
-      config.try = true;
-      config.meta.db = {
-        client: 'sqlite3',
-        connection: ':memory:',
-        pool: {
-          min: 1,
-          max: 1,
-          // disposeTimeout: 360000*1000,
-          idleTimeoutMillis: 360000 * 1000,
-        },
-      } as any;
-    }
-
-    if (process.env.NC_MAILER) {
-      config.mailer = {
-        from: process.env.NC_MAILER_FROM,
-        options: {
-          host: process.env.NC_MAILER_HOST,
-          port: parseInt(process.env.NC_MAILER_PORT, 10),
-          secure: process.env.NC_MAILER_SECURE === 'true',
-          auth: {
-            user: process.env.NC_MAILER_USER,
-            pass: process.env.NC_MAILER_PASS,
-          },
-        },
-      };
-    }
-
-    if (process.env.NC_PUBLIC_URL) {
-      // config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
-      config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
-      config.publicUrl = process.env.NC_PUBLIC_URL;
-    }
-
-    config.port = +(process?.env?.PORT ?? 8080);
-    // config.env = process.env?.NODE_ENV || 'dev';
-    // config.workingEnv = process.env?.NODE_ENV || 'dev';
-    config.env = '_noco';
-    config.workingEnv = '_noco';
-    config.toolDir = this.getToolDir();
-    config.projectType =
-      type ||
-      config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type ||
-      'rest';
-
-    return config;
-  }
-
-  public static async makeProjectConfigFromConnection(
-    dbConnectionConfig: any,
-    type?: string,
-  ): Promise<NcConfig> {
-    const config = new NcConfigFactory();
-    let dbConfig = dbConnectionConfig;
-
-    if (dbConfig.client === 'sqlite3') {
-      dbConfig = {
-        client: 'sqlite3',
-        connection: {
-          ...dbConnectionConfig,
-          database: dbConnectionConfig.connection.filename,
-        },
-      };
-    }
-
-    // todo:
-    const key = '';
-    Object.assign(dbConfig, {
-      meta: {
-        tn: 'nc_evolutions',
-        api: {
-          prefix: '',
-          swagger: true,
-          type: type || 'rest',
-        },
-        dbAlias: `db${key}`,
-        metaTables: 'db',
-        migrations: {
-          disabled: false,
-          name: 'nc_evolutions',
-        },
-      },
-    });
-
-    // config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
-    config.envs['_noco'].db.push(dbConfig);
-
-    if (process.env.NC_AUTH_ADMIN_SECRET) {
-      config.auth = {
-        masterKey: {
-          secret: process.env.NC_AUTH_ADMIN_SECRET,
-        },
-      };
-    } else if (process.env.NC_NO_AUTH) {
-      config.auth = {
-        disabled: true,
-      };
-      // } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
-    } else if (config?.envs?.['_noco']?.db?.[0]) {
-      config.auth = {
-        jwt: {
-          // dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
-          dbAlias:
-            process.env.NC_AUTH_JWT_DB_ALIAS ||
-            config.envs['_noco'].db[0].meta.dbAlias,
-          secret: process.env.NC_AUTH_JWT_SECRET,
-        },
-      };
-    }
-
-    if (process.env.NC_DB) {
-      config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
-    }
-
-    if (process.env.NC_TRY) {
-      config.try = true;
-      config.meta.db = {
-        client: 'sqlite3',
-        connection: ':memory:',
-        pool: {
-          min: 1,
-          max: 1,
-          // disposeTimeout: 360000*1000,
-          idleTimeoutMillis: 360000 * 1000,
-        },
-      } as any;
-    }
-
-    if (process.env.NC_PUBLIC_URL) {
-      // config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
-      config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
-      config.publicUrl = process.env.NC_PUBLIC_URL;
-    }
-
-    config.port = +(process?.env?.PORT ?? 8080);
-    // config.env = process.env?.NODE_ENV || 'dev';
-    // config.workingEnv = process.env?.NODE_ENV || 'dev';
-    config.env = '_noco';
-    config.workingEnv = '_noco';
-    config.toolDir = process.env.NC_TOOL_DIR || process.cwd();
-    config.projectType =
-      type ||
-      config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type ||
-      'rest';
-
-    return config;
-  }
-
-  public static async metaDbCreateIfNotExist(args: NcConfig) {
-    if (args.meta?.db?.client === 'sqlite3') {
-      const metaSqlClient = await SqlClientFactory.create({
-        ...args.meta.db,
-        connection: args.meta.db,
-      });
-      await metaSqlClient.createDatabaseIfNotExists({
-        database: args.meta.db?.connection?.filename,
-      });
-    } else {
-      const metaSqlClient = await SqlClientFactory.create(args.meta.db);
-      await metaSqlClient.createDatabaseIfNotExists(args.meta.db?.connection);
-      await metaSqlClient.knex.destroy();
-    }
-
-    /*    const dbPath = path.join(args.toolDir, 'xc.db')
-        const exists = fs.existsSync(dbPath);
-        if (!exists) {
-          const fd = fs.openSync(dbPath, "w");
-          fs.closeSync(fd);
-        }
-        */
-  }
-
-  public version = '0.6';
-  public port: number;
-  public auth?: any;
-  public env: 'production' | 'dev' | 'test' | string;
-  public workingEnv: string;
-  public toolDir: string;
-  public envs: {
-    [p: string]: { db: DbConfig[]; api?: any; publicUrl?: string };
-  };
-  // public projectType: "rest" | "graphql" | "grpc";
-  public queriesFolder: string | string[] = '';
-  public seedsFolder: string | string[];
-  public title: string;
-  public publicUrl: string;
-  public projectType;
-  public meta = {
-    db: {
-      client: 'sqlite3',
-      connection: {
-        filename: 'noco.db',
-      },
-    },
-  };
-  public mailer: any;
-  public try = false;
-
-  public dashboardPath = '/dashboard';
-
-  constructor() {
-    this.envs = { _noco: { db: [] } };
-  }
-
-  public static async jdbcToXcUrl() {
-    if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) {
-      const database_url = await promisify(fs.readFile)(
-        process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
-        'utf-8',
-      );
-      process.env.NC_DB = this.extractXcUrlFromJdbc(database_url);
-    } else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) {
-      process.env.NC_DB = this.extractXcUrlFromJdbc(
-        process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
-      );
-    }
-  }
-
-  public static extractXcUrlFromJdbc(url: string, rtConfig = false) {
-    // drop the jdbc prefix
-    if (url.startsWith('jdbc:')) {
-      url = url.substring(5);
-    }
-
-    const config = parseDbUrl(url);
-
-    const parsedConfig: {
-      driver?: string;
-      host?: string;
-      port?: string;
-      database?: string;
-      user?: string;
-      password?: string;
-      ssl?: string;
-    } = {};
-    for (const [key, value] of Object.entries(config)) {
-      const fnd = knownQueryParams.find(
-        (param) => param.parameter === key || param.aliases.includes(key),
-      );
-      if (fnd) {
-        parsedConfig[fnd.parameter] = value;
-      } else {
-        parsedConfig[key] = value;
-      }
-    }
-
-    if (!parsedConfig?.port)
-      parsedConfig.port =
-        defaultClientPortMapping[
-          driverClientMapping[parsedConfig.driver] || parsedConfig.driver
-        ];
-
-    if (rtConfig) {
-      const { driver, ...connectionConfig } = parsedConfig;
-
-      const client = driverClientMapping[driver] || driver;
-
-      const avoidSSL = [
-        'localhost',
-        '127.0.0.1',
-        'host.docker.internal',
-        '172.17.0.1',
-      ];
-
-      if (
-        client === 'pg' &&
-        !connectionConfig?.ssl &&
-        !avoidSSL.includes(connectionConfig.host)
-      ) {
-        connectionConfig.ssl = 'true';
-      }
-
-      return {
-        client: client,
-        connection: {
-          ...connectionConfig,
-        },
-      } as any;
-    }
-
-    const { driver, host, port, database, user, password, ...extra } =
-      parsedConfig;
-
-    const extraParams = [];
-
-    for (const [key, value] of Object.entries(extra)) {
-      extraParams.push(`${key}=${value}`);
-    }
-
-    const res = `${driverClientMapping[driver] || driver}://${host}${
-      port ? `:${port}` : ''
-    }?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${
-      database ? `d=${database}&` : ''
-    }${extraParams.join('&')}`;
-
-    return res;
-  }
-
-  // public static initOneClickDeployment() {
-  //   if (process.env.NC_ONE_CLICK) {
-  //     const url = NcConfigFactory.extractXcUrlFromJdbc(process.env.DATABASE_URL);
-  //     process.env.NC_DB = url;
-  //   }
-  // }
-}
-
-export { defaultConnectionConfig, defaultConnectionOptions };

packages/nocodb/src/utils/common/NcConnectionMgr.ts

@@ -2,7 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import SqlClientFactory from '../../db/sql-client/lib/SqlClientFactory';
 import { XKnex } from '../../db/CustomKnex';
-import { defaultConnectionConfig } from '../NcConfigFactory';
+import { defaultConnectionConfig } from '../nc-config';
 // import type { NcConfig } from '../../../interface/config';
 import type { Knex } from 'knex';
 // import type NcMetaIO from '../../meta/NcMetaIO';

packages/nocodb/src/utils/common/NcConnectionMgrv2.ts

@@ -3,7 +3,7 @@ import { XKnex } from '../../db/CustomKnex';
 import {
   defaultConnectionConfig,
   defaultConnectionOptions,
-} from '../NcConfigFactory';
+} from '../nc-config';
 import Noco from '../../Noco';
 import type Base from '../../models/Base';
 

packages/nocodb/src/utils/nc-config/NcConfig.ts

@@ -0,0 +1,182 @@
+import * as path from 'path';
+import fs from 'fs';
+import { promisify } from 'util';
+import { SqlClientFactory } from '../../db/sql-client/lib/SqlClientFactory';
+import { getToolDir, metaUrlToDbConfig } from './helpers';
+import { DriverClient } from './interfaces';
+import type { DbConfig } from './interfaces';
+
+export class NcConfig {
+  version: string;
+  meta: {
+    db: DbConfig;
+  } = {
+    db: {
+      client: DriverClient.SQLITE,
+      connection: {
+        filename: 'noco.db',
+      },
+    },
+  };
+  auth: {
+    jwt: {
+      secret: string;
+      options?: any;
+    };
+  };
+
+  // if this is true, port is not exposed
+  worker: boolean;
+
+  toolDir: string;
+
+  // exposed instance port
+  port: number;
+
+  // if this is true, use sqlite3 :memory: as meta db
+  try: boolean;
+
+  // optional
+  publicUrl?: string;
+  dashboardPath?: string;
+
+  // TODO what is this?
+  envs: any;
+
+  queriesFolder: string;
+  env: string;
+  workingEnv: string;
+  projectType: string;
+
+  private constructor() {}
+
+  public static async create(param: {
+    meta: {
+      metaUrl?: string;
+      metaJson?: string;
+      metaJsonFile?: string;
+    };
+    secret?: string;
+    port?: string | number;
+    tryMode?: boolean;
+    worker?: boolean;
+    dashboardPath?: string;
+    publicUrl?: string;
+  }): Promise<NcConfig> {
+    const { meta, secret, port, worker, tryMode, publicUrl, dashboardPath } =
+      param;
+
+    const ncConfig = new NcConfig();
+
+    ncConfig.auth = {
+      jwt: {
+        secret: secret,
+      },
+    };
+
+    ncConfig.port = +(port ?? 8080);
+    ncConfig.toolDir = getToolDir();
+    ncConfig.worker = worker ?? false;
+
+    ncConfig.env = '_noco';
+    ncConfig.workingEnv = '_noco';
+
+    ncConfig.projectType =
+      ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest';
+
+    if (ncConfig.meta?.db?.connection?.filename) {
+      ncConfig.meta.db.connection.filename = path.join(
+        ncConfig.toolDir,
+        ncConfig.meta.db.connection.filename,
+      );
+    }
+
+    if (tryMode) {
+      ncConfig.try = true;
+      ncConfig.meta.db = {
+        client: DriverClient.SQLITE,
+        connection: ':memory:' as any,
+        pool: {
+          min: 1,
+          max: 1,
+          // disposeTimeout: 360000*1000,
+          idleTimeoutMillis: 360000 * 1000,
+        },
+      };
+    } else {
+      if (meta?.metaUrl) {
+        ncConfig.meta.db = await metaUrlToDbConfig(meta.metaUrl);
+      } else if (meta?.metaJson) {
+        ncConfig.meta.db = JSON.parse(meta.metaJson);
+      } else if (meta?.metaJsonFile) {
+        if (!(await promisify(fs.exists)(meta.metaJsonFile))) {
+          throw new Error(`NC_DB_JSON_FILE not found: ${meta.metaJsonFile}`);
+        }
+        const fileContent = await promisify(fs.readFile)(meta.metaJsonFile, {
+          encoding: 'utf8',
+        });
+        ncConfig.meta.db = JSON.parse(fileContent);
+      }
+    }
+
+    if (publicUrl) {
+      ncConfig.envs['_noco'].publicUrl = publicUrl;
+      ncConfig.publicUrl = publicUrl;
+    }
+
+    if (dashboardPath) {
+      ncConfig.dashboardPath = dashboardPath;
+    }
+
+    try {
+      // make sure meta db exists
+      await ncConfig.metaDbCreateIfNotExist();
+    } catch (e) {
+      throw new Error(e);
+    }
+
+    return ncConfig;
+  }
+
+  public static async createByEnv(): Promise<NcConfig> {
+    return NcConfig.create({
+      meta: {
+        metaUrl: process.env.NC_DB,
+        metaJson: process.env.NC_DB_JSON,
+        metaJsonFile: process.env.NC_DB_JSON_FILE,
+      },
+      secret: process.env.NC_AUTH_JWT_SECRET,
+      port: process.env.NC_PORT,
+      tryMode: !!process.env.NC_TRY,
+      worker: !!process.env.NC_WORKER,
+      dashboardPath: process.env.NC_DASHBOARD_PATH,
+      publicUrl: process.env.NC_PUBLIC_URL,
+    });
+  }
+
+  private async metaDbCreateIfNotExist() {
+    if (this.meta?.db?.client === 'sqlite3') {
+      const metaSqlClient = await SqlClientFactory.create({
+        ...this.meta.db,
+        connection: this.meta.db,
+      });
+      if (this.meta.db?.connection?.filename) {
+        await metaSqlClient.createDatabaseIfNotExists({
+          database: this.meta.db?.connection?.filename,
+        });
+      } else {
+        throw new Error('Configuration missing meta db connection');
+      }
+    } else {
+      const metaSqlClient = await SqlClientFactory.create(this.meta.db);
+      if (this.meta.db?.connection?.database) {
+        await metaSqlClient.createDatabaseIfNotExists(
+          (this.meta.db as any).connection,
+        );
+        await metaSqlClient.knex.destroy();
+      } else {
+        throw new Error('Configuration missing meta db connection');
+      }
+    }
+  }
+}

packages/nocodb/src/utils/nc-config/constants.ts

@@ -0,0 +1,84 @@
+export const driverClientMapping = {
+  mysql: 'mysql2',
+  mariadb: 'mysql2',
+  postgres: 'pg',
+  postgresql: 'pg',
+  sqlite: 'sqlite3',
+  mssql: 'mssql',
+};
+
+export const defaultClientPortMapping = {
+  mysql: 3306,
+  mysql2: 3306,
+  postgres: 5432,
+  pg: 5432,
+  mssql: 1433,
+};
+
+export const defaultConnectionConfig: any = {
+  // https://github.com/knex/knex/issues/97
+  // timezone: process.env.NC_TIMEZONE || 'UTC',
+  dateStrings: true,
+};
+
+// default knex options
+export const defaultConnectionOptions = {
+  pool: {
+    min: 0,
+    max: 10,
+  },
+};
+
+export const avoidSSL = [
+  'localhost',
+  '127.0.0.1',
+  'host.docker.internal',
+  '172.17.0.1',
+];
+
+export const knownQueryParams = [
+  {
+    parameter: 'database',
+    aliases: ['d', 'db'],
+  },
+  {
+    parameter: 'password',
+    aliases: ['p'],
+  },
+  {
+    parameter: 'user',
+    aliases: ['u'],
+  },
+  {
+    parameter: 'title',
+    aliases: ['t'],
+  },
+  {
+    parameter: 'keyFilePath',
+    aliases: [],
+  },
+  {
+    parameter: 'certFilePath',
+    aliases: [],
+  },
+  {
+    parameter: 'caFilePath',
+    aliases: [],
+  },
+  {
+    parameter: 'ssl',
+    aliases: [],
+  },
+  {
+    parameter: 'options',
+    aliases: ['opt', 'opts'],
+  },
+];
+
+export enum DriverClient {
+  MYSQL = 'mysql2',
+  MSSQL = 'mssql',
+  PG = 'pg',
+  SQLITE = 'sqlite3',
+  SNOWFLAKE = 'snowflake',
+}

packages/nocodb/src/utils/nc-config/helpers.ts

@@ -0,0 +1,324 @@
+import fs from 'fs';
+import { URL } from 'url';
+import { promisify } from 'util';
+import parseDbUrl from 'parse-database-url';
+import {
+  avoidSSL,
+  defaultClientPortMapping,
+  defaultConnectionConfig,
+  defaultConnectionOptions,
+  driverClientMapping,
+  knownQueryParams,
+} from './constants';
+import { DriverClient } from './interfaces';
+import type { Connection, DbConfig } from './interfaces';
+
+export async function prepareEnv() {
+  if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) {
+    const database_url = await promisify(fs.readFile)(
+      process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
+      'utf-8',
+    );
+    process.env.NC_DB = jdbcToXcUrl(database_url);
+  } else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) {
+    process.env.NC_DB = jdbcToXcUrl(
+      process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
+    );
+  }
+}
+
+export function getToolDir() {
+  return process.env.NC_TOOL_DIR || process.cwd();
+}
+
+export function jdbcToXcConfig(url: string): DbConfig {
+  // drop the jdbc prefix
+  url.replace(/^jdbc:/, '');
+
+  const config = parseDbUrl(url);
+
+  const parsedConfig: Connection = {};
+
+  for (const [key, value] of Object.entries(config)) {
+    const fnd = knownQueryParams.find(
+      (param) => param.parameter === key || param.aliases.includes(key),
+    );
+    if (fnd) {
+      parsedConfig[fnd.parameter] = value;
+    } else {
+      parsedConfig[key] = value;
+    }
+  }
+
+  if (!parsedConfig?.port) {
+    parsedConfig.port =
+      defaultClientPortMapping[
+        driverClientMapping[parsedConfig.driver] || parsedConfig.driver
+      ];
+  }
+
+  const { driver, ...connectionConfig } = parsedConfig;
+
+  const client = driverClientMapping[driver] || driver;
+
+  if (
+    client === 'pg' &&
+    !connectionConfig?.ssl &&
+    !avoidSSL.includes(connectionConfig.host)
+  ) {
+    connectionConfig.ssl = true;
+  }
+
+  return {
+    client: client,
+    connection: {
+      ...connectionConfig,
+    },
+  } as DbConfig;
+}
+
+export function jdbcToXcUrl(url: string): string {
+  // drop the jdbc prefix
+  url.replace(/^jdbc:/, '');
+
+  const config = parseDbUrl(url);
+
+  const parsedConfig: Connection = {};
+
+  for (const [key, value] of Object.entries(config)) {
+    const fnd = knownQueryParams.find(
+      (param) => param.parameter === key || param.aliases.includes(key),
+    );
+    if (fnd) {
+      parsedConfig[fnd.parameter] = value;
+    } else {
+      parsedConfig[key] = value;
+    }
+  }
+
+  if (!parsedConfig?.port) {
+    parsedConfig.port =
+      defaultClientPortMapping[
+        driverClientMapping[parsedConfig.driver] || parsedConfig.driver
+      ];
+  }
+
+  const { driver, host, port, database, user, password, ...extra } =
+    parsedConfig;
+
+  const extraParams = [];
+
+  for (const [key, value] of Object.entries(extra)) {
+    extraParams.push(`${key}=${value}`);
+  }
+
+  const res = `${driverClientMapping[driver] || driver}://${host}${
+    port ? `:${port}` : ''
+  }?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${
+    database ? `d=${database}&` : ''
+  }${extraParams.join('&')}`;
+
+  return res;
+}
+
+export function xcUrlToDbConfig(
+  urlString: string,
+  key = '',
+  type?: string,
+): DbConfig {
+  const url = new URL(urlString);
+
+  let dbConfig: DbConfig;
+
+  if (url.protocol.startsWith('sqlite3')) {
+    dbConfig = {
+      client: 'sqlite3',
+      connection: {
+        client: 'sqlite3',
+        connection: {
+          filename:
+            url.searchParams.get('d') || url.searchParams.get('database'),
+        },
+        database: url.searchParams.get('d') || url.searchParams.get('database'),
+      },
+    } as any;
+  } else {
+    const parsedQuery = {};
+    for (const [key, value] of url.searchParams.entries()) {
+      const fnd = knownQueryParams.find(
+        (param) => param.parameter === key || param.aliases.includes(key),
+      );
+      if (fnd) {
+        parsedQuery[fnd.parameter] = value;
+      } else {
+        parsedQuery[key] = value;
+      }
+    }
+
+    dbConfig = {
+      client: url.protocol.replace(':', '') as DriverClient,
+      connection: {
+        ...parsedQuery,
+        host: url.hostname,
+        port: +url.port,
+      },
+      acquireConnectionTimeout: 600000,
+    };
+
+    if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
+      dbConfig.connection.ssl = true;
+    }
+
+    if (
+      url.searchParams.get('keyFilePath') &&
+      url.searchParams.get('certFilePath') &&
+      url.searchParams.get('caFilePath')
+    ) {
+      dbConfig.connection.ssl = {
+        keyFilePath: url.searchParams.get('keyFilePath'),
+        certFilePath: url.searchParams.get('certFilePath'),
+        caFilePath: url.searchParams.get('caFilePath'),
+      };
+    }
+  }
+
+  /* TODO check if this is needed
+    if (config && !config.title) {
+      config.title =
+        url.searchParams.get('t') ||
+        url.searchParams.get('title') ||
+        this.generateRandomTitle();
+    }
+  */
+
+  Object.assign(dbConfig, {
+    meta: {
+      tn: 'nc_evolutions',
+      allSchemas:
+        !!url.searchParams.get('allSchemas') ||
+        !(url.searchParams.get('d') || url.searchParams.get('database')),
+      api: {
+        prefix: url.searchParams.get('apiPrefix') || '',
+        swagger: true,
+        type:
+          type ||
+          ((url.searchParams.get('api') || url.searchParams.get('a')) as any) ||
+          'rest',
+      },
+      dbAlias: url.searchParams.get('dbAlias') || `db${key}`,
+      metaTables: 'db',
+      migrations: {
+        disabled: false,
+        name: 'nc_evolutions',
+      },
+    },
+  });
+
+  return dbConfig;
+}
+
+export async function metaUrlToDbConfig(urlString): Promise<DbConfig> {
+  const url = new URL(urlString);
+
+  let dbConfig: DbConfig;
+
+  if (url.protocol.startsWith('sqlite3')) {
+    const db = url.searchParams.get('d') || url.searchParams.get('database');
+    dbConfig = {
+      client: DriverClient.SQLITE,
+      connection: {
+        filename: db,
+      },
+      ...(db === ':memory:'
+        ? {
+            pool: {
+              min: 1,
+              max: 1,
+              // disposeTimeout: 360000*1000,
+              idleTimeoutMillis: 360000 * 1000,
+            },
+          }
+        : {}),
+    };
+  } else {
+    const parsedQuery = {};
+    for (const [key, value] of url.searchParams.entries()) {
+      const fnd = knownQueryParams.find(
+        (param) => param.parameter === key || param.aliases.includes(key),
+      );
+      if (fnd) {
+        parsedQuery[fnd.parameter] = value;
+      } else {
+        parsedQuery[key] = value;
+      }
+    }
+
+    dbConfig = {
+      client: url.protocol.replace(':', '') as DriverClient,
+      connection: {
+        ...defaultConnectionConfig,
+        ...parsedQuery,
+        host: url.hostname,
+        port: +url.port,
+      },
+      acquireConnectionTimeout: 600000,
+      ...defaultConnectionOptions,
+      ...(url.searchParams.has('search_path')
+        ? {
+            searchPath: url.searchParams.get('search_path').split(','),
+          }
+        : {}),
+    };
+
+    if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
+      dbConfig.connection.ssl = true;
+    }
+  }
+
+  url.searchParams.forEach((_value, key) => {
+    let value: any = _value;
+    if (value === 'true') {
+      value = true;
+    } else if (value === 'false') {
+      value = false;
+    } else if (/^\d+$/.test(value)) {
+      value = +value;
+    }
+    // todo: implement config read from JSON file or JSON env val read
+    if (
+      !['password', 'p', 'database', 'd', 'user', 'u', 'search_path'].includes(
+        key,
+      )
+    ) {
+      key.split('.').reduce((obj, k, i, arr) => {
+        return (obj[k] = i === arr.length - 1 ? value : obj[k] || {});
+      }, dbConfig);
+    }
+  });
+
+  if (
+    dbConfig?.connection?.ssl &&
+    typeof dbConfig?.connection?.ssl === 'object'
+  ) {
+    if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) {
+      dbConfig.connection.ssl.ca = (
+        await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath)
+      ).toString();
+      delete dbConfig.connection.ssl.caFilePath;
+    }
+    if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) {
+      dbConfig.connection.ssl.key = (
+        await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath)
+      ).toString();
+      delete dbConfig.connection.ssl.keyFilePath;
+    }
+    if (dbConfig.connection.ssl.certFilePath && !dbConfig.connection.ssl.cert) {
+      dbConfig.connection.ssl.cert = (
+        await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath)
+      ).toString();
+      delete dbConfig.connection.ssl.certFilePath;
+    }
+  }
+
+  return dbConfig;
+}

packages/nocodb/src/utils/nc-config/index.ts

@@ -0,0 +1,4 @@
+export * from './helpers';
+export * from './interfaces';
+export * from './constants';
+export * from './NcConfig';

packages/nocodb/src/utils/nc-config/interfaces.ts

@@ -0,0 +1,39 @@
+import { DriverClient } from './constants';
+
+interface Connection {
+  driver?: DriverClient;
+  host?: string;
+  port?: number;
+  database?: string;
+  user?: string;
+  password?: string;
+  ssl?:
+    | boolean
+    | {
+        ca?: string;
+        cert?: string;
+        key?: string;
+        caFilePath?: string;
+        certFilePath?: string;
+        keyFilePath?: string;
+      };
+  filename?: string;
+}
+
+interface DbConfig {
+  client: DriverClient;
+  connection: Connection;
+  acquireConnectionTimeout?: number;
+  useNullAsDefault?: boolean;
+  pool?: {
+    min?: number;
+    max?: number;
+    idleTimeoutMillis?: number;
+  };
+  migrations?: {
+    directory?: string;
+    tableName?: string;
+  };
+}
+
+export { DriverClient, Connection, DbConfig };

packages/nocodb/tests/unit/TestDbMngr.ts

@@ -2,9 +2,9 @@ import fs from 'fs';
 import process from 'process';
 import { knex } from 'knex';
 import SqlMgrv2 from '../../src/db/sql-mgr/v2/SqlMgrv2';
+import { jdbcToXcUrl, xcUrlToDbConfig } from '../../src/utils/nc-config';
 import type { Knex } from 'knex';
 import type { DbConfig } from '../../src/interface/config';
-import NcConfigFactory from '../../src/utils/NcConfigFactory'
 
 export default class TestDbMngr {
   public static readonly dbName = 'test_meta';
@@ -75,7 +75,7 @@ export default class TestDbMngr {
 
   private static async isDbConfigured() {
     const { user, password, host, port, client } = TestDbMngr.connection;
-    const config = NcConfigFactory.urlToDbConfig(
+    const config = xcUrlToDbConfig(
       `${client}://${user}:${password}@${host}:${port}`,
     );
     config.connection = {
@@ -84,7 +84,7 @@ export default class TestDbMngr {
       host,
       port,
     };
-    const result = await TestDbMngr.testConnection(config);
+    const result = await TestDbMngr.testConnection(config as any);
     return result.code !== -1;
   }
   static async connectDb() {
@@ -95,9 +95,9 @@ export default class TestDbMngr {
       ] = `${client}://${user}:${password}@${host}:${port}/${TestDbMngr.dbName}`;
     }
 
-    TestDbMngr.dbConfig = NcConfigFactory.urlToDbConfig(
+    TestDbMngr.dbConfig = xcUrlToDbConfig(
-      NcConfigFactory.extractXcUrlFromJdbc(process.env[`DATABASE_URL`]),
+      jdbcToXcUrl(process.env[`DATABASE_URL`]),
-    );
+    ) as any;
     this.dbConfig.meta = {
       tn: 'nc_evolutions',
       dbAlias: 'db',

tests/playwright/pages/Dashboard/TreeView.ts

@@ -216,7 +216,6 @@ export class TreeViewPage extends BasePage {
       uiAction: () => this.rootPage.getByRole('button', { name: 'Confirm' }).click(),
       httpMethodsToMatch: ['POST'],
       requestUrlPathToMatch: `/api/v1/db/meta/duplicate/`,
-      responseJsonMatcher: json => json.name === 'duplicate-model',
     });
     await this.get().locator(`[data-testid="tree-view-table-${title} copy"]`).waitFor();
   }