Browse Source

Merge pull request #5711 from nocodb/feat/worker

feat: worker container
pull/5752/head
mertmit 2 years ago committed by GitHub
parent
commit
6dc26295ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      packages/nc-gui/components/dashboard/TreeView.vue
  2. 1
      packages/nc-gui/nuxt-shim.d.ts
  3. 4
      packages/nc-gui/pages/index/index/index.vue
  4. 17
      packages/nc-gui/plugins/jobs.ts
  5. 10
      packages/nocodb/src/app.module.ts
  6. 8
      packages/nocodb/src/cache/RedisCacheMgr.ts
  7. 19
      packages/nocodb/src/connection/connection.spec.ts
  8. 37
      packages/nocodb/src/connection/connection.ts
  9. 12
      packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts
  10. 3
      packages/nocodb/src/index.ts
  11. 6
      packages/nocodb/src/init.ts
  12. 24
      packages/nocodb/src/main.ts
  13. 45
      packages/nocodb/src/meta/meta.service.ts
  14. 20
      packages/nocodb/src/modules/datas/datas.module.ts
  15. 27
      packages/nocodb/src/modules/global/global.module.ts
  16. 65
      packages/nocodb/src/modules/global/init-meta-service.provider.ts
  17. 24
      packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts
  18. 15
      packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts
  19. 51
      packages/nocodb/src/modules/jobs/fallback/jobs.service.ts
  20. 48
      packages/nocodb/src/modules/jobs/jobs.gateway.ts
  21. 60
      packages/nocodb/src/modules/jobs/jobs.module.ts
  22. 59
      packages/nocodb/src/modules/jobs/jobs.service.ts
  23. 32
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts
  24. 40
      packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts
  25. 0
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/EntityMap.ts
  26. 0
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts
  27. 4
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts
  28. 0
      packages/nocodb/src/modules/jobs/jobs/at-import/helpers/syncMap.ts
  29. 45
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts
  30. 12
      packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts
  31. 20
      packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts
  32. 40
      packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts
  33. 16
      packages/nocodb/src/modules/jobs/jobs/jobs-log.service.ts
  34. 107
      packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts
  35. 53
      packages/nocodb/src/modules/jobs/redis/jobs-redis.service.ts
  36. 98
      packages/nocodb/src/modules/jobs/redis/jobs.service.ts
  37. 69
      packages/nocodb/src/modules/metas/metas.module.ts
  38. 4
      packages/nocodb/src/modules/test/test.module.ts
  39. 4
      packages/nocodb/src/modules/users/users.module.ts
  40. 17
      packages/nocodb/src/plugins/storage/Local.ts
  41. 19
      packages/nocodb/src/services/app-init.service.spec.ts
  42. 79
      packages/nocodb/src/services/app-init.service.ts
  43. 7
      packages/nocodb/src/services/auth.service.ts
  44. 4
      packages/nocodb/src/services/projects.service.ts
  45. 4
      packages/nocodb/src/services/utils.service.ts
  46. 755
      packages/nocodb/src/utils/NcConfigFactory.ts
  47. 2
      packages/nocodb/src/utils/common/NcConnectionMgr.ts
  48. 2
      packages/nocodb/src/utils/common/NcConnectionMgrv2.ts
  49. 182
      packages/nocodb/src/utils/nc-config/NcConfig.ts
  50. 84
      packages/nocodb/src/utils/nc-config/constants.ts
  51. 324
      packages/nocodb/src/utils/nc-config/helpers.ts
  52. 4
      packages/nocodb/src/utils/nc-config/index.ts
  53. 39
      packages/nocodb/src/utils/nc-config/interfaces.ts
  54. 12
      packages/nocodb/tests/unit/TestDbMngr.ts
  55. 1
      tests/playwright/pages/Dashboard/TreeView.ts

4
packages/nc-gui/components/dashboard/TreeView.vue

@ -399,8 +399,8 @@ const duplicateTable = async (table: TableType) => {
const { close } = useDialog(resolveComponent('DlgTableDuplicate'), {
'modelValue': isOpen,
'table': table,
'onOk': async (jobData: { name: string; id: string }) => {
$jobs.subscribe({ name: jobData.name, id: jobData.id }, undefined, async (status: string, data?: any) => {
'onOk': async (jobData: { id: string }) => {
$jobs.subscribe({ id: jobData.id }, undefined, async (status: string, data?: any) => {
if (status === JobStatus.COMPLETED) {
await loadTables()
const newTable = tables.value.find((el) => el.id === data?.result?.id)

1
packages/nc-gui/nuxt-shim.d.ts vendored

@ -18,7 +18,6 @@ declare module '#app/nuxt' {
job:
| {
id: string
name: string
}
| any,
subscribedCb?: () => void,

4
packages/nc-gui/pages/index/index/index.vue

@ -90,10 +90,10 @@ const duplicateProject = (project: ProjectType) => {
const { close } = useDialog(resolveComponent('DlgProjectDuplicate'), {
'modelValue': isOpen,
'project': project,
'onOk': async (jobData: { name: string; id: string }) => {
'onOk': async (jobData: { id: string }) => {
await loadProjects()
$jobs.subscribe({ name: jobData.name, id: jobData.id }, undefined, async (status: string) => {
$jobs.subscribe({ id: jobData.id }, undefined, async (status: string) => {
if (status === JobStatus.COMPLETED) {
await loadProjects()
} else if (status === JobStatus.FAILED) {

17
packages/nc-gui/plugins/jobs.ts

@ -29,22 +29,22 @@ export default defineNuxtPlugin(async (nuxtApp) => {
await init(nuxtApp.$state.token.value)
}
const send = (name: string, data: any) => {
const send = (evt: string, data: any) => {
if (socket) {
const _id = messageIndex++
socket.emit(name, { _id, data })
socket.emit(evt, { _id, data })
return _id
}
}
const jobs = {
subscribe(
job: { id: string; name: string } | any,
job: { id: string } | any,
subscribedCb?: () => void,
statusCb?: (status: JobStatus, data?: any) => void,
logCb?: (data: { message: string }) => void,
) {
const logFn = (data: { id: string; name: string; data: { message: string } }) => {
const logFn = (data: { id: string; data: { message: string } }) => {
if (data.id === job.id) {
if (logCb) logCb(data.data)
}
@ -61,11 +61,10 @@ export default defineNuxtPlugin(async (nuxtApp) => {
const _id = send('subscribe', job)
const subscribeFn = (data: { _id: number; name: string; id: string }) => {
const subscribeFn = (data: { _id: number; id: string }) => {
if (data._id === _id) {
if (data.id !== job.id || data.name !== job.name) {
if (data.id !== job.id) {
job.id = data.id
job.name = data.name
}
if (subscribedCb) subscribedCb()
socket?.on('log', logFn)
@ -75,10 +74,10 @@ export default defineNuxtPlugin(async (nuxtApp) => {
}
socket?.on('subscribed', subscribeFn)
},
getStatus(name: string, id: string): Promise<string> {
getStatus(id: string): Promise<string> {
return new Promise((resolve) => {
if (socket) {
const _id = send('status', { name, id })
const _id = send('status', { id })
const tempFn = (data: any) => {
if (data._id === _id) {
resolve(data.status)

10
packages/nocodb/src/app.module.ts

@ -1,6 +1,5 @@
import { Module, RequestMethod } from '@nestjs/common';
import { APP_FILTER } from '@nestjs/core';
import { BullModule } from '@nestjs/bull';
import { EventEmitterModule as NestJsEventEmitter } from '@nestjs/event-emitter';
import { GlobalExceptionFilter } from './filters/global-exception/global-exception.filter';
import { GlobalMiddleware } from './middlewares/global/global.middleware';
@ -17,7 +16,6 @@ import { AuthTokenStrategy } from './strategies/authtoken.strategy/authtoken.str
import { BaseViewStrategy } from './strategies/base-view.strategy/base-view.strategy';
import { MetasModule } from './modules/metas/metas.module';
import { JobsModule } from './modules/jobs/jobs.module';
import { AppInitService } from './services/app-init.service';
import type { MiddlewareConsumer } from '@nestjs/common';
@Module({
@ -30,13 +28,6 @@ import type { MiddlewareConsumer } from '@nestjs/common';
EventEmitterModule,
JobsModule,
NestJsEventEmitter.forRoot(),
...(process.env['NC_REDIS_URL']
? [
BullModule.forRoot({
redis: process.env.NC_REDIS_URL,
}),
]
: []),
],
controllers: [],
providers: [
@ -49,7 +40,6 @@ import type { MiddlewareConsumer } from '@nestjs/common';
AuthTokenStrategy,
BaseViewStrategy,
HookHandlerService,
AppInitService,
],
})
export class AppModule {

8
packages/nocodb/src/cache/RedisCacheMgr.ts vendored

@ -12,8 +12,12 @@ export default class RedisCacheMgr extends CacheMgr {
constructor(config: any) {
super();
this.client = new Redis(config);
// flush the existing db with selected key (Default: 0)
this.client.flushdb();
// avoid flushing db in worker container
if (process.env.NC_WORKER_CONTAINER !== 'true') {
// flush the existing db with selected key (Default: 0)
this.client.flushdb();
}
// TODO(cache): fetch orgs once it's implemented
const orgs = 'noco';

19
packages/nocodb/src/connection/connection.spec.ts

@ -1,19 +0,0 @@
import { Test } from '@nestjs/testing';
import { Connection } from './knex';
import type { TestingModule } from '@nestjs/testing';
describe('Knex', () => {
let provider: Connection;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [Connection],
}).compile();
provider = module.get<Connection>(Connection);
});
it('should be defined', () => {
expect(provider).toBeDefined();
});
});

37
packages/nocodb/src/connection/connection.ts

@ -1,37 +0,0 @@
import { Global, Injectable, Scope } from '@nestjs/common';
import { XKnex } from '../db/CustomKnex';
import NcConfigFactory from '../utils/NcConfigFactory';
import type * as knex from 'knex';
@Injectable({
scope: Scope.DEFAULT,
})
export class Connection {
public static knex: knex.Knex;
public static _config: any;
get knexInstance(): knex.Knex {
return Connection.knex;
}
get config(): knex.Knex {
return Connection._config;
}
// init metadb connection
static async init(): Promise<void> {
Connection._config = await NcConfigFactory.make();
if (!Connection.knex) {
Connection.knex = XKnex({
...this._config.meta.db,
useNullAsDefault: true,
});
}
}
// init metadb connection
async init(): Promise<void> {
return await Connection.init();
}
}

12
packages/nocodb/src/db/sql-migrator/lib/KnexMigrator.ts

@ -10,7 +10,7 @@ import Debug from '../../util/Debug';
import Emit from '../../util/emit';
import Result from '../../util/Result';
import * as fileHelp from '../../util/file.help';
import NcConfigFactory from '../../../utils/NcConfigFactory';
import { getToolDir, NcConfig } from '../../../utils/nc-config';
import SqlMigrator from './SqlMigrator';
const evt = new Emit();
@ -39,7 +39,7 @@ export default class KnexMigrator extends SqlMigrator {
this.project_id = projectObj?.project_id;
this.project = projectObj?.config;
this.metaDb = projectObj?.metaDb;
this.toolDir = NcConfigFactory.getToolDir();
this.toolDir = getToolDir();
}
emit(data, _args?) {
@ -312,8 +312,12 @@ export default class KnexMigrator extends SqlMigrator {
if (exists) {
await this._readProjectJson(projJsonFilePath);
this.emit('Migrator for project initalised successfully');
} else if (NcConfigFactory.hasDbUrl()) {
this.project = await NcConfigFactory.make();
} else if (
Object.keys(process.env).some((envKey) =>
envKey.startsWith('NC_DB_URL'),
)
) {
this.project = await NcConfig.createByEnv();
} else {
args.type = args.type || 'sqlite';

3
packages/nocodb/src/index.ts

@ -1,6 +1,5 @@
import Noco from './Noco';
import NcConfigFactory from './utils/NcConfigFactory';
export default Noco;
export { Noco, NcConfigFactory };
export { Noco };

6
packages/nocodb/src/init.ts

@ -1,12 +1,12 @@
import { Connection } from './connection/connection';
import { MetaService } from './meta/meta.service';
import { NcConfig } from './utils/nc-config';
import Noco from './Noco';
// run upgrader
import NcUpgrader from './version-upgrader/NcUpgrader';
export default async () => {
await Connection.init();
Noco._ncMeta = new MetaService(new Connection());
const config = await NcConfig.createByEnv();
Noco._ncMeta = new MetaService(config);
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
};

24
packages/nocodb/src/main.ts

@ -5,12 +5,22 @@ import { AppModule } from './app.module';
async function bootstrap() {
const app = await NestFactory.create(AppModule);
app.use(express.json({ limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' }));
app.use(
cors({
exposedHeaders: 'xc-db-response',
}),
);
await app.listen(8080);
if (process.env.NC_WORKER_CONTAINER !== 'true') {
app.use(
express.json({ limit: process.env.NC_REQUEST_BODY_SIZE || '50mb' }),
);
app.use(
cors({
exposedHeaders: 'xc-db-response',
}),
);
await app.listen(8080);
} else {
if (!process.env.NC_REDIS_URL) {
throw new Error('NC_REDIS_URL is required');
}
process.env.NC_DISABLE_TELE = 'true';
await app.init();
}
}
bootstrap();

45
packages/nocodb/src/meta/meta.service.ts

@ -1,22 +1,15 @@
import {
Global,
Inject,
Injectable,
OnApplicationBootstrap,
OnModuleInit,
Optional,
} from '@nestjs/common';
import { Injectable, Optional } from '@nestjs/common';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc';
import timezone from 'dayjs/plugin/timezone';
import { customAlphabet } from 'nanoid';
import CryptoJS from 'crypto-js';
import { Connection } from '../connection/connection';
import Noco from '../Noco';
import NocoCache from '../cache/NocoCache';
import { XKnex } from '../db/CustomKnex';
import { NcConfig } from '../utils/nc-config';
import XcMigrationSourcev2 from './migrations/XcMigrationSourcev2';
import XcMigrationSource from './migrations/XcMigrationSource';
import type { Knex } from 'knex';
import type * as knex from 'knex';
dayjs.extend(utc);
dayjs.extend(timezone);
@ -192,18 +185,38 @@ const nanoidv2 = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 14);
@Injectable()
export class MetaService {
constructor(private metaConnection: Connection, @Optional() trx = null) {
private _knex: knex.Knex;
private _config: any;
constructor(config: NcConfig, @Optional() trx = null) {
this._config = config;
this._knex = XKnex({
...this._config.meta.db,
useNullAsDefault: true,
});
this.trx = trx;
}
get knexInstance(): knex.Knex {
return this._knex;
}
get config(): NcConfig {
return this._config;
}
public get connection() {
return this.trx ?? this.metaConnection.knexInstance;
return this.trx ?? this.knexInstance;
}
get knexConnection() {
return this.connection;
}
public get knex(): any {
return this.knexConnection;
}
public async metaGet(
project_id: string,
dbAlias: string,
@ -758,7 +771,7 @@ export class MetaService {
});
// todo: tobe done
return new MetaService(this.metaConnection, trx);
return new MetaService(this.config, trx);
}
async metaReset(
@ -1027,10 +1040,6 @@ export class MetaService {
.delete();
}
public get knex(): any {
return this.knexConnection;
}
private getNanoId() {
return nanoid();
}

20
packages/nocodb/src/modules/datas/datas.module.ts

@ -27,14 +27,18 @@ import { PublicDatasService } from '../../services/public-datas.service';
}),
],
controllers: [
DatasController,
BulkDataAliasController,
DataAliasController,
DataAliasNestedController,
DataAliasExportController,
OldDatasController,
PublicDatasController,
PublicDatasExportController,
...(process.env.NC_WORKER_CONTAINER !== 'true'
? [
DatasController,
BulkDataAliasController,
DataAliasController,
DataAliasNestedController,
DataAliasExportController,
OldDatasController,
PublicDatasController,
PublicDatasExportController,
]
: []),
],
providers: [
DatasService,

27
packages/nocodb/src/modules/global/global.module.ts

@ -1,25 +1,18 @@
import { Global, Module } from '@nestjs/common';
import { ExtractJwt } from 'passport-jwt';
import {
AppInitService,
appInitServiceProvider,
} from '../../services/app-init.service';
import { SocketGateway } from '../../gateways/socket.gateway';
import { Connection } from '../../connection/connection';
import { GlobalGuard } from '../../guards/global/global.guard';
import { MetaService } from '../../meta/meta.service';
import Noco from '../../Noco';
import { JwtStrategy } from '../../strategies/jwt.strategy';
import { UsersService } from '../../services/users/users.service';
import Noco from '../../Noco';
import { InitMetaServiceProvider } from './init-meta-service.provider';
import type { Provider } from '@nestjs/common';
export const JwtStrategyProvider: Provider = {
provide: JwtStrategy,
useFactory: async (
usersService: UsersService,
appInitService: AppInitService,
) => {
const config = appInitService.appConfig;
useFactory: async (usersService: UsersService, metaService: MetaService) => {
const config = metaService.config;
await Noco.initJwt();
@ -34,29 +27,25 @@ export const JwtStrategyProvider: Provider = {
return new JwtStrategy(options, usersService);
},
inject: [UsersService, AppInitService],
inject: [UsersService, MetaService],
};
@Global()
@Module({
imports: [],
providers: [
appInitServiceProvider,
Connection,
MetaService,
InitMetaServiceProvider,
UsersService,
JwtStrategyProvider,
GlobalGuard,
SocketGateway,
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [SocketGateway] : []),
],
exports: [
AppInitService,
Connection,
MetaService,
JwtStrategyProvider,
UsersService,
GlobalGuard,
SocketGateway,
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [SocketGateway] : []),
],
})
export class GlobalModule {}

65
packages/nocodb/src/modules/global/init-meta-service.provider.ts

@ -0,0 +1,65 @@
import { T } from 'nc-help';
import { MetaService } from '../../meta/meta.service';
import Noco from '../../Noco';
import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2';
import NcUpgrader from '../../version-upgrader/NcUpgrader';
import NocoCache from '../../cache/NocoCache';
import getInstance from '../../utils/getInstance';
import initAdminFromEnv from '../../helpers/initAdminFromEnv';
import { User } from '../../models';
import { NcConfig, prepareEnv } from '../../utils/nc-config';
import type { Provider } from '@nestjs/common';
import type { IEventEmitter } from '../event-emitter/event-emitter.interface';
export const InitMetaServiceProvider: Provider = {
// initialize app,
// 1. init cache
// 2. init db connection and create if not exist
// 3. init meta and set to Noco
// 4. init jwt
// 5. init plugin manager
// 6. run upgrader
useFactory: async (eventEmitter: IEventEmitter) => {
// NC_DATABASE_URL_FILE, DATABASE_URL_FILE, DATABASE_URL, NC_DATABASE_URL to NC_DB
await prepareEnv();
const config = await NcConfig.createByEnv();
// set version
process.env.NC_VERSION = '0107004';
// init cache
await NocoCache.init();
// init meta service
const metaService = new MetaService(config);
await metaService.init();
// provide meta and config to Noco
Noco._ncMeta = metaService;
Noco.config = config;
Noco.eventEmitter = eventEmitter;
// init jwt secret
await Noco.initJwt();
// load super admin user from env if env is set
await initAdminFromEnv(metaService);
// init plugin manager
await NcPluginMgrv2.init(Noco.ncMeta);
await Noco.loadEEState();
// run upgrader
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
T.init({
instance: getInstance,
});
T.emit('evt_app_started', await User.count());
return metaService;
},
provide: MetaService,
inject: ['IEventEmitter'],
};

24
packages/nocodb/src/modules/jobs/fallback-queue.service.ts → packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts

@ -1,12 +1,12 @@
import { Injectable } from '@nestjs/common';
import PQueue from 'p-queue';
import Emittery from 'emittery';
import { JobStatus, JobTypes } from '../../interface/Jobs';
import { DuplicateProcessor } from './export-import/duplicate.processor';
import { JobStatus, JobTypes } from '../../../interface/Jobs';
import { DuplicateProcessor } from '../jobs/export-import/duplicate.processor';
import { AtImportProcessor } from '../jobs/at-import/at-import.processor';
import { JobsEventService } from './jobs-event.service';
import { AtImportProcessor } from './at-import/at-import.processor';
interface Job {
export interface Job {
id: string;
name: string;
status: string;
@ -27,16 +27,12 @@ export class QueueService {
private readonly atImportProcessor: AtImportProcessor,
) {
this.emitter.on(JobStatus.ACTIVE, (data: { job: Job }) => {
const job = this.queueMemory.find(
(job) => job.id === data.job.id && job.name === data.job.name,
);
const job = this.queueMemory.find((job) => job.id === data.job.id);
job.status = JobStatus.ACTIVE;
this.jobsEventService.onActive.apply(this.jobsEventService, [job as any]);
});
this.emitter.on(JobStatus.COMPLETED, (data: { job: Job; result: any }) => {
const job = this.queueMemory.find(
(job) => job.id === data.job.id && job.name === data.job.name,
);
const job = this.queueMemory.find((job) => job.id === data.job.id);
job.status = JobStatus.COMPLETED;
this.jobsEventService.onCompleted.apply(this.jobsEventService, [
job,
@ -46,9 +42,7 @@ export class QueueService {
this.removeJob(job);
});
this.emitter.on(JobStatus.FAILED, (data: { job: Job; error: Error }) => {
const job = this.queueMemory.find(
(job) => job.id === data.job.id && job.name === data.job.name,
);
const job = this.queueMemory.find((job) => job.id === data.job.id);
job.status = JobStatus.FAILED;
this.jobsEventService.onFailed.apply(this.jobsEventService, [
job,
@ -126,9 +120,7 @@ export class QueueService {
// remove job from memory
private removeJob(job: Job) {
const fIndex = this.queueMemory.findIndex(
(q) => q.id === job.id && q.name === job.name,
);
const fIndex = this.queueMemory.findIndex((q) => q.id === job.id);
if (fIndex) {
this.queueMemory.splice(fIndex, 1);
}

15
packages/nocodb/src/modules/jobs/jobs-event.service.ts → packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts

@ -7,7 +7,7 @@ import {
import { Job } from 'bull';
import boxen from 'boxen';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../interface/Jobs';
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
@Processor(JOBS_QUEUE)
export class JobsEventService {
@ -16,7 +16,6 @@ export class JobsEventService {
@OnQueueActive()
onActive(job: Job) {
this.eventEmitter.emit(JobEvents.STATUS, {
name: job.name,
id: job.id.toString(),
status: JobStatus.ACTIVE,
});
@ -26,7 +25,7 @@ export class JobsEventService {
onFailed(job: Job, error: Error) {
console.error(
boxen(
`---- !! JOB FAILED !! ----\nname: ${job.name}\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
`---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
{
padding: 1,
borderStyle: 'double',
@ -36,7 +35,6 @@ export class JobsEventService {
);
this.eventEmitter.emit(JobEvents.STATUS, {
name: job.name,
id: job.id.toString(),
status: JobStatus.FAILED,
data: {
@ -50,7 +48,6 @@ export class JobsEventService {
@OnQueueCompleted()
onCompleted(job: Job, data: any) {
this.eventEmitter.emit(JobEvents.STATUS, {
name: job.name,
id: job.id.toString(),
status: JobStatus.COMPLETED,
data: {
@ -58,12 +55,4 @@ export class JobsEventService {
},
});
}
sendLog(job: Job, data: { message: string }) {
this.eventEmitter.emit(JobEvents.LOG, {
name: job.name,
id: job.id.toString(),
data,
});
}
}

51
packages/nocodb/src/modules/jobs/fallback/jobs.service.ts

@ -0,0 +1,51 @@
import { Injectable } from '@nestjs/common';
import { JobStatus } from '../../../interface/Jobs';
import { QueueService } from './fallback-queue.service';
@Injectable()
export class JobsService {
constructor(private readonly fallbackQueueService: QueueService) {}
async add(name: string, data: any) {
return this.fallbackQueueService.add(name, data);
}
async jobStatus(jobId: string) {
return await (
await this.fallbackQueueService.getJob(jobId)
).status;
}
async jobList() {
return await this.fallbackQueueService.getJobs([
JobStatus.ACTIVE,
JobStatus.WAITING,
JobStatus.DELAYED,
JobStatus.PAUSED,
]);
}
async getJobWithData(data: any) {
const jobs = await this.fallbackQueueService.getJobs([
// 'completed',
JobStatus.WAITING,
JobStatus.ACTIVE,
JobStatus.DELAYED,
// 'failed',
JobStatus.PAUSED,
]);
const job = jobs.find((j) => {
for (const key in data) {
if (j.data[key]) {
if (j.data[key] !== data[key]) return false;
} else {
return false;
}
}
return true;
});
return job;
}
}

48
packages/nocodb/src/modules/jobs/jobs.gateway.ts

@ -9,10 +9,10 @@ import { Server, Socket } from 'socket.io';
import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host';
import { AuthGuard } from '@nestjs/passport';
import { OnEvent } from '@nestjs/event-emitter';
import { Inject } from '@nestjs/common';
import { JobEvents } from '../../interface/Jobs';
import { JobsService } from './jobs.service';
import type { JobStatus } from '../../interface/Jobs';
import type { OnModuleInit } from '@nestjs/common';
import type { JobStatus } from '../../interface/Jobs';
@WebSocketGateway({
cors: {
@ -23,7 +23,7 @@ import type { OnModuleInit } from '@nestjs/common';
namespace: 'jobs',
})
export class JobsGateway implements OnModuleInit {
constructor(private readonly jobsService: JobsService) {}
constructor(@Inject('JobsService') private readonly jobsService) {}
@WebSocketServer()
server: Server;
@ -43,34 +43,28 @@ export class JobsGateway implements OnModuleInit {
@SubscribeMessage('subscribe')
async subscribe(
@MessageBody()
body: { _id: number; data: { id: string; name: string } | any },
body: { _id: number; data: { id: string } | any },
@ConnectedSocket() client: Socket,
): Promise<void> {
const { _id, data } = body;
if (
Object.keys(data).every((k) => ['name', 'id'].includes(k)) &&
data?.name &&
data?.id
) {
const rooms = (await this.jobsService.jobList(data.name)).map(
(j) => `${j.name}-${j.id}`,
if (Object.keys(data).every((k) => ['id'].includes(k)) && data?.id) {
const rooms = (await this.jobsService.jobList()).map(
(j) => `jobs-${j.id}`,
);
const room = rooms.find((r) => r === `${data.name}-${data.id}`);
const room = rooms.find((r) => r === `jobs-${data.id}`);
if (room) {
client.join(`${data.name}-${data.id}`);
client.join(`jobs-${data.id}`);
client.emit('subscribed', {
_id,
name: data.name,
id: data.id,
});
}
} else {
const job = await this.jobsService.getJobWithData(data);
if (job) {
client.join(`${job.name}-${job.id}`);
client.join(`jobs-${job.id}`);
client.emit('subscribed', {
_id,
name: job.name,
id: job.id,
});
}
@ -79,42 +73,30 @@ export class JobsGateway implements OnModuleInit {
@SubscribeMessage('status')
async status(
@MessageBody() body: { _id: number; data: { id: string; name: string } },
@MessageBody() body: { _id: number; data: { id: string } },
@ConnectedSocket() client: Socket,
): Promise<void> {
const { _id, data } = body;
client.emit('status', {
_id,
id: data.id,
name: data.name,
status: await this.jobsService.jobStatus(data.id),
});
}
@OnEvent(JobEvents.STATUS)
async sendJobStatus(data: {
name: string;
id: string;
status: JobStatus;
data?: any;
}): Promise<void> {
this.server.to(`${data.name}-${data.id}`).emit('status', {
sendJobStatus(data: { id: string; status: JobStatus; data?: any }): void {
this.server.to(`jobs-${data.id}`).emit('status', {
id: data.id,
name: data.name,
status: data.status,
data: data.data,
});
}
@OnEvent(JobEvents.LOG)
async sendJobLog(data: {
name: string;
id: string;
data: { message: string };
}): Promise<void> {
this.server.to(`${data.name}-${data.id}`).emit('log', {
sendJobLog(data: { id: string; data: { message: string } }): void {
this.server.to(`jobs-${data.id}`).emit('log', {
id: data.id,
name: data.name,
data: data.data,
});
}

60
packages/nocodb/src/modules/jobs/jobs.module.ts

@ -4,35 +4,59 @@ import { GlobalModule } from '../global/global.module';
import { DatasModule } from '../datas/datas.module';
import { MetasModule } from '../metas/metas.module';
import { JOBS_QUEUE } from '../../interface/Jobs';
import { JobsService } from './jobs.service';
import { ExportService } from './export-import/export.service';
import { ImportService } from './export-import/import.service';
import { DuplicateController } from './export-import/duplicate.controller';
import { DuplicateProcessor } from './export-import/duplicate.processor';
import { ExportService } from './jobs/export-import/export.service';
import { ImportService } from './jobs/export-import/import.service';
import { AtImportController } from './jobs/at-import/at-import.controller';
import { AtImportProcessor } from './jobs/at-import/at-import.processor';
import { DuplicateController } from './jobs/export-import/duplicate.controller';
import { DuplicateProcessor } from './jobs/export-import/duplicate.processor';
import { JobsLogService } from './jobs/jobs-log.service';
import { JobsGateway } from './jobs.gateway';
import { QueueService } from './fallback-queue.service';
import { JobsEventService } from './jobs-event.service';
import { AtImportController } from './at-import/at-import.controller';
import { AtImportProcessor } from './at-import/at-import.processor';
// Redis
import { JobsService } from './redis/jobs.service';
import { JobsRedisService } from './redis/jobs-redis.service';
import { JobsEventService } from './redis/jobs-event.service';
// Fallback
import { JobsService as FallbackJobsService } from './fallback/jobs.service';
import { QueueService as FallbackQueueService } from './fallback/fallback-queue.service';
import { JobsEventService as FallbackJobsEventService } from './fallback/jobs-event.service';
@Module({
imports: [
GlobalModule,
DatasModule,
MetasModule,
BullModule.registerQueue({
name: JOBS_QUEUE,
}),
...(process.env.NC_REDIS_URL
? [
BullModule.forRoot({
url: process.env.NC_REDIS_URL,
}),
BullModule.registerQueue({
name: JOBS_QUEUE,
}),
]
: []),
],
controllers: [
...(process.env.NC_WORKER_CONTAINER !== 'true'
? [DuplicateController, AtImportController]
: []),
],
controllers: [DuplicateController, AtImportController],
providers: [
QueueService,
JobsGateway,
JobsService,
JobsEventService,
DuplicateProcessor,
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [JobsGateway] : []),
...(process.env.NC_REDIS_URL
? [JobsRedisService, JobsEventService]
: [FallbackQueueService, FallbackJobsEventService]),
{
provide: 'JobsService',
useClass: process.env.NC_REDIS_URL ? JobsService : FallbackJobsService,
},
JobsLogService,
ExportService,
ImportService,
DuplicateProcessor,
AtImportProcessor,
],
})

59
packages/nocodb/src/modules/jobs/jobs.service.ts

@ -1,59 +0,0 @@
import { InjectQueue } from '@nestjs/bull';
import { Injectable } from '@nestjs/common';
import { Queue } from 'bull';
import { JOBS_QUEUE, JobStatus } from '../../interface/Jobs';
import { QueueService } from './fallback-queue.service';
@Injectable()
export class JobsService {
public activeQueue;
constructor(
@InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue,
private readonly fallbackQueueService: QueueService,
) {
this.activeQueue = this.fallbackQueueService;
/* process.env.NC_REDIS_URL
? this.jobsQueue
: this.fallbackQueueService;
*/
}
async jobStatus(jobId: string) {
return await (await this.activeQueue.getJob(jobId)).getState();
}
async jobList(jobType: string) {
return (
await this.activeQueue.getJobs([
JobStatus.ACTIVE,
JobStatus.WAITING,
JobStatus.DELAYED,
JobStatus.PAUSED,
])
).filter((j) => j.name === jobType);
}
async getJobWithData(data: any) {
const jobs = await this.activeQueue.getJobs([
// 'completed',
JobStatus.WAITING,
JobStatus.ACTIVE,
JobStatus.DELAYED,
// 'failed',
JobStatus.PAUSED,
]);
const job = jobs.find((j) => {
for (const key in data) {
if (j.data[key]) {
if (j.data[key] !== data[key]) return false;
} else {
return false;
}
}
return true;
});
return job;
}
}

32
packages/nocodb/src/modules/jobs/at-import/at-import.controller.ts → packages/nocodb/src/modules/jobs/jobs/at-import/at-import.controller.ts

@ -1,30 +1,36 @@
import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common';
import { GlobalGuard } from '../../../guards/global/global.guard';
import { ExtractProjectIdMiddleware } from '../../../middlewares/extract-project-id/extract-project-id.middleware';
import { SyncSource } from '../../../models';
import { NcError } from '../../../helpers/catchError';
import { JobsService } from '../jobs.service';
import { JobTypes } from '../../../interface/Jobs';
import {
Controller,
HttpCode,
Inject,
Post,
Request,
UseGuards,
} from '@nestjs/common';
import { GlobalGuard } from '../../../../guards/global/global.guard';
import { ExtractProjectIdMiddleware } from '../../../../middlewares/extract-project-id/extract-project-id.middleware';
import { SyncSource } from '../../../../models';
import { NcError } from '../../../../helpers/catchError';
import { JobTypes } from '../../../../interface/Jobs';
@Controller()
@UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
export class AtImportController {
constructor(private readonly jobsService: JobsService) {}
constructor(@Inject('JobsService') private readonly jobsService) {}
@Post('/api/v1/db/meta/import/airtable')
@HttpCode(200)
async importAirtable(@Request() req) {
const job = await this.jobsService.activeQueue.add(JobTypes.AtImport, {
const job = await this.jobsService.add(JobTypes.AtImport, {
...req.body,
});
return { id: job.id, name: job.name };
return { id: job.id };
}
@Post('/api/v1/db/meta/syncs/:syncId/trigger')
@HttpCode(200)
async triggerSync(@Request() req) {
const jobs = await this.jobsService.jobList(JobTypes.AtImport);
const jobs = await this.jobsService.jobList();
const fnd = jobs.find((j) => j.data.syncId === req.params.syncId);
if (fnd) {
@ -44,7 +50,7 @@ export class AtImportController {
baseURL = `http://localhost:${process.env.PORT || 8080}`;
}
const job = await this.jobsService.activeQueue.add(JobTypes.AtImport, {
const job = await this.jobsService.add(JobTypes.AtImport, {
syncId: req.params.syncId,
...(syncSource?.details || {}),
projectId: syncSource.project_id,
@ -54,7 +60,7 @@ export class AtImportController {
user: user,
});
return { id: job.id, name: job.name };
return { id: job.id };
}
@Post('/api/v1/db/meta/syncs/:syncId/abort')

40
packages/nocodb/src/modules/jobs/at-import/at-import.processor.ts → packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@ -9,23 +9,23 @@ import utc from 'dayjs/plugin/utc';
import tinycolor from 'tinycolor2';
import { Process, Processor } from '@nestjs/bull';
import { Job } from 'bull';
import extractRolesObj from '../../../utils/extractRolesObj';
import { AttachmentsService } from '../../../services/attachments.service';
import { ColumnsService } from '../../../services/columns.service';
import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
import { FiltersService } from '../../../services/filters.service';
import { FormColumnsService } from '../../../services/form-columns.service';
import { GalleriesService } from '../../../services/galleries.service';
import { GridsService } from '../../../services/grids.service';
import { ProjectUsersService } from '../../../services/project-users/project-users.service';
import { ProjectsService } from '../../../services/projects.service';
import { SortsService } from '../../../services/sorts.service';
import { TablesService } from '../../../services/tables.service';
import { ViewColumnsService } from '../../../services/view-columns.service';
import { ViewsService } from '../../../services/views.service';
import { FormsService } from '../../../services/forms.service';
import { JobsEventService } from '../jobs-event.service';
import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs';
import extractRolesObj from '../../../../utils/extractRolesObj';
import { AttachmentsService } from '../../../../services/attachments.service';
import { ColumnsService } from '../../../../services/columns.service';
import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
import { FiltersService } from '../../../../services/filters.service';
import { FormColumnsService } from '../../../../services/form-columns.service';
import { GalleriesService } from '../../../../services/galleries.service';
import { GridsService } from '../../../../services/grids.service';
import { ProjectUsersService } from '../../../../services/project-users/project-users.service';
import { ProjectsService } from '../../../../services/projects.service';
import { SortsService } from '../../../../services/sorts.service';
import { TablesService } from '../../../../services/tables.service';
import { ViewColumnsService } from '../../../../services/view-columns.service';
import { ViewsService } from '../../../../services/views.service';
import { FormsService } from '../../../../services/forms.service';
import { JOBS_QUEUE, JobTypes } from '../../../../interface/Jobs';
import { JobsLogService } from '../jobs-log.service';
import FetchAT from './helpers/fetchAT';
import { importData, importLTARData } from './helpers/readAndProcessData';
import EntityMap from './helpers/EntityMap';
@ -99,7 +99,7 @@ export class AtImportProcessor {
private readonly viewColumnsService: ViewColumnsService,
private readonly sortsService: SortsService,
private readonly bulkDataAliasService: BulkDataAliasService,
private readonly jobsEventService: JobsEventService,
private readonly jobsLogService: JobsLogService,
) {}
@Process(JobTypes.AtImport)
@ -135,11 +135,11 @@ export class AtImportProcessor {
};
const logBasic = (log) => {
this.jobsEventService.sendLog(job, { message: log });
this.jobsLogService.sendLog(job, { message: log });
};
const logDetailed = (log) => {
if (debugMode) this.jobsEventService.sendLog(job, { message: log });
if (debugMode) this.jobsLogService.sendLog(job, { message: log });
};
const perfStats = [];

0
packages/nocodb/src/modules/jobs/at-import/helpers/EntityMap.ts → packages/nocodb/src/modules/jobs/jobs/at-import/helpers/EntityMap.ts

0
packages/nocodb/src/modules/jobs/at-import/helpers/fetchAT.ts → packages/nocodb/src/modules/jobs/jobs/at-import/helpers/fetchAT.ts

4
packages/nocodb/src/modules/jobs/at-import/helpers/readAndProcessData.ts → packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts

@ -1,8 +1,8 @@
/* eslint-disable no-async-promise-executor */
import { RelationTypes, UITypes } from 'nocodb-sdk';
import EntityMap from './EntityMap';
import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
import type { TablesService } from '../../../../services/tables.service';
import type { BulkDataAliasService } from '../../../../../services/bulk-data-alias.service';
import type { TablesService } from '../../../../../services/tables.service';
// @ts-ignore
import type { AirtableBase } from 'airtable/lib/airtable_base';
import type { TableType } from 'nocodb-sdk';

0
packages/nocodb/src/modules/jobs/at-import/helpers/syncMap.ts → packages/nocodb/src/modules/jobs/jobs/at-import/helpers/syncMap.ts

45
packages/nocodb/src/modules/jobs/export-import/duplicate.controller.ts → packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts

@ -2,28 +2,28 @@ import {
Body,
Controller,
HttpCode,
Inject,
Param,
Post,
Request,
UseGuards,
} from '@nestjs/common';
import { ProjectStatus } from 'nocodb-sdk';
import { GlobalGuard } from '../../../guards/global/global.guard';
import { GlobalGuard } from '../../../../guards/global/global.guard';
import {
Acl,
ExtractProjectIdMiddleware,
} from '../../../middlewares/extract-project-id/extract-project-id.middleware';
import { ProjectsService } from '../../../services/projects.service';
import { Base, Model, Project } from '../../../models';
import { generateUniqueName } from '../../../helpers/exportImportHelpers';
import { JobsService } from '../jobs.service';
import { JobTypes } from '../../../interface/Jobs';
} from '../../../../middlewares/extract-project-id/extract-project-id.middleware';
import { ProjectsService } from '../../../../services/projects.service';
import { Base, Model, Project } from '../../../../models';
import { generateUniqueName } from '../../../../helpers/exportImportHelpers';
import { JobTypes } from '../../../../interface/Jobs';
@Controller()
@UseGuards(ExtractProjectIdMiddleware, GlobalGuard)
export class DuplicateController {
constructor(
private readonly jobsService: JobsService,
@Inject('JobsService') private readonly jobsService,
private readonly projectsService: ProjectsService,
) {}
@ -67,7 +67,7 @@ export class DuplicateController {
user: { id: req.user.id },
});
const job = await this.jobsService.activeQueue.add(JobTypes.DuplicateBase, {
const job = await this.jobsService.add(JobTypes.DuplicateBase, {
projectId: project.id,
baseId: base.id,
dupProjectId: dupProject.id,
@ -78,7 +78,7 @@ export class DuplicateController {
},
});
return { id: job.id, name: job.name };
return { id: job.id };
}
@Post('/api/v1/db/meta/duplicate/:projectId/table/:modelId')
@ -116,21 +116,18 @@ export class DuplicateController {
models.map((p) => p.title),
);
const job = await this.jobsService.activeQueue.add(
JobTypes.DuplicateModel,
{
projectId: project.id,
baseId: base.id,
modelId: model.id,
title: uniqueTitle,
options,
req: {
user: req.user,
clientIp: req.clientIp,
},
const job = await this.jobsService.add(JobTypes.DuplicateModel, {
projectId: project.id,
baseId: base.id,
modelId: model.id,
title: uniqueTitle,
options,
req: {
user: req.user,
clientIp: req.clientIp,
},
);
});
return { id: job.id, name: job.name };
return { id: job.id };
}
}

12
packages/nocodb/src/modules/jobs/export-import/duplicate.processor.ts → packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@ -4,12 +4,12 @@ import { Job } from 'bull';
import papaparse from 'papaparse';
import { UITypes } from 'nocodb-sdk';
import { Logger } from '@nestjs/common';
import { Base, Column, Model, Project } from '../../../models';
import { ProjectsService } from '../../../services/projects.service';
import { findWithIdentifier } from '../../../helpers/exportImportHelpers';
import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
import { JOBS_QUEUE, JobTypes } from '../../../interface/Jobs';
import { elapsedTime, initTime } from '../helpers';
import { Base, Column, Model, Project } from '../../../../models';
import { ProjectsService } from '../../../../services/projects.service';
import { findWithIdentifier } from '../../../../helpers/exportImportHelpers';
import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
import { JOBS_QUEUE, JobTypes } from '../../../../interface/Jobs';
import { elapsedTime, initTime } from '../../helpers';
import { ExportService } from './export.service';
import { ImportService } from './import.service';

20
packages/nocodb/src/modules/jobs/export-import/export.service.ts → packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@ -2,19 +2,19 @@ import { Readable } from 'stream';
import { UITypes, ViewTypes } from 'nocodb-sdk';
import { unparse } from 'papaparse';
import { Injectable, Logger } from '@nestjs/common';
import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2';
import { getViewAndModelByAliasOrId } from '../../../modules/datas/helpers';
import NcConnectionMgrv2 from '../../../../utils/common/NcConnectionMgrv2';
import { getViewAndModelByAliasOrId } from '../../../datas/helpers';
import {
clearPrefix,
generateBaseIdMap,
} from '../../../helpers/exportImportHelpers';
import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2';
import { NcError } from '../../../helpers/catchError';
import { Base, Hook, Model, Project } from '../../../models';
import { DatasService } from '../../../services/datas.service';
import { elapsedTime, initTime } from '../helpers';
import type { BaseModelSqlv2 } from '../../../db/BaseModelSqlv2';
import type { View } from '../../../models';
} from '../../../../helpers/exportImportHelpers';
import NcPluginMgrv2 from '../../../../helpers/NcPluginMgrv2';
import { NcError } from '../../../../helpers/catchError';
import { Base, Hook, Model, Project } from '../../../../models';
import { DatasService } from '../../../../services/datas.service';
import { elapsedTime, initTime } from '../../helpers';
import type { BaseModelSqlv2 } from '../../../../db/BaseModelSqlv2';
import type { View } from '../../../../models';
@Injectable()
export class ExportService {

40
packages/nocodb/src/modules/jobs/export-import/import.service.ts → packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@ -9,28 +9,28 @@ import {
reverseGet,
withoutId,
withoutNull,
} from '../../../helpers/exportImportHelpers';
import { NcError } from '../../../helpers/catchError';
import { Base, Column, Model, Project } from '../../../models';
import { TablesService } from '../../../services/tables.service';
import { ColumnsService } from '../../../services/columns.service';
import { FiltersService } from '../../../services/filters.service';
import { SortsService } from '../../../services/sorts.service';
import { ViewColumnsService } from '../../../services/view-columns.service';
import { GridColumnsService } from '../../../services/grid-columns.service';
import { FormColumnsService } from '../../../services/form-columns.service';
import { GridsService } from '../../../services/grids.service';
import { FormsService } from '../../../services/forms.service';
import { GalleriesService } from '../../../services/galleries.service';
import { KanbansService } from '../../../services/kanbans.service';
import { HooksService } from '../../../services/hooks.service';
import { ViewsService } from '../../../services/views.service';
import NcPluginMgrv2 from '../../../helpers/NcPluginMgrv2';
import { BulkDataAliasService } from '../../../services/bulk-data-alias.service';
import { elapsedTime, initTime } from '../helpers';
} from '../../../../helpers/exportImportHelpers';
import { NcError } from '../../../../helpers/catchError';
import { Base, Column, Model, Project } from '../../../../models';
import { TablesService } from '../../../../services/tables.service';
import { ColumnsService } from '../../../../services/columns.service';
import { FiltersService } from '../../../../services/filters.service';
import { SortsService } from '../../../../services/sorts.service';
import { ViewColumnsService } from '../../../../services/view-columns.service';
import { GridColumnsService } from '../../../../services/grid-columns.service';
import { FormColumnsService } from '../../../../services/form-columns.service';
import { GridsService } from '../../../../services/grids.service';
import { FormsService } from '../../../../services/forms.service';
import { GalleriesService } from '../../../../services/galleries.service';
import { KanbansService } from '../../../../services/kanbans.service';
import { HooksService } from '../../../../services/hooks.service';
import { ViewsService } from '../../../../services/views.service';
import NcPluginMgrv2 from '../../../../helpers/NcPluginMgrv2';
import { BulkDataAliasService } from '../../../../services/bulk-data-alias.service';
import { elapsedTime, initTime } from '../../helpers';
import type { Readable } from 'stream';
import type { ViewCreateReqType } from 'nocodb-sdk';
import type { LinkToAnotherRecordColumn, User, View } from '../../../models';
import type { LinkToAnotherRecordColumn, User, View } from '../../../../models';
@Injectable()
export class ImportService {

16
packages/nocodb/src/modules/jobs/jobs/jobs-log.service.ts

@ -0,0 +1,16 @@
import { Injectable } from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { JobEvents } from '../../../interface/Jobs';
import type { Job } from 'bull';
@Injectable()
export class JobsLogService {
constructor(private eventEmitter: EventEmitter2) {}
sendLog(job: Job, data: { message: string }) {
this.eventEmitter.emit(JobEvents.LOG, {
id: job.id.toString(),
data,
});
}
}

107
packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts

@ -0,0 +1,107 @@
import {
OnQueueActive,
OnQueueCompleted,
OnQueueFailed,
Processor,
} from '@nestjs/bull';
import { Job } from 'bull';
import boxen from 'boxen';
import { EventEmitter2, OnEvent } from '@nestjs/event-emitter';
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
import { JobsRedisService } from './jobs-redis.service';
@Processor(JOBS_QUEUE)
export class JobsEventService {
constructor(
private jobsRedisService: JobsRedisService,
private eventEmitter: EventEmitter2,
) {}
@OnQueueActive()
onActive(job: Job) {
if (process.env.NC_WORKER_CONTAINER === 'true') {
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
cmd: JobEvents.STATUS,
id: job.id.toString(),
status: JobStatus.ACTIVE,
});
} else {
this.eventEmitter.emit(JobEvents.STATUS, {
id: job.id.toString(),
status: JobStatus.ACTIVE,
});
}
}
@OnQueueFailed()
onFailed(job: Job, error: Error) {
console.error(
boxen(
`---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`,
{
padding: 1,
borderStyle: 'double',
borderColor: 'yellow',
},
),
);
if (process.env.NC_WORKER_CONTAINER === 'true') {
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
cmd: JobEvents.STATUS,
id: job.id.toString(),
status: JobStatus.FAILED,
data: {
error: {
message: error?.message,
},
},
});
} else {
this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`);
this.eventEmitter.emit(JobEvents.STATUS, {
id: job.id.toString(),
status: JobStatus.FAILED,
data: {
error: {
message: error?.message,
},
},
});
}
}
@OnQueueCompleted()
onCompleted(job: Job, data: any) {
if (process.env.NC_WORKER_CONTAINER === 'true') {
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, {
cmd: JobEvents.STATUS,
id: job.id.toString(),
status: JobStatus.COMPLETED,
data: {
result: data,
},
});
} else {
this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`);
this.eventEmitter.emit(JobEvents.STATUS, {
id: job.id.toString(),
status: JobStatus.COMPLETED,
data: {
result: data,
},
});
}
}
@OnEvent(JobEvents.LOG)
onLog(data: { id: string; data: { message: string } }) {
if (process.env.NC_WORKER_CONTAINER === 'true') {
this.jobsRedisService.publish(`jobs-${data.id}`, {
cmd: JobEvents.LOG,
id: data.id,
data: data.data,
});
}
}
}

53
packages/nocodb/src/modules/jobs/redis/jobs-redis.service.ts

@ -0,0 +1,53 @@
import { Injectable } from '@nestjs/common';
import Redis from 'ioredis';
@Injectable()
export class JobsRedisService {
private redisClient: Redis;
private redisSubscriber: Redis;
private unsubscribeCallbacks: { [key: string]: () => void } = {};
constructor() {
if (process.env.NC_WORKER_CONTAINER === 'true') {
this.redisClient = new Redis(process.env.NC_REDIS_URL);
return;
}
this.redisSubscriber = new Redis(process.env.NC_REDIS_URL);
}
publish(channel: string, message: string | any) {
if (typeof message === 'string') {
this.redisClient.publish(channel, message);
} else {
try {
this.redisClient.publish(channel, JSON.stringify(message));
} catch (e) {
console.error(e);
}
}
}
subscribe(channel: string, callback: (message: any) => void) {
this.redisSubscriber.subscribe(channel);
const onMessage = (_channel, message) => {
try {
message = JSON.parse(message);
} catch (e) {}
callback(message);
};
this.redisSubscriber.on('message', onMessage);
this.unsubscribeCallbacks[channel] = () => {
this.redisSubscriber.unsubscribe(channel);
this.redisSubscriber.off('message', onMessage);
};
}
unsubscribe(channel: string) {
if (this.unsubscribeCallbacks[channel]) {
this.unsubscribeCallbacks[channel]();
delete this.unsubscribeCallbacks[channel];
}
}
}

98
packages/nocodb/src/modules/jobs/redis/jobs.service.ts

@ -0,0 +1,98 @@
import { InjectQueue } from '@nestjs/bull';
import { Injectable } from '@nestjs/common';
import { Queue } from 'bull';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs';
import { JobsRedisService } from './jobs-redis.service';
import type { OnModuleInit } from '@nestjs/common';
@Injectable()
export class JobsService implements OnModuleInit {
constructor(
@InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue,
private jobsRedisService: JobsRedisService,
private eventEmitter: EventEmitter2,
) {}
// pause primary instance queue
async onModuleInit() {
if (process.env.NC_WORKER_CONTAINER !== 'true') {
await this.jobsQueue.pause(true);
}
}
async add(name: string, data: any) {
// resume primary instance queue if there is no worker
const workerCount = (await this.jobsQueue.getWorkers()).length;
const localWorkerPaused = await this.jobsQueue.isPaused(true);
// if there is no worker and primary instance queue is paused, resume it
// if there is any worker and primary instance queue is not paused, pause it
if (workerCount < 1 && localWorkerPaused) {
await this.jobsQueue.resume(true);
} else if (workerCount > 0 && !localWorkerPaused) {
await this.jobsQueue.pause(true);
}
const job = await this.jobsQueue.add(name, data);
// subscribe to job events
this.jobsRedisService.subscribe(`jobs-${job.id.toString()}`, (data) => {
const cmd = data.cmd;
delete data.cmd;
switch (cmd) {
case JobEvents.STATUS:
this.eventEmitter.emit(JobEvents.STATUS, data);
if ([JobStatus.COMPLETED, JobStatus.FAILED].includes(data.status)) {
this.jobsRedisService.unsubscribe(`jobs-${data.id.toString()}`);
}
break;
case JobEvents.LOG:
this.eventEmitter.emit(JobEvents.LOG, data);
break;
}
});
return job;
}
async jobStatus(jobId: string) {
const job = await this.jobsQueue.getJob(jobId);
if (job) {
return await job.getState();
}
}
async jobList() {
return await this.jobsQueue.getJobs([
JobStatus.ACTIVE,
JobStatus.WAITING,
JobStatus.DELAYED,
JobStatus.PAUSED,
]);
}
async getJobWithData(data: any) {
const jobs = await this.jobsQueue.getJobs([
// 'completed',
JobStatus.WAITING,
JobStatus.ACTIVE,
JobStatus.DELAYED,
// 'failed',
JobStatus.PAUSED,
]);
const job = jobs.find((j) => {
for (const key in data) {
if (j.data[key]) {
if (j.data[key] !== data[key]) return false;
} else {
return false;
}
}
return true;
});
return job;
}
}

69
packages/nocodb/src/modules/metas/metas.module.ts

@ -67,7 +67,6 @@ import { UtilsService } from '../../services/utils.service';
import { ViewColumnsService } from '../../services/view-columns.service';
import { ViewsService } from '../../services/views.service';
import { ApiDocsService } from '../../services/api-docs/api-docs.service';
import { EventEmitterModule } from '../event-emitter/event-emitter.module';
import { GlobalModule } from '../global/global.module';
import { ProjectUsersController } from '../../controllers/project-users.controller';
import { ProjectUsersService } from '../../services/project-users/project-users.service';
@ -83,38 +82,42 @@ import { ProjectUsersService } from '../../services/project-users/project-users.
GlobalModule,
],
controllers: [
ApiDocsController,
ApiTokensController,
AttachmentsController,
AuditsController,
BasesController,
CachesController,
ColumnsController,
FiltersController,
FormColumnsController,
FormsController,
GalleriesController,
GridColumnsController,
GridsController,
HooksController,
KanbansController,
MapsController,
MetaDiffsController,
ModelVisibilitiesController,
OrgLcenseController,
OrgTokensController,
OrgUsersController,
PluginsController,
ProjectUsersController,
ProjectsController,
PublicMetasController,
ViewsController,
ViewColumnsController,
UtilsController,
TablesController,
SyncController,
SortsController,
SharedBasesController,
...(process.env.NC_WORKER_CONTAINER !== 'true'
? [
ApiDocsController,
ApiTokensController,
AttachmentsController,
AuditsController,
BasesController,
CachesController,
ColumnsController,
FiltersController,
FormColumnsController,
FormsController,
GalleriesController,
GridColumnsController,
GridsController,
HooksController,
KanbansController,
MapsController,
MetaDiffsController,
ModelVisibilitiesController,
OrgLcenseController,
OrgTokensController,
OrgUsersController,
PluginsController,
ProjectUsersController,
ProjectsController,
PublicMetasController,
ViewsController,
ViewColumnsController,
UtilsController,
TablesController,
SyncController,
SortsController,
SharedBasesController,
]
: []),
],
providers: [
ApiDocsService,

4
packages/nocodb/src/modules/test/test.module.ts

@ -2,6 +2,8 @@ import { Module } from '@nestjs/common';
import { TestController } from '../../controllers/test/test.controller';
@Module({
controllers: [TestController],
controllers: [
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [TestController] : []),
],
})
export class TestModule {}

4
packages/nocodb/src/modules/users/users.module.ts

@ -10,7 +10,9 @@ import { UsersController } from '../../controllers/users/users.controller';
@Module({
imports: [GlobalModule, PassportModule],
controllers: [UsersController],
controllers: [
...(process.env.NC_WORKER_CONTAINER !== 'true' ? [UsersController] : []),
],
providers: [UsersService, GoogleStrategyProvider],
exports: [UsersService],
})

17
packages/nocodb/src/plugins/storage/Local.ts

@ -3,7 +3,7 @@ import path from 'path';
import { promisify } from 'util';
import mkdirp from 'mkdirp';
import axios from 'axios';
import NcConfigFactory from '../../utils/NcConfigFactory';
import { getToolDir } from '../../utils/nc-config';
import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
import type { Readable } from 'stream';
@ -11,7 +11,7 @@ export default class Local implements IStorageAdapterV2 {
constructor() {}
public async fileCreate(key: string, file: XcFile): Promise<any> {
const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
const destPath = path.join(getToolDir(), ...key.split('/'));
try {
await mkdirp(path.dirname(destPath));
const data = await promisify(fs.readFile)(file.path);
@ -24,7 +24,7 @@ export default class Local implements IStorageAdapterV2 {
}
async fileCreateByUrl(key: string, url: string): Promise<any> {
const destPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
const destPath = path.join(getToolDir(), ...key.split('/'));
return new Promise((resolve, reject) => {
axios
.get(url, {
@ -71,10 +71,7 @@ export default class Local implements IStorageAdapterV2 {
stream: Readable,
): Promise<void> {
return new Promise((resolve, reject) => {
const destPath = path.join(
NcConfigFactory.getToolDir(),
...key.split('/'),
);
const destPath = path.join(getToolDir(), ...key.split('/'));
try {
mkdirp(path.dirname(destPath)).then(() => {
const writableStream = fs.createWriteStream(destPath);
@ -89,12 +86,12 @@ export default class Local implements IStorageAdapterV2 {
}
public async fileReadByStream(key: string): Promise<Readable> {
const srcPath = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
const srcPath = path.join(getToolDir(), ...key.split('/'));
return fs.createReadStream(srcPath, { encoding: 'utf8' });
}
public async getDirectoryList(key: string): Promise<string[]> {
const destDir = path.join(NcConfigFactory.getToolDir(), ...key.split('/'));
const destDir = path.join(getToolDir(), ...key.split('/'));
return fs.promises.readdir(destDir);
}
@ -106,7 +103,7 @@ export default class Local implements IStorageAdapterV2 {
public async fileRead(filePath: string): Promise<any> {
try {
const fileData = await fs.promises.readFile(
path.join(NcConfigFactory.getToolDir(), ...filePath.split('/')),
path.join(getToolDir(), ...filePath.split('/')),
);
return fileData;
} catch (e) {

19
packages/nocodb/src/services/app-init.service.spec.ts

@ -1,19 +0,0 @@
import { Test } from '@nestjs/testing';
import { AppInitService } from './app-init.service';
import type { TestingModule } from '@nestjs/testing';
describe('AppInitService', () => {
let service: AppInitService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [AppInitService],
}).compile();
service = module.get<AppInitService>(AppInitService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

79
packages/nocodb/src/services/app-init.service.ts

@ -1,79 +0,0 @@
import { T } from 'nc-help';
import NocoCache from '../cache/NocoCache';
import { Connection } from '../connection/connection';
import initAdminFromEnv from '../helpers/initAdminFromEnv';
import NcPluginMgrv2 from '../helpers/NcPluginMgrv2';
import { MetaService } from '../meta/meta.service';
import { User } from '../models';
import Noco from '../Noco';
import getInstance from '../utils/getInstance';
import NcConfigFactory from '../utils/NcConfigFactory';
import NcUpgrader from '../version-upgrader/NcUpgrader';
import type { IEventEmitter } from '../modules/event-emitter/event-emitter.interface';
import type { Provider } from '@nestjs/common';
export class AppInitService {
private readonly config: any;
constructor(config) {
this.config = config;
}
get appConfig(): any {
return this.config;
}
}
export const appInitServiceProvider: Provider = {
provide: AppInitService,
// initialize app,
// 1. init cache
// 2. init db connection and create if not exist
// 3. init meta and set to Noco
// 4. init jwt
// 5. init plugin manager
// 6. run upgrader
useFactory: async (
connection: Connection,
metaService: MetaService,
eventEmitter: IEventEmitter,
) => {
process.env.NC_VERSION = '0107004';
await NocoCache.init();
await connection.init();
await NcConfigFactory.metaDbCreateIfNotExist(connection.config);
await metaService.init();
// todo: remove
// temporary hack
Noco._ncMeta = metaService;
Noco.config = connection.config;
Noco.eventEmitter = eventEmitter;
// init jwt secret
await Noco.initJwt();
// load super admin user from env if env is set
await initAdminFromEnv(metaService);
// init plugin manager
await NcPluginMgrv2.init(Noco.ncMeta);
await Noco.loadEEState();
// run upgrader
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
T.init({
instance: getInstance,
});
T.emit('evt_app_started', await User.count());
// todo: move app config to app-init service
return new AppInitService(connection.config);
},
inject: [Connection, MetaService, 'IEventEmitter'],
};

7
packages/nocodb/src/services/auth.service.ts

@ -5,18 +5,13 @@ import * as bcrypt from 'bcryptjs';
import { v4 as uuidv4 } from 'uuid';
import Noco from '../Noco';
import { Connection } from '../connection/connection';
import { genJwt } from './users/helpers';
import { UsersService } from './users/users.service';
import type { CreateUserDto } from '../controllers/auth.controller';
@Injectable()
export class AuthService {
constructor(
private usersService: UsersService,
// private jwtService: JwtService,
private connection: Connection,
) {}
constructor(private usersService: UsersService) {}
async validateUser(email: string, pass: string): Promise<any> {
const user = await this.usersService.findOne(email);

4
packages/nocodb/src/services/projects.service.ts

@ -11,7 +11,7 @@ import syncMigration from '../helpers/syncMigration';
import { Project, ProjectUser } from '../models';
import Noco from '../Noco';
import extractRolesObj from '../utils/extractRolesObj';
import NcConfigFactory from '../utils/NcConfigFactory';
import { getToolDir } from '../utils/nc-config';
import type { ProjectUpdateReqType } from 'nocodb-sdk';
import type { ProjectReqType } from 'nocodb-sdk';
@ -96,7 +96,7 @@ export class ProjectsService {
// if env variable NC_MINIMAL_DBS is set, then create a SQLite file/connection for each project
// each file will be named as nc_<random_id>.db
const fs = require('fs');
const toolDir = NcConfigFactory.getToolDir();
const toolDir = getToolDir();
const nanoidv2 = customAlphabet(
'1234567890abcdefghijklmnopqrstuvwxyz',
14,

4
packages/nocodb/src/services/utils.service.ts

@ -9,7 +9,7 @@ import { Project, User } from '../models';
import Noco from '../Noco';
import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2';
import { MetaTable } from '../utils/globals';
import NcConfigFactory from '../utils/NcConfigFactory';
import { jdbcToXcConfig } from '../utils/nc-config/helpers';
import { packageVersion } from '../utils/packageVersion';
const versionCache = {
@ -186,7 +186,7 @@ export class UtilsService {
}) {
const { url } = param.body;
try {
const connectionConfig = NcConfigFactory.extractXcUrlFromJdbc(url, true);
const connectionConfig = jdbcToXcConfig(url);
return connectionConfig;
} catch (error) {
return NcError.internalServerError(

755
packages/nocodb/src/utils/NcConfigFactory.ts

@ -1,755 +0,0 @@
import fs from 'fs';
import { URL } from 'url';
import { promisify } from 'util';
import * as path from 'path';
import parseDbUrl from 'parse-database-url';
import { SqlClientFactory } from '../db/sql-client/lib/SqlClientFactory';
// import SqlClientFactory from '../db/sql-client/lib/SqlClientFactory';
// import type {
// AuthConfig,
// DbConfig,
// MailerConfig,
// NcConfig,
// } from '../../interface/config';
// const {
// uniqueNamesGenerator,
// starWars,
// adjectives,
// animals,
// } = require('unique-names-generator');
type NcConfig = any;
type DbConfig = any;
const driverClientMapping = {
mysql: 'mysql2',
mariadb: 'mysql2',
postgres: 'pg',
postgresql: 'pg',
sqlite: 'sqlite3',
mssql: 'mssql',
};
const defaultClientPortMapping = {
mysql: 3306,
mysql2: 3306,
postgres: 5432,
pg: 5432,
mssql: 1433,
};
const defaultConnectionConfig: any = {
// https://github.com/knex/knex/issues/97
// timezone: process.env.NC_TIMEZONE || 'UTC',
dateStrings: true,
};
// default knex options
const defaultConnectionOptions = {
pool: {
min: 0,
max: 10,
},
};
const knownQueryParams = [
{
parameter: 'database',
aliases: ['d', 'db'],
},
{
parameter: 'password',
aliases: ['p'],
},
{
parameter: 'user',
aliases: ['u'],
},
{
parameter: 'title',
aliases: ['t'],
},
{
parameter: 'keyFilePath',
aliases: [],
},
{
parameter: 'certFilePath',
aliases: [],
},
{
parameter: 'caFilePath',
aliases: [],
},
{
parameter: 'ssl',
aliases: [],
},
{
parameter: 'options',
aliases: ['opt', 'opts'],
},
];
export default class NcConfigFactory {
public static async make(): Promise<any> {
await this.jdbcToXcUrl();
const ncConfig = new NcConfigFactory();
ncConfig.auth = {
jwt: {
secret: process.env.NC_AUTH_JWT_SECRET,
},
};
ncConfig.port = +(process?.env?.PORT ?? 8080);
ncConfig.env = '_noco'; // process.env?.NODE_ENV || 'dev';
ncConfig.workingEnv = '_noco'; // process.env?.NODE_ENV || 'dev';
// ncConfig.toolDir = this.getToolDir();
ncConfig.projectType =
ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest';
if (ncConfig.meta?.db?.connection?.filename) {
ncConfig.meta.db.connection.filename = path.join(
this.getToolDir(),
ncConfig.meta.db.connection.filename,
);
}
if (process.env.NC_DB) {
ncConfig.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
} else if (process.env.NC_DB_JSON) {
ncConfig.meta.db = JSON.parse(process.env.NC_DB_JSON);
} else if (process.env.NC_DB_JSON_FILE) {
const filePath = process.env.NC_DB_JSON_FILE;
if (!(await promisify(fs.exists)(filePath))) {
throw new Error(`NC_DB_JSON_FILE not found: ${filePath}`);
}
const fileContent = await promisify(fs.readFile)(filePath, {
encoding: 'utf8',
});
ncConfig.meta.db = JSON.parse(fileContent);
}
if (process.env.NC_TRY) {
ncConfig.try = true;
ncConfig.meta.db = {
client: 'sqlite3',
connection: ':memory:',
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
} as any;
}
if (process.env.NC_PUBLIC_URL) {
ncConfig.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
// ncConfig.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
ncConfig.publicUrl = process.env.NC_PUBLIC_URL;
}
if (process.env.NC_DASHBOARD_URL) {
ncConfig.dashboardPath = process.env.NC_DASHBOARD_URL;
}
return ncConfig;
}
public static getToolDir() {
return process.env.NC_TOOL_DIR || process.cwd();
}
public static hasDbUrl(): boolean {
return Object.keys(process.env).some((envKey) =>
envKey.startsWith('NC_DB_URL'),
);
}
public static makeFromUrls(urls: string[]): NcConfig {
const config = new NcConfigFactory();
// config.envs[process.env.NODE_ENV || 'dev'].db = [];
config.envs['_noco'].db = [];
for (const [i, url] of Object.entries(urls)) {
// config.envs[process.env.NODE_ENV || 'dev'].db.push(this.urlToDbConfig(url, i));
config.envs['_noco'].db.push(this.urlToDbConfig(url, i));
}
return config;
}
public static urlToDbConfig(
urlString: string,
key = '',
config?: NcConfigFactory,
type?: string,
): DbConfig {
const url = new URL(urlString);
let dbConfig: DbConfig;
if (url.protocol.startsWith('sqlite3')) {
dbConfig = {
client: 'sqlite3',
connection: {
client: 'sqlite3',
connection: {
filename:
url.searchParams.get('d') || url.searchParams.get('database'),
},
database:
url.searchParams.get('d') || url.searchParams.get('database'),
},
} as any;
} else {
const parsedQuery = {};
for (const [key, value] of url.searchParams.entries()) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedQuery[fnd.parameter] = value;
} else {
parsedQuery[key] = value;
}
}
dbConfig = {
client: url.protocol.replace(':', ''),
connection: {
...defaultConnectionConfig,
...parsedQuery,
host: url.hostname,
port: +url.port,
},
// pool: {
// min: 1,
// max: 1
// },
acquireConnectionTimeout: 600000,
} as any;
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
dbConfig.connection.ssl = true;
}
if (
url.searchParams.get('keyFilePath') &&
url.searchParams.get('certFilePath') &&
url.searchParams.get('caFilePath')
) {
dbConfig.connection.ssl = {
keyFilePath: url.searchParams.get('keyFilePath'),
certFilePath: url.searchParams.get('certFilePath'),
caFilePath: url.searchParams.get('caFilePath'),
};
}
}
if (config && !config.title) {
config.title =
url.searchParams.get('t') ||
url.searchParams.get('title') ||
this.generateRandomTitle();
}
Object.assign(dbConfig, {
meta: {
tn: 'nc_evolutions',
allSchemas:
!!url.searchParams.get('allSchemas') ||
!(url.searchParams.get('d') || url.searchParams.get('database')),
api: {
prefix: url.searchParams.get('apiPrefix') || '',
swagger: true,
type:
type ||
((url.searchParams.get('api') ||
url.searchParams.get('a')) as any) ||
'rest',
},
dbAlias: url.searchParams.get('dbAlias') || `db${key}`,
metaTables: 'db',
migrations: {
disabled: false,
name: 'nc_evolutions',
},
},
});
return dbConfig;
}
private static generateRandomTitle(): string {
return ''; /*uniqueNamesGenerator({
dictionaries: [[starWars], [adjectives, animals]][
Math.floor(Math.random() * 2)
],
})
.toLowerCase()
.replace(/[ -]/g, '_');*/
}
static async metaUrlToDbConfig(urlString) {
const url = new URL(urlString);
let dbConfig;
if (url.protocol.startsWith('sqlite3')) {
const db = url.searchParams.get('d') || url.searchParams.get('database');
dbConfig = {
client: 'sqlite3',
connection: {
filename: db,
},
...(db === ':memory:'
? {
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
}
: {}),
};
} else {
const parsedQuery = {};
for (const [key, value] of url.searchParams.entries()) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedQuery[fnd.parameter] = value;
} else {
parsedQuery[key] = value;
}
}
dbConfig = {
client: url.protocol.replace(':', ''),
connection: {
...defaultConnectionConfig,
...parsedQuery,
host: url.hostname,
port: +url.port,
},
acquireConnectionTimeout: 600000,
...(url.searchParams.has('search_path')
? {
searchPath: url.searchParams.get('search_path').split(','),
}
: {}),
};
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
dbConfig.connection.ssl = true;
}
}
url.searchParams.forEach((_value, key) => {
let value: any = _value;
if (value === 'true') {
value = true;
} else if (value === 'false') {
value = false;
} else if (/^\d+$/.test(value)) {
value = +value;
}
// todo: implement config read from JSON file or JSON env val read
if (
![
'password',
'p',
'database',
'd',
'user',
'u',
'search_path',
].includes(key)
) {
key.split('.').reduce((obj, k, i, arr) => {
return (obj[k] = i === arr.length - 1 ? value : obj[k] || {});
}, dbConfig);
}
});
if (
dbConfig?.connection?.ssl &&
typeof dbConfig?.connection?.ssl === 'object'
) {
if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) {
dbConfig.connection.ssl.ca = (
await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath)
).toString();
delete dbConfig.connection.ssl.caFilePath;
}
if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) {
dbConfig.connection.ssl.key = (
await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath)
).toString();
delete dbConfig.connection.ssl.keyFilePath;
}
if (
dbConfig.connection.ssl.certFilePath &&
!dbConfig.connection.ssl.cert
) {
dbConfig.connection.ssl.cert = (
await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath)
).toString();
delete dbConfig.connection.ssl.certFilePath;
}
}
return dbConfig;
}
public static async makeProjectConfigFromUrl(
url,
type?: string,
): Promise<NcConfig> {
const config = new NcConfigFactory();
const dbConfig = this.urlToDbConfig(url, '', config, type);
// config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
config.envs['_noco'].db.push(dbConfig);
if (process.env.NC_AUTH_ADMIN_SECRET) {
config.auth = {
masterKey: {
secret: process.env.NC_AUTH_ADMIN_SECRET,
},
};
} else if (process.env.NC_NO_AUTH) {
config.auth = {
disabled: true,
};
// } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
} else if (config?.envs?.['_noco']?.db?.[0]) {
config.auth = {
jwt: {
// dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
dbAlias:
process.env.NC_AUTH_JWT_DB_ALIAS ||
config.envs['_noco'].db[0].meta.dbAlias,
secret: process.env.NC_AUTH_JWT_SECRET,
},
};
}
if (process.env.NC_DB) {
config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
}
if (process.env.NC_TRY) {
config.try = true;
config.meta.db = {
client: 'sqlite3',
connection: ':memory:',
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
} as any;
}
if (process.env.NC_MAILER) {
config.mailer = {
from: process.env.NC_MAILER_FROM,
options: {
host: process.env.NC_MAILER_HOST,
port: parseInt(process.env.NC_MAILER_PORT, 10),
secure: process.env.NC_MAILER_SECURE === 'true',
auth: {
user: process.env.NC_MAILER_USER,
pass: process.env.NC_MAILER_PASS,
},
},
};
}
if (process.env.NC_PUBLIC_URL) {
// config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
config.publicUrl = process.env.NC_PUBLIC_URL;
}
config.port = +(process?.env?.PORT ?? 8080);
// config.env = process.env?.NODE_ENV || 'dev';
// config.workingEnv = process.env?.NODE_ENV || 'dev';
config.env = '_noco';
config.workingEnv = '_noco';
config.toolDir = this.getToolDir();
config.projectType =
type ||
config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type ||
'rest';
return config;
}
public static async makeProjectConfigFromConnection(
dbConnectionConfig: any,
type?: string,
): Promise<NcConfig> {
const config = new NcConfigFactory();
let dbConfig = dbConnectionConfig;
if (dbConfig.client === 'sqlite3') {
dbConfig = {
client: 'sqlite3',
connection: {
...dbConnectionConfig,
database: dbConnectionConfig.connection.filename,
},
};
}
// todo:
const key = '';
Object.assign(dbConfig, {
meta: {
tn: 'nc_evolutions',
api: {
prefix: '',
swagger: true,
type: type || 'rest',
},
dbAlias: `db${key}`,
metaTables: 'db',
migrations: {
disabled: false,
name: 'nc_evolutions',
},
},
});
// config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
config.envs['_noco'].db.push(dbConfig);
if (process.env.NC_AUTH_ADMIN_SECRET) {
config.auth = {
masterKey: {
secret: process.env.NC_AUTH_ADMIN_SECRET,
},
};
} else if (process.env.NC_NO_AUTH) {
config.auth = {
disabled: true,
};
// } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
} else if (config?.envs?.['_noco']?.db?.[0]) {
config.auth = {
jwt: {
// dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
dbAlias:
process.env.NC_AUTH_JWT_DB_ALIAS ||
config.envs['_noco'].db[0].meta.dbAlias,
secret: process.env.NC_AUTH_JWT_SECRET,
},
};
}
if (process.env.NC_DB) {
config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB);
}
if (process.env.NC_TRY) {
config.try = true;
config.meta.db = {
client: 'sqlite3',
connection: ':memory:',
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
} as any;
}
if (process.env.NC_PUBLIC_URL) {
// config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL;
config.publicUrl = process.env.NC_PUBLIC_URL;
}
config.port = +(process?.env?.PORT ?? 8080);
// config.env = process.env?.NODE_ENV || 'dev';
// config.workingEnv = process.env?.NODE_ENV || 'dev';
config.env = '_noco';
config.workingEnv = '_noco';
config.toolDir = process.env.NC_TOOL_DIR || process.cwd();
config.projectType =
type ||
config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type ||
'rest';
return config;
}
public static async metaDbCreateIfNotExist(args: NcConfig) {
if (args.meta?.db?.client === 'sqlite3') {
const metaSqlClient = await SqlClientFactory.create({
...args.meta.db,
connection: args.meta.db,
});
await metaSqlClient.createDatabaseIfNotExists({
database: args.meta.db?.connection?.filename,
});
} else {
const metaSqlClient = await SqlClientFactory.create(args.meta.db);
await metaSqlClient.createDatabaseIfNotExists(args.meta.db?.connection);
await metaSqlClient.knex.destroy();
}
/* const dbPath = path.join(args.toolDir, 'xc.db')
const exists = fs.existsSync(dbPath);
if (!exists) {
const fd = fs.openSync(dbPath, "w");
fs.closeSync(fd);
}
*/
}
public version = '0.6';
public port: number;
public auth?: any;
public env: 'production' | 'dev' | 'test' | string;
public workingEnv: string;
public toolDir: string;
public envs: {
[p: string]: { db: DbConfig[]; api?: any; publicUrl?: string };
};
// public projectType: "rest" | "graphql" | "grpc";
public queriesFolder: string | string[] = '';
public seedsFolder: string | string[];
public title: string;
public publicUrl: string;
public projectType;
public meta = {
db: {
client: 'sqlite3',
connection: {
filename: 'noco.db',
},
},
};
public mailer: any;
public try = false;
public dashboardPath = '/dashboard';
constructor() {
this.envs = { _noco: { db: [] } };
}
public static async jdbcToXcUrl() {
if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) {
const database_url = await promisify(fs.readFile)(
process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
'utf-8',
);
process.env.NC_DB = this.extractXcUrlFromJdbc(database_url);
} else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) {
process.env.NC_DB = this.extractXcUrlFromJdbc(
process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
);
}
}
public static extractXcUrlFromJdbc(url: string, rtConfig = false) {
// drop the jdbc prefix
if (url.startsWith('jdbc:')) {
url = url.substring(5);
}
const config = parseDbUrl(url);
const parsedConfig: {
driver?: string;
host?: string;
port?: string;
database?: string;
user?: string;
password?: string;
ssl?: string;
} = {};
for (const [key, value] of Object.entries(config)) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedConfig[fnd.parameter] = value;
} else {
parsedConfig[key] = value;
}
}
if (!parsedConfig?.port)
parsedConfig.port =
defaultClientPortMapping[
driverClientMapping[parsedConfig.driver] || parsedConfig.driver
];
if (rtConfig) {
const { driver, ...connectionConfig } = parsedConfig;
const client = driverClientMapping[driver] || driver;
const avoidSSL = [
'localhost',
'127.0.0.1',
'host.docker.internal',
'172.17.0.1',
];
if (
client === 'pg' &&
!connectionConfig?.ssl &&
!avoidSSL.includes(connectionConfig.host)
) {
connectionConfig.ssl = 'true';
}
return {
client: client,
connection: {
...connectionConfig,
},
} as any;
}
const { driver, host, port, database, user, password, ...extra } =
parsedConfig;
const extraParams = [];
for (const [key, value] of Object.entries(extra)) {
extraParams.push(`${key}=${value}`);
}
const res = `${driverClientMapping[driver] || driver}://${host}${
port ? `:${port}` : ''
}?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${
database ? `d=${database}&` : ''
}${extraParams.join('&')}`;
return res;
}
// public static initOneClickDeployment() {
// if (process.env.NC_ONE_CLICK) {
// const url = NcConfigFactory.extractXcUrlFromJdbc(process.env.DATABASE_URL);
// process.env.NC_DB = url;
// }
// }
}
export { defaultConnectionConfig, defaultConnectionOptions };

2
packages/nocodb/src/utils/common/NcConnectionMgr.ts

@ -2,7 +2,7 @@ import fs from 'fs';
import { promisify } from 'util';
import SqlClientFactory from '../../db/sql-client/lib/SqlClientFactory';
import { XKnex } from '../../db/CustomKnex';
import { defaultConnectionConfig } from '../NcConfigFactory';
import { defaultConnectionConfig } from '../nc-config';
// import type { NcConfig } from '../../../interface/config';
import type { Knex } from 'knex';
// import type NcMetaIO from '../../meta/NcMetaIO';

2
packages/nocodb/src/utils/common/NcConnectionMgrv2.ts

@ -3,7 +3,7 @@ import { XKnex } from '../../db/CustomKnex';
import {
defaultConnectionConfig,
defaultConnectionOptions,
} from '../NcConfigFactory';
} from '../nc-config';
import Noco from '../../Noco';
import type Base from '../../models/Base';

182
packages/nocodb/src/utils/nc-config/NcConfig.ts

@ -0,0 +1,182 @@
import * as path from 'path';
import fs from 'fs';
import { promisify } from 'util';
import { SqlClientFactory } from '../../db/sql-client/lib/SqlClientFactory';
import { getToolDir, metaUrlToDbConfig } from './helpers';
import { DriverClient } from './interfaces';
import type { DbConfig } from './interfaces';
export class NcConfig {
version: string;
meta: {
db: DbConfig;
} = {
db: {
client: DriverClient.SQLITE,
connection: {
filename: 'noco.db',
},
},
};
auth: {
jwt: {
secret: string;
options?: any;
};
};
// if this is true, port is not exposed
worker: boolean;
toolDir: string;
// exposed instance port
port: number;
// if this is true, use sqlite3 :memory: as meta db
try: boolean;
// optional
publicUrl?: string;
dashboardPath?: string;
// TODO what is this?
envs: any;
queriesFolder: string;
env: string;
workingEnv: string;
projectType: string;
private constructor() {}
public static async create(param: {
meta: {
metaUrl?: string;
metaJson?: string;
metaJsonFile?: string;
};
secret?: string;
port?: string | number;
tryMode?: boolean;
worker?: boolean;
dashboardPath?: string;
publicUrl?: string;
}): Promise<NcConfig> {
const { meta, secret, port, worker, tryMode, publicUrl, dashboardPath } =
param;
const ncConfig = new NcConfig();
ncConfig.auth = {
jwt: {
secret: secret,
},
};
ncConfig.port = +(port ?? 8080);
ncConfig.toolDir = getToolDir();
ncConfig.worker = worker ?? false;
ncConfig.env = '_noco';
ncConfig.workingEnv = '_noco';
ncConfig.projectType =
ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest';
if (ncConfig.meta?.db?.connection?.filename) {
ncConfig.meta.db.connection.filename = path.join(
ncConfig.toolDir,
ncConfig.meta.db.connection.filename,
);
}
if (tryMode) {
ncConfig.try = true;
ncConfig.meta.db = {
client: DriverClient.SQLITE,
connection: ':memory:' as any,
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
};
} else {
if (meta?.metaUrl) {
ncConfig.meta.db = await metaUrlToDbConfig(meta.metaUrl);
} else if (meta?.metaJson) {
ncConfig.meta.db = JSON.parse(meta.metaJson);
} else if (meta?.metaJsonFile) {
if (!(await promisify(fs.exists)(meta.metaJsonFile))) {
throw new Error(`NC_DB_JSON_FILE not found: ${meta.metaJsonFile}`);
}
const fileContent = await promisify(fs.readFile)(meta.metaJsonFile, {
encoding: 'utf8',
});
ncConfig.meta.db = JSON.parse(fileContent);
}
}
if (publicUrl) {
ncConfig.envs['_noco'].publicUrl = publicUrl;
ncConfig.publicUrl = publicUrl;
}
if (dashboardPath) {
ncConfig.dashboardPath = dashboardPath;
}
try {
// make sure meta db exists
await ncConfig.metaDbCreateIfNotExist();
} catch (e) {
throw new Error(e);
}
return ncConfig;
}
public static async createByEnv(): Promise<NcConfig> {
return NcConfig.create({
meta: {
metaUrl: process.env.NC_DB,
metaJson: process.env.NC_DB_JSON,
metaJsonFile: process.env.NC_DB_JSON_FILE,
},
secret: process.env.NC_AUTH_JWT_SECRET,
port: process.env.NC_PORT,
tryMode: !!process.env.NC_TRY,
worker: !!process.env.NC_WORKER,
dashboardPath: process.env.NC_DASHBOARD_PATH,
publicUrl: process.env.NC_PUBLIC_URL,
});
}
private async metaDbCreateIfNotExist() {
if (this.meta?.db?.client === 'sqlite3') {
const metaSqlClient = await SqlClientFactory.create({
...this.meta.db,
connection: this.meta.db,
});
if (this.meta.db?.connection?.filename) {
await metaSqlClient.createDatabaseIfNotExists({
database: this.meta.db?.connection?.filename,
});
} else {
throw new Error('Configuration missing meta db connection');
}
} else {
const metaSqlClient = await SqlClientFactory.create(this.meta.db);
if (this.meta.db?.connection?.database) {
await metaSqlClient.createDatabaseIfNotExists(
(this.meta.db as any).connection,
);
await metaSqlClient.knex.destroy();
} else {
throw new Error('Configuration missing meta db connection');
}
}
}
}

84
packages/nocodb/src/utils/nc-config/constants.ts

@ -0,0 +1,84 @@
export const driverClientMapping = {
mysql: 'mysql2',
mariadb: 'mysql2',
postgres: 'pg',
postgresql: 'pg',
sqlite: 'sqlite3',
mssql: 'mssql',
};
export const defaultClientPortMapping = {
mysql: 3306,
mysql2: 3306,
postgres: 5432,
pg: 5432,
mssql: 1433,
};
export const defaultConnectionConfig: any = {
// https://github.com/knex/knex/issues/97
// timezone: process.env.NC_TIMEZONE || 'UTC',
dateStrings: true,
};
// default knex options
export const defaultConnectionOptions = {
pool: {
min: 0,
max: 10,
},
};
export const avoidSSL = [
'localhost',
'127.0.0.1',
'host.docker.internal',
'172.17.0.1',
];
export const knownQueryParams = [
{
parameter: 'database',
aliases: ['d', 'db'],
},
{
parameter: 'password',
aliases: ['p'],
},
{
parameter: 'user',
aliases: ['u'],
},
{
parameter: 'title',
aliases: ['t'],
},
{
parameter: 'keyFilePath',
aliases: [],
},
{
parameter: 'certFilePath',
aliases: [],
},
{
parameter: 'caFilePath',
aliases: [],
},
{
parameter: 'ssl',
aliases: [],
},
{
parameter: 'options',
aliases: ['opt', 'opts'],
},
];
export enum DriverClient {
MYSQL = 'mysql2',
MSSQL = 'mssql',
PG = 'pg',
SQLITE = 'sqlite3',
SNOWFLAKE = 'snowflake',
}

324
packages/nocodb/src/utils/nc-config/helpers.ts

@ -0,0 +1,324 @@
import fs from 'fs';
import { URL } from 'url';
import { promisify } from 'util';
import parseDbUrl from 'parse-database-url';
import {
avoidSSL,
defaultClientPortMapping,
defaultConnectionConfig,
defaultConnectionOptions,
driverClientMapping,
knownQueryParams,
} from './constants';
import { DriverClient } from './interfaces';
import type { Connection, DbConfig } from './interfaces';
export async function prepareEnv() {
if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) {
const database_url = await promisify(fs.readFile)(
process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE,
'utf-8',
);
process.env.NC_DB = jdbcToXcUrl(database_url);
} else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) {
process.env.NC_DB = jdbcToXcUrl(
process.env.NC_DATABASE_URL || process.env.DATABASE_URL,
);
}
}
export function getToolDir() {
return process.env.NC_TOOL_DIR || process.cwd();
}
export function jdbcToXcConfig(url: string): DbConfig {
// drop the jdbc prefix
url.replace(/^jdbc:/, '');
const config = parseDbUrl(url);
const parsedConfig: Connection = {};
for (const [key, value] of Object.entries(config)) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedConfig[fnd.parameter] = value;
} else {
parsedConfig[key] = value;
}
}
if (!parsedConfig?.port) {
parsedConfig.port =
defaultClientPortMapping[
driverClientMapping[parsedConfig.driver] || parsedConfig.driver
];
}
const { driver, ...connectionConfig } = parsedConfig;
const client = driverClientMapping[driver] || driver;
if (
client === 'pg' &&
!connectionConfig?.ssl &&
!avoidSSL.includes(connectionConfig.host)
) {
connectionConfig.ssl = true;
}
return {
client: client,
connection: {
...connectionConfig,
},
} as DbConfig;
}
export function jdbcToXcUrl(url: string): string {
// drop the jdbc prefix
url.replace(/^jdbc:/, '');
const config = parseDbUrl(url);
const parsedConfig: Connection = {};
for (const [key, value] of Object.entries(config)) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedConfig[fnd.parameter] = value;
} else {
parsedConfig[key] = value;
}
}
if (!parsedConfig?.port) {
parsedConfig.port =
defaultClientPortMapping[
driverClientMapping[parsedConfig.driver] || parsedConfig.driver
];
}
const { driver, host, port, database, user, password, ...extra } =
parsedConfig;
const extraParams = [];
for (const [key, value] of Object.entries(extra)) {
extraParams.push(`${key}=${value}`);
}
const res = `${driverClientMapping[driver] || driver}://${host}${
port ? `:${port}` : ''
}?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${
database ? `d=${database}&` : ''
}${extraParams.join('&')}`;
return res;
}
export function xcUrlToDbConfig(
urlString: string,
key = '',
type?: string,
): DbConfig {
const url = new URL(urlString);
let dbConfig: DbConfig;
if (url.protocol.startsWith('sqlite3')) {
dbConfig = {
client: 'sqlite3',
connection: {
client: 'sqlite3',
connection: {
filename:
url.searchParams.get('d') || url.searchParams.get('database'),
},
database: url.searchParams.get('d') || url.searchParams.get('database'),
},
} as any;
} else {
const parsedQuery = {};
for (const [key, value] of url.searchParams.entries()) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedQuery[fnd.parameter] = value;
} else {
parsedQuery[key] = value;
}
}
dbConfig = {
client: url.protocol.replace(':', '') as DriverClient,
connection: {
...parsedQuery,
host: url.hostname,
port: +url.port,
},
acquireConnectionTimeout: 600000,
};
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
dbConfig.connection.ssl = true;
}
if (
url.searchParams.get('keyFilePath') &&
url.searchParams.get('certFilePath') &&
url.searchParams.get('caFilePath')
) {
dbConfig.connection.ssl = {
keyFilePath: url.searchParams.get('keyFilePath'),
certFilePath: url.searchParams.get('certFilePath'),
caFilePath: url.searchParams.get('caFilePath'),
};
}
}
/* TODO check if this is needed
if (config && !config.title) {
config.title =
url.searchParams.get('t') ||
url.searchParams.get('title') ||
this.generateRandomTitle();
}
*/
Object.assign(dbConfig, {
meta: {
tn: 'nc_evolutions',
allSchemas:
!!url.searchParams.get('allSchemas') ||
!(url.searchParams.get('d') || url.searchParams.get('database')),
api: {
prefix: url.searchParams.get('apiPrefix') || '',
swagger: true,
type:
type ||
((url.searchParams.get('api') || url.searchParams.get('a')) as any) ||
'rest',
},
dbAlias: url.searchParams.get('dbAlias') || `db${key}`,
metaTables: 'db',
migrations: {
disabled: false,
name: 'nc_evolutions',
},
},
});
return dbConfig;
}
export async function metaUrlToDbConfig(urlString): Promise<DbConfig> {
const url = new URL(urlString);
let dbConfig: DbConfig;
if (url.protocol.startsWith('sqlite3')) {
const db = url.searchParams.get('d') || url.searchParams.get('database');
dbConfig = {
client: DriverClient.SQLITE,
connection: {
filename: db,
},
...(db === ':memory:'
? {
pool: {
min: 1,
max: 1,
// disposeTimeout: 360000*1000,
idleTimeoutMillis: 360000 * 1000,
},
}
: {}),
};
} else {
const parsedQuery = {};
for (const [key, value] of url.searchParams.entries()) {
const fnd = knownQueryParams.find(
(param) => param.parameter === key || param.aliases.includes(key),
);
if (fnd) {
parsedQuery[fnd.parameter] = value;
} else {
parsedQuery[key] = value;
}
}
dbConfig = {
client: url.protocol.replace(':', '') as DriverClient,
connection: {
...defaultConnectionConfig,
...parsedQuery,
host: url.hostname,
port: +url.port,
},
acquireConnectionTimeout: 600000,
...defaultConnectionOptions,
...(url.searchParams.has('search_path')
? {
searchPath: url.searchParams.get('search_path').split(','),
}
: {}),
};
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) {
dbConfig.connection.ssl = true;
}
}
url.searchParams.forEach((_value, key) => {
let value: any = _value;
if (value === 'true') {
value = true;
} else if (value === 'false') {
value = false;
} else if (/^\d+$/.test(value)) {
value = +value;
}
// todo: implement config read from JSON file or JSON env val read
if (
!['password', 'p', 'database', 'd', 'user', 'u', 'search_path'].includes(
key,
)
) {
key.split('.').reduce((obj, k, i, arr) => {
return (obj[k] = i === arr.length - 1 ? value : obj[k] || {});
}, dbConfig);
}
});
if (
dbConfig?.connection?.ssl &&
typeof dbConfig?.connection?.ssl === 'object'
) {
if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) {
dbConfig.connection.ssl.ca = (
await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath)
).toString();
delete dbConfig.connection.ssl.caFilePath;
}
if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) {
dbConfig.connection.ssl.key = (
await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath)
).toString();
delete dbConfig.connection.ssl.keyFilePath;
}
if (dbConfig.connection.ssl.certFilePath && !dbConfig.connection.ssl.cert) {
dbConfig.connection.ssl.cert = (
await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath)
).toString();
delete dbConfig.connection.ssl.certFilePath;
}
}
return dbConfig;
}

4
packages/nocodb/src/utils/nc-config/index.ts

@ -0,0 +1,4 @@
export * from './helpers';
export * from './interfaces';
export * from './constants';
export * from './NcConfig';

39
packages/nocodb/src/utils/nc-config/interfaces.ts

@ -0,0 +1,39 @@
import { DriverClient } from './constants';
interface Connection {
driver?: DriverClient;
host?: string;
port?: number;
database?: string;
user?: string;
password?: string;
ssl?:
| boolean
| {
ca?: string;
cert?: string;
key?: string;
caFilePath?: string;
certFilePath?: string;
keyFilePath?: string;
};
filename?: string;
}
interface DbConfig {
client: DriverClient;
connection: Connection;
acquireConnectionTimeout?: number;
useNullAsDefault?: boolean;
pool?: {
min?: number;
max?: number;
idleTimeoutMillis?: number;
};
migrations?: {
directory?: string;
tableName?: string;
};
}
export { DriverClient, Connection, DbConfig };

12
packages/nocodb/tests/unit/TestDbMngr.ts

@ -2,9 +2,9 @@ import fs from 'fs';
import process from 'process';
import { knex } from 'knex';
import SqlMgrv2 from '../../src/db/sql-mgr/v2/SqlMgrv2';
import { jdbcToXcUrl, xcUrlToDbConfig } from '../../src/utils/nc-config';
import type { Knex } from 'knex';
import type { DbConfig } from '../../src/interface/config';
import NcConfigFactory from '../../src/utils/NcConfigFactory'
export default class TestDbMngr {
public static readonly dbName = 'test_meta';
@ -75,7 +75,7 @@ export default class TestDbMngr {
private static async isDbConfigured() {
const { user, password, host, port, client } = TestDbMngr.connection;
const config = NcConfigFactory.urlToDbConfig(
const config = xcUrlToDbConfig(
`${client}://${user}:${password}@${host}:${port}`,
);
config.connection = {
@ -84,7 +84,7 @@ export default class TestDbMngr {
host,
port,
};
const result = await TestDbMngr.testConnection(config);
const result = await TestDbMngr.testConnection(config as any);
return result.code !== -1;
}
static async connectDb() {
@ -95,9 +95,9 @@ export default class TestDbMngr {
] = `${client}://${user}:${password}@${host}:${port}/${TestDbMngr.dbName}`;
}
TestDbMngr.dbConfig = NcConfigFactory.urlToDbConfig(
NcConfigFactory.extractXcUrlFromJdbc(process.env[`DATABASE_URL`]),
);
TestDbMngr.dbConfig = xcUrlToDbConfig(
jdbcToXcUrl(process.env[`DATABASE_URL`]),
) as any;
this.dbConfig.meta = {
tn: 'nc_evolutions',
dbAlias: 'db',

1
tests/playwright/pages/Dashboard/TreeView.ts

@ -216,7 +216,6 @@ export class TreeViewPage extends BasePage {
uiAction: () => this.rootPage.getByRole('button', { name: 'Confirm' }).click(),
httpMethodsToMatch: ['POST'],
requestUrlPathToMatch: `/api/v1/db/meta/duplicate/`,
responseJsonMatcher: json => json.name === 'duplicate-model',
});
await this.get().locator(`[data-testid="tree-view-table-${title} copy"]`).waitFor();
}

Loading…
Cancel
Save