mirror of https://github.com/nocodb/nocodb
mertmit
2 years ago
committed by
GitHub
55 changed files with 1344 additions and 1294 deletions
@ -1,19 +0,0 @@
|
||||
import { Test } from '@nestjs/testing'; |
||||
import { Connection } from './knex'; |
||||
import type { TestingModule } from '@nestjs/testing'; |
||||
|
||||
describe('Knex', () => { |
||||
let provider: Connection; |
||||
|
||||
beforeEach(async () => { |
||||
const module: TestingModule = await Test.createTestingModule({ |
||||
providers: [Connection], |
||||
}).compile(); |
||||
|
||||
provider = module.get<Connection>(Connection); |
||||
}); |
||||
|
||||
it('should be defined', () => { |
||||
expect(provider).toBeDefined(); |
||||
}); |
||||
}); |
||||
@ -1,37 +0,0 @@
|
||||
import { Global, Injectable, Scope } from '@nestjs/common'; |
||||
|
||||
import { XKnex } from '../db/CustomKnex'; |
||||
import NcConfigFactory from '../utils/NcConfigFactory'; |
||||
import type * as knex from 'knex'; |
||||
|
||||
@Injectable({ |
||||
scope: Scope.DEFAULT, |
||||
}) |
||||
export class Connection { |
||||
public static knex: knex.Knex; |
||||
public static _config: any; |
||||
|
||||
get knexInstance(): knex.Knex { |
||||
return Connection.knex; |
||||
} |
||||
|
||||
get config(): knex.Knex { |
||||
return Connection._config; |
||||
} |
||||
|
||||
// init metadb connection
|
||||
static async init(): Promise<void> { |
||||
Connection._config = await NcConfigFactory.make(); |
||||
if (!Connection.knex) { |
||||
Connection.knex = XKnex({ |
||||
...this._config.meta.db, |
||||
useNullAsDefault: true, |
||||
}); |
||||
} |
||||
} |
||||
|
||||
// init metadb connection
|
||||
async init(): Promise<void> { |
||||
return await Connection.init(); |
||||
} |
||||
} |
||||
@ -1,6 +1,5 @@
|
||||
import Noco from './Noco'; |
||||
import NcConfigFactory from './utils/NcConfigFactory'; |
||||
|
||||
export default Noco; |
||||
|
||||
export { Noco, NcConfigFactory }; |
||||
export { Noco }; |
||||
|
||||
@ -1,12 +1,12 @@
|
||||
import { Connection } from './connection/connection'; |
||||
import { MetaService } from './meta/meta.service'; |
||||
import { NcConfig } from './utils/nc-config'; |
||||
import Noco from './Noco'; |
||||
|
||||
// run upgrader
|
||||
import NcUpgrader from './version-upgrader/NcUpgrader'; |
||||
|
||||
export default async () => { |
||||
await Connection.init(); |
||||
Noco._ncMeta = new MetaService(new Connection()); |
||||
const config = await NcConfig.createByEnv(); |
||||
Noco._ncMeta = new MetaService(config); |
||||
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta }); |
||||
}; |
||||
|
||||
@ -0,0 +1,65 @@
|
||||
import { T } from 'nc-help'; |
||||
import { MetaService } from '../../meta/meta.service'; |
||||
import Noco from '../../Noco'; |
||||
import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2'; |
||||
import NcUpgrader from '../../version-upgrader/NcUpgrader'; |
||||
import NocoCache from '../../cache/NocoCache'; |
||||
import getInstance from '../../utils/getInstance'; |
||||
import initAdminFromEnv from '../../helpers/initAdminFromEnv'; |
||||
import { User } from '../../models'; |
||||
import { NcConfig, prepareEnv } from '../../utils/nc-config'; |
||||
import type { Provider } from '@nestjs/common'; |
||||
import type { IEventEmitter } from '../event-emitter/event-emitter.interface'; |
||||
|
||||
export const InitMetaServiceProvider: Provider = { |
||||
// initialize app,
|
||||
// 1. init cache
|
||||
// 2. init db connection and create if not exist
|
||||
// 3. init meta and set to Noco
|
||||
// 4. init jwt
|
||||
// 5. init plugin manager
|
||||
// 6. run upgrader
|
||||
useFactory: async (eventEmitter: IEventEmitter) => { |
||||
// NC_DATABASE_URL_FILE, DATABASE_URL_FILE, DATABASE_URL, NC_DATABASE_URL to NC_DB
|
||||
await prepareEnv(); |
||||
|
||||
const config = await NcConfig.createByEnv(); |
||||
|
||||
// set version
|
||||
process.env.NC_VERSION = '0107004'; |
||||
|
||||
// init cache
|
||||
await NocoCache.init(); |
||||
|
||||
// init meta service
|
||||
const metaService = new MetaService(config); |
||||
await metaService.init(); |
||||
|
||||
// provide meta and config to Noco
|
||||
Noco._ncMeta = metaService; |
||||
Noco.config = config; |
||||
Noco.eventEmitter = eventEmitter; |
||||
|
||||
// init jwt secret
|
||||
await Noco.initJwt(); |
||||
|
||||
// load super admin user from env if env is set
|
||||
await initAdminFromEnv(metaService); |
||||
|
||||
// init plugin manager
|
||||
await NcPluginMgrv2.init(Noco.ncMeta); |
||||
await Noco.loadEEState(); |
||||
|
||||
// run upgrader
|
||||
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta }); |
||||
|
||||
T.init({ |
||||
instance: getInstance, |
||||
}); |
||||
T.emit('evt_app_started', await User.count()); |
||||
|
||||
return metaService; |
||||
}, |
||||
provide: MetaService, |
||||
inject: ['IEventEmitter'], |
||||
}; |
||||
@ -0,0 +1,51 @@
|
||||
import { Injectable } from '@nestjs/common'; |
||||
import { JobStatus } from '../../../interface/Jobs'; |
||||
import { QueueService } from './fallback-queue.service'; |
||||
|
||||
@Injectable() |
||||
export class JobsService { |
||||
constructor(private readonly fallbackQueueService: QueueService) {} |
||||
|
||||
async add(name: string, data: any) { |
||||
return this.fallbackQueueService.add(name, data); |
||||
} |
||||
|
||||
async jobStatus(jobId: string) { |
||||
return await ( |
||||
await this.fallbackQueueService.getJob(jobId) |
||||
).status; |
||||
} |
||||
|
||||
async jobList() { |
||||
return await this.fallbackQueueService.getJobs([ |
||||
JobStatus.ACTIVE, |
||||
JobStatus.WAITING, |
||||
JobStatus.DELAYED, |
||||
JobStatus.PAUSED, |
||||
]); |
||||
} |
||||
|
||||
async getJobWithData(data: any) { |
||||
const jobs = await this.fallbackQueueService.getJobs([ |
||||
// 'completed',
|
||||
JobStatus.WAITING, |
||||
JobStatus.ACTIVE, |
||||
JobStatus.DELAYED, |
||||
// 'failed',
|
||||
JobStatus.PAUSED, |
||||
]); |
||||
|
||||
const job = jobs.find((j) => { |
||||
for (const key in data) { |
||||
if (j.data[key]) { |
||||
if (j.data[key] !== data[key]) return false; |
||||
} else { |
||||
return false; |
||||
} |
||||
} |
||||
return true; |
||||
}); |
||||
|
||||
return job; |
||||
} |
||||
} |
||||
@ -1,59 +0,0 @@
|
||||
import { InjectQueue } from '@nestjs/bull'; |
||||
import { Injectable } from '@nestjs/common'; |
||||
import { Queue } from 'bull'; |
||||
import { JOBS_QUEUE, JobStatus } from '../../interface/Jobs'; |
||||
import { QueueService } from './fallback-queue.service'; |
||||
|
||||
@Injectable() |
||||
export class JobsService { |
||||
public activeQueue; |
||||
constructor( |
||||
@InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue, |
||||
private readonly fallbackQueueService: QueueService, |
||||
) { |
||||
this.activeQueue = this.fallbackQueueService; |
||||
/* process.env.NC_REDIS_URL |
||||
? this.jobsQueue |
||||
: this.fallbackQueueService; |
||||
*/ |
||||
} |
||||
|
||||
async jobStatus(jobId: string) { |
||||
return await (await this.activeQueue.getJob(jobId)).getState(); |
||||
} |
||||
|
||||
async jobList(jobType: string) { |
||||
return ( |
||||
await this.activeQueue.getJobs([ |
||||
JobStatus.ACTIVE, |
||||
JobStatus.WAITING, |
||||
JobStatus.DELAYED, |
||||
JobStatus.PAUSED, |
||||
]) |
||||
).filter((j) => j.name === jobType); |
||||
} |
||||
|
||||
async getJobWithData(data: any) { |
||||
const jobs = await this.activeQueue.getJobs([ |
||||
// 'completed',
|
||||
JobStatus.WAITING, |
||||
JobStatus.ACTIVE, |
||||
JobStatus.DELAYED, |
||||
// 'failed',
|
||||
JobStatus.PAUSED, |
||||
]); |
||||
|
||||
const job = jobs.find((j) => { |
||||
for (const key in data) { |
||||
if (j.data[key]) { |
||||
if (j.data[key] !== data[key]) return false; |
||||
} else { |
||||
return false; |
||||
} |
||||
} |
||||
return true; |
||||
}); |
||||
|
||||
return job; |
||||
} |
||||
} |
||||
@ -1,8 +1,8 @@
|
||||
/* eslint-disable no-async-promise-executor */ |
||||
import { RelationTypes, UITypes } from 'nocodb-sdk'; |
||||
import EntityMap from './EntityMap'; |
||||
import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service'; |
||||
import type { TablesService } from '../../../../services/tables.service'; |
||||
import type { BulkDataAliasService } from '../../../../../services/bulk-data-alias.service'; |
||||
import type { TablesService } from '../../../../../services/tables.service'; |
||||
// @ts-ignore
|
||||
import type { AirtableBase } from 'airtable/lib/airtable_base'; |
||||
import type { TableType } from 'nocodb-sdk'; |
||||
@ -0,0 +1,16 @@
|
||||
import { Injectable } from '@nestjs/common'; |
||||
import { EventEmitter2 } from '@nestjs/event-emitter'; |
||||
import { JobEvents } from '../../../interface/Jobs'; |
||||
import type { Job } from 'bull'; |
||||
|
||||
@Injectable() |
||||
export class JobsLogService { |
||||
constructor(private eventEmitter: EventEmitter2) {} |
||||
|
||||
sendLog(job: Job, data: { message: string }) { |
||||
this.eventEmitter.emit(JobEvents.LOG, { |
||||
id: job.id.toString(), |
||||
data, |
||||
}); |
||||
} |
||||
} |
||||
@ -0,0 +1,107 @@
|
||||
import { |
||||
OnQueueActive, |
||||
OnQueueCompleted, |
||||
OnQueueFailed, |
||||
Processor, |
||||
} from '@nestjs/bull'; |
||||
import { Job } from 'bull'; |
||||
import boxen from 'boxen'; |
||||
import { EventEmitter2, OnEvent } from '@nestjs/event-emitter'; |
||||
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs'; |
||||
import { JobsRedisService } from './jobs-redis.service'; |
||||
|
||||
@Processor(JOBS_QUEUE) |
||||
export class JobsEventService { |
||||
constructor( |
||||
private jobsRedisService: JobsRedisService, |
||||
private eventEmitter: EventEmitter2, |
||||
) {} |
||||
|
||||
@OnQueueActive() |
||||
onActive(job: Job) { |
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, { |
||||
cmd: JobEvents.STATUS, |
||||
id: job.id.toString(), |
||||
status: JobStatus.ACTIVE, |
||||
}); |
||||
} else { |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.ACTIVE, |
||||
}); |
||||
} |
||||
} |
||||
|
||||
@OnQueueFailed() |
||||
onFailed(job: Job, error: Error) { |
||||
console.error( |
||||
boxen( |
||||
`---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, |
||||
{ |
||||
padding: 1, |
||||
borderStyle: 'double', |
||||
borderColor: 'yellow', |
||||
}, |
||||
), |
||||
); |
||||
|
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, { |
||||
cmd: JobEvents.STATUS, |
||||
id: job.id.toString(), |
||||
status: JobStatus.FAILED, |
||||
data: { |
||||
error: { |
||||
message: error?.message, |
||||
}, |
||||
}, |
||||
}); |
||||
} else { |
||||
this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`); |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.FAILED, |
||||
data: { |
||||
error: { |
||||
message: error?.message, |
||||
}, |
||||
}, |
||||
}); |
||||
} |
||||
} |
||||
|
||||
@OnQueueCompleted() |
||||
onCompleted(job: Job, data: any) { |
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, { |
||||
cmd: JobEvents.STATUS, |
||||
id: job.id.toString(), |
||||
status: JobStatus.COMPLETED, |
||||
data: { |
||||
result: data, |
||||
}, |
||||
}); |
||||
} else { |
||||
this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`); |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.COMPLETED, |
||||
data: { |
||||
result: data, |
||||
}, |
||||
}); |
||||
} |
||||
} |
||||
|
||||
@OnEvent(JobEvents.LOG) |
||||
onLog(data: { id: string; data: { message: string } }) { |
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.jobsRedisService.publish(`jobs-${data.id}`, { |
||||
cmd: JobEvents.LOG, |
||||
id: data.id, |
||||
data: data.data, |
||||
}); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,53 @@
|
||||
import { Injectable } from '@nestjs/common'; |
||||
import Redis from 'ioredis'; |
||||
|
||||
@Injectable() |
||||
export class JobsRedisService { |
||||
private redisClient: Redis; |
||||
private redisSubscriber: Redis; |
||||
private unsubscribeCallbacks: { [key: string]: () => void } = {}; |
||||
|
||||
constructor() { |
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.redisClient = new Redis(process.env.NC_REDIS_URL); |
||||
return; |
||||
} |
||||
this.redisSubscriber = new Redis(process.env.NC_REDIS_URL); |
||||
} |
||||
|
||||
publish(channel: string, message: string | any) { |
||||
if (typeof message === 'string') { |
||||
this.redisClient.publish(channel, message); |
||||
} else { |
||||
try { |
||||
this.redisClient.publish(channel, JSON.stringify(message)); |
||||
} catch (e) { |
||||
console.error(e); |
||||
} |
||||
} |
||||
} |
||||
|
||||
subscribe(channel: string, callback: (message: any) => void) { |
||||
this.redisSubscriber.subscribe(channel); |
||||
|
||||
const onMessage = (_channel, message) => { |
||||
try { |
||||
message = JSON.parse(message); |
||||
} catch (e) {} |
||||
callback(message); |
||||
}; |
||||
|
||||
this.redisSubscriber.on('message', onMessage); |
||||
this.unsubscribeCallbacks[channel] = () => { |
||||
this.redisSubscriber.unsubscribe(channel); |
||||
this.redisSubscriber.off('message', onMessage); |
||||
}; |
||||
} |
||||
|
||||
unsubscribe(channel: string) { |
||||
if (this.unsubscribeCallbacks[channel]) { |
||||
this.unsubscribeCallbacks[channel](); |
||||
delete this.unsubscribeCallbacks[channel]; |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,98 @@
|
||||
import { InjectQueue } from '@nestjs/bull'; |
||||
import { Injectable } from '@nestjs/common'; |
||||
import { Queue } from 'bull'; |
||||
import { EventEmitter2 } from '@nestjs/event-emitter'; |
||||
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs'; |
||||
import { JobsRedisService } from './jobs-redis.service'; |
||||
import type { OnModuleInit } from '@nestjs/common'; |
||||
|
||||
@Injectable() |
||||
export class JobsService implements OnModuleInit { |
||||
constructor( |
||||
@InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue, |
||||
private jobsRedisService: JobsRedisService, |
||||
private eventEmitter: EventEmitter2, |
||||
) {} |
||||
|
||||
// pause primary instance queue
|
||||
async onModuleInit() { |
||||
if (process.env.NC_WORKER_CONTAINER !== 'true') { |
||||
await this.jobsQueue.pause(true); |
||||
} |
||||
} |
||||
|
||||
async add(name: string, data: any) { |
||||
// resume primary instance queue if there is no worker
|
||||
const workerCount = (await this.jobsQueue.getWorkers()).length; |
||||
const localWorkerPaused = await this.jobsQueue.isPaused(true); |
||||
|
||||
// if there is no worker and primary instance queue is paused, resume it
|
||||
// if there is any worker and primary instance queue is not paused, pause it
|
||||
if (workerCount < 1 && localWorkerPaused) { |
||||
await this.jobsQueue.resume(true); |
||||
} else if (workerCount > 0 && !localWorkerPaused) { |
||||
await this.jobsQueue.pause(true); |
||||
} |
||||
|
||||
const job = await this.jobsQueue.add(name, data); |
||||
|
||||
// subscribe to job events
|
||||
this.jobsRedisService.subscribe(`jobs-${job.id.toString()}`, (data) => { |
||||
const cmd = data.cmd; |
||||
delete data.cmd; |
||||
switch (cmd) { |
||||
case JobEvents.STATUS: |
||||
this.eventEmitter.emit(JobEvents.STATUS, data); |
||||
if ([JobStatus.COMPLETED, JobStatus.FAILED].includes(data.status)) { |
||||
this.jobsRedisService.unsubscribe(`jobs-${data.id.toString()}`); |
||||
} |
||||
break; |
||||
case JobEvents.LOG: |
||||
this.eventEmitter.emit(JobEvents.LOG, data); |
||||
break; |
||||
} |
||||
}); |
||||
|
||||
return job; |
||||
} |
||||
|
||||
async jobStatus(jobId: string) { |
||||
const job = await this.jobsQueue.getJob(jobId); |
||||
if (job) { |
||||
return await job.getState(); |
||||
} |
||||
} |
||||
|
||||
async jobList() { |
||||
return await this.jobsQueue.getJobs([ |
||||
JobStatus.ACTIVE, |
||||
JobStatus.WAITING, |
||||
JobStatus.DELAYED, |
||||
JobStatus.PAUSED, |
||||
]); |
||||
} |
||||
|
||||
async getJobWithData(data: any) { |
||||
const jobs = await this.jobsQueue.getJobs([ |
||||
// 'completed',
|
||||
JobStatus.WAITING, |
||||
JobStatus.ACTIVE, |
||||
JobStatus.DELAYED, |
||||
// 'failed',
|
||||
JobStatus.PAUSED, |
||||
]); |
||||
|
||||
const job = jobs.find((j) => { |
||||
for (const key in data) { |
||||
if (j.data[key]) { |
||||
if (j.data[key] !== data[key]) return false; |
||||
} else { |
||||
return false; |
||||
} |
||||
} |
||||
return true; |
||||
}); |
||||
|
||||
return job; |
||||
} |
||||
} |
||||
@ -1,19 +0,0 @@
|
||||
import { Test } from '@nestjs/testing'; |
||||
import { AppInitService } from './app-init.service'; |
||||
import type { TestingModule } from '@nestjs/testing'; |
||||
|
||||
describe('AppInitService', () => { |
||||
let service: AppInitService; |
||||
|
||||
beforeEach(async () => { |
||||
const module: TestingModule = await Test.createTestingModule({ |
||||
providers: [AppInitService], |
||||
}).compile(); |
||||
|
||||
service = module.get<AppInitService>(AppInitService); |
||||
}); |
||||
|
||||
it('should be defined', () => { |
||||
expect(service).toBeDefined(); |
||||
}); |
||||
}); |
||||
@ -1,79 +0,0 @@
|
||||
import { T } from 'nc-help'; |
||||
import NocoCache from '../cache/NocoCache'; |
||||
import { Connection } from '../connection/connection'; |
||||
import initAdminFromEnv from '../helpers/initAdminFromEnv'; |
||||
import NcPluginMgrv2 from '../helpers/NcPluginMgrv2'; |
||||
import { MetaService } from '../meta/meta.service'; |
||||
import { User } from '../models'; |
||||
import Noco from '../Noco'; |
||||
import getInstance from '../utils/getInstance'; |
||||
import NcConfigFactory from '../utils/NcConfigFactory'; |
||||
import NcUpgrader from '../version-upgrader/NcUpgrader'; |
||||
import type { IEventEmitter } from '../modules/event-emitter/event-emitter.interface'; |
||||
import type { Provider } from '@nestjs/common'; |
||||
|
||||
export class AppInitService { |
||||
private readonly config: any; |
||||
|
||||
constructor(config) { |
||||
this.config = config; |
||||
} |
||||
|
||||
get appConfig(): any { |
||||
return this.config; |
||||
} |
||||
} |
||||
|
||||
export const appInitServiceProvider: Provider = { |
||||
provide: AppInitService, |
||||
// initialize app,
|
||||
// 1. init cache
|
||||
// 2. init db connection and create if not exist
|
||||
// 3. init meta and set to Noco
|
||||
// 4. init jwt
|
||||
// 5. init plugin manager
|
||||
// 6. run upgrader
|
||||
useFactory: async ( |
||||
connection: Connection, |
||||
metaService: MetaService, |
||||
eventEmitter: IEventEmitter, |
||||
) => { |
||||
process.env.NC_VERSION = '0107004'; |
||||
|
||||
await NocoCache.init(); |
||||
|
||||
await connection.init(); |
||||
|
||||
await NcConfigFactory.metaDbCreateIfNotExist(connection.config); |
||||
|
||||
await metaService.init(); |
||||
|
||||
// todo: remove
|
||||
// temporary hack
|
||||
Noco._ncMeta = metaService; |
||||
Noco.config = connection.config; |
||||
Noco.eventEmitter = eventEmitter; |
||||
|
||||
// init jwt secret
|
||||
await Noco.initJwt(); |
||||
|
||||
// load super admin user from env if env is set
|
||||
await initAdminFromEnv(metaService); |
||||
|
||||
// init plugin manager
|
||||
await NcPluginMgrv2.init(Noco.ncMeta); |
||||
await Noco.loadEEState(); |
||||
|
||||
// run upgrader
|
||||
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta }); |
||||
|
||||
T.init({ |
||||
instance: getInstance, |
||||
}); |
||||
T.emit('evt_app_started', await User.count()); |
||||
|
||||
// todo: move app config to app-init service
|
||||
return new AppInitService(connection.config); |
||||
}, |
||||
inject: [Connection, MetaService, 'IEventEmitter'], |
||||
}; |
||||
@ -1,755 +0,0 @@
|
||||
import fs from 'fs'; |
||||
import { URL } from 'url'; |
||||
import { promisify } from 'util'; |
||||
import * as path from 'path'; |
||||
import parseDbUrl from 'parse-database-url'; |
||||
import { SqlClientFactory } from '../db/sql-client/lib/SqlClientFactory'; |
||||
// import SqlClientFactory from '../db/sql-client/lib/SqlClientFactory';
|
||||
// import type {
|
||||
// AuthConfig,
|
||||
// DbConfig,
|
||||
// MailerConfig,
|
||||
// NcConfig,
|
||||
// } from '../../interface/config';
|
||||
|
||||
// const {
|
||||
// uniqueNamesGenerator,
|
||||
// starWars,
|
||||
// adjectives,
|
||||
// animals,
|
||||
// } = require('unique-names-generator');
|
||||
|
||||
type NcConfig = any; |
||||
type DbConfig = any; |
||||
|
||||
const driverClientMapping = { |
||||
mysql: 'mysql2', |
||||
mariadb: 'mysql2', |
||||
postgres: 'pg', |
||||
postgresql: 'pg', |
||||
sqlite: 'sqlite3', |
||||
mssql: 'mssql', |
||||
}; |
||||
|
||||
const defaultClientPortMapping = { |
||||
mysql: 3306, |
||||
mysql2: 3306, |
||||
postgres: 5432, |
||||
pg: 5432, |
||||
mssql: 1433, |
||||
}; |
||||
|
||||
const defaultConnectionConfig: any = { |
||||
// https://github.com/knex/knex/issues/97
|
||||
// timezone: process.env.NC_TIMEZONE || 'UTC',
|
||||
dateStrings: true, |
||||
}; |
||||
|
||||
// default knex options
|
||||
const defaultConnectionOptions = { |
||||
pool: { |
||||
min: 0, |
||||
max: 10, |
||||
}, |
||||
}; |
||||
|
||||
const knownQueryParams = [ |
||||
{ |
||||
parameter: 'database', |
||||
aliases: ['d', 'db'], |
||||
}, |
||||
{ |
||||
parameter: 'password', |
||||
aliases: ['p'], |
||||
}, |
||||
{ |
||||
parameter: 'user', |
||||
aliases: ['u'], |
||||
}, |
||||
{ |
||||
parameter: 'title', |
||||
aliases: ['t'], |
||||
}, |
||||
{ |
||||
parameter: 'keyFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'certFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'caFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'ssl', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'options', |
||||
aliases: ['opt', 'opts'], |
||||
}, |
||||
]; |
||||
|
||||
export default class NcConfigFactory { |
||||
public static async make(): Promise<any> { |
||||
await this.jdbcToXcUrl(); |
||||
|
||||
const ncConfig = new NcConfigFactory(); |
||||
|
||||
ncConfig.auth = { |
||||
jwt: { |
||||
secret: process.env.NC_AUTH_JWT_SECRET, |
||||
}, |
||||
}; |
||||
|
||||
ncConfig.port = +(process?.env?.PORT ?? 8080); |
||||
ncConfig.env = '_noco'; // process.env?.NODE_ENV || 'dev';
|
||||
ncConfig.workingEnv = '_noco'; // process.env?.NODE_ENV || 'dev';
|
||||
// ncConfig.toolDir = this.getToolDir();
|
||||
ncConfig.projectType = |
||||
ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest'; |
||||
|
||||
if (ncConfig.meta?.db?.connection?.filename) { |
||||
ncConfig.meta.db.connection.filename = path.join( |
||||
this.getToolDir(), |
||||
ncConfig.meta.db.connection.filename, |
||||
); |
||||
} |
||||
|
||||
if (process.env.NC_DB) { |
||||
ncConfig.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB); |
||||
} else if (process.env.NC_DB_JSON) { |
||||
ncConfig.meta.db = JSON.parse(process.env.NC_DB_JSON); |
||||
} else if (process.env.NC_DB_JSON_FILE) { |
||||
const filePath = process.env.NC_DB_JSON_FILE; |
||||
|
||||
if (!(await promisify(fs.exists)(filePath))) { |
||||
throw new Error(`NC_DB_JSON_FILE not found: ${filePath}`); |
||||
} |
||||
|
||||
const fileContent = await promisify(fs.readFile)(filePath, { |
||||
encoding: 'utf8', |
||||
}); |
||||
ncConfig.meta.db = JSON.parse(fileContent); |
||||
} |
||||
|
||||
if (process.env.NC_TRY) { |
||||
ncConfig.try = true; |
||||
ncConfig.meta.db = { |
||||
client: 'sqlite3', |
||||
connection: ':memory:', |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
} as any; |
||||
} |
||||
|
||||
if (process.env.NC_PUBLIC_URL) { |
||||
ncConfig.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL; |
||||
// ncConfig.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
|
||||
ncConfig.publicUrl = process.env.NC_PUBLIC_URL; |
||||
} |
||||
|
||||
if (process.env.NC_DASHBOARD_URL) { |
||||
ncConfig.dashboardPath = process.env.NC_DASHBOARD_URL; |
||||
} |
||||
|
||||
return ncConfig; |
||||
} |
||||
|
||||
public static getToolDir() { |
||||
return process.env.NC_TOOL_DIR || process.cwd(); |
||||
} |
||||
|
||||
public static hasDbUrl(): boolean { |
||||
return Object.keys(process.env).some((envKey) => |
||||
envKey.startsWith('NC_DB_URL'), |
||||
); |
||||
} |
||||
|
||||
public static makeFromUrls(urls: string[]): NcConfig { |
||||
const config = new NcConfigFactory(); |
||||
|
||||
// config.envs[process.env.NODE_ENV || 'dev'].db = [];
|
||||
config.envs['_noco'].db = []; |
||||
for (const [i, url] of Object.entries(urls)) { |
||||
// config.envs[process.env.NODE_ENV || 'dev'].db.push(this.urlToDbConfig(url, i));
|
||||
config.envs['_noco'].db.push(this.urlToDbConfig(url, i)); |
||||
} |
||||
|
||||
return config; |
||||
} |
||||
|
||||
public static urlToDbConfig( |
||||
urlString: string, |
||||
key = '', |
||||
config?: NcConfigFactory, |
||||
type?: string, |
||||
): DbConfig { |
||||
const url = new URL(urlString); |
||||
|
||||
let dbConfig: DbConfig; |
||||
|
||||
if (url.protocol.startsWith('sqlite3')) { |
||||
dbConfig = { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
filename: |
||||
url.searchParams.get('d') || url.searchParams.get('database'), |
||||
}, |
||||
database: |
||||
url.searchParams.get('d') || url.searchParams.get('database'), |
||||
}, |
||||
} as any; |
||||
} else { |
||||
const parsedQuery = {}; |
||||
for (const [key, value] of url.searchParams.entries()) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedQuery[fnd.parameter] = value; |
||||
} else { |
||||
parsedQuery[key] = value; |
||||
} |
||||
} |
||||
|
||||
dbConfig = { |
||||
client: url.protocol.replace(':', ''), |
||||
connection: { |
||||
...defaultConnectionConfig, |
||||
...parsedQuery, |
||||
host: url.hostname, |
||||
port: +url.port, |
||||
}, |
||||
// pool: {
|
||||
// min: 1,
|
||||
// max: 1
|
||||
// },
|
||||
acquireConnectionTimeout: 600000, |
||||
} as any; |
||||
|
||||
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { |
||||
dbConfig.connection.ssl = true; |
||||
} |
||||
|
||||
if ( |
||||
url.searchParams.get('keyFilePath') && |
||||
url.searchParams.get('certFilePath') && |
||||
url.searchParams.get('caFilePath') |
||||
) { |
||||
dbConfig.connection.ssl = { |
||||
keyFilePath: url.searchParams.get('keyFilePath'), |
||||
certFilePath: url.searchParams.get('certFilePath'), |
||||
caFilePath: url.searchParams.get('caFilePath'), |
||||
}; |
||||
} |
||||
} |
||||
|
||||
if (config && !config.title) { |
||||
config.title = |
||||
url.searchParams.get('t') || |
||||
url.searchParams.get('title') || |
||||
this.generateRandomTitle(); |
||||
} |
||||
|
||||
Object.assign(dbConfig, { |
||||
meta: { |
||||
tn: 'nc_evolutions', |
||||
allSchemas: |
||||
!!url.searchParams.get('allSchemas') || |
||||
!(url.searchParams.get('d') || url.searchParams.get('database')), |
||||
api: { |
||||
prefix: url.searchParams.get('apiPrefix') || '', |
||||
swagger: true, |
||||
type: |
||||
type || |
||||
((url.searchParams.get('api') || |
||||
url.searchParams.get('a')) as any) || |
||||
'rest', |
||||
}, |
||||
dbAlias: url.searchParams.get('dbAlias') || `db${key}`, |
||||
metaTables: 'db', |
||||
migrations: { |
||||
disabled: false, |
||||
name: 'nc_evolutions', |
||||
}, |
||||
}, |
||||
}); |
||||
|
||||
return dbConfig; |
||||
} |
||||
|
||||
private static generateRandomTitle(): string { |
||||
return ''; /*uniqueNamesGenerator({ |
||||
dictionaries: [[starWars], [adjectives, animals]][ |
||||
Math.floor(Math.random() * 2) |
||||
], |
||||
}) |
||||
.toLowerCase() |
||||
.replace(/[ -]/g, '_');*/ |
||||
} |
||||
|
||||
static async metaUrlToDbConfig(urlString) { |
||||
const url = new URL(urlString); |
||||
|
||||
let dbConfig; |
||||
|
||||
if (url.protocol.startsWith('sqlite3')) { |
||||
const db = url.searchParams.get('d') || url.searchParams.get('database'); |
||||
dbConfig = { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
filename: db, |
||||
}, |
||||
...(db === ':memory:' |
||||
? { |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
} |
||||
: {}), |
||||
}; |
||||
} else { |
||||
const parsedQuery = {}; |
||||
for (const [key, value] of url.searchParams.entries()) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedQuery[fnd.parameter] = value; |
||||
} else { |
||||
parsedQuery[key] = value; |
||||
} |
||||
} |
||||
|
||||
dbConfig = { |
||||
client: url.protocol.replace(':', ''), |
||||
connection: { |
||||
...defaultConnectionConfig, |
||||
...parsedQuery, |
||||
host: url.hostname, |
||||
port: +url.port, |
||||
}, |
||||
acquireConnectionTimeout: 600000, |
||||
...(url.searchParams.has('search_path') |
||||
? { |
||||
searchPath: url.searchParams.get('search_path').split(','), |
||||
} |
||||
: {}), |
||||
}; |
||||
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { |
||||
dbConfig.connection.ssl = true; |
||||
} |
||||
} |
||||
url.searchParams.forEach((_value, key) => { |
||||
let value: any = _value; |
||||
if (value === 'true') { |
||||
value = true; |
||||
} else if (value === 'false') { |
||||
value = false; |
||||
} else if (/^\d+$/.test(value)) { |
||||
value = +value; |
||||
} |
||||
// todo: implement config read from JSON file or JSON env val read
|
||||
if ( |
||||
![ |
||||
'password', |
||||
'p', |
||||
'database', |
||||
'd', |
||||
'user', |
||||
'u', |
||||
'search_path', |
||||
].includes(key) |
||||
) { |
||||
key.split('.').reduce((obj, k, i, arr) => { |
||||
return (obj[k] = i === arr.length - 1 ? value : obj[k] || {}); |
||||
}, dbConfig); |
||||
} |
||||
}); |
||||
|
||||
if ( |
||||
dbConfig?.connection?.ssl && |
||||
typeof dbConfig?.connection?.ssl === 'object' |
||||
) { |
||||
if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) { |
||||
dbConfig.connection.ssl.ca = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.caFilePath; |
||||
} |
||||
if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) { |
||||
dbConfig.connection.ssl.key = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.keyFilePath; |
||||
} |
||||
if ( |
||||
dbConfig.connection.ssl.certFilePath && |
||||
!dbConfig.connection.ssl.cert |
||||
) { |
||||
dbConfig.connection.ssl.cert = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.certFilePath; |
||||
} |
||||
} |
||||
|
||||
return dbConfig; |
||||
} |
||||
|
||||
public static async makeProjectConfigFromUrl( |
||||
url, |
||||
type?: string, |
||||
): Promise<NcConfig> { |
||||
const config = new NcConfigFactory(); |
||||
const dbConfig = this.urlToDbConfig(url, '', config, type); |
||||
// config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
|
||||
config.envs['_noco'].db.push(dbConfig); |
||||
|
||||
if (process.env.NC_AUTH_ADMIN_SECRET) { |
||||
config.auth = { |
||||
masterKey: { |
||||
secret: process.env.NC_AUTH_ADMIN_SECRET, |
||||
}, |
||||
}; |
||||
} else if (process.env.NC_NO_AUTH) { |
||||
config.auth = { |
||||
disabled: true, |
||||
}; |
||||
// } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
|
||||
} else if (config?.envs?.['_noco']?.db?.[0]) { |
||||
config.auth = { |
||||
jwt: { |
||||
// dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
|
||||
dbAlias: |
||||
process.env.NC_AUTH_JWT_DB_ALIAS || |
||||
config.envs['_noco'].db[0].meta.dbAlias, |
||||
secret: process.env.NC_AUTH_JWT_SECRET, |
||||
}, |
||||
}; |
||||
} |
||||
|
||||
if (process.env.NC_DB) { |
||||
config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB); |
||||
} |
||||
|
||||
if (process.env.NC_TRY) { |
||||
config.try = true; |
||||
config.meta.db = { |
||||
client: 'sqlite3', |
||||
connection: ':memory:', |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
} as any; |
||||
} |
||||
|
||||
if (process.env.NC_MAILER) { |
||||
config.mailer = { |
||||
from: process.env.NC_MAILER_FROM, |
||||
options: { |
||||
host: process.env.NC_MAILER_HOST, |
||||
port: parseInt(process.env.NC_MAILER_PORT, 10), |
||||
secure: process.env.NC_MAILER_SECURE === 'true', |
||||
auth: { |
||||
user: process.env.NC_MAILER_USER, |
||||
pass: process.env.NC_MAILER_PASS, |
||||
}, |
||||
}, |
||||
}; |
||||
} |
||||
|
||||
if (process.env.NC_PUBLIC_URL) { |
||||
// config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
|
||||
config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL; |
||||
config.publicUrl = process.env.NC_PUBLIC_URL; |
||||
} |
||||
|
||||
config.port = +(process?.env?.PORT ?? 8080); |
||||
// config.env = process.env?.NODE_ENV || 'dev';
|
||||
// config.workingEnv = process.env?.NODE_ENV || 'dev';
|
||||
config.env = '_noco'; |
||||
config.workingEnv = '_noco'; |
||||
config.toolDir = this.getToolDir(); |
||||
config.projectType = |
||||
type || |
||||
config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type || |
||||
'rest'; |
||||
|
||||
return config; |
||||
} |
||||
|
||||
public static async makeProjectConfigFromConnection( |
||||
dbConnectionConfig: any, |
||||
type?: string, |
||||
): Promise<NcConfig> { |
||||
const config = new NcConfigFactory(); |
||||
let dbConfig = dbConnectionConfig; |
||||
|
||||
if (dbConfig.client === 'sqlite3') { |
||||
dbConfig = { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
...dbConnectionConfig, |
||||
database: dbConnectionConfig.connection.filename, |
||||
}, |
||||
}; |
||||
} |
||||
|
||||
// todo:
|
||||
const key = ''; |
||||
Object.assign(dbConfig, { |
||||
meta: { |
||||
tn: 'nc_evolutions', |
||||
api: { |
||||
prefix: '', |
||||
swagger: true, |
||||
type: type || 'rest', |
||||
}, |
||||
dbAlias: `db${key}`, |
||||
metaTables: 'db', |
||||
migrations: { |
||||
disabled: false, |
||||
name: 'nc_evolutions', |
||||
}, |
||||
}, |
||||
}); |
||||
|
||||
// config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
|
||||
config.envs['_noco'].db.push(dbConfig); |
||||
|
||||
if (process.env.NC_AUTH_ADMIN_SECRET) { |
||||
config.auth = { |
||||
masterKey: { |
||||
secret: process.env.NC_AUTH_ADMIN_SECRET, |
||||
}, |
||||
}; |
||||
} else if (process.env.NC_NO_AUTH) { |
||||
config.auth = { |
||||
disabled: true, |
||||
}; |
||||
// } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
|
||||
} else if (config?.envs?.['_noco']?.db?.[0]) { |
||||
config.auth = { |
||||
jwt: { |
||||
// dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
|
||||
dbAlias: |
||||
process.env.NC_AUTH_JWT_DB_ALIAS || |
||||
config.envs['_noco'].db[0].meta.dbAlias, |
||||
secret: process.env.NC_AUTH_JWT_SECRET, |
||||
}, |
||||
}; |
||||
} |
||||
|
||||
if (process.env.NC_DB) { |
||||
config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB); |
||||
} |
||||
|
||||
if (process.env.NC_TRY) { |
||||
config.try = true; |
||||
config.meta.db = { |
||||
client: 'sqlite3', |
||||
connection: ':memory:', |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
} as any; |
||||
} |
||||
|
||||
if (process.env.NC_PUBLIC_URL) { |
||||
// config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
|
||||
config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL; |
||||
config.publicUrl = process.env.NC_PUBLIC_URL; |
||||
} |
||||
|
||||
config.port = +(process?.env?.PORT ?? 8080); |
||||
// config.env = process.env?.NODE_ENV || 'dev';
|
||||
// config.workingEnv = process.env?.NODE_ENV || 'dev';
|
||||
config.env = '_noco'; |
||||
config.workingEnv = '_noco'; |
||||
config.toolDir = process.env.NC_TOOL_DIR || process.cwd(); |
||||
config.projectType = |
||||
type || |
||||
config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type || |
||||
'rest'; |
||||
|
||||
return config; |
||||
} |
||||
|
||||
public static async metaDbCreateIfNotExist(args: NcConfig) { |
||||
if (args.meta?.db?.client === 'sqlite3') { |
||||
const metaSqlClient = await SqlClientFactory.create({ |
||||
...args.meta.db, |
||||
connection: args.meta.db, |
||||
}); |
||||
await metaSqlClient.createDatabaseIfNotExists({ |
||||
database: args.meta.db?.connection?.filename, |
||||
}); |
||||
} else { |
||||
const metaSqlClient = await SqlClientFactory.create(args.meta.db); |
||||
await metaSqlClient.createDatabaseIfNotExists(args.meta.db?.connection); |
||||
await metaSqlClient.knex.destroy(); |
||||
} |
||||
|
||||
/* const dbPath = path.join(args.toolDir, 'xc.db') |
||||
const exists = fs.existsSync(dbPath); |
||||
if (!exists) { |
||||
const fd = fs.openSync(dbPath, "w"); |
||||
fs.closeSync(fd); |
||||
} |
||||
*/ |
||||
} |
||||
|
||||
public version = '0.6'; |
||||
public port: number; |
||||
public auth?: any; |
||||
public env: 'production' | 'dev' | 'test' | string; |
||||
public workingEnv: string; |
||||
public toolDir: string; |
||||
public envs: { |
||||
[p: string]: { db: DbConfig[]; api?: any; publicUrl?: string }; |
||||
}; |
||||
// public projectType: "rest" | "graphql" | "grpc";
|
||||
public queriesFolder: string | string[] = ''; |
||||
public seedsFolder: string | string[]; |
||||
public title: string; |
||||
public publicUrl: string; |
||||
public projectType; |
||||
public meta = { |
||||
db: { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
filename: 'noco.db', |
||||
}, |
||||
}, |
||||
}; |
||||
public mailer: any; |
||||
public try = false; |
||||
|
||||
public dashboardPath = '/dashboard'; |
||||
|
||||
constructor() { |
||||
this.envs = { _noco: { db: [] } }; |
||||
} |
||||
|
||||
public static async jdbcToXcUrl() { |
||||
if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) { |
||||
const database_url = await promisify(fs.readFile)( |
||||
process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE, |
||||
'utf-8', |
||||
); |
||||
process.env.NC_DB = this.extractXcUrlFromJdbc(database_url); |
||||
} else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) { |
||||
process.env.NC_DB = this.extractXcUrlFromJdbc( |
||||
process.env.NC_DATABASE_URL || process.env.DATABASE_URL, |
||||
); |
||||
} |
||||
} |
||||
|
||||
public static extractXcUrlFromJdbc(url: string, rtConfig = false) { |
||||
// drop the jdbc prefix
|
||||
if (url.startsWith('jdbc:')) { |
||||
url = url.substring(5); |
||||
} |
||||
|
||||
const config = parseDbUrl(url); |
||||
|
||||
const parsedConfig: { |
||||
driver?: string; |
||||
host?: string; |
||||
port?: string; |
||||
database?: string; |
||||
user?: string; |
||||
password?: string; |
||||
ssl?: string; |
||||
} = {}; |
||||
for (const [key, value] of Object.entries(config)) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedConfig[fnd.parameter] = value; |
||||
} else { |
||||
parsedConfig[key] = value; |
||||
} |
||||
} |
||||
|
||||
if (!parsedConfig?.port) |
||||
parsedConfig.port = |
||||
defaultClientPortMapping[ |
||||
driverClientMapping[parsedConfig.driver] || parsedConfig.driver |
||||
]; |
||||
|
||||
if (rtConfig) { |
||||
const { driver, ...connectionConfig } = parsedConfig; |
||||
|
||||
const client = driverClientMapping[driver] || driver; |
||||
|
||||
const avoidSSL = [ |
||||
'localhost', |
||||
'127.0.0.1', |
||||
'host.docker.internal', |
||||
'172.17.0.1', |
||||
]; |
||||
|
||||
if ( |
||||
client === 'pg' && |
||||
!connectionConfig?.ssl && |
||||
!avoidSSL.includes(connectionConfig.host) |
||||
) { |
||||
connectionConfig.ssl = 'true'; |
||||
} |
||||
|
||||
return { |
||||
client: client, |
||||
connection: { |
||||
...connectionConfig, |
||||
}, |
||||
} as any; |
||||
} |
||||
|
||||
const { driver, host, port, database, user, password, ...extra } = |
||||
parsedConfig; |
||||
|
||||
const extraParams = []; |
||||
|
||||
for (const [key, value] of Object.entries(extra)) { |
||||
extraParams.push(`${key}=${value}`); |
||||
} |
||||
|
||||
const res = `${driverClientMapping[driver] || driver}://${host}${ |
||||
port ? `:${port}` : '' |
||||
}?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${ |
||||
database ? `d=${database}&` : '' |
||||
}${extraParams.join('&')}`;
|
||||
|
||||
return res; |
||||
} |
||||
|
||||
// public static initOneClickDeployment() {
|
||||
// if (process.env.NC_ONE_CLICK) {
|
||||
// const url = NcConfigFactory.extractXcUrlFromJdbc(process.env.DATABASE_URL);
|
||||
// process.env.NC_DB = url;
|
||||
// }
|
||||
// }
|
||||
} |
||||
|
||||
export { defaultConnectionConfig, defaultConnectionOptions }; |
||||
@ -0,0 +1,182 @@
|
||||
import * as path from 'path'; |
||||
import fs from 'fs'; |
||||
import { promisify } from 'util'; |
||||
import { SqlClientFactory } from '../../db/sql-client/lib/SqlClientFactory'; |
||||
import { getToolDir, metaUrlToDbConfig } from './helpers'; |
||||
import { DriverClient } from './interfaces'; |
||||
import type { DbConfig } from './interfaces'; |
||||
|
||||
export class NcConfig { |
||||
version: string; |
||||
meta: { |
||||
db: DbConfig; |
||||
} = { |
||||
db: { |
||||
client: DriverClient.SQLITE, |
||||
connection: { |
||||
filename: 'noco.db', |
||||
}, |
||||
}, |
||||
}; |
||||
auth: { |
||||
jwt: { |
||||
secret: string; |
||||
options?: any; |
||||
}; |
||||
}; |
||||
|
||||
// if this is true, port is not exposed
|
||||
worker: boolean; |
||||
|
||||
toolDir: string; |
||||
|
||||
// exposed instance port
|
||||
port: number; |
||||
|
||||
// if this is true, use sqlite3 :memory: as meta db
|
||||
try: boolean; |
||||
|
||||
// optional
|
||||
publicUrl?: string; |
||||
dashboardPath?: string; |
||||
|
||||
// TODO what is this?
|
||||
envs: any; |
||||
|
||||
queriesFolder: string; |
||||
env: string; |
||||
workingEnv: string; |
||||
projectType: string; |
||||
|
||||
private constructor() {} |
||||
|
||||
public static async create(param: { |
||||
meta: { |
||||
metaUrl?: string; |
||||
metaJson?: string; |
||||
metaJsonFile?: string; |
||||
}; |
||||
secret?: string; |
||||
port?: string | number; |
||||
tryMode?: boolean; |
||||
worker?: boolean; |
||||
dashboardPath?: string; |
||||
publicUrl?: string; |
||||
}): Promise<NcConfig> { |
||||
const { meta, secret, port, worker, tryMode, publicUrl, dashboardPath } = |
||||
param; |
||||
|
||||
const ncConfig = new NcConfig(); |
||||
|
||||
ncConfig.auth = { |
||||
jwt: { |
||||
secret: secret, |
||||
}, |
||||
}; |
||||
|
||||
ncConfig.port = +(port ?? 8080); |
||||
ncConfig.toolDir = getToolDir(); |
||||
ncConfig.worker = worker ?? false; |
||||
|
||||
ncConfig.env = '_noco'; |
||||
ncConfig.workingEnv = '_noco'; |
||||
|
||||
ncConfig.projectType = |
||||
ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest'; |
||||
|
||||
if (ncConfig.meta?.db?.connection?.filename) { |
||||
ncConfig.meta.db.connection.filename = path.join( |
||||
ncConfig.toolDir, |
||||
ncConfig.meta.db.connection.filename, |
||||
); |
||||
} |
||||
|
||||
if (tryMode) { |
||||
ncConfig.try = true; |
||||
ncConfig.meta.db = { |
||||
client: DriverClient.SQLITE, |
||||
connection: ':memory:' as any, |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
}; |
||||
} else { |
||||
if (meta?.metaUrl) { |
||||
ncConfig.meta.db = await metaUrlToDbConfig(meta.metaUrl); |
||||
} else if (meta?.metaJson) { |
||||
ncConfig.meta.db = JSON.parse(meta.metaJson); |
||||
} else if (meta?.metaJsonFile) { |
||||
if (!(await promisify(fs.exists)(meta.metaJsonFile))) { |
||||
throw new Error(`NC_DB_JSON_FILE not found: ${meta.metaJsonFile}`); |
||||
} |
||||
const fileContent = await promisify(fs.readFile)(meta.metaJsonFile, { |
||||
encoding: 'utf8', |
||||
}); |
||||
ncConfig.meta.db = JSON.parse(fileContent); |
||||
} |
||||
} |
||||
|
||||
if (publicUrl) { |
||||
ncConfig.envs['_noco'].publicUrl = publicUrl; |
||||
ncConfig.publicUrl = publicUrl; |
||||
} |
||||
|
||||
if (dashboardPath) { |
||||
ncConfig.dashboardPath = dashboardPath; |
||||
} |
||||
|
||||
try { |
||||
// make sure meta db exists
|
||||
await ncConfig.metaDbCreateIfNotExist(); |
||||
} catch (e) { |
||||
throw new Error(e); |
||||
} |
||||
|
||||
return ncConfig; |
||||
} |
||||
|
||||
public static async createByEnv(): Promise<NcConfig> { |
||||
return NcConfig.create({ |
||||
meta: { |
||||
metaUrl: process.env.NC_DB, |
||||
metaJson: process.env.NC_DB_JSON, |
||||
metaJsonFile: process.env.NC_DB_JSON_FILE, |
||||
}, |
||||
secret: process.env.NC_AUTH_JWT_SECRET, |
||||
port: process.env.NC_PORT, |
||||
tryMode: !!process.env.NC_TRY, |
||||
worker: !!process.env.NC_WORKER, |
||||
dashboardPath: process.env.NC_DASHBOARD_PATH, |
||||
publicUrl: process.env.NC_PUBLIC_URL, |
||||
}); |
||||
} |
||||
|
||||
private async metaDbCreateIfNotExist() { |
||||
if (this.meta?.db?.client === 'sqlite3') { |
||||
const metaSqlClient = await SqlClientFactory.create({ |
||||
...this.meta.db, |
||||
connection: this.meta.db, |
||||
}); |
||||
if (this.meta.db?.connection?.filename) { |
||||
await metaSqlClient.createDatabaseIfNotExists({ |
||||
database: this.meta.db?.connection?.filename, |
||||
}); |
||||
} else { |
||||
throw new Error('Configuration missing meta db connection'); |
||||
} |
||||
} else { |
||||
const metaSqlClient = await SqlClientFactory.create(this.meta.db); |
||||
if (this.meta.db?.connection?.database) { |
||||
await metaSqlClient.createDatabaseIfNotExists( |
||||
(this.meta.db as any).connection, |
||||
); |
||||
await metaSqlClient.knex.destroy(); |
||||
} else { |
||||
throw new Error('Configuration missing meta db connection'); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,84 @@
|
||||
export const driverClientMapping = { |
||||
mysql: 'mysql2', |
||||
mariadb: 'mysql2', |
||||
postgres: 'pg', |
||||
postgresql: 'pg', |
||||
sqlite: 'sqlite3', |
||||
mssql: 'mssql', |
||||
}; |
||||
|
||||
export const defaultClientPortMapping = { |
||||
mysql: 3306, |
||||
mysql2: 3306, |
||||
postgres: 5432, |
||||
pg: 5432, |
||||
mssql: 1433, |
||||
}; |
||||
|
||||
export const defaultConnectionConfig: any = { |
||||
// https://github.com/knex/knex/issues/97
|
||||
// timezone: process.env.NC_TIMEZONE || 'UTC',
|
||||
dateStrings: true, |
||||
}; |
||||
|
||||
// default knex options
|
||||
export const defaultConnectionOptions = { |
||||
pool: { |
||||
min: 0, |
||||
max: 10, |
||||
}, |
||||
}; |
||||
|
||||
export const avoidSSL = [ |
||||
'localhost', |
||||
'127.0.0.1', |
||||
'host.docker.internal', |
||||
'172.17.0.1', |
||||
]; |
||||
|
||||
export const knownQueryParams = [ |
||||
{ |
||||
parameter: 'database', |
||||
aliases: ['d', 'db'], |
||||
}, |
||||
{ |
||||
parameter: 'password', |
||||
aliases: ['p'], |
||||
}, |
||||
{ |
||||
parameter: 'user', |
||||
aliases: ['u'], |
||||
}, |
||||
{ |
||||
parameter: 'title', |
||||
aliases: ['t'], |
||||
}, |
||||
{ |
||||
parameter: 'keyFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'certFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'caFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'ssl', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'options', |
||||
aliases: ['opt', 'opts'], |
||||
}, |
||||
]; |
||||
|
||||
export enum DriverClient { |
||||
MYSQL = 'mysql2', |
||||
MSSQL = 'mssql', |
||||
PG = 'pg', |
||||
SQLITE = 'sqlite3', |
||||
SNOWFLAKE = 'snowflake', |
||||
} |
||||
@ -0,0 +1,324 @@
|
||||
import fs from 'fs'; |
||||
import { URL } from 'url'; |
||||
import { promisify } from 'util'; |
||||
import parseDbUrl from 'parse-database-url'; |
||||
import { |
||||
avoidSSL, |
||||
defaultClientPortMapping, |
||||
defaultConnectionConfig, |
||||
defaultConnectionOptions, |
||||
driverClientMapping, |
||||
knownQueryParams, |
||||
} from './constants'; |
||||
import { DriverClient } from './interfaces'; |
||||
import type { Connection, DbConfig } from './interfaces'; |
||||
|
||||
export async function prepareEnv() { |
||||
if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) { |
||||
const database_url = await promisify(fs.readFile)( |
||||
process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE, |
||||
'utf-8', |
||||
); |
||||
process.env.NC_DB = jdbcToXcUrl(database_url); |
||||
} else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) { |
||||
process.env.NC_DB = jdbcToXcUrl( |
||||
process.env.NC_DATABASE_URL || process.env.DATABASE_URL, |
||||
); |
||||
} |
||||
} |
||||
|
||||
export function getToolDir() { |
||||
return process.env.NC_TOOL_DIR || process.cwd(); |
||||
} |
||||
|
||||
export function jdbcToXcConfig(url: string): DbConfig { |
||||
// drop the jdbc prefix
|
||||
url.replace(/^jdbc:/, ''); |
||||
|
||||
const config = parseDbUrl(url); |
||||
|
||||
const parsedConfig: Connection = {}; |
||||
|
||||
for (const [key, value] of Object.entries(config)) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedConfig[fnd.parameter] = value; |
||||
} else { |
||||
parsedConfig[key] = value; |
||||
} |
||||
} |
||||
|
||||
if (!parsedConfig?.port) { |
||||
parsedConfig.port = |
||||
defaultClientPortMapping[ |
||||
driverClientMapping[parsedConfig.driver] || parsedConfig.driver |
||||
]; |
||||
} |
||||
|
||||
const { driver, ...connectionConfig } = parsedConfig; |
||||
|
||||
const client = driverClientMapping[driver] || driver; |
||||
|
||||
if ( |
||||
client === 'pg' && |
||||
!connectionConfig?.ssl && |
||||
!avoidSSL.includes(connectionConfig.host) |
||||
) { |
||||
connectionConfig.ssl = true; |
||||
} |
||||
|
||||
return { |
||||
client: client, |
||||
connection: { |
||||
...connectionConfig, |
||||
}, |
||||
} as DbConfig; |
||||
} |
||||
|
||||
export function jdbcToXcUrl(url: string): string { |
||||
// drop the jdbc prefix
|
||||
url.replace(/^jdbc:/, ''); |
||||
|
||||
const config = parseDbUrl(url); |
||||
|
||||
const parsedConfig: Connection = {}; |
||||
|
||||
for (const [key, value] of Object.entries(config)) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedConfig[fnd.parameter] = value; |
||||
} else { |
||||
parsedConfig[key] = value; |
||||
} |
||||
} |
||||
|
||||
if (!parsedConfig?.port) { |
||||
parsedConfig.port = |
||||
defaultClientPortMapping[ |
||||
driverClientMapping[parsedConfig.driver] || parsedConfig.driver |
||||
]; |
||||
} |
||||
|
||||
const { driver, host, port, database, user, password, ...extra } = |
||||
parsedConfig; |
||||
|
||||
const extraParams = []; |
||||
|
||||
for (const [key, value] of Object.entries(extra)) { |
||||
extraParams.push(`${key}=${value}`); |
||||
} |
||||
|
||||
const res = `${driverClientMapping[driver] || driver}://${host}${ |
||||
port ? `:${port}` : '' |
||||
}?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${ |
||||
database ? `d=${database}&` : '' |
||||
}${extraParams.join('&')}`;
|
||||
|
||||
return res; |
||||
} |
||||
|
||||
export function xcUrlToDbConfig( |
||||
urlString: string, |
||||
key = '', |
||||
type?: string, |
||||
): DbConfig { |
||||
const url = new URL(urlString); |
||||
|
||||
let dbConfig: DbConfig; |
||||
|
||||
if (url.protocol.startsWith('sqlite3')) { |
||||
dbConfig = { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
filename: |
||||
url.searchParams.get('d') || url.searchParams.get('database'), |
||||
}, |
||||
database: url.searchParams.get('d') || url.searchParams.get('database'), |
||||
}, |
||||
} as any; |
||||
} else { |
||||
const parsedQuery = {}; |
||||
for (const [key, value] of url.searchParams.entries()) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedQuery[fnd.parameter] = value; |
||||
} else { |
||||
parsedQuery[key] = value; |
||||
} |
||||
} |
||||
|
||||
dbConfig = { |
||||
client: url.protocol.replace(':', '') as DriverClient, |
||||
connection: { |
||||
...parsedQuery, |
||||
host: url.hostname, |
||||
port: +url.port, |
||||
}, |
||||
acquireConnectionTimeout: 600000, |
||||
}; |
||||
|
||||
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { |
||||
dbConfig.connection.ssl = true; |
||||
} |
||||
|
||||
if ( |
||||
url.searchParams.get('keyFilePath') && |
||||
url.searchParams.get('certFilePath') && |
||||
url.searchParams.get('caFilePath') |
||||
) { |
||||
dbConfig.connection.ssl = { |
||||
keyFilePath: url.searchParams.get('keyFilePath'), |
||||
certFilePath: url.searchParams.get('certFilePath'), |
||||
caFilePath: url.searchParams.get('caFilePath'), |
||||
}; |
||||
} |
||||
} |
||||
|
||||
/* TODO check if this is needed |
||||
if (config && !config.title) { |
||||
config.title = |
||||
url.searchParams.get('t') || |
||||
url.searchParams.get('title') || |
||||
this.generateRandomTitle(); |
||||
} |
||||
*/ |
||||
|
||||
Object.assign(dbConfig, { |
||||
meta: { |
||||
tn: 'nc_evolutions', |
||||
allSchemas: |
||||
!!url.searchParams.get('allSchemas') || |
||||
!(url.searchParams.get('d') || url.searchParams.get('database')), |
||||
api: { |
||||
prefix: url.searchParams.get('apiPrefix') || '', |
||||
swagger: true, |
||||
type: |
||||
type || |
||||
((url.searchParams.get('api') || url.searchParams.get('a')) as any) || |
||||
'rest', |
||||
}, |
||||
dbAlias: url.searchParams.get('dbAlias') || `db${key}`, |
||||
metaTables: 'db', |
||||
migrations: { |
||||
disabled: false, |
||||
name: 'nc_evolutions', |
||||
}, |
||||
}, |
||||
}); |
||||
|
||||
return dbConfig; |
||||
} |
||||
|
||||
export async function metaUrlToDbConfig(urlString): Promise<DbConfig> { |
||||
const url = new URL(urlString); |
||||
|
||||
let dbConfig: DbConfig; |
||||
|
||||
if (url.protocol.startsWith('sqlite3')) { |
||||
const db = url.searchParams.get('d') || url.searchParams.get('database'); |
||||
dbConfig = { |
||||
client: DriverClient.SQLITE, |
||||
connection: { |
||||
filename: db, |
||||
}, |
||||
...(db === ':memory:' |
||||
? { |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
} |
||||
: {}), |
||||
}; |
||||
} else { |
||||
const parsedQuery = {}; |
||||
for (const [key, value] of url.searchParams.entries()) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedQuery[fnd.parameter] = value; |
||||
} else { |
||||
parsedQuery[key] = value; |
||||
} |
||||
} |
||||
|
||||
dbConfig = { |
||||
client: url.protocol.replace(':', '') as DriverClient, |
||||
connection: { |
||||
...defaultConnectionConfig, |
||||
...parsedQuery, |
||||
host: url.hostname, |
||||
port: +url.port, |
||||
}, |
||||
acquireConnectionTimeout: 600000, |
||||
...defaultConnectionOptions, |
||||
...(url.searchParams.has('search_path') |
||||
? { |
||||
searchPath: url.searchParams.get('search_path').split(','), |
||||
} |
||||
: {}), |
||||
}; |
||||
|
||||
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { |
||||
dbConfig.connection.ssl = true; |
||||
} |
||||
} |
||||
|
||||
url.searchParams.forEach((_value, key) => { |
||||
let value: any = _value; |
||||
if (value === 'true') { |
||||
value = true; |
||||
} else if (value === 'false') { |
||||
value = false; |
||||
} else if (/^\d+$/.test(value)) { |
||||
value = +value; |
||||
} |
||||
// todo: implement config read from JSON file or JSON env val read
|
||||
if ( |
||||
!['password', 'p', 'database', 'd', 'user', 'u', 'search_path'].includes( |
||||
key, |
||||
) |
||||
) { |
||||
key.split('.').reduce((obj, k, i, arr) => { |
||||
return (obj[k] = i === arr.length - 1 ? value : obj[k] || {}); |
||||
}, dbConfig); |
||||
} |
||||
}); |
||||
|
||||
if ( |
||||
dbConfig?.connection?.ssl && |
||||
typeof dbConfig?.connection?.ssl === 'object' |
||||
) { |
||||
if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) { |
||||
dbConfig.connection.ssl.ca = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.caFilePath; |
||||
} |
||||
if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) { |
||||
dbConfig.connection.ssl.key = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.keyFilePath; |
||||
} |
||||
if (dbConfig.connection.ssl.certFilePath && !dbConfig.connection.ssl.cert) { |
||||
dbConfig.connection.ssl.cert = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.certFilePath; |
||||
} |
||||
} |
||||
|
||||
return dbConfig; |
||||
} |
||||
@ -0,0 +1,4 @@
|
||||
export * from './helpers'; |
||||
export * from './interfaces'; |
||||
export * from './constants'; |
||||
export * from './NcConfig'; |
||||
@ -0,0 +1,39 @@
|
||||
import { DriverClient } from './constants'; |
||||
|
||||
interface Connection { |
||||
driver?: DriverClient; |
||||
host?: string; |
||||
port?: number; |
||||
database?: string; |
||||
user?: string; |
||||
password?: string; |
||||
ssl?: |
||||
| boolean |
||||
| { |
||||
ca?: string; |
||||
cert?: string; |
||||
key?: string; |
||||
caFilePath?: string; |
||||
certFilePath?: string; |
||||
keyFilePath?: string; |
||||
}; |
||||
filename?: string; |
||||
} |
||||
|
||||
interface DbConfig { |
||||
client: DriverClient; |
||||
connection: Connection; |
||||
acquireConnectionTimeout?: number; |
||||
useNullAsDefault?: boolean; |
||||
pool?: { |
||||
min?: number; |
||||
max?: number; |
||||
idleTimeoutMillis?: number; |
||||
}; |
||||
migrations?: { |
||||
directory?: string; |
||||
tableName?: string; |
||||
}; |
||||
} |
||||
|
||||
export { DriverClient, Connection, DbConfig }; |
||||
Loading…
Reference in new issue