mirror of https://github.com/nocodb/nocodb
github-actions[bot]
2 years ago
committed by
GitHub
102 changed files with 14891 additions and 8419 deletions
@ -1,19 +0,0 @@
|
||||
import { Test } from '@nestjs/testing'; |
||||
import { Connection } from './knex'; |
||||
import type { TestingModule } from '@nestjs/testing'; |
||||
|
||||
describe('Knex', () => { |
||||
let provider: Connection; |
||||
|
||||
beforeEach(async () => { |
||||
const module: TestingModule = await Test.createTestingModule({ |
||||
providers: [Connection], |
||||
}).compile(); |
||||
|
||||
provider = module.get<Connection>(Connection); |
||||
}); |
||||
|
||||
it('should be defined', () => { |
||||
expect(provider).toBeDefined(); |
||||
}); |
||||
}); |
||||
@ -1,37 +0,0 @@
|
||||
import { Global, Injectable, Scope } from '@nestjs/common'; |
||||
|
||||
import { XKnex } from '../db/CustomKnex'; |
||||
import NcConfigFactory from '../utils/NcConfigFactory'; |
||||
import type * as knex from 'knex'; |
||||
|
||||
@Injectable({ |
||||
scope: Scope.DEFAULT, |
||||
}) |
||||
export class Connection { |
||||
public static knex: knex.Knex; |
||||
public static _config: any; |
||||
|
||||
get knexInstance(): knex.Knex { |
||||
return Connection.knex; |
||||
} |
||||
|
||||
get config(): knex.Knex { |
||||
return Connection._config; |
||||
} |
||||
|
||||
// init metadb connection
|
||||
static async init(): Promise<void> { |
||||
Connection._config = await NcConfigFactory.make(); |
||||
if (!Connection.knex) { |
||||
Connection.knex = XKnex({ |
||||
...this._config.meta.db, |
||||
useNullAsDefault: true, |
||||
}); |
||||
} |
||||
} |
||||
|
||||
// init metadb connection
|
||||
async init(): Promise<void> { |
||||
return await Connection.init(); |
||||
} |
||||
} |
||||
@ -1,6 +1,5 @@
|
||||
import Noco from './Noco'; |
||||
import NcConfigFactory from './utils/NcConfigFactory'; |
||||
|
||||
export default Noco; |
||||
|
||||
export { Noco, NcConfigFactory }; |
||||
export { Noco }; |
||||
|
||||
@ -1,12 +1,12 @@
|
||||
import { Connection } from './connection/connection'; |
||||
import { MetaService } from './meta/meta.service'; |
||||
import { NcConfig } from './utils/nc-config'; |
||||
import Noco from './Noco'; |
||||
|
||||
// run upgrader
|
||||
import NcUpgrader from './version-upgrader/NcUpgrader'; |
||||
|
||||
export default async () => { |
||||
await Connection.init(); |
||||
Noco._ncMeta = new MetaService(new Connection()); |
||||
const config = await NcConfig.createByEnv(); |
||||
Noco._ncMeta = new MetaService(config); |
||||
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta }); |
||||
}; |
||||
|
||||
@ -0,0 +1,65 @@
|
||||
import { T } from 'nc-help'; |
||||
import { MetaService } from '../../meta/meta.service'; |
||||
import Noco from '../../Noco'; |
||||
import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2'; |
||||
import NcUpgrader from '../../version-upgrader/NcUpgrader'; |
||||
import NocoCache from '../../cache/NocoCache'; |
||||
import getInstance from '../../utils/getInstance'; |
||||
import initAdminFromEnv from '../../helpers/initAdminFromEnv'; |
||||
import { User } from '../../models'; |
||||
import { NcConfig, prepareEnv } from '../../utils/nc-config'; |
||||
import type { Provider } from '@nestjs/common'; |
||||
import type { IEventEmitter } from '../event-emitter/event-emitter.interface'; |
||||
|
||||
export const InitMetaServiceProvider: Provider = { |
||||
// initialize app,
|
||||
// 1. init cache
|
||||
// 2. init db connection and create if not exist
|
||||
// 3. init meta and set to Noco
|
||||
// 4. init jwt
|
||||
// 5. init plugin manager
|
||||
// 6. run upgrader
|
||||
useFactory: async (eventEmitter: IEventEmitter) => { |
||||
// NC_DATABASE_URL_FILE, DATABASE_URL_FILE, DATABASE_URL, NC_DATABASE_URL to NC_DB
|
||||
await prepareEnv(); |
||||
|
||||
const config = await NcConfig.createByEnv(); |
||||
|
||||
// set version
|
||||
process.env.NC_VERSION = '0107004'; |
||||
|
||||
// init cache
|
||||
await NocoCache.init(); |
||||
|
||||
// init meta service
|
||||
const metaService = new MetaService(config); |
||||
await metaService.init(); |
||||
|
||||
// provide meta and config to Noco
|
||||
Noco._ncMeta = metaService; |
||||
Noco.config = config; |
||||
Noco.eventEmitter = eventEmitter; |
||||
|
||||
// init jwt secret
|
||||
await Noco.initJwt(); |
||||
|
||||
// load super admin user from env if env is set
|
||||
await initAdminFromEnv(metaService); |
||||
|
||||
// init plugin manager
|
||||
await NcPluginMgrv2.init(Noco.ncMeta); |
||||
await Noco.loadEEState(); |
||||
|
||||
// run upgrader
|
||||
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta }); |
||||
|
||||
T.init({ |
||||
instance: getInstance, |
||||
}); |
||||
T.emit('evt_app_started', await User.count()); |
||||
|
||||
return metaService; |
||||
}, |
||||
provide: MetaService, |
||||
inject: ['IEventEmitter'], |
||||
}; |
||||
@ -0,0 +1,51 @@
|
||||
import { Injectable } from '@nestjs/common'; |
||||
import { JobStatus } from '../../../interface/Jobs'; |
||||
import { QueueService } from './fallback-queue.service'; |
||||
|
||||
@Injectable() |
||||
export class JobsService { |
||||
constructor(private readonly fallbackQueueService: QueueService) {} |
||||
|
||||
async add(name: string, data: any) { |
||||
return this.fallbackQueueService.add(name, data); |
||||
} |
||||
|
||||
async jobStatus(jobId: string) { |
||||
return await ( |
||||
await this.fallbackQueueService.getJob(jobId) |
||||
).status; |
||||
} |
||||
|
||||
async jobList() { |
||||
return await this.fallbackQueueService.getJobs([ |
||||
JobStatus.ACTIVE, |
||||
JobStatus.WAITING, |
||||
JobStatus.DELAYED, |
||||
JobStatus.PAUSED, |
||||
]); |
||||
} |
||||
|
||||
async getJobWithData(data: any) { |
||||
const jobs = await this.fallbackQueueService.getJobs([ |
||||
// 'completed',
|
||||
JobStatus.WAITING, |
||||
JobStatus.ACTIVE, |
||||
JobStatus.DELAYED, |
||||
// 'failed',
|
||||
JobStatus.PAUSED, |
||||
]); |
||||
|
||||
const job = jobs.find((j) => { |
||||
for (const key in data) { |
||||
if (j.data[key]) { |
||||
if (j.data[key] !== data[key]) return false; |
||||
} else { |
||||
return false; |
||||
} |
||||
} |
||||
return true; |
||||
}); |
||||
|
||||
return job; |
||||
} |
||||
} |
||||
@ -1,59 +0,0 @@
|
||||
import { InjectQueue } from '@nestjs/bull'; |
||||
import { Injectable } from '@nestjs/common'; |
||||
import { Queue } from 'bull'; |
||||
import { JOBS_QUEUE, JobStatus } from '../../interface/Jobs'; |
||||
import { QueueService } from './fallback-queue.service'; |
||||
|
||||
@Injectable() |
||||
export class JobsService { |
||||
public activeQueue; |
||||
constructor( |
||||
@InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue, |
||||
private readonly fallbackQueueService: QueueService, |
||||
) { |
||||
this.activeQueue = this.fallbackQueueService; |
||||
/* process.env.NC_REDIS_URL |
||||
? this.jobsQueue |
||||
: this.fallbackQueueService; |
||||
*/ |
||||
} |
||||
|
||||
async jobStatus(jobId: string) { |
||||
return await (await this.activeQueue.getJob(jobId)).getState(); |
||||
} |
||||
|
||||
async jobList(jobType: string) { |
||||
return ( |
||||
await this.activeQueue.getJobs([ |
||||
JobStatus.ACTIVE, |
||||
JobStatus.WAITING, |
||||
JobStatus.DELAYED, |
||||
JobStatus.PAUSED, |
||||
]) |
||||
).filter((j) => j.name === jobType); |
||||
} |
||||
|
||||
async getJobWithData(data: any) { |
||||
const jobs = await this.activeQueue.getJobs([ |
||||
// 'completed',
|
||||
JobStatus.WAITING, |
||||
JobStatus.ACTIVE, |
||||
JobStatus.DELAYED, |
||||
// 'failed',
|
||||
JobStatus.PAUSED, |
||||
]); |
||||
|
||||
const job = jobs.find((j) => { |
||||
for (const key in data) { |
||||
if (j.data[key]) { |
||||
if (j.data[key] !== data[key]) return false; |
||||
} else { |
||||
return false; |
||||
} |
||||
} |
||||
return true; |
||||
}); |
||||
|
||||
return job; |
||||
} |
||||
} |
||||
@ -1,8 +1,8 @@
|
||||
/* eslint-disable no-async-promise-executor */ |
||||
import { RelationTypes, UITypes } from 'nocodb-sdk'; |
||||
import EntityMap from './EntityMap'; |
||||
import type { BulkDataAliasService } from '../../../../services/bulk-data-alias.service'; |
||||
import type { TablesService } from '../../../../services/tables.service'; |
||||
import type { BulkDataAliasService } from '../../../../../services/bulk-data-alias.service'; |
||||
import type { TablesService } from '../../../../../services/tables.service'; |
||||
// @ts-ignore
|
||||
import type { AirtableBase } from 'airtable/lib/airtable_base'; |
||||
import type { TableType } from 'nocodb-sdk'; |
||||
@ -0,0 +1,16 @@
|
||||
import { Injectable } from '@nestjs/common'; |
||||
import { EventEmitter2 } from '@nestjs/event-emitter'; |
||||
import { JobEvents } from '../../../interface/Jobs'; |
||||
import type { Job } from 'bull'; |
||||
|
||||
@Injectable() |
||||
export class JobsLogService { |
||||
constructor(private eventEmitter: EventEmitter2) {} |
||||
|
||||
sendLog(job: Job, data: { message: string }) { |
||||
this.eventEmitter.emit(JobEvents.LOG, { |
||||
id: job.id.toString(), |
||||
data, |
||||
}); |
||||
} |
||||
} |
||||
@ -0,0 +1,107 @@
|
||||
import { |
||||
OnQueueActive, |
||||
OnQueueCompleted, |
||||
OnQueueFailed, |
||||
Processor, |
||||
} from '@nestjs/bull'; |
||||
import { Job } from 'bull'; |
||||
import boxen from 'boxen'; |
||||
import { EventEmitter2, OnEvent } from '@nestjs/event-emitter'; |
||||
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs'; |
||||
import { JobsRedisService } from './jobs-redis.service'; |
||||
|
||||
@Processor(JOBS_QUEUE) |
||||
export class JobsEventService { |
||||
constructor( |
||||
private jobsRedisService: JobsRedisService, |
||||
private eventEmitter: EventEmitter2, |
||||
) {} |
||||
|
||||
@OnQueueActive() |
||||
onActive(job: Job) { |
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, { |
||||
cmd: JobEvents.STATUS, |
||||
id: job.id.toString(), |
||||
status: JobStatus.ACTIVE, |
||||
}); |
||||
} else { |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.ACTIVE, |
||||
}); |
||||
} |
||||
} |
||||
|
||||
@OnQueueFailed() |
||||
onFailed(job: Job, error: Error) { |
||||
console.error( |
||||
boxen( |
||||
`---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, |
||||
{ |
||||
padding: 1, |
||||
borderStyle: 'double', |
||||
borderColor: 'yellow', |
||||
}, |
||||
), |
||||
); |
||||
|
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, { |
||||
cmd: JobEvents.STATUS, |
||||
id: job.id.toString(), |
||||
status: JobStatus.FAILED, |
||||
data: { |
||||
error: { |
||||
message: error?.message, |
||||
}, |
||||
}, |
||||
}); |
||||
} else { |
||||
this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`); |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.FAILED, |
||||
data: { |
||||
error: { |
||||
message: error?.message, |
||||
}, |
||||
}, |
||||
}); |
||||
} |
||||
} |
||||
|
||||
@OnQueueCompleted() |
||||
onCompleted(job: Job, data: any) { |
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.jobsRedisService.publish(`jobs-${job.id.toString()}`, { |
||||
cmd: JobEvents.STATUS, |
||||
id: job.id.toString(), |
||||
status: JobStatus.COMPLETED, |
||||
data: { |
||||
result: data, |
||||
}, |
||||
}); |
||||
} else { |
||||
this.jobsRedisService.unsubscribe(`jobs-${job.id.toString()}`); |
||||
this.eventEmitter.emit(JobEvents.STATUS, { |
||||
id: job.id.toString(), |
||||
status: JobStatus.COMPLETED, |
||||
data: { |
||||
result: data, |
||||
}, |
||||
}); |
||||
} |
||||
} |
||||
|
||||
@OnEvent(JobEvents.LOG) |
||||
onLog(data: { id: string; data: { message: string } }) { |
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.jobsRedisService.publish(`jobs-${data.id}`, { |
||||
cmd: JobEvents.LOG, |
||||
id: data.id, |
||||
data: data.data, |
||||
}); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,53 @@
|
||||
import { Injectable } from '@nestjs/common'; |
||||
import Redis from 'ioredis'; |
||||
|
||||
@Injectable() |
||||
export class JobsRedisService { |
||||
private redisClient: Redis; |
||||
private redisSubscriber: Redis; |
||||
private unsubscribeCallbacks: { [key: string]: () => void } = {}; |
||||
|
||||
constructor() { |
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.redisClient = new Redis(process.env.NC_REDIS_URL); |
||||
return; |
||||
} |
||||
this.redisSubscriber = new Redis(process.env.NC_REDIS_URL); |
||||
} |
||||
|
||||
publish(channel: string, message: string | any) { |
||||
if (typeof message === 'string') { |
||||
this.redisClient.publish(channel, message); |
||||
} else { |
||||
try { |
||||
this.redisClient.publish(channel, JSON.stringify(message)); |
||||
} catch (e) { |
||||
console.error(e); |
||||
} |
||||
} |
||||
} |
||||
|
||||
subscribe(channel: string, callback: (message: any) => void) { |
||||
this.redisSubscriber.subscribe(channel); |
||||
|
||||
const onMessage = (_channel, message) => { |
||||
try { |
||||
message = JSON.parse(message); |
||||
} catch (e) {} |
||||
callback(message); |
||||
}; |
||||
|
||||
this.redisSubscriber.on('message', onMessage); |
||||
this.unsubscribeCallbacks[channel] = () => { |
||||
this.redisSubscriber.unsubscribe(channel); |
||||
this.redisSubscriber.off('message', onMessage); |
||||
}; |
||||
} |
||||
|
||||
unsubscribe(channel: string) { |
||||
if (this.unsubscribeCallbacks[channel]) { |
||||
this.unsubscribeCallbacks[channel](); |
||||
delete this.unsubscribeCallbacks[channel]; |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,98 @@
|
||||
import { InjectQueue } from '@nestjs/bull'; |
||||
import { Injectable } from '@nestjs/common'; |
||||
import { Queue } from 'bull'; |
||||
import { EventEmitter2 } from '@nestjs/event-emitter'; |
||||
import { JobEvents, JOBS_QUEUE, JobStatus } from '../../../interface/Jobs'; |
||||
import { JobsRedisService } from './jobs-redis.service'; |
||||
import type { OnModuleInit } from '@nestjs/common'; |
||||
|
||||
@Injectable() |
||||
export class JobsService implements OnModuleInit { |
||||
constructor( |
||||
@InjectQueue(JOBS_QUEUE) private readonly jobsQueue: Queue, |
||||
private jobsRedisService: JobsRedisService, |
||||
private eventEmitter: EventEmitter2, |
||||
) {} |
||||
|
||||
// pause primary instance queue
|
||||
async onModuleInit() { |
||||
if (process.env.NC_WORKER_CONTAINER !== 'true') { |
||||
await this.jobsQueue.pause(true); |
||||
} |
||||
} |
||||
|
||||
async add(name: string, data: any) { |
||||
// resume primary instance queue if there is no worker
|
||||
const workerCount = (await this.jobsQueue.getWorkers()).length; |
||||
const localWorkerPaused = await this.jobsQueue.isPaused(true); |
||||
|
||||
// if there is no worker and primary instance queue is paused, resume it
|
||||
// if there is any worker and primary instance queue is not paused, pause it
|
||||
if (workerCount < 1 && localWorkerPaused) { |
||||
await this.jobsQueue.resume(true); |
||||
} else if (workerCount > 0 && !localWorkerPaused) { |
||||
await this.jobsQueue.pause(true); |
||||
} |
||||
|
||||
const job = await this.jobsQueue.add(name, data); |
||||
|
||||
// subscribe to job events
|
||||
this.jobsRedisService.subscribe(`jobs-${job.id.toString()}`, (data) => { |
||||
const cmd = data.cmd; |
||||
delete data.cmd; |
||||
switch (cmd) { |
||||
case JobEvents.STATUS: |
||||
this.eventEmitter.emit(JobEvents.STATUS, data); |
||||
if ([JobStatus.COMPLETED, JobStatus.FAILED].includes(data.status)) { |
||||
this.jobsRedisService.unsubscribe(`jobs-${data.id.toString()}`); |
||||
} |
||||
break; |
||||
case JobEvents.LOG: |
||||
this.eventEmitter.emit(JobEvents.LOG, data); |
||||
break; |
||||
} |
||||
}); |
||||
|
||||
return job; |
||||
} |
||||
|
||||
async jobStatus(jobId: string) { |
||||
const job = await this.jobsQueue.getJob(jobId); |
||||
if (job) { |
||||
return await job.getState(); |
||||
} |
||||
} |
||||
|
||||
async jobList() { |
||||
return await this.jobsQueue.getJobs([ |
||||
JobStatus.ACTIVE, |
||||
JobStatus.WAITING, |
||||
JobStatus.DELAYED, |
||||
JobStatus.PAUSED, |
||||
]); |
||||
} |
||||
|
||||
async getJobWithData(data: any) { |
||||
const jobs = await this.jobsQueue.getJobs([ |
||||
// 'completed',
|
||||
JobStatus.WAITING, |
||||
JobStatus.ACTIVE, |
||||
JobStatus.DELAYED, |
||||
// 'failed',
|
||||
JobStatus.PAUSED, |
||||
]); |
||||
|
||||
const job = jobs.find((j) => { |
||||
for (const key in data) { |
||||
if (j.data[key]) { |
||||
if (j.data[key] !== data[key]) return false; |
||||
} else { |
||||
return false; |
||||
} |
||||
} |
||||
return true; |
||||
}); |
||||
|
||||
return job; |
||||
} |
||||
} |
||||
@ -1,19 +0,0 @@
|
||||
import { Test } from '@nestjs/testing'; |
||||
import { AppInitService } from './app-init.service'; |
||||
import type { TestingModule } from '@nestjs/testing'; |
||||
|
||||
describe('AppInitService', () => { |
||||
let service: AppInitService; |
||||
|
||||
beforeEach(async () => { |
||||
const module: TestingModule = await Test.createTestingModule({ |
||||
providers: [AppInitService], |
||||
}).compile(); |
||||
|
||||
service = module.get<AppInitService>(AppInitService); |
||||
}); |
||||
|
||||
it('should be defined', () => { |
||||
expect(service).toBeDefined(); |
||||
}); |
||||
}); |
||||
@ -1,79 +0,0 @@
|
||||
import { T } from 'nc-help'; |
||||
import NocoCache from '../cache/NocoCache'; |
||||
import { Connection } from '../connection/connection'; |
||||
import initAdminFromEnv from '../helpers/initAdminFromEnv'; |
||||
import NcPluginMgrv2 from '../helpers/NcPluginMgrv2'; |
||||
import { MetaService } from '../meta/meta.service'; |
||||
import { User } from '../models'; |
||||
import Noco from '../Noco'; |
||||
import getInstance from '../utils/getInstance'; |
||||
import NcConfigFactory from '../utils/NcConfigFactory'; |
||||
import NcUpgrader from '../version-upgrader/NcUpgrader'; |
||||
import type { IEventEmitter } from '../modules/event-emitter/event-emitter.interface'; |
||||
import type { Provider } from '@nestjs/common'; |
||||
|
||||
export class AppInitService { |
||||
private readonly config: any; |
||||
|
||||
constructor(config) { |
||||
this.config = config; |
||||
} |
||||
|
||||
get appConfig(): any { |
||||
return this.config; |
||||
} |
||||
} |
||||
|
||||
export const appInitServiceProvider: Provider = { |
||||
provide: AppInitService, |
||||
// initialize app,
|
||||
// 1. init cache
|
||||
// 2. init db connection and create if not exist
|
||||
// 3. init meta and set to Noco
|
||||
// 4. init jwt
|
||||
// 5. init plugin manager
|
||||
// 6. run upgrader
|
||||
useFactory: async ( |
||||
connection: Connection, |
||||
metaService: MetaService, |
||||
eventEmitter: IEventEmitter, |
||||
) => { |
||||
process.env.NC_VERSION = '0107004'; |
||||
|
||||
await NocoCache.init(); |
||||
|
||||
await connection.init(); |
||||
|
||||
await NcConfigFactory.metaDbCreateIfNotExist(connection.config); |
||||
|
||||
await metaService.init(); |
||||
|
||||
// todo: remove
|
||||
// temporary hack
|
||||
Noco._ncMeta = metaService; |
||||
Noco.config = connection.config; |
||||
Noco.eventEmitter = eventEmitter; |
||||
|
||||
// init jwt secret
|
||||
await Noco.initJwt(); |
||||
|
||||
// load super admin user from env if env is set
|
||||
await initAdminFromEnv(metaService); |
||||
|
||||
// init plugin manager
|
||||
await NcPluginMgrv2.init(Noco.ncMeta); |
||||
await Noco.loadEEState(); |
||||
|
||||
// run upgrader
|
||||
await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta }); |
||||
|
||||
T.init({ |
||||
instance: getInstance, |
||||
}); |
||||
T.emit('evt_app_started', await User.count()); |
||||
|
||||
// todo: move app config to app-init service
|
||||
return new AppInitService(connection.config); |
||||
}, |
||||
inject: [Connection, MetaService, 'IEventEmitter'], |
||||
}; |
||||
@ -1,755 +0,0 @@
|
||||
import fs from 'fs'; |
||||
import { URL } from 'url'; |
||||
import { promisify } from 'util'; |
||||
import * as path from 'path'; |
||||
import parseDbUrl from 'parse-database-url'; |
||||
import { SqlClientFactory } from '../db/sql-client/lib/SqlClientFactory'; |
||||
// import SqlClientFactory from '../db/sql-client/lib/SqlClientFactory';
|
||||
// import type {
|
||||
// AuthConfig,
|
||||
// DbConfig,
|
||||
// MailerConfig,
|
||||
// NcConfig,
|
||||
// } from '../../interface/config';
|
||||
|
||||
// const {
|
||||
// uniqueNamesGenerator,
|
||||
// starWars,
|
||||
// adjectives,
|
||||
// animals,
|
||||
// } = require('unique-names-generator');
|
||||
|
||||
type NcConfig = any; |
||||
type DbConfig = any; |
||||
|
||||
const driverClientMapping = { |
||||
mysql: 'mysql2', |
||||
mariadb: 'mysql2', |
||||
postgres: 'pg', |
||||
postgresql: 'pg', |
||||
sqlite: 'sqlite3', |
||||
mssql: 'mssql', |
||||
}; |
||||
|
||||
const defaultClientPortMapping = { |
||||
mysql: 3306, |
||||
mysql2: 3306, |
||||
postgres: 5432, |
||||
pg: 5432, |
||||
mssql: 1433, |
||||
}; |
||||
|
||||
const defaultConnectionConfig: any = { |
||||
// https://github.com/knex/knex/issues/97
|
||||
// timezone: process.env.NC_TIMEZONE || 'UTC',
|
||||
dateStrings: true, |
||||
}; |
||||
|
||||
// default knex options
|
||||
const defaultConnectionOptions = { |
||||
pool: { |
||||
min: 0, |
||||
max: 10, |
||||
}, |
||||
}; |
||||
|
||||
const knownQueryParams = [ |
||||
{ |
||||
parameter: 'database', |
||||
aliases: ['d', 'db'], |
||||
}, |
||||
{ |
||||
parameter: 'password', |
||||
aliases: ['p'], |
||||
}, |
||||
{ |
||||
parameter: 'user', |
||||
aliases: ['u'], |
||||
}, |
||||
{ |
||||
parameter: 'title', |
||||
aliases: ['t'], |
||||
}, |
||||
{ |
||||
parameter: 'keyFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'certFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'caFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'ssl', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'options', |
||||
aliases: ['opt', 'opts'], |
||||
}, |
||||
]; |
||||
|
||||
export default class NcConfigFactory { |
||||
public static async make(): Promise<any> { |
||||
await this.jdbcToXcUrl(); |
||||
|
||||
const ncConfig = new NcConfigFactory(); |
||||
|
||||
ncConfig.auth = { |
||||
jwt: { |
||||
secret: process.env.NC_AUTH_JWT_SECRET, |
||||
}, |
||||
}; |
||||
|
||||
ncConfig.port = +(process?.env?.PORT ?? 8080); |
||||
ncConfig.env = '_noco'; // process.env?.NODE_ENV || 'dev';
|
||||
ncConfig.workingEnv = '_noco'; // process.env?.NODE_ENV || 'dev';
|
||||
// ncConfig.toolDir = this.getToolDir();
|
||||
ncConfig.projectType = |
||||
ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest'; |
||||
|
||||
if (ncConfig.meta?.db?.connection?.filename) { |
||||
ncConfig.meta.db.connection.filename = path.join( |
||||
this.getToolDir(), |
||||
ncConfig.meta.db.connection.filename, |
||||
); |
||||
} |
||||
|
||||
if (process.env.NC_DB) { |
||||
ncConfig.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB); |
||||
} else if (process.env.NC_DB_JSON) { |
||||
ncConfig.meta.db = JSON.parse(process.env.NC_DB_JSON); |
||||
} else if (process.env.NC_DB_JSON_FILE) { |
||||
const filePath = process.env.NC_DB_JSON_FILE; |
||||
|
||||
if (!(await promisify(fs.exists)(filePath))) { |
||||
throw new Error(`NC_DB_JSON_FILE not found: ${filePath}`); |
||||
} |
||||
|
||||
const fileContent = await promisify(fs.readFile)(filePath, { |
||||
encoding: 'utf8', |
||||
}); |
||||
ncConfig.meta.db = JSON.parse(fileContent); |
||||
} |
||||
|
||||
if (process.env.NC_TRY) { |
||||
ncConfig.try = true; |
||||
ncConfig.meta.db = { |
||||
client: 'sqlite3', |
||||
connection: ':memory:', |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
} as any; |
||||
} |
||||
|
||||
if (process.env.NC_PUBLIC_URL) { |
||||
ncConfig.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL; |
||||
// ncConfig.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
|
||||
ncConfig.publicUrl = process.env.NC_PUBLIC_URL; |
||||
} |
||||
|
||||
if (process.env.NC_DASHBOARD_URL) { |
||||
ncConfig.dashboardPath = process.env.NC_DASHBOARD_URL; |
||||
} |
||||
|
||||
return ncConfig; |
||||
} |
||||
|
||||
public static getToolDir() { |
||||
return process.env.NC_TOOL_DIR || process.cwd(); |
||||
} |
||||
|
||||
public static hasDbUrl(): boolean { |
||||
return Object.keys(process.env).some((envKey) => |
||||
envKey.startsWith('NC_DB_URL'), |
||||
); |
||||
} |
||||
|
||||
public static makeFromUrls(urls: string[]): NcConfig { |
||||
const config = new NcConfigFactory(); |
||||
|
||||
// config.envs[process.env.NODE_ENV || 'dev'].db = [];
|
||||
config.envs['_noco'].db = []; |
||||
for (const [i, url] of Object.entries(urls)) { |
||||
// config.envs[process.env.NODE_ENV || 'dev'].db.push(this.urlToDbConfig(url, i));
|
||||
config.envs['_noco'].db.push(this.urlToDbConfig(url, i)); |
||||
} |
||||
|
||||
return config; |
||||
} |
||||
|
||||
public static urlToDbConfig( |
||||
urlString: string, |
||||
key = '', |
||||
config?: NcConfigFactory, |
||||
type?: string, |
||||
): DbConfig { |
||||
const url = new URL(urlString); |
||||
|
||||
let dbConfig: DbConfig; |
||||
|
||||
if (url.protocol.startsWith('sqlite3')) { |
||||
dbConfig = { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
filename: |
||||
url.searchParams.get('d') || url.searchParams.get('database'), |
||||
}, |
||||
database: |
||||
url.searchParams.get('d') || url.searchParams.get('database'), |
||||
}, |
||||
} as any; |
||||
} else { |
||||
const parsedQuery = {}; |
||||
for (const [key, value] of url.searchParams.entries()) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedQuery[fnd.parameter] = value; |
||||
} else { |
||||
parsedQuery[key] = value; |
||||
} |
||||
} |
||||
|
||||
dbConfig = { |
||||
client: url.protocol.replace(':', ''), |
||||
connection: { |
||||
...defaultConnectionConfig, |
||||
...parsedQuery, |
||||
host: url.hostname, |
||||
port: +url.port, |
||||
}, |
||||
// pool: {
|
||||
// min: 1,
|
||||
// max: 1
|
||||
// },
|
||||
acquireConnectionTimeout: 600000, |
||||
} as any; |
||||
|
||||
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { |
||||
dbConfig.connection.ssl = true; |
||||
} |
||||
|
||||
if ( |
||||
url.searchParams.get('keyFilePath') && |
||||
url.searchParams.get('certFilePath') && |
||||
url.searchParams.get('caFilePath') |
||||
) { |
||||
dbConfig.connection.ssl = { |
||||
keyFilePath: url.searchParams.get('keyFilePath'), |
||||
certFilePath: url.searchParams.get('certFilePath'), |
||||
caFilePath: url.searchParams.get('caFilePath'), |
||||
}; |
||||
} |
||||
} |
||||
|
||||
if (config && !config.title) { |
||||
config.title = |
||||
url.searchParams.get('t') || |
||||
url.searchParams.get('title') || |
||||
this.generateRandomTitle(); |
||||
} |
||||
|
||||
Object.assign(dbConfig, { |
||||
meta: { |
||||
tn: 'nc_evolutions', |
||||
allSchemas: |
||||
!!url.searchParams.get('allSchemas') || |
||||
!(url.searchParams.get('d') || url.searchParams.get('database')), |
||||
api: { |
||||
prefix: url.searchParams.get('apiPrefix') || '', |
||||
swagger: true, |
||||
type: |
||||
type || |
||||
((url.searchParams.get('api') || |
||||
url.searchParams.get('a')) as any) || |
||||
'rest', |
||||
}, |
||||
dbAlias: url.searchParams.get('dbAlias') || `db${key}`, |
||||
metaTables: 'db', |
||||
migrations: { |
||||
disabled: false, |
||||
name: 'nc_evolutions', |
||||
}, |
||||
}, |
||||
}); |
||||
|
||||
return dbConfig; |
||||
} |
||||
|
||||
private static generateRandomTitle(): string { |
||||
return ''; /*uniqueNamesGenerator({ |
||||
dictionaries: [[starWars], [adjectives, animals]][ |
||||
Math.floor(Math.random() * 2) |
||||
], |
||||
}) |
||||
.toLowerCase() |
||||
.replace(/[ -]/g, '_');*/ |
||||
} |
||||
|
||||
static async metaUrlToDbConfig(urlString) { |
||||
const url = new URL(urlString); |
||||
|
||||
let dbConfig; |
||||
|
||||
if (url.protocol.startsWith('sqlite3')) { |
||||
const db = url.searchParams.get('d') || url.searchParams.get('database'); |
||||
dbConfig = { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
filename: db, |
||||
}, |
||||
...(db === ':memory:' |
||||
? { |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
} |
||||
: {}), |
||||
}; |
||||
} else { |
||||
const parsedQuery = {}; |
||||
for (const [key, value] of url.searchParams.entries()) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedQuery[fnd.parameter] = value; |
||||
} else { |
||||
parsedQuery[key] = value; |
||||
} |
||||
} |
||||
|
||||
dbConfig = { |
||||
client: url.protocol.replace(':', ''), |
||||
connection: { |
||||
...defaultConnectionConfig, |
||||
...parsedQuery, |
||||
host: url.hostname, |
||||
port: +url.port, |
||||
}, |
||||
acquireConnectionTimeout: 600000, |
||||
...(url.searchParams.has('search_path') |
||||
? { |
||||
searchPath: url.searchParams.get('search_path').split(','), |
||||
} |
||||
: {}), |
||||
}; |
||||
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { |
||||
dbConfig.connection.ssl = true; |
||||
} |
||||
} |
||||
url.searchParams.forEach((_value, key) => { |
||||
let value: any = _value; |
||||
if (value === 'true') { |
||||
value = true; |
||||
} else if (value === 'false') { |
||||
value = false; |
||||
} else if (/^\d+$/.test(value)) { |
||||
value = +value; |
||||
} |
||||
// todo: implement config read from JSON file or JSON env val read
|
||||
if ( |
||||
![ |
||||
'password', |
||||
'p', |
||||
'database', |
||||
'd', |
||||
'user', |
||||
'u', |
||||
'search_path', |
||||
].includes(key) |
||||
) { |
||||
key.split('.').reduce((obj, k, i, arr) => { |
||||
return (obj[k] = i === arr.length - 1 ? value : obj[k] || {}); |
||||
}, dbConfig); |
||||
} |
||||
}); |
||||
|
||||
if ( |
||||
dbConfig?.connection?.ssl && |
||||
typeof dbConfig?.connection?.ssl === 'object' |
||||
) { |
||||
if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) { |
||||
dbConfig.connection.ssl.ca = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.caFilePath; |
||||
} |
||||
if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) { |
||||
dbConfig.connection.ssl.key = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.keyFilePath; |
||||
} |
||||
if ( |
||||
dbConfig.connection.ssl.certFilePath && |
||||
!dbConfig.connection.ssl.cert |
||||
) { |
||||
dbConfig.connection.ssl.cert = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.certFilePath; |
||||
} |
||||
} |
||||
|
||||
return dbConfig; |
||||
} |
||||
|
||||
public static async makeProjectConfigFromUrl( |
||||
url, |
||||
type?: string, |
||||
): Promise<NcConfig> { |
||||
const config = new NcConfigFactory(); |
||||
const dbConfig = this.urlToDbConfig(url, '', config, type); |
||||
// config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
|
||||
config.envs['_noco'].db.push(dbConfig); |
||||
|
||||
if (process.env.NC_AUTH_ADMIN_SECRET) { |
||||
config.auth = { |
||||
masterKey: { |
||||
secret: process.env.NC_AUTH_ADMIN_SECRET, |
||||
}, |
||||
}; |
||||
} else if (process.env.NC_NO_AUTH) { |
||||
config.auth = { |
||||
disabled: true, |
||||
}; |
||||
// } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
|
||||
} else if (config?.envs?.['_noco']?.db?.[0]) { |
||||
config.auth = { |
||||
jwt: { |
||||
// dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
|
||||
dbAlias: |
||||
process.env.NC_AUTH_JWT_DB_ALIAS || |
||||
config.envs['_noco'].db[0].meta.dbAlias, |
||||
secret: process.env.NC_AUTH_JWT_SECRET, |
||||
}, |
||||
}; |
||||
} |
||||
|
||||
if (process.env.NC_DB) { |
||||
config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB); |
||||
} |
||||
|
||||
if (process.env.NC_TRY) { |
||||
config.try = true; |
||||
config.meta.db = { |
||||
client: 'sqlite3', |
||||
connection: ':memory:', |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
} as any; |
||||
} |
||||
|
||||
if (process.env.NC_MAILER) { |
||||
config.mailer = { |
||||
from: process.env.NC_MAILER_FROM, |
||||
options: { |
||||
host: process.env.NC_MAILER_HOST, |
||||
port: parseInt(process.env.NC_MAILER_PORT, 10), |
||||
secure: process.env.NC_MAILER_SECURE === 'true', |
||||
auth: { |
||||
user: process.env.NC_MAILER_USER, |
||||
pass: process.env.NC_MAILER_PASS, |
||||
}, |
||||
}, |
||||
}; |
||||
} |
||||
|
||||
if (process.env.NC_PUBLIC_URL) { |
||||
// config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
|
||||
config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL; |
||||
config.publicUrl = process.env.NC_PUBLIC_URL; |
||||
} |
||||
|
||||
config.port = +(process?.env?.PORT ?? 8080); |
||||
// config.env = process.env?.NODE_ENV || 'dev';
|
||||
// config.workingEnv = process.env?.NODE_ENV || 'dev';
|
||||
config.env = '_noco'; |
||||
config.workingEnv = '_noco'; |
||||
config.toolDir = this.getToolDir(); |
||||
config.projectType = |
||||
type || |
||||
config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type || |
||||
'rest'; |
||||
|
||||
return config; |
||||
} |
||||
|
||||
public static async makeProjectConfigFromConnection( |
||||
dbConnectionConfig: any, |
||||
type?: string, |
||||
): Promise<NcConfig> { |
||||
const config = new NcConfigFactory(); |
||||
let dbConfig = dbConnectionConfig; |
||||
|
||||
if (dbConfig.client === 'sqlite3') { |
||||
dbConfig = { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
...dbConnectionConfig, |
||||
database: dbConnectionConfig.connection.filename, |
||||
}, |
||||
}; |
||||
} |
||||
|
||||
// todo:
|
||||
const key = ''; |
||||
Object.assign(dbConfig, { |
||||
meta: { |
||||
tn: 'nc_evolutions', |
||||
api: { |
||||
prefix: '', |
||||
swagger: true, |
||||
type: type || 'rest', |
||||
}, |
||||
dbAlias: `db${key}`, |
||||
metaTables: 'db', |
||||
migrations: { |
||||
disabled: false, |
||||
name: 'nc_evolutions', |
||||
}, |
||||
}, |
||||
}); |
||||
|
||||
// config.envs[process.env.NODE_ENV || 'dev'].db.push(dbConfig);
|
||||
config.envs['_noco'].db.push(dbConfig); |
||||
|
||||
if (process.env.NC_AUTH_ADMIN_SECRET) { |
||||
config.auth = { |
||||
masterKey: { |
||||
secret: process.env.NC_AUTH_ADMIN_SECRET, |
||||
}, |
||||
}; |
||||
} else if (process.env.NC_NO_AUTH) { |
||||
config.auth = { |
||||
disabled: true, |
||||
}; |
||||
// } else if (config?.envs?.[process.env.NODE_ENV || 'dev']?.db?.[0]) {
|
||||
} else if (config?.envs?.['_noco']?.db?.[0]) { |
||||
config.auth = { |
||||
jwt: { |
||||
// dbAlias: process.env.NC_AUTH_JWT_DB_ALIAS || config.envs[process.env.NODE_ENV || 'dev'].db[0].meta.dbAlias,
|
||||
dbAlias: |
||||
process.env.NC_AUTH_JWT_DB_ALIAS || |
||||
config.envs['_noco'].db[0].meta.dbAlias, |
||||
secret: process.env.NC_AUTH_JWT_SECRET, |
||||
}, |
||||
}; |
||||
} |
||||
|
||||
if (process.env.NC_DB) { |
||||
config.meta.db = await this.metaUrlToDbConfig(process.env.NC_DB); |
||||
} |
||||
|
||||
if (process.env.NC_TRY) { |
||||
config.try = true; |
||||
config.meta.db = { |
||||
client: 'sqlite3', |
||||
connection: ':memory:', |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
} as any; |
||||
} |
||||
|
||||
if (process.env.NC_PUBLIC_URL) { |
||||
// config.envs[process.env.NODE_ENV || 'dev'].publicUrl = process.env.NC_PUBLIC_URL;
|
||||
config.envs['_noco'].publicUrl = process.env.NC_PUBLIC_URL; |
||||
config.publicUrl = process.env.NC_PUBLIC_URL; |
||||
} |
||||
|
||||
config.port = +(process?.env?.PORT ?? 8080); |
||||
// config.env = process.env?.NODE_ENV || 'dev';
|
||||
// config.workingEnv = process.env?.NODE_ENV || 'dev';
|
||||
config.env = '_noco'; |
||||
config.workingEnv = '_noco'; |
||||
config.toolDir = process.env.NC_TOOL_DIR || process.cwd(); |
||||
config.projectType = |
||||
type || |
||||
config?.envs?.[config.workingEnv]?.db?.[0]?.meta?.api?.type || |
||||
'rest'; |
||||
|
||||
return config; |
||||
} |
||||
|
||||
public static async metaDbCreateIfNotExist(args: NcConfig) { |
||||
if (args.meta?.db?.client === 'sqlite3') { |
||||
const metaSqlClient = await SqlClientFactory.create({ |
||||
...args.meta.db, |
||||
connection: args.meta.db, |
||||
}); |
||||
await metaSqlClient.createDatabaseIfNotExists({ |
||||
database: args.meta.db?.connection?.filename, |
||||
}); |
||||
} else { |
||||
const metaSqlClient = await SqlClientFactory.create(args.meta.db); |
||||
await metaSqlClient.createDatabaseIfNotExists(args.meta.db?.connection); |
||||
await metaSqlClient.knex.destroy(); |
||||
} |
||||
|
||||
/* const dbPath = path.join(args.toolDir, 'xc.db') |
||||
const exists = fs.existsSync(dbPath); |
||||
if (!exists) { |
||||
const fd = fs.openSync(dbPath, "w"); |
||||
fs.closeSync(fd); |
||||
} |
||||
*/ |
||||
} |
||||
|
||||
public version = '0.6'; |
||||
public port: number; |
||||
public auth?: any; |
||||
public env: 'production' | 'dev' | 'test' | string; |
||||
public workingEnv: string; |
||||
public toolDir: string; |
||||
public envs: { |
||||
[p: string]: { db: DbConfig[]; api?: any; publicUrl?: string }; |
||||
}; |
||||
// public projectType: "rest" | "graphql" | "grpc";
|
||||
public queriesFolder: string | string[] = ''; |
||||
public seedsFolder: string | string[]; |
||||
public title: string; |
||||
public publicUrl: string; |
||||
public projectType; |
||||
public meta = { |
||||
db: { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
filename: 'noco.db', |
||||
}, |
||||
}, |
||||
}; |
||||
public mailer: any; |
||||
public try = false; |
||||
|
||||
public dashboardPath = '/dashboard'; |
||||
|
||||
constructor() { |
||||
this.envs = { _noco: { db: [] } }; |
||||
} |
||||
|
||||
public static async jdbcToXcUrl() { |
||||
if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) { |
||||
const database_url = await promisify(fs.readFile)( |
||||
process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE, |
||||
'utf-8', |
||||
); |
||||
process.env.NC_DB = this.extractXcUrlFromJdbc(database_url); |
||||
} else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) { |
||||
process.env.NC_DB = this.extractXcUrlFromJdbc( |
||||
process.env.NC_DATABASE_URL || process.env.DATABASE_URL, |
||||
); |
||||
} |
||||
} |
||||
|
||||
public static extractXcUrlFromJdbc(url: string, rtConfig = false) { |
||||
// drop the jdbc prefix
|
||||
if (url.startsWith('jdbc:')) { |
||||
url = url.substring(5); |
||||
} |
||||
|
||||
const config = parseDbUrl(url); |
||||
|
||||
const parsedConfig: { |
||||
driver?: string; |
||||
host?: string; |
||||
port?: string; |
||||
database?: string; |
||||
user?: string; |
||||
password?: string; |
||||
ssl?: string; |
||||
} = {}; |
||||
for (const [key, value] of Object.entries(config)) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedConfig[fnd.parameter] = value; |
||||
} else { |
||||
parsedConfig[key] = value; |
||||
} |
||||
} |
||||
|
||||
if (!parsedConfig?.port) |
||||
parsedConfig.port = |
||||
defaultClientPortMapping[ |
||||
driverClientMapping[parsedConfig.driver] || parsedConfig.driver |
||||
]; |
||||
|
||||
if (rtConfig) { |
||||
const { driver, ...connectionConfig } = parsedConfig; |
||||
|
||||
const client = driverClientMapping[driver] || driver; |
||||
|
||||
const avoidSSL = [ |
||||
'localhost', |
||||
'127.0.0.1', |
||||
'host.docker.internal', |
||||
'172.17.0.1', |
||||
]; |
||||
|
||||
if ( |
||||
client === 'pg' && |
||||
!connectionConfig?.ssl && |
||||
!avoidSSL.includes(connectionConfig.host) |
||||
) { |
||||
connectionConfig.ssl = 'true'; |
||||
} |
||||
|
||||
return { |
||||
client: client, |
||||
connection: { |
||||
...connectionConfig, |
||||
}, |
||||
} as any; |
||||
} |
||||
|
||||
const { driver, host, port, database, user, password, ...extra } = |
||||
parsedConfig; |
||||
|
||||
const extraParams = []; |
||||
|
||||
for (const [key, value] of Object.entries(extra)) { |
||||
extraParams.push(`${key}=${value}`); |
||||
} |
||||
|
||||
const res = `${driverClientMapping[driver] || driver}://${host}${ |
||||
port ? `:${port}` : '' |
||||
}?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${ |
||||
database ? `d=${database}&` : '' |
||||
}${extraParams.join('&')}`;
|
||||
|
||||
return res; |
||||
} |
||||
|
||||
// public static initOneClickDeployment() {
|
||||
// if (process.env.NC_ONE_CLICK) {
|
||||
// const url = NcConfigFactory.extractXcUrlFromJdbc(process.env.DATABASE_URL);
|
||||
// process.env.NC_DB = url;
|
||||
// }
|
||||
// }
|
||||
} |
||||
|
||||
export { defaultConnectionConfig, defaultConnectionOptions }; |
||||
@ -0,0 +1,182 @@
|
||||
import * as path from 'path'; |
||||
import fs from 'fs'; |
||||
import { promisify } from 'util'; |
||||
import { SqlClientFactory } from '../../db/sql-client/lib/SqlClientFactory'; |
||||
import { getToolDir, metaUrlToDbConfig } from './helpers'; |
||||
import { DriverClient } from './interfaces'; |
||||
import type { DbConfig } from './interfaces'; |
||||
|
||||
export class NcConfig { |
||||
version: string; |
||||
meta: { |
||||
db: DbConfig; |
||||
} = { |
||||
db: { |
||||
client: DriverClient.SQLITE, |
||||
connection: { |
||||
filename: 'noco.db', |
||||
}, |
||||
}, |
||||
}; |
||||
auth: { |
||||
jwt: { |
||||
secret: string; |
||||
options?: any; |
||||
}; |
||||
}; |
||||
|
||||
// if this is true, port is not exposed
|
||||
worker: boolean; |
||||
|
||||
toolDir: string; |
||||
|
||||
// exposed instance port
|
||||
port: number; |
||||
|
||||
// if this is true, use sqlite3 :memory: as meta db
|
||||
try: boolean; |
||||
|
||||
// optional
|
||||
publicUrl?: string; |
||||
dashboardPath?: string; |
||||
|
||||
// TODO what is this?
|
||||
envs: any; |
||||
|
||||
queriesFolder: string; |
||||
env: string; |
||||
workingEnv: string; |
||||
projectType: string; |
||||
|
||||
private constructor() {} |
||||
|
||||
public static async create(param: { |
||||
meta: { |
||||
metaUrl?: string; |
||||
metaJson?: string; |
||||
metaJsonFile?: string; |
||||
}; |
||||
secret?: string; |
||||
port?: string | number; |
||||
tryMode?: boolean; |
||||
worker?: boolean; |
||||
dashboardPath?: string; |
||||
publicUrl?: string; |
||||
}): Promise<NcConfig> { |
||||
const { meta, secret, port, worker, tryMode, publicUrl, dashboardPath } = |
||||
param; |
||||
|
||||
const ncConfig = new NcConfig(); |
||||
|
||||
ncConfig.auth = { |
||||
jwt: { |
||||
secret: secret, |
||||
}, |
||||
}; |
||||
|
||||
ncConfig.port = +(port ?? 8080); |
||||
ncConfig.toolDir = getToolDir(); |
||||
ncConfig.worker = worker ?? false; |
||||
|
||||
ncConfig.env = '_noco'; |
||||
ncConfig.workingEnv = '_noco'; |
||||
|
||||
ncConfig.projectType = |
||||
ncConfig?.envs?.[ncConfig.workingEnv]?.db?.[0]?.meta?.api?.type || 'rest'; |
||||
|
||||
if (ncConfig.meta?.db?.connection?.filename) { |
||||
ncConfig.meta.db.connection.filename = path.join( |
||||
ncConfig.toolDir, |
||||
ncConfig.meta.db.connection.filename, |
||||
); |
||||
} |
||||
|
||||
if (tryMode) { |
||||
ncConfig.try = true; |
||||
ncConfig.meta.db = { |
||||
client: DriverClient.SQLITE, |
||||
connection: ':memory:' as any, |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
}; |
||||
} else { |
||||
if (meta?.metaUrl) { |
||||
ncConfig.meta.db = await metaUrlToDbConfig(meta.metaUrl); |
||||
} else if (meta?.metaJson) { |
||||
ncConfig.meta.db = JSON.parse(meta.metaJson); |
||||
} else if (meta?.metaJsonFile) { |
||||
if (!(await promisify(fs.exists)(meta.metaJsonFile))) { |
||||
throw new Error(`NC_DB_JSON_FILE not found: ${meta.metaJsonFile}`); |
||||
} |
||||
const fileContent = await promisify(fs.readFile)(meta.metaJsonFile, { |
||||
encoding: 'utf8', |
||||
}); |
||||
ncConfig.meta.db = JSON.parse(fileContent); |
||||
} |
||||
} |
||||
|
||||
if (publicUrl) { |
||||
ncConfig.envs['_noco'].publicUrl = publicUrl; |
||||
ncConfig.publicUrl = publicUrl; |
||||
} |
||||
|
||||
if (dashboardPath) { |
||||
ncConfig.dashboardPath = dashboardPath; |
||||
} |
||||
|
||||
try { |
||||
// make sure meta db exists
|
||||
await ncConfig.metaDbCreateIfNotExist(); |
||||
} catch (e) { |
||||
throw new Error(e); |
||||
} |
||||
|
||||
return ncConfig; |
||||
} |
||||
|
||||
public static async createByEnv(): Promise<NcConfig> { |
||||
return NcConfig.create({ |
||||
meta: { |
||||
metaUrl: process.env.NC_DB, |
||||
metaJson: process.env.NC_DB_JSON, |
||||
metaJsonFile: process.env.NC_DB_JSON_FILE, |
||||
}, |
||||
secret: process.env.NC_AUTH_JWT_SECRET, |
||||
port: process.env.NC_PORT, |
||||
tryMode: !!process.env.NC_TRY, |
||||
worker: !!process.env.NC_WORKER, |
||||
dashboardPath: process.env.NC_DASHBOARD_PATH, |
||||
publicUrl: process.env.NC_PUBLIC_URL, |
||||
}); |
||||
} |
||||
|
||||
private async metaDbCreateIfNotExist() { |
||||
if (this.meta?.db?.client === 'sqlite3') { |
||||
const metaSqlClient = await SqlClientFactory.create({ |
||||
...this.meta.db, |
||||
connection: this.meta.db, |
||||
}); |
||||
if (this.meta.db?.connection?.filename) { |
||||
await metaSqlClient.createDatabaseIfNotExists({ |
||||
database: this.meta.db?.connection?.filename, |
||||
}); |
||||
} else { |
||||
throw new Error('Configuration missing meta db connection'); |
||||
} |
||||
} else { |
||||
const metaSqlClient = await SqlClientFactory.create(this.meta.db); |
||||
if (this.meta.db?.connection?.database) { |
||||
await metaSqlClient.createDatabaseIfNotExists( |
||||
(this.meta.db as any).connection, |
||||
); |
||||
await metaSqlClient.knex.destroy(); |
||||
} else { |
||||
throw new Error('Configuration missing meta db connection'); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,84 @@
|
||||
export const driverClientMapping = { |
||||
mysql: 'mysql2', |
||||
mariadb: 'mysql2', |
||||
postgres: 'pg', |
||||
postgresql: 'pg', |
||||
sqlite: 'sqlite3', |
||||
mssql: 'mssql', |
||||
}; |
||||
|
||||
export const defaultClientPortMapping = { |
||||
mysql: 3306, |
||||
mysql2: 3306, |
||||
postgres: 5432, |
||||
pg: 5432, |
||||
mssql: 1433, |
||||
}; |
||||
|
||||
export const defaultConnectionConfig: any = { |
||||
// https://github.com/knex/knex/issues/97
|
||||
// timezone: process.env.NC_TIMEZONE || 'UTC',
|
||||
dateStrings: true, |
||||
}; |
||||
|
||||
// default knex options
|
||||
export const defaultConnectionOptions = { |
||||
pool: { |
||||
min: 0, |
||||
max: 10, |
||||
}, |
||||
}; |
||||
|
||||
export const avoidSSL = [ |
||||
'localhost', |
||||
'127.0.0.1', |
||||
'host.docker.internal', |
||||
'172.17.0.1', |
||||
]; |
||||
|
||||
export const knownQueryParams = [ |
||||
{ |
||||
parameter: 'database', |
||||
aliases: ['d', 'db'], |
||||
}, |
||||
{ |
||||
parameter: 'password', |
||||
aliases: ['p'], |
||||
}, |
||||
{ |
||||
parameter: 'user', |
||||
aliases: ['u'], |
||||
}, |
||||
{ |
||||
parameter: 'title', |
||||
aliases: ['t'], |
||||
}, |
||||
{ |
||||
parameter: 'keyFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'certFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'caFilePath', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'ssl', |
||||
aliases: [], |
||||
}, |
||||
{ |
||||
parameter: 'options', |
||||
aliases: ['opt', 'opts'], |
||||
}, |
||||
]; |
||||
|
||||
export enum DriverClient { |
||||
MYSQL = 'mysql2', |
||||
MSSQL = 'mssql', |
||||
PG = 'pg', |
||||
SQLITE = 'sqlite3', |
||||
SNOWFLAKE = 'snowflake', |
||||
} |
||||
@ -0,0 +1,324 @@
|
||||
import fs from 'fs'; |
||||
import { URL } from 'url'; |
||||
import { promisify } from 'util'; |
||||
import parseDbUrl from 'parse-database-url'; |
||||
import { |
||||
avoidSSL, |
||||
defaultClientPortMapping, |
||||
defaultConnectionConfig, |
||||
defaultConnectionOptions, |
||||
driverClientMapping, |
||||
knownQueryParams, |
||||
} from './constants'; |
||||
import { DriverClient } from './interfaces'; |
||||
import type { Connection, DbConfig } from './interfaces'; |
||||
|
||||
export async function prepareEnv() { |
||||
if (process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE) { |
||||
const database_url = await promisify(fs.readFile)( |
||||
process.env.NC_DATABASE_URL_FILE || process.env.DATABASE_URL_FILE, |
||||
'utf-8', |
||||
); |
||||
process.env.NC_DB = jdbcToXcUrl(database_url); |
||||
} else if (process.env.NC_DATABASE_URL || process.env.DATABASE_URL) { |
||||
process.env.NC_DB = jdbcToXcUrl( |
||||
process.env.NC_DATABASE_URL || process.env.DATABASE_URL, |
||||
); |
||||
} |
||||
} |
||||
|
||||
export function getToolDir() { |
||||
return process.env.NC_TOOL_DIR || process.cwd(); |
||||
} |
||||
|
||||
export function jdbcToXcConfig(url: string): DbConfig { |
||||
// drop the jdbc prefix
|
||||
url.replace(/^jdbc:/, ''); |
||||
|
||||
const config = parseDbUrl(url); |
||||
|
||||
const parsedConfig: Connection = {}; |
||||
|
||||
for (const [key, value] of Object.entries(config)) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedConfig[fnd.parameter] = value; |
||||
} else { |
||||
parsedConfig[key] = value; |
||||
} |
||||
} |
||||
|
||||
if (!parsedConfig?.port) { |
||||
parsedConfig.port = |
||||
defaultClientPortMapping[ |
||||
driverClientMapping[parsedConfig.driver] || parsedConfig.driver |
||||
]; |
||||
} |
||||
|
||||
const { driver, ...connectionConfig } = parsedConfig; |
||||
|
||||
const client = driverClientMapping[driver] || driver; |
||||
|
||||
if ( |
||||
client === 'pg' && |
||||
!connectionConfig?.ssl && |
||||
!avoidSSL.includes(connectionConfig.host) |
||||
) { |
||||
connectionConfig.ssl = true; |
||||
} |
||||
|
||||
return { |
||||
client: client, |
||||
connection: { |
||||
...connectionConfig, |
||||
}, |
||||
} as DbConfig; |
||||
} |
||||
|
||||
export function jdbcToXcUrl(url: string): string { |
||||
// drop the jdbc prefix
|
||||
url.replace(/^jdbc:/, ''); |
||||
|
||||
const config = parseDbUrl(url); |
||||
|
||||
const parsedConfig: Connection = {}; |
||||
|
||||
for (const [key, value] of Object.entries(config)) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedConfig[fnd.parameter] = value; |
||||
} else { |
||||
parsedConfig[key] = value; |
||||
} |
||||
} |
||||
|
||||
if (!parsedConfig?.port) { |
||||
parsedConfig.port = |
||||
defaultClientPortMapping[ |
||||
driverClientMapping[parsedConfig.driver] || parsedConfig.driver |
||||
]; |
||||
} |
||||
|
||||
const { driver, host, port, database, user, password, ...extra } = |
||||
parsedConfig; |
||||
|
||||
const extraParams = []; |
||||
|
||||
for (const [key, value] of Object.entries(extra)) { |
||||
extraParams.push(`${key}=${value}`); |
||||
} |
||||
|
||||
const res = `${driverClientMapping[driver] || driver}://${host}${ |
||||
port ? `:${port}` : '' |
||||
}?${user ? `u=${user}&` : ''}${password ? `p=${password}&` : ''}${ |
||||
database ? `d=${database}&` : '' |
||||
}${extraParams.join('&')}`;
|
||||
|
||||
return res; |
||||
} |
||||
|
||||
export function xcUrlToDbConfig( |
||||
urlString: string, |
||||
key = '', |
||||
type?: string, |
||||
): DbConfig { |
||||
const url = new URL(urlString); |
||||
|
||||
let dbConfig: DbConfig; |
||||
|
||||
if (url.protocol.startsWith('sqlite3')) { |
||||
dbConfig = { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
client: 'sqlite3', |
||||
connection: { |
||||
filename: |
||||
url.searchParams.get('d') || url.searchParams.get('database'), |
||||
}, |
||||
database: url.searchParams.get('d') || url.searchParams.get('database'), |
||||
}, |
||||
} as any; |
||||
} else { |
||||
const parsedQuery = {}; |
||||
for (const [key, value] of url.searchParams.entries()) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedQuery[fnd.parameter] = value; |
||||
} else { |
||||
parsedQuery[key] = value; |
||||
} |
||||
} |
||||
|
||||
dbConfig = { |
||||
client: url.protocol.replace(':', '') as DriverClient, |
||||
connection: { |
||||
...parsedQuery, |
||||
host: url.hostname, |
||||
port: +url.port, |
||||
}, |
||||
acquireConnectionTimeout: 600000, |
||||
}; |
||||
|
||||
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { |
||||
dbConfig.connection.ssl = true; |
||||
} |
||||
|
||||
if ( |
||||
url.searchParams.get('keyFilePath') && |
||||
url.searchParams.get('certFilePath') && |
||||
url.searchParams.get('caFilePath') |
||||
) { |
||||
dbConfig.connection.ssl = { |
||||
keyFilePath: url.searchParams.get('keyFilePath'), |
||||
certFilePath: url.searchParams.get('certFilePath'), |
||||
caFilePath: url.searchParams.get('caFilePath'), |
||||
}; |
||||
} |
||||
} |
||||
|
||||
/* TODO check if this is needed |
||||
if (config && !config.title) { |
||||
config.title = |
||||
url.searchParams.get('t') || |
||||
url.searchParams.get('title') || |
||||
this.generateRandomTitle(); |
||||
} |
||||
*/ |
||||
|
||||
Object.assign(dbConfig, { |
||||
meta: { |
||||
tn: 'nc_evolutions', |
||||
allSchemas: |
||||
!!url.searchParams.get('allSchemas') || |
||||
!(url.searchParams.get('d') || url.searchParams.get('database')), |
||||
api: { |
||||
prefix: url.searchParams.get('apiPrefix') || '', |
||||
swagger: true, |
||||
type: |
||||
type || |
||||
((url.searchParams.get('api') || url.searchParams.get('a')) as any) || |
||||
'rest', |
||||
}, |
||||
dbAlias: url.searchParams.get('dbAlias') || `db${key}`, |
||||
metaTables: 'db', |
||||
migrations: { |
||||
disabled: false, |
||||
name: 'nc_evolutions', |
||||
}, |
||||
}, |
||||
}); |
||||
|
||||
return dbConfig; |
||||
} |
||||
|
||||
export async function metaUrlToDbConfig(urlString): Promise<DbConfig> { |
||||
const url = new URL(urlString); |
||||
|
||||
let dbConfig: DbConfig; |
||||
|
||||
if (url.protocol.startsWith('sqlite3')) { |
||||
const db = url.searchParams.get('d') || url.searchParams.get('database'); |
||||
dbConfig = { |
||||
client: DriverClient.SQLITE, |
||||
connection: { |
||||
filename: db, |
||||
}, |
||||
...(db === ':memory:' |
||||
? { |
||||
pool: { |
||||
min: 1, |
||||
max: 1, |
||||
// disposeTimeout: 360000*1000,
|
||||
idleTimeoutMillis: 360000 * 1000, |
||||
}, |
||||
} |
||||
: {}), |
||||
}; |
||||
} else { |
||||
const parsedQuery = {}; |
||||
for (const [key, value] of url.searchParams.entries()) { |
||||
const fnd = knownQueryParams.find( |
||||
(param) => param.parameter === key || param.aliases.includes(key), |
||||
); |
||||
if (fnd) { |
||||
parsedQuery[fnd.parameter] = value; |
||||
} else { |
||||
parsedQuery[key] = value; |
||||
} |
||||
} |
||||
|
||||
dbConfig = { |
||||
client: url.protocol.replace(':', '') as DriverClient, |
||||
connection: { |
||||
...defaultConnectionConfig, |
||||
...parsedQuery, |
||||
host: url.hostname, |
||||
port: +url.port, |
||||
}, |
||||
acquireConnectionTimeout: 600000, |
||||
...defaultConnectionOptions, |
||||
...(url.searchParams.has('search_path') |
||||
? { |
||||
searchPath: url.searchParams.get('search_path').split(','), |
||||
} |
||||
: {}), |
||||
}; |
||||
|
||||
if (process.env.NODE_TLS_REJECT_UNAUTHORIZED) { |
||||
dbConfig.connection.ssl = true; |
||||
} |
||||
} |
||||
|
||||
url.searchParams.forEach((_value, key) => { |
||||
let value: any = _value; |
||||
if (value === 'true') { |
||||
value = true; |
||||
} else if (value === 'false') { |
||||
value = false; |
||||
} else if (/^\d+$/.test(value)) { |
||||
value = +value; |
||||
} |
||||
// todo: implement config read from JSON file or JSON env val read
|
||||
if ( |
||||
!['password', 'p', 'database', 'd', 'user', 'u', 'search_path'].includes( |
||||
key, |
||||
) |
||||
) { |
||||
key.split('.').reduce((obj, k, i, arr) => { |
||||
return (obj[k] = i === arr.length - 1 ? value : obj[k] || {}); |
||||
}, dbConfig); |
||||
} |
||||
}); |
||||
|
||||
if ( |
||||
dbConfig?.connection?.ssl && |
||||
typeof dbConfig?.connection?.ssl === 'object' |
||||
) { |
||||
if (dbConfig.connection.ssl.caFilePath && !dbConfig.connection.ssl.ca) { |
||||
dbConfig.connection.ssl.ca = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.caFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.caFilePath; |
||||
} |
||||
if (dbConfig.connection.ssl.keyFilePath && !dbConfig.connection.ssl.key) { |
||||
dbConfig.connection.ssl.key = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.keyFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.keyFilePath; |
||||
} |
||||
if (dbConfig.connection.ssl.certFilePath && !dbConfig.connection.ssl.cert) { |
||||
dbConfig.connection.ssl.cert = ( |
||||
await promisify(fs.readFile)(dbConfig.connection.ssl.certFilePath) |
||||
).toString(); |
||||
delete dbConfig.connection.ssl.certFilePath; |
||||
} |
||||
} |
||||
|
||||
return dbConfig; |
||||
} |
||||
@ -0,0 +1,4 @@
|
||||
export * from './helpers'; |
||||
export * from './interfaces'; |
||||
export * from './constants'; |
||||
export * from './NcConfig'; |
||||
@ -0,0 +1,39 @@
|
||||
import { DriverClient } from './constants'; |
||||
|
||||
interface Connection { |
||||
driver?: DriverClient; |
||||
host?: string; |
||||
port?: number; |
||||
database?: string; |
||||
user?: string; |
||||
password?: string; |
||||
ssl?: |
||||
| boolean |
||||
| { |
||||
ca?: string; |
||||
cert?: string; |
||||
key?: string; |
||||
caFilePath?: string; |
||||
certFilePath?: string; |
||||
keyFilePath?: string; |
||||
}; |
||||
filename?: string; |
||||
} |
||||
|
||||
interface DbConfig { |
||||
client: DriverClient; |
||||
connection: Connection; |
||||
acquireConnectionTimeout?: number; |
||||
useNullAsDefault?: boolean; |
||||
pool?: { |
||||
min?: number; |
||||
max?: number; |
||||
idleTimeoutMillis?: number; |
||||
}; |
||||
migrations?: { |
||||
directory?: string; |
||||
tableName?: string; |
||||
}; |
||||
} |
||||
|
||||
export { DriverClient, Connection, DbConfig }; |
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue