diff --git a/packages/nocodb/package-lock.json b/packages/nocodb/package-lock.json index 91d1f8deef..1921b5128d 100644 --- a/packages/nocodb/package-lock.json +++ b/packages/nocodb/package-lock.json @@ -11800,9 +11800,9 @@ "integrity": "sha512-3AryS9uwa5NfISLxMciUonrH7YfXp+nlahB9T7girXIsLQrmwX4MdnuKs32akduCOGpKmjTJSWmATULbuMkbfw==" }, "nc-help": { - "version": "0.2.8", - "resolved": "https://registry.npmjs.org/nc-help/-/nc-help-0.2.8.tgz", - "integrity": "sha512-h6HUWb2trYEVm7x2/hWxifeq+ZALCIFxZFISgpI/IQPhe/BSoGUNpcRIhWg8wt1y87kxZ7/VCkvTQUhW9OTOGg==", + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/nc-help/-/nc-help-0.2.10.tgz", + "integrity": "sha512-XkLvKf3clpewn576RGX84nsNBWDjvFJFsaNPKw0n77zcjlC6zeBOX6fGJsTVEBg56usr0jaK+LfOtJUUEc9D+A==", "requires": { "axios": "^0.21.1", "boxen": "^4.2.0", diff --git a/packages/nocodb/package.json b/packages/nocodb/package.json index d4908b8a4e..c1059dfec3 100644 --- a/packages/nocodb/package.json +++ b/packages/nocodb/package.json @@ -144,7 +144,7 @@ "mysql2": "^2.2.5", "nanoid": "^3.1.20", "nc-common": "0.0.6", - "nc-help": "^0.2.8", + "nc-help": "^0.2.10", "nc-lib-gui": "^0.2.9", "nc-plugin": "^0.1.1", "nodemailer": "^6.4.10", diff --git a/packages/nocodb/src/example/docker.ts b/packages/nocodb/src/example/docker.ts index e5ed8ecc09..6e56ea8e0e 100644 --- a/packages/nocodb/src/example/docker.ts +++ b/packages/nocodb/src/example/docker.ts @@ -19,8 +19,8 @@ server.set('view engine', 'ejs'); process.env[`DEBUG`] = 'xc*'; +process.env[`NODE_TLS_REJECT_UNAUTHORIZED`] = '0'; -// process.env[`DATABASE_URL`] = 'postgres://postgres:xgene@localhost:32768/chinook4'; // process.env[`NC_ONE_CLICK`] = 'true'; // process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0" // diff --git a/packages/nocodb/src/lib/migrator/SqlMigrator/lib/KnexMigrator.ts b/packages/nocodb/src/lib/migrator/SqlMigrator/lib/KnexMigrator.ts index 3fd9cca7a3..74e901a849 100644 --- a/packages/nocodb/src/lib/migrator/SqlMigrator/lib/KnexMigrator.ts +++ b/packages/nocodb/src/lib/migrator/SqlMigrator/lib/KnexMigrator.ts @@ -298,7 +298,7 @@ export default class KnexMigrator extends SqlMigrator { } async _initDbWithSql(connectionConfig) { - const sqlClient = SqlClientFactory.create(connectionConfig); + const sqlClient = connectionConfig.sqlClient || SqlClientFactory.create(connectionConfig); if (connectionConfig.client === "oracledb") { this.emit( `${connectionConfig.client}: Creating DB if not exists ${ @@ -375,7 +375,7 @@ export default class KnexMigrator extends SqlMigrator { // await sqlClient.createTableIfNotExists({tn: connectionConfig.meta.tn}); } - async _initEnvDbsWithSql(env, dbAlias = null) { + async _initEnvDbsWithSql(env, dbAlias = null, sqlClient =null) { const {envs} = this.project; @@ -387,7 +387,7 @@ export default class KnexMigrator extends SqlMigrator { /* if no dbAlias - init all dbs in env. Else check if it matches the one sent in args */ if (!dbAlias || dbAlias === envs[env].db[i].meta.dbAlias) { - await this._initDbWithSql(connectionConfig); + await this._initDbWithSql({...connectionConfig, sqlClient}); } } } @@ -500,7 +500,7 @@ export default class KnexMigrator extends SqlMigrator { args.dbAlias, args.env ); - const sqlClient = SqlClientFactory.create(connection); + const sqlClient = args.sqlClient || SqlClientFactory.create(connection); const migrations = await sqlClient.selectAll(connection.meta.tn); /** ************** END : get files and migrations *************** */ @@ -901,10 +901,10 @@ export default class KnexMigrator extends SqlMigrator { /* happens when creating the project */ for (const env in this.project.envs) { args.env = env; - await this._initEnvDbsWithSql(args.env, args.dbAlias); + await this._initEnvDbsWithSql(args.env, args.dbAlias, args.sqlClient); } } else { - await this._initEnvDbsWithSql(args.env, args.dbAlias); + await this._initEnvDbsWithSql(args.env, args.dbAlias, args.sqlClient); } } catch (e) { @@ -1069,7 +1069,7 @@ export default class KnexMigrator extends SqlMigrator { * and only filenames are migrated to _evolution table * @memberof KnexMigrator */ - async migrationsUp(args: any = {}) { + async migrationsUp(args: any = {}) { const func = this.migrationsUp.name; // const result = new Result(); @@ -1092,7 +1092,8 @@ export default class KnexMigrator extends SqlMigrator { upFilesPattern: path.join(this.toolDir, 'nc', this.project.id, args.dbAlias, 'migrations', '*.up.sql'), downFilesPattern: path.join(this.toolDir, 'nc', this.project.id, args.dbAlias, 'migrations', '*.down.sql'), tn: this._getEvolutionsTablename(args),//`${this.toolDir}`, - sqlContentMigrate: args.sqlContentMigrate + sqlContentMigrate: args.sqlContentMigrate, + sqlClient: args.sqlClient }); } diff --git a/packages/nocodb/src/lib/noco/NcProjectBuilder.ts b/packages/nocodb/src/lib/noco/NcProjectBuilder.ts index ba834b2a77..aab9d94e6b 100644 --- a/packages/nocodb/src/lib/noco/NcProjectBuilder.ts +++ b/packages/nocodb/src/lib/noco/NcProjectBuilder.ts @@ -12,6 +12,7 @@ import Noco from "./Noco"; import {GqlApiBuilder} from "./gql/GqlApiBuilder"; import {XCEeError} from "./meta/NcMetaMgr"; import {RestApiBuilder} from "./rest/RestApiBuilder"; +import NcConnectionMgr from "./common/NcConnectionMgr"; export default class NcProjectBuilder { @@ -617,6 +618,12 @@ export default class NcProjectBuilder { try { + const sqlClient = NcConnectionMgr.getSqlClient({ + dbAlias: connectionConfig?.mets?.dbAlias, + env: this.config.env, + config: this.config, + projectId: this.id + }) /* create migrator */ const migrator = new Migrator({ project_id: this.id, @@ -638,7 +645,8 @@ export default class NcProjectBuilder { await migrator.sync({ folder: this.config?.toolDir, env: this.appConfig.workingEnv, - dbAlias: connectionConfig.meta.dbAlias + dbAlias: connectionConfig.meta.dbAlias, + sqlClient }); await migrator.migrationsUp({ @@ -647,9 +655,9 @@ export default class NcProjectBuilder { dbAlias: connectionConfig.meta.dbAlias, migrationSteps: 99999, sqlContentMigrate: 1, + sqlClient }); - } catch (e) { console.log(e); // throw e; diff --git a/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts b/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts index 8bf420628e..10a4ce2af4 100644 --- a/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts +++ b/packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts @@ -1,12 +1,12 @@ -import * as fs from "fs"; +// import * as fs from "fs"; import debug from 'debug'; import {Router} from "express"; import inflection from "inflection"; -import Knex from "knex"; +// import Knex from "knex"; import { MysqlClient, PgClient, SqlClient, - SqlClientFactory, + // SqlClientFactory, Tele } from 'nc-help'; @@ -23,6 +23,7 @@ import XcCache from "../plugins/adapters/cache/XcCache"; import BaseModel from "./BaseModel"; import {XcCron} from "./XcCron"; +import NcConnectionMgr from "./NcConnectionMgr"; const log = debug('nc:api:base'); @@ -951,57 +952,69 @@ export default abstract class BaseApiBuilder implements XcDynami protected initDbDriver(): void { - if (!this.dbDriver) { - if(this.projectBuilder?.prefix){ - this.dbDriver = this.xcMeta.knex - }else { - if (this.connectionConfig?.connection?.ssl && typeof this.connectionConfig?.connection?.ssl === 'object') { - if (this.connectionConfig.connection.ssl.caFilePath) { - this.connectionConfig.connection.ssl.ca = fs - .readFileSync(this.connectionConfig.connection.ssl.caFilePath) - .toString(); - } - if (this.connectionConfig.connection.ssl.keyFilePath) { - this.connectionConfig.connection.ssl.key = fs - .readFileSync(this.connectionConfig.connection.ssl.keyFilePath) - .toString(); - } - if (this.connectionConfig.connection.ssl.certFilePath) { - this.connectionConfig.connection.ssl.cert = fs - .readFileSync(this.connectionConfig.connection.ssl.certFilePath) - .toString(); - } - } - - const isSqlite = this.connectionConfig.client === 'sqlite3'; - this.baseLog(`initDbDriver : initializing db driver first time`) - this.dbDriver = XKnex(isSqlite ? - this.connectionConfig.connection as Knex.Config : - { - ...this.connectionConfig, connection: { - ...this.connectionConfig.connection, - typeCast(_field, next) { - const res = next(); - if (res instanceof Buffer) { - return [...res].map(v => ('00' + v.toString(16)).slice(-2)).join(''); - } - return res; - } - } - } as any); - if (isSqlite) { - this.dbDriver.raw(`PRAGMA journal_mode=WAL;`).then(() => { - }) - } - } - } - if (!this.sqlClient) { - this.sqlClient = SqlClientFactory.create(this.connectionConfig) as MysqlClient; - // close knex connection in sqlclient and reuse existing connection - this.sqlClient.knex.destroy(); - this.sqlClient.knex = this.getDbDriver(); - this.sqlClient.sqlClient = this.getDbDriver(); - } + this.dbDriver = NcConnectionMgr.get({ + dbAlias:this.dbAlias, + env:this.config.env, + config:this.config, + projectId:this.projectId + }); + this.sqlClient = NcConnectionMgr.getSqlClient({ + dbAlias:this.dbAlias, + env:this.config.env, + config:this.config, + projectId:this.projectId + }) + // if (!this.dbDriver) { + // if(this.projectBuilder?.prefix){ + // this.dbDriver = this.xcMeta.knex + // }else { + // if (this.connectionConfig?.connection?.ssl && typeof this.connectionConfig?.connection?.ssl === 'object') { + // if (this.connectionConfig.connection.ssl.caFilePath) { + // this.connectionConfig.connection.ssl.ca = fs + // .readFileSync(this.connectionConfig.connection.ssl.caFilePath) + // .toString(); + // } + // if (this.connectionConfig.connection.ssl.keyFilePath) { + // this.connectionConfig.connection.ssl.key = fs + // .readFileSync(this.connectionConfig.connection.ssl.keyFilePath) + // .toString(); + // } + // if (this.connectionConfig.connection.ssl.certFilePath) { + // this.connectionConfig.connection.ssl.cert = fs + // .readFileSync(this.connectionConfig.connection.ssl.certFilePath) + // .toString(); + // } + // } + // + // const isSqlite = this.connectionConfig.client === 'sqlite3'; + // this.baseLog(`initDbDriver : initializing db driver first time`) + // this.dbDriver = XKnex(isSqlite ? + // this.connectionConfig.connection as Knex.Config : + // { + // ...this.connectionConfig, connection: { + // ...this.connectionConfig.connection, + // typeCast(_field, next) { + // const res = next(); + // if (res instanceof Buffer) { + // return [...res].map(v => ('00' + v.toString(16)).slice(-2)).join(''); + // } + // return res; + // } + // } + // } as any); + // if (isSqlite) { + // this.dbDriver.raw(`PRAGMA journal_mode=WAL;`).then(() => { + // }) + // } + // } + // } + // if (!this.sqlClient) { + // this.sqlClient = SqlClientFactory.create(this.connectionConfig) as MysqlClient; + // // close knex connection in sqlclient and reuse existing connection + // this.sqlClient.knex.destroy(); + // this.sqlClient.knex = this.getDbDriver(); + // this.sqlClient.sqlClient = this.getDbDriver(); + // } } diff --git a/packages/nocodb/src/lib/noco/common/NcConnectionMgr.ts b/packages/nocodb/src/lib/noco/common/NcConnectionMgr.ts new file mode 100644 index 0000000000..d0e98b5e7c --- /dev/null +++ b/packages/nocodb/src/lib/noco/common/NcConnectionMgr.ts @@ -0,0 +1,115 @@ +import {XKnex} from "../../dataMapper"; +import {NcConfig} from "../../../interface/config"; +import fs from "fs"; +import Knex from "knex"; + +import {SqlClientFactory} from 'nc-help'; +import NcMetaIO from "../meta/NcMetaIO"; + +export default class NcConnectionMgr { + private static connectionRefs: { + [projectId: string]: { + [env: string]: { + [dbAlias: string]: XKnex + } + } + } = {}; + + private static metaKnex: NcMetaIO; + + public static setXcMeta(ncMeta: NcMetaIO) { + this.metaKnex = ncMeta; + } + + public static get({ + dbAlias = 'db', + env = 'dev', + config, + projectId + }: { + dbAlias: string, + env: string, + config: NcConfig, + projectId: string + }): XKnex { + if (this.connectionRefs?.[projectId]?.[env]?.[dbAlias]) { + return this.connectionRefs?.[projectId]?.[env]?.[dbAlias]; + } + this.connectionRefs[projectId] = this.connectionRefs[projectId] || {}; + this.connectionRefs[projectId][env] = this.connectionRefs[projectId] [env] || {}; + if (config?.prefix && this.metaKnex) { + this.connectionRefs[projectId][env][dbAlias] = this.metaKnex?.knex; + } else { + const connectionConfig = this.getConnectionConfig(config, env, dbAlias) + + if (connectionConfig?.connection?.ssl && typeof connectionConfig?.connection?.ssl === 'object') { + if (connectionConfig.connection.ssl.caFilePath) { + connectionConfig.connection.ssl.ca = fs + .readFileSync(connectionConfig.connection.ssl.caFilePath) + .toString(); + } + if (connectionConfig.connection.ssl.keyFilePath) { + connectionConfig.connection.ssl.key = fs + .readFileSync(connectionConfig.connection.ssl.keyFilePath) + .toString(); + } + if (connectionConfig.connection.ssl.certFilePath) { + connectionConfig.connection.ssl.cert = fs + .readFileSync(connectionConfig.connection.ssl.certFilePath) + .toString(); + } + } + + const isSqlite = connectionConfig.client === 'sqlite3'; + + this.connectionRefs[projectId][env][dbAlias] = XKnex(isSqlite ? + connectionConfig.connection as Knex.Config : + { + ...connectionConfig, + connection: { + ...connectionConfig.connection, + typeCast(_field, next) { + const res = next(); + if (res instanceof Buffer) { + return [...res].map(v => ('00' + v.toString(16)).slice(-2)).join(''); + } + return res; + } + } + } as any); + if (isSqlite) { + this.connectionRefs[projectId][env][dbAlias].raw(`PRAGMA journal_mode=WAL;`).then(() => { + }) + } + } + return this.connectionRefs[projectId][env][dbAlias]; + } + + + private static getConnectionConfig(config: NcConfig, env: string, dbAlias: string) { + return config?.envs?.[env]?.db?.find(db => db?.meta?.dbAlias === dbAlias); + } + + public static getSqlClient({ + projectId, + dbAlias = 'db', + env = 'dev', + config + }: { + dbAlias: string, + env: string, + config: NcConfig, + projectId: string + }): any { + const knex = this.get({ + dbAlias, + env, + config, + projectId + }) + return SqlClientFactory.create({knex, ...this.getConnectionConfig(config, env, dbAlias)}) + } +} + + + diff --git a/packages/nocodb/src/lib/noco/meta/NcMetaIOImpl.ts b/packages/nocodb/src/lib/noco/meta/NcMetaIOImpl.ts index fe1331f814..ee70496df8 100644 --- a/packages/nocodb/src/lib/noco/meta/NcMetaIOImpl.ts +++ b/packages/nocodb/src/lib/noco/meta/NcMetaIOImpl.ts @@ -8,6 +8,7 @@ import Noco from "../Noco"; import XcMigrationSource from "../common/XcMigrationSource"; import NcMetaIO, {META_TABLES} from "./NcMetaIO"; +import NcConnectionMgr from "../common/NcConnectionMgr"; export default class NcMetaIOImpl extends NcMetaIO { @@ -76,6 +77,7 @@ export default class NcMetaIOImpl extends NcMetaIO { dbIndex = dbIndex === -1 ? 0 : dbIndex; this.connection = XKnex(this.config.envs?.[this.config.workingEnv]?.db[dbIndex] as any); } + NcConnectionMgr.setXcMeta(this); } private get knexConnection(): XKnex { diff --git a/packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts b/packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts index 32f114ded0..1ef2d97499 100644 --- a/packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts +++ b/packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts @@ -36,6 +36,7 @@ import {RestApiBuilder} from "../rest/RestApiBuilder"; import RestAuthCtrl from "../rest/RestAuthCtrlEE"; import {packageVersion} from 'nc-help'; import NcMetaIO, {META_TABLES} from "./NcMetaIO"; +// import NcConnectionMgr from "../common/NcConnectionMgr"; const XC_PLUGIN_DET = 'XC_PLUGIN_DET'; @@ -81,6 +82,10 @@ export default class NcMetaMgr { for (const project of await this.xcMeta.projectList()) { const config = JSON.parse(project.config); this.projectConfigs[project.id] = config; + // const knexRefs = (this.app?.projectBuilders?.find(p => p.id === project.id)?.apiBuilders || []).reduce((ref, ab) => ({ + // ...ref, + // [ab.dbAlias]: ab.getDbDriver() + // }), {}) this.projectMgr.getSqlMgr({...project, config, metaDb: this.xcMeta?.knex}).projectOpenByWeb(config); } diff --git a/packages/nocodb/src/lib/sqlMgr/SqlMgr.ts b/packages/nocodb/src/lib/sqlMgr/SqlMgr.ts index f395903039..2883810864 100644 --- a/packages/nocodb/src/lib/sqlMgr/SqlMgr.ts +++ b/packages/nocodb/src/lib/sqlMgr/SqlMgr.ts @@ -11,6 +11,8 @@ import slash from 'slash'; const log = new Debug("SqlMgr"); import KnexMigrator from '../migrator/SqlMigrator/lib/KnexMigrator' +// import {XKnex} from "../dataMapper"; +import NcConnectionMgr from "../noco/common/NcConnectionMgr"; const ToolOps = { @@ -268,18 +270,23 @@ export default class SqlMgr { args.folder = slash(this.currentProjectFolder); // create connections for each db connection - for (const key in this.currentProjectJson.envs) { - for (let i = 0; i < this.currentProjectJson.envs[key].db.length; i++) { + for (const env in this.currentProjectJson.envs) { + for (let i = 0; i < this.currentProjectJson.envs[env].db.length; i++) { const connectionConfig = JSON.parse( - JSON.stringify(this.currentProjectJson.envs[key].db[i]) + JSON.stringify(this.currentProjectJson.envs[env].db[i]) ); - const connectionKey = `${key}_${ - this.currentProjectJson.envs[key].db[i].meta.dbAlias + const connectionKey = `${env}_${ + this.currentProjectJson.envs[env].db[i].meta.dbAlias }`; this.currentProjectConnections[connectionKey] = SqlClientFactory.create( - connectionConfig + {...connectionConfig, knex: NcConnectionMgr.get({ + dbAlias:this.currentProjectJson.envs[env].db[i].meta.dbAlias, + env:env, + config:args, + projectId:args.id + })} ); this.currentProjectServers[connectionKey] = { @@ -784,10 +791,10 @@ export default class SqlMgr { console.log(args); // create sql client for this operation - const client = await this.projectGetSqlClient(args); + const sqlClient = await this.projectGetSqlClient(args); // do sql operation - const sqlMigrationStatements = await client[op](opArgs); + const sqlMigrationStatements = await sqlClient[op](opArgs); console.log( `Sql Migration Statement for '${op}'`, sqlMigrationStatements.data.object @@ -824,7 +831,8 @@ export default class SqlMgr { ...args, sqlContentMigrate: 0, migrationSteps: 9999, - folder: this.currentProjectFolder + folder: this.currentProjectFolder, + sqlClient }; // console.log(`Migration up args for '${op}'`, migrationArgs); await this.migrator().migrationsUp(migrationArgs); diff --git a/packages/nocodb/src/lib/utils/NcConfigFactory.ts b/packages/nocodb/src/lib/utils/NcConfigFactory.ts index e4c8236dd6..f1195df72c 100644 --- a/packages/nocodb/src/lib/utils/NcConfigFactory.ts +++ b/packages/nocodb/src/lib/utils/NcConfigFactory.ts @@ -217,10 +217,10 @@ export default class NcConfigFactory implements NcConfig { "port": +url.port, 'user': url.searchParams.get('u') || url.searchParams.get('user'), }, - pool: { - min: 1, - max: 1 - }, + // pool: { + // min: 1, + // max: 1 + // }, acquireConnectionTimeout: 600000, } as any; @@ -303,10 +303,10 @@ export default class NcConfigFactory implements NcConfig { "port": +url.port, 'user': url.searchParams.get('u') || url.searchParams.get('user'), }, - pool: { - min: 1, - max: 2 - }, + // pool: { + // min: 1, + // max: 2 + // }, acquireConnectionTimeout: 600000, ...(url.searchParams.has('search_path') ? { searchPath: url.searchParams.get('search_path').split(',') @@ -521,6 +521,7 @@ export default class NcConfigFactory implements NcConfig { args.meta.db ); await metaSqlClient.createDatabaseIfNotExists(args.meta.db?.connection); + await metaSqlClient.knex.destroy(); }