Browse Source

fix: Avoid creating duplicate knex connection

Signed-off-by: Pranav C <61551451+pranavxc@users.noreply.github.com>
pull/401/head
Pranav C 3 years ago
parent
commit
f91ac04f0b
  1. 6
      packages/nocodb/package-lock.json
  2. 2
      packages/nocodb/package.json
  3. 2
      packages/nocodb/src/example/docker.ts
  4. 17
      packages/nocodb/src/lib/migrator/SqlMigrator/lib/KnexMigrator.ts
  5. 12
      packages/nocodb/src/lib/noco/NcProjectBuilder.ts
  6. 121
      packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts
  7. 115
      packages/nocodb/src/lib/noco/common/NcConnectionMgr.ts
  8. 2
      packages/nocodb/src/lib/noco/meta/NcMetaIOImpl.ts
  9. 5
      packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts
  10. 26
      packages/nocodb/src/lib/sqlMgr/SqlMgr.ts
  11. 17
      packages/nocodb/src/lib/utils/NcConfigFactory.ts

6
packages/nocodb/package-lock.json generated

@ -11800,9 +11800,9 @@
"integrity": "sha512-3AryS9uwa5NfISLxMciUonrH7YfXp+nlahB9T7girXIsLQrmwX4MdnuKs32akduCOGpKmjTJSWmATULbuMkbfw=="
},
"nc-help": {
"version": "0.2.8",
"resolved": "https://registry.npmjs.org/nc-help/-/nc-help-0.2.8.tgz",
"integrity": "sha512-h6HUWb2trYEVm7x2/hWxifeq+ZALCIFxZFISgpI/IQPhe/BSoGUNpcRIhWg8wt1y87kxZ7/VCkvTQUhW9OTOGg==",
"version": "0.2.10",
"resolved": "https://registry.npmjs.org/nc-help/-/nc-help-0.2.10.tgz",
"integrity": "sha512-XkLvKf3clpewn576RGX84nsNBWDjvFJFsaNPKw0n77zcjlC6zeBOX6fGJsTVEBg56usr0jaK+LfOtJUUEc9D+A==",
"requires": {
"axios": "^0.21.1",
"boxen": "^4.2.0",

2
packages/nocodb/package.json

@ -144,7 +144,7 @@
"mysql2": "^2.2.5",
"nanoid": "^3.1.20",
"nc-common": "0.0.6",
"nc-help": "^0.2.8",
"nc-help": "^0.2.10",
"nc-lib-gui": "^0.2.9",
"nc-plugin": "^0.1.1",
"nodemailer": "^6.4.10",

2
packages/nocodb/src/example/docker.ts

@ -19,8 +19,8 @@ server.set('view engine', 'ejs');
process.env[`DEBUG`] = 'xc*';
process.env[`NODE_TLS_REJECT_UNAUTHORIZED`] = '0';
// process.env[`DATABASE_URL`] = 'postgres://postgres:xgene@localhost:32768/chinook4';
// process.env[`NC_ONE_CLICK`] = 'true';
// process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"
//

17
packages/nocodb/src/lib/migrator/SqlMigrator/lib/KnexMigrator.ts

@ -298,7 +298,7 @@ export default class KnexMigrator extends SqlMigrator {
}
async _initDbWithSql(connectionConfig) {
const sqlClient = SqlClientFactory.create(connectionConfig);
const sqlClient = connectionConfig.sqlClient || SqlClientFactory.create(connectionConfig);
if (connectionConfig.client === "oracledb") {
this.emit(
`${connectionConfig.client}: Creating DB if not exists ${
@ -375,7 +375,7 @@ export default class KnexMigrator extends SqlMigrator {
// await sqlClient.createTableIfNotExists({tn: connectionConfig.meta.tn});
}
async _initEnvDbsWithSql(env, dbAlias = null) {
async _initEnvDbsWithSql(env, dbAlias = null, sqlClient =null) {
const {envs} = this.project;
@ -387,7 +387,7 @@ export default class KnexMigrator extends SqlMigrator {
/* if no dbAlias - init all dbs in env. Else check if it matches the one sent in args */
if (!dbAlias || dbAlias === envs[env].db[i].meta.dbAlias) {
await this._initDbWithSql(connectionConfig);
await this._initDbWithSql({...connectionConfig, sqlClient});
}
}
}
@ -500,7 +500,7 @@ export default class KnexMigrator extends SqlMigrator {
args.dbAlias,
args.env
);
const sqlClient = SqlClientFactory.create(connection);
const sqlClient = args.sqlClient || SqlClientFactory.create(connection);
const migrations = await sqlClient.selectAll(connection.meta.tn);
/** ************** END : get files and migrations *************** */
@ -901,10 +901,10 @@ export default class KnexMigrator extends SqlMigrator {
/* happens when creating the project */
for (const env in this.project.envs) {
args.env = env;
await this._initEnvDbsWithSql(args.env, args.dbAlias);
await this._initEnvDbsWithSql(args.env, args.dbAlias, args.sqlClient);
}
} else {
await this._initEnvDbsWithSql(args.env, args.dbAlias);
await this._initEnvDbsWithSql(args.env, args.dbAlias, args.sqlClient);
}
} catch (e) {
@ -1069,7 +1069,7 @@ export default class KnexMigrator extends SqlMigrator {
* and only filenames are migrated to _evolution table
* @memberof KnexMigrator
*/
async migrationsUp(args: any = {}) {
async migrationsUp(args: any = {}) {
const func = this.migrationsUp.name;
// const result = new Result();
@ -1092,7 +1092,8 @@ export default class KnexMigrator extends SqlMigrator {
upFilesPattern: path.join(this.toolDir, 'nc', this.project.id, args.dbAlias, 'migrations', '*.up.sql'),
downFilesPattern: path.join(this.toolDir, 'nc', this.project.id, args.dbAlias, 'migrations', '*.down.sql'),
tn: this._getEvolutionsTablename(args),//`${this.toolDir}`,
sqlContentMigrate: args.sqlContentMigrate
sqlContentMigrate: args.sqlContentMigrate,
sqlClient: args.sqlClient
});
}

12
packages/nocodb/src/lib/noco/NcProjectBuilder.ts

@ -12,6 +12,7 @@ import Noco from "./Noco";
import {GqlApiBuilder} from "./gql/GqlApiBuilder";
import {XCEeError} from "./meta/NcMetaMgr";
import {RestApiBuilder} from "./rest/RestApiBuilder";
import NcConnectionMgr from "./common/NcConnectionMgr";
export default class NcProjectBuilder {
@ -617,6 +618,12 @@ export default class NcProjectBuilder {
try {
const sqlClient = NcConnectionMgr.getSqlClient({
dbAlias: connectionConfig?.mets?.dbAlias,
env: this.config.env,
config: this.config,
projectId: this.id
})
/* create migrator */
const migrator = new Migrator({
project_id: this.id,
@ -638,7 +645,8 @@ export default class NcProjectBuilder {
await migrator.sync({
folder: this.config?.toolDir,
env: this.appConfig.workingEnv,
dbAlias: connectionConfig.meta.dbAlias
dbAlias: connectionConfig.meta.dbAlias,
sqlClient
});
await migrator.migrationsUp({
@ -647,9 +655,9 @@ export default class NcProjectBuilder {
dbAlias: connectionConfig.meta.dbAlias,
migrationSteps: 99999,
sqlContentMigrate: 1,
sqlClient
});
} catch (e) {
console.log(e);
// throw e;

121
packages/nocodb/src/lib/noco/common/BaseApiBuilder.ts

@ -1,12 +1,12 @@
import * as fs from "fs";
// import * as fs from "fs";
import debug from 'debug';
import {Router} from "express";
import inflection from "inflection";
import Knex from "knex";
// import Knex from "knex";
import {
MysqlClient, PgClient, SqlClient,
SqlClientFactory,
// SqlClientFactory,
Tele
} from 'nc-help';
@ -23,6 +23,7 @@ import XcCache from "../plugins/adapters/cache/XcCache";
import BaseModel from "./BaseModel";
import {XcCron} from "./XcCron";
import NcConnectionMgr from "./NcConnectionMgr";
const log = debug('nc:api:base');
@ -951,57 +952,69 @@ export default abstract class BaseApiBuilder<T extends Noco> implements XcDynami
protected initDbDriver(): void {
if (!this.dbDriver) {
if(this.projectBuilder?.prefix){
this.dbDriver = this.xcMeta.knex
}else {
if (this.connectionConfig?.connection?.ssl && typeof this.connectionConfig?.connection?.ssl === 'object') {
if (this.connectionConfig.connection.ssl.caFilePath) {
this.connectionConfig.connection.ssl.ca = fs
.readFileSync(this.connectionConfig.connection.ssl.caFilePath)
.toString();
}
if (this.connectionConfig.connection.ssl.keyFilePath) {
this.connectionConfig.connection.ssl.key = fs
.readFileSync(this.connectionConfig.connection.ssl.keyFilePath)
.toString();
}
if (this.connectionConfig.connection.ssl.certFilePath) {
this.connectionConfig.connection.ssl.cert = fs
.readFileSync(this.connectionConfig.connection.ssl.certFilePath)
.toString();
}
}
const isSqlite = this.connectionConfig.client === 'sqlite3';
this.baseLog(`initDbDriver : initializing db driver first time`)
this.dbDriver = XKnex(isSqlite ?
this.connectionConfig.connection as Knex.Config :
{
...this.connectionConfig, connection: {
...this.connectionConfig.connection,
typeCast(_field, next) {
const res = next();
if (res instanceof Buffer) {
return [...res].map(v => ('00' + v.toString(16)).slice(-2)).join('');
}
return res;
}
}
} as any);
if (isSqlite) {
this.dbDriver.raw(`PRAGMA journal_mode=WAL;`).then(() => {
})
}
}
}
if (!this.sqlClient) {
this.sqlClient = SqlClientFactory.create(this.connectionConfig) as MysqlClient;
// close knex connection in sqlclient and reuse existing connection
this.sqlClient.knex.destroy();
this.sqlClient.knex = this.getDbDriver();
this.sqlClient.sqlClient = this.getDbDriver();
}
this.dbDriver = NcConnectionMgr.get({
dbAlias:this.dbAlias,
env:this.config.env,
config:this.config,
projectId:this.projectId
});
this.sqlClient = NcConnectionMgr.getSqlClient({
dbAlias:this.dbAlias,
env:this.config.env,
config:this.config,
projectId:this.projectId
})
// if (!this.dbDriver) {
// if(this.projectBuilder?.prefix){
// this.dbDriver = this.xcMeta.knex
// }else {
// if (this.connectionConfig?.connection?.ssl && typeof this.connectionConfig?.connection?.ssl === 'object') {
// if (this.connectionConfig.connection.ssl.caFilePath) {
// this.connectionConfig.connection.ssl.ca = fs
// .readFileSync(this.connectionConfig.connection.ssl.caFilePath)
// .toString();
// }
// if (this.connectionConfig.connection.ssl.keyFilePath) {
// this.connectionConfig.connection.ssl.key = fs
// .readFileSync(this.connectionConfig.connection.ssl.keyFilePath)
// .toString();
// }
// if (this.connectionConfig.connection.ssl.certFilePath) {
// this.connectionConfig.connection.ssl.cert = fs
// .readFileSync(this.connectionConfig.connection.ssl.certFilePath)
// .toString();
// }
// }
//
// const isSqlite = this.connectionConfig.client === 'sqlite3';
// this.baseLog(`initDbDriver : initializing db driver first time`)
// this.dbDriver = XKnex(isSqlite ?
// this.connectionConfig.connection as Knex.Config :
// {
// ...this.connectionConfig, connection: {
// ...this.connectionConfig.connection,
// typeCast(_field, next) {
// const res = next();
// if (res instanceof Buffer) {
// return [...res].map(v => ('00' + v.toString(16)).slice(-2)).join('');
// }
// return res;
// }
// }
// } as any);
// if (isSqlite) {
// this.dbDriver.raw(`PRAGMA journal_mode=WAL;`).then(() => {
// })
// }
// }
// }
// if (!this.sqlClient) {
// this.sqlClient = SqlClientFactory.create(this.connectionConfig) as MysqlClient;
// // close knex connection in sqlclient and reuse existing connection
// this.sqlClient.knex.destroy();
// this.sqlClient.knex = this.getDbDriver();
// this.sqlClient.sqlClient = this.getDbDriver();
// }
}

115
packages/nocodb/src/lib/noco/common/NcConnectionMgr.ts

@ -0,0 +1,115 @@
import {XKnex} from "../../dataMapper";
import {NcConfig} from "../../../interface/config";
import fs from "fs";
import Knex from "knex";
import {SqlClientFactory} from 'nc-help';
import NcMetaIO from "../meta/NcMetaIO";
export default class NcConnectionMgr {
private static connectionRefs: {
[projectId: string]: {
[env: string]: {
[dbAlias: string]: XKnex
}
}
} = {};
private static metaKnex: NcMetaIO;
public static setXcMeta(ncMeta: NcMetaIO) {
this.metaKnex = ncMeta;
}
public static get({
dbAlias = 'db',
env = 'dev',
config,
projectId
}: {
dbAlias: string,
env: string,
config: NcConfig,
projectId: string
}): XKnex {
if (this.connectionRefs?.[projectId]?.[env]?.[dbAlias]) {
return this.connectionRefs?.[projectId]?.[env]?.[dbAlias];
}
this.connectionRefs[projectId] = this.connectionRefs[projectId] || {};
this.connectionRefs[projectId][env] = this.connectionRefs[projectId] [env] || {};
if (config?.prefix && this.metaKnex) {
this.connectionRefs[projectId][env][dbAlias] = this.metaKnex?.knex;
} else {
const connectionConfig = this.getConnectionConfig(config, env, dbAlias)
if (connectionConfig?.connection?.ssl && typeof connectionConfig?.connection?.ssl === 'object') {
if (connectionConfig.connection.ssl.caFilePath) {
connectionConfig.connection.ssl.ca = fs
.readFileSync(connectionConfig.connection.ssl.caFilePath)
.toString();
}
if (connectionConfig.connection.ssl.keyFilePath) {
connectionConfig.connection.ssl.key = fs
.readFileSync(connectionConfig.connection.ssl.keyFilePath)
.toString();
}
if (connectionConfig.connection.ssl.certFilePath) {
connectionConfig.connection.ssl.cert = fs
.readFileSync(connectionConfig.connection.ssl.certFilePath)
.toString();
}
}
const isSqlite = connectionConfig.client === 'sqlite3';
this.connectionRefs[projectId][env][dbAlias] = XKnex(isSqlite ?
connectionConfig.connection as Knex.Config :
{
...connectionConfig,
connection: {
...connectionConfig.connection,
typeCast(_field, next) {
const res = next();
if (res instanceof Buffer) {
return [...res].map(v => ('00' + v.toString(16)).slice(-2)).join('');
}
return res;
}
}
} as any);
if (isSqlite) {
this.connectionRefs[projectId][env][dbAlias].raw(`PRAGMA journal_mode=WAL;`).then(() => {
})
}
}
return this.connectionRefs[projectId][env][dbAlias];
}
private static getConnectionConfig(config: NcConfig, env: string, dbAlias: string) {
return config?.envs?.[env]?.db?.find(db => db?.meta?.dbAlias === dbAlias);
}
public static getSqlClient({
projectId,
dbAlias = 'db',
env = 'dev',
config
}: {
dbAlias: string,
env: string,
config: NcConfig,
projectId: string
}): any {
const knex = this.get({
dbAlias,
env,
config,
projectId
})
return SqlClientFactory.create({knex, ...this.getConnectionConfig(config, env, dbAlias)})
}
}

2
packages/nocodb/src/lib/noco/meta/NcMetaIOImpl.ts

@ -8,6 +8,7 @@ import Noco from "../Noco";
import XcMigrationSource from "../common/XcMigrationSource";
import NcMetaIO, {META_TABLES} from "./NcMetaIO";
import NcConnectionMgr from "../common/NcConnectionMgr";
export default class NcMetaIOImpl extends NcMetaIO {
@ -76,6 +77,7 @@ export default class NcMetaIOImpl extends NcMetaIO {
dbIndex = dbIndex === -1 ? 0 : dbIndex;
this.connection = XKnex(this.config.envs?.[this.config.workingEnv]?.db[dbIndex] as any);
}
NcConnectionMgr.setXcMeta(this);
}
private get knexConnection(): XKnex {

5
packages/nocodb/src/lib/noco/meta/NcMetaMgr.ts

@ -36,6 +36,7 @@ import {RestApiBuilder} from "../rest/RestApiBuilder";
import RestAuthCtrl from "../rest/RestAuthCtrlEE";
import {packageVersion} from 'nc-help';
import NcMetaIO, {META_TABLES} from "./NcMetaIO";
// import NcConnectionMgr from "../common/NcConnectionMgr";
const XC_PLUGIN_DET = 'XC_PLUGIN_DET';
@ -81,6 +82,10 @@ export default class NcMetaMgr {
for (const project of await this.xcMeta.projectList()) {
const config = JSON.parse(project.config);
this.projectConfigs[project.id] = config;
// const knexRefs = (this.app?.projectBuilders?.find(p => p.id === project.id)?.apiBuilders || []).reduce((ref, ab) => ({
// ...ref,
// [ab.dbAlias]: ab.getDbDriver()
// }), {})
this.projectMgr.getSqlMgr({...project, config, metaDb: this.xcMeta?.knex}).projectOpenByWeb(config);
}

26
packages/nocodb/src/lib/sqlMgr/SqlMgr.ts

@ -11,6 +11,8 @@ import slash from 'slash';
const log = new Debug("SqlMgr");
import KnexMigrator from '../migrator/SqlMigrator/lib/KnexMigrator'
// import {XKnex} from "../dataMapper";
import NcConnectionMgr from "../noco/common/NcConnectionMgr";
const ToolOps = {
@ -268,18 +270,23 @@ export default class SqlMgr {
args.folder = slash(this.currentProjectFolder);
// create connections for each db connection
for (const key in this.currentProjectJson.envs) {
for (let i = 0; i < this.currentProjectJson.envs[key].db.length; i++) {
for (const env in this.currentProjectJson.envs) {
for (let i = 0; i < this.currentProjectJson.envs[env].db.length; i++) {
const connectionConfig = JSON.parse(
JSON.stringify(this.currentProjectJson.envs[key].db[i])
JSON.stringify(this.currentProjectJson.envs[env].db[i])
);
const connectionKey = `${key}_${
this.currentProjectJson.envs[key].db[i].meta.dbAlias
const connectionKey = `${env}_${
this.currentProjectJson.envs[env].db[i].meta.dbAlias
}`;
this.currentProjectConnections[connectionKey] = SqlClientFactory.create(
connectionConfig
{...connectionConfig, knex: NcConnectionMgr.get({
dbAlias:this.currentProjectJson.envs[env].db[i].meta.dbAlias,
env:env,
config:args,
projectId:args.id
})}
);
this.currentProjectServers[connectionKey] = {
@ -784,10 +791,10 @@ export default class SqlMgr {
console.log(args);
// create sql client for this operation
const client = await this.projectGetSqlClient(args);
const sqlClient = await this.projectGetSqlClient(args);
// do sql operation
const sqlMigrationStatements = await client[op](opArgs);
const sqlMigrationStatements = await sqlClient[op](opArgs);
console.log(
`Sql Migration Statement for '${op}'`,
sqlMigrationStatements.data.object
@ -824,7 +831,8 @@ export default class SqlMgr {
...args,
sqlContentMigrate: 0,
migrationSteps: 9999,
folder: this.currentProjectFolder
folder: this.currentProjectFolder,
sqlClient
};
// console.log(`Migration up args for '${op}'`, migrationArgs);
await this.migrator().migrationsUp(migrationArgs);

17
packages/nocodb/src/lib/utils/NcConfigFactory.ts

@ -217,10 +217,10 @@ export default class NcConfigFactory implements NcConfig {
"port": +url.port,
'user': url.searchParams.get('u') || url.searchParams.get('user'),
},
pool: {
min: 1,
max: 1
},
// pool: {
// min: 1,
// max: 1
// },
acquireConnectionTimeout: 600000,
} as any;
@ -303,10 +303,10 @@ export default class NcConfigFactory implements NcConfig {
"port": +url.port,
'user': url.searchParams.get('u') || url.searchParams.get('user'),
},
pool: {
min: 1,
max: 2
},
// pool: {
// min: 1,
// max: 2
// },
acquireConnectionTimeout: 600000,
...(url.searchParams.has('search_path') ? {
searchPath: url.searchParams.get('search_path').split(',')
@ -521,6 +521,7 @@ export default class NcConfigFactory implements NcConfig {
args.meta.db
);
await metaSqlClient.createDatabaseIfNotExists(args.meta.db?.connection);
await metaSqlClient.knex.destroy();
}

Loading…
Cancel
Save