From a13f26cf6069b8e931d4201a4940decdc6add030 Mon Sep 17 00:00:00 2001 From: o1lab Date: Fri, 15 Jun 2018 17:24:02 +0100 Subject: [PATCH] Feature : option to support multiple cpu cores (using cluster) --- README.md | 3 ++ bin/index.js | 100 +++++++++++++++++++++++----------- examples/aws-lambda/index.js | 78 +++++++++++++-------------- index.js | 101 ++++++++++++++++++++++++----------- lib/util/cmd.helper.js | 7 +++ lib/xsql.js | 2 +- package-lock.json | 14 +++++ package.json | 1 + 8 files changed, 202 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index 08813b10e5..92afa180e1 100644 --- a/README.md +++ b/README.md @@ -88,9 +88,11 @@ Powered by popular node packages : ([express](https://github.com/expressjs/expre * Upload multiple files * Download file * Health and version apis +* Use more than one CPU Cores * [Docker support](#docker) and [Nginx reverse proxy config](#nginx-reverse-proxy-config-with-docker) :fire::fire::fire: - Thanks to [@markuman](https://github.com/markuman) * AWS Lambda Example - Thanks to [@bertyhell](https://github.com/bertyhell) :fire::fire::fire: + Use HTTP clients like [Postman](https://www.getpostman.com/) or [similar tools](https://chrome.google.com/webstore/search/http%20client?_category=apps) to invoke REST API calls ____ @@ -939,6 +941,7 @@ http://localhost:3000/_version -a, --apiPrefix Api url prefix -> /api/ by default -s, --storageFolder Storage folder -> current working dir by default (available only with local) -i, --ignoreTables Comma separated table names to ignore + -c, --useCpuCores Specify number of cpu cores to use / 1 by default / 0 to use max -y, --readOnly readonly apis -> false by default -h, --help Output usage information diff --git a/bin/index.js b/bin/index.js index 09ec15101e..cb81fc7c26 100755 --- a/bin/index.js +++ b/bin/index.js @@ -9,50 +9,86 @@ const cors = require('cors'); const dataHelp = require('../lib/util/data.helper.js'); const Xapi = require('../lib/xapi.js'); const cmdargs = require('../lib/util/cmd.helper.js'); +const cluster = require('cluster'); +const numCPUs = require('os').cpus().length; -cmdargs.handle(sqlConfig) +function startXmysql(sqlConfig) { + /**************** START : setup express ****************/ + let app = express(); + app.use(morgan('tiny')); + app.use(cors()); + app.use(bodyParser.json()); + app.use(bodyParser.urlencoded({ + extended: true + })); + /**************** END : setup express ****************/ -/**************** START : setup express ****************/ -let app = express(); -app.use(morgan('tiny')) -app.use(cors()) -app.use(bodyParser.json()) -app.use(bodyParser.urlencoded({ - extended: true -})) -/**************** END : setup express ****************/ + /**************** START : setup mysql ****************/ + let mysqlPool = mysql.createPool(sqlConfig); + /**************** END : setup mysql ****************/ -/**************** START : setup mysql ****************/ -let mysqlPool = mysql.createPool(sqlConfig); -/**************** END : setup mysql ****************/ + /**************** START : setup Xapi ****************/ + console.log(''); + console.log(''); + console.log(''); + console.log(' Generating REST APIs at the speed of your thought.. '); + console.log(''); -/**************** START : setup Xapi ****************/ -console.log(''); -console.log(''); -console.log(''); -console.log(' Generating REST APIs at the speed of your thought.. '); -console.log(''); + let t = process.hrtime(); + let moreApis = new Xapi(sqlConfig, mysqlPool, app); -let t = process.hrtime(); -let moreApis = new Xapi(sqlConfig,mysqlPool,app); + moreApis.init((err, results) => { -moreApis.init((err, results) => { + app.listen(sqlConfig.portNumber, sqlConfig.ipAddress); + var t1 = process.hrtime(t); + var t2 = t1[0] + t1[1] / 1000000000; - app.listen(sqlConfig.portNumber,sqlConfig.ipAddress) - var t1 = process.hrtime(t); - var t2 = t1[0]+t1[1]/1000000000; - console.log(' '); - console.log(" Xmysql took : %d seconds",dataHelp.round(t2,1)); - console.log(" API's base URL : " + "localhost:"+sqlConfig.portNumber); - console.log(' '); - console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - '); + console.log(" Xmysql took : %d seconds", dataHelp.round(t2, 1)); + console.log(" API's base URL : " + "localhost:" + sqlConfig.portNumber); + console.log(' '); + console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - '); + + }); + /**************** END : setup Xapi ****************/ + +} + +function start(sqlConfig) { + + //handle cmd line arguments + cmdargs.handle(sqlConfig); + + if (cluster.isMaster && sqlConfig.useCpuCores > 1) { + + console.log(`Master ${process.pid} is running`); + + for (let i = 0; i < numCPUs && i < sqlConfig.useCpuCores; i++) { + console.log(`Forking process number ${i}...`); + cluster.fork(); + } + + cluster.on('exit', function(worker, code, signal) { + console.log('Worker ' + worker.process.pid + ' died with code: ' + + code + ', and signal: ' + signal); + console.log('Starting a new worker'); + cluster.fork(); + }); + + } else { + + startXmysql(sqlConfig); + + } +} + + +start(sqlConfig); + -}) -/**************** END : setup Xapi ****************/ diff --git a/examples/aws-lambda/index.js b/examples/aws-lambda/index.js index 98a455376d..0b6ac515f9 100644 --- a/examples/aws-lambda/index.js +++ b/examples/aws-lambda/index.js @@ -31,51 +31,51 @@ var morgan = require('morgan'); var app = express(); var onXapiInitialized = new Promise(function (resolve, reject) { - try { - // /**************** START : setup express ****************/ - app.use(morgan('tiny')); - app.use(cors()); - app.use(bodyParser.json()); - app.use(bodyParser.urlencoded({ - extended: true - })); - // /**************** END : setup express ****************/ + try { + // /**************** START : setup express ****************/ + app.use(morgan('tiny')); + app.use(cors()); + app.use(bodyParser.json()); + app.use(bodyParser.urlencoded({ + extended: true + })); + // /**************** END : setup express ****************/ - app.use(function (req, res, next) { - // You can add authentication here - console.log('Received request for: ' + req.url, req); - next(); - }); + app.use(function (req, res, next) { + // You can add authentication here + console.log('Received request for: ' + req.url, req); + next(); + }); - var mysqlConfig = { - host: config.mysql.host, - port: 3306, - database: config.mysql.database, - user: config.mysql.user, - password: config.mysql.password, - apiPrefix: '/', - ipAddress: 'localhost', - portNumber: 3000, - ignoreTables: [], - storageFolder: __dirname - }; + var mysqlConfig = { + host: config.mysql.host, + port: 3306, + database: config.mysql.database, + user: config.mysql.user, + password: config.mysql.password, + apiPrefix: '/', + ipAddress: 'localhost', + portNumber: 3000, + ignoreTables: [], + storageFolder: __dirname + }; - var mysqlPool = mysql.createPool(mysqlConfig); - var xapi = new Xapi(mysqlConfig, mysqlPool, app); - xapi.init(function (err, results) { - app.listen(3000); - resolve(); - }); - } - catch (err) { - reject(err); - } + var mysqlPool = mysql.createPool(mysqlConfig); + var xapi = new Xapi(mysqlConfig, mysqlPool, app); + xapi.init(function (err, results) { + app.listen(3000); + resolve(); + }); + } + catch (err) { + reject(err); + } }); function handler(event, context, callback) { - onXapiInitialized.then(function () { - serverless(app)(event, context, callback); - }); + onXapiInitialized.then(function () { + serverless(app)(event, context, callback); + }); } exports.handler = handler; diff --git a/index.js b/index.js index 2d8501598c..aa0f9cbb76 100644 --- a/index.js +++ b/index.js @@ -7,51 +7,88 @@ const sqlConfig = require('commander'); const mysql = require('mysql'); const cors = require('cors'); const dataHelp = require('./lib/util/data.helper.js'); - const Xapi = require('./lib/xapi.js'); const cmdargs = require('./lib/util/cmd.helper.js'); +const cluster = require('cluster'); +const numCPUs = require('os').cpus().length; + + + +function startXmysql(sqlConfig) { + /**************** START : setup express ****************/ + let app = express(); + app.use(morgan('tiny')); + app.use(cors()); + app.use(bodyParser.json()); + app.use(bodyParser.urlencoded({ + extended: true + })); + /**************** END : setup express ****************/ + + + /**************** START : setup mysql ****************/ + let mysqlPool = mysql.createPool(sqlConfig); + /**************** END : setup mysql ****************/ + + + /**************** START : setup Xapi ****************/ + console.log(''); + console.log(''); + console.log(''); + console.log(' Generating REST APIs at the speed of your thought.. '); + console.log(''); + + let t = process.hrtime(); + let moreApis = new Xapi(sqlConfig, mysqlPool, app); + + moreApis.init((err, results) => { + + app.listen(sqlConfig.portNumber, sqlConfig.ipAddress); + var t1 = process.hrtime(t); + var t2 = t1[0] + t1[1] / 1000000000; + + + console.log(" Xmysql took : %d seconds", dataHelp.round(t2, 1)); + console.log(" API's base URL : " + "localhost:" + sqlConfig.portNumber); + console.log(' '); + console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - '); + + }); + /**************** END : setup Xapi ****************/ + +} + +function start(sqlConfig) { + + cmdargs.handle(sqlConfig); -cmdargs.handle(sqlConfig); + if (cluster.isMaster && sqlConfig.useCpuCores > 1) { + console.log(`Master ${process.pid} is running`); + for (let i = 0; i < numCPUs && i < sqlConfig.useCpuCores; i++) { + console.log(`Forking process number ${i}...`); + cluster.fork(); + } -/**************** START : setup express ****************/ -let app = express(); -app.use(morgan('tiny')); -app.use(cors()); -app.use(bodyParser.json()); -app.use(bodyParser.urlencoded({ - extended: true -})); -/**************** END : setup express ****************/ + cluster.on('exit', function(worker, code, signal) { + console.log('Worker ' + worker.process.pid + ' died with code: ' + + code + ', and signal: ' + signal); + console.log('Starting a new worker'); + cluster.fork(); + }); -/**************** START : setup mysql ****************/ -let mysqlPool = mysql.createPool(sqlConfig); -/**************** END : setup mysql ****************/ + } else { + startXmysql(sqlConfig); -/**************** START : setup Xapi ****************/ -console.log(''); -console.log(''); -console.log(''); -console.log(' Generating REST APIs at the speed of your thought.. '); -console.log(''); + } +} -let t = process.hrtime(); -let moreApis = new Xapi(sqlConfig,mysqlPool,app); -moreApis.init((err, results) => { +start(sqlConfig); - app.listen(sqlConfig.portNumber,sqlConfig.ipAddress); - var t1 = process.hrtime(t); - var t2 = t1[0]+t1[1]/1000000000; - console.log(" Xmysql took : %d seconds",dataHelp.round(t2,1)); - console.log(" API's base URL : " + "localhost:"+sqlConfig.portNumber); - console.log(' '); - console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - '); -}); -/**************** END : setup Xapi ****************/ diff --git a/lib/util/cmd.helper.js b/lib/util/cmd.helper.js index e93d67683d..b5dc1f41ab 100644 --- a/lib/util/cmd.helper.js +++ b/lib/util/cmd.helper.js @@ -2,6 +2,7 @@ const program = require('commander'); const colors = require('colors'); const pkginfo = require('pkginfo')(module); +const maxCpus = require('os').cpus().length; program.on('--help', () => { @@ -25,6 +26,7 @@ program .option('-i, --ignoreTables ', 'comma separated table names to ignore') .option('-a, --apiPrefix ', 'api url prefix / "/api/" by default') .option('-y, --readOnly', 'readonly apis / false by default') + .option('-c, --useCpuCores ', 'use number of CPU cores (using cluster) / 1 by default') .parse(process.argv) @@ -63,6 +65,11 @@ exports.handle = program => { program.storageFolder = program.storageFolder || process.cwd() program.apiPrefix = program.apiPrefix || '/api/' program.readOnly = program.readOnly || false; + program.useCpuCores = program.useCpuCores || 1; + + if(program.useCpuCores === '0') { + program.useCpuCores = maxCpus; + } if (program.ignoreTables) { let ignoreTables = program.ignoreTables.split(',') diff --git a/lib/xsql.js b/lib/xsql.js index 0e555867cd..8e59f114d1 100644 --- a/lib/xsql.js +++ b/lib/xsql.js @@ -255,7 +255,7 @@ class Xsql { /**************** START : prepare value object in prepared statement ****************/ - // iterate over sent object array + // iterate over sent object array let arrOfArr = [] for (var i = 0; i < objectArray.length; ++i) { let arrValues = [] diff --git a/package-lock.json b/package-lock.json index 538260bae6..59fdc7615d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -126,6 +126,15 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" }, + "cluster": { + "version": "0.7.7", + "resolved": "https://registry.npmjs.org/cluster/-/cluster-0.7.7.tgz", + "integrity": "sha1-5JfiZ8yVa9CwUTrbSqOTNX0Ahe8=", + "requires": { + "log": "1.4.0", + "mkdirp": "0.5.1" + } + }, "colors": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", @@ -472,6 +481,11 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "log": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/log/-/log-1.4.0.tgz", + "integrity": "sha1-S6HYkP3iSbAx3KA7w36q8yVlbxw=" + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", diff --git a/package.json b/package.json index 4aef991381..97229f139a 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "dependencies": { "assert": "^1.4.1", "body-parser": "^1.18.2", + "cluster": "^0.7.7", "colors": "^1.1.2", "commander": "^2.11.0", "cors": "^2.8.4",