github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Feature : option to support multiple cpu cores (using cluster)

pull/39/head
o1lab 6 years ago
parent
ad42700863
commit
a13f26cf60
8 changed files with 202 additions and 104 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      README.md
  2. 100
      bin/index.js
  3. 78
      examples/aws-lambda/index.js
  4. 101
      index.js
  5. 7
      lib/util/cmd.helper.js
  6. 2
      lib/xsql.js
  7. 14
      package-lock.json
  8. 1
      package.json

3
README.md
Unescape View File

@ -88,9 +88,11 @@ Powered by popular node packages : ([express](https://github.com/expressjs/expre
* Upload multiple files
* Download file
* Health and version apis
* Use more than one CPU Cores
* [Docker support](#docker) and [Nginx reverse proxy config](#nginx-reverse-proxy-config-with-docker) :fire::fire::fire: - Thanks to [@markuman](https://github.com/markuman)
* AWS Lambda Example - Thanks to [@bertyhell](https://github.com/bertyhell) :fire::fire::fire:
Use HTTP clients like [Postman](https://www.getpostman.com/) or [similar tools](https://chrome.google.com/webstore/search/http%20client?_category=apps) to invoke REST API calls
____
@ -939,6 +941,7 @@ http://localhost:3000/_version
-a, --apiPrefix <n> Api url prefix -> /api/ by default
-s, --storageFolder <n> Storage folder -> current working dir by default (available only with local)
-i, --ignoreTables <n> Comma separated table names to ignore
-c, --useCpuCores <n> Specify number of cpu cores to use / 1 by default / 0 to use max
-y, --readOnly readonly apis -> false by default
-h, --help Output usage information

100
bin/index.js
Unescape View File

@ -9,50 +9,86 @@ const cors = require('cors');
const dataHelp = require('../lib/util/data.helper.js');
const Xapi = require('../lib/xapi.js');
const cmdargs = require('../lib/util/cmd.helper.js');
const cluster = require('cluster');
const numCPUs = require('os').cpus().length;
cmdargs.handle(sqlConfig)
function startXmysql(sqlConfig) {
/**************** START : setup express ****************/
let app = express();
app.use(morgan('tiny'));
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
/**************** END : setup express ****************/
/**************** START : setup express ****************/
let app = express();
app.use(morgan('tiny'))
app.use(cors())
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({
extended: true
}))
/**************** END : setup express ****************/
/**************** START : setup mysql ****************/
let mysqlPool = mysql.createPool(sqlConfig);
/**************** END : setup mysql ****************/
/**************** START : setup mysql ****************/
let mysqlPool = mysql.createPool(sqlConfig);
/**************** END : setup mysql ****************/
/**************** START : setup Xapi ****************/
console.log('');
console.log('');
console.log('');
console.log(' Generating REST APIs at the speed of your thought.. ');
console.log('');
/**************** START : setup Xapi ****************/
console.log('');
console.log('');
console.log('');
console.log(' Generating REST APIs at the speed of your thought.. ');
console.log('');
let t = process.hrtime();
let moreApis = new Xapi(sqlConfig, mysqlPool, app);
let t = process.hrtime();
let moreApis = new Xapi(sqlConfig,mysqlPool,app);
moreApis.init((err, results) => {
moreApis.init((err, results) => {
app.listen(sqlConfig.portNumber, sqlConfig.ipAddress);
var t1 = process.hrtime(t);
var t2 = t1[0] + t1[1] / 1000000000;
app.listen(sqlConfig.portNumber,sqlConfig.ipAddress)
var t1 = process.hrtime(t);
var t2 = t1[0]+t1[1]/1000000000;
console.log(' ');
console.log(" Xmysql took : %d seconds",dataHelp.round(t2,1));
console.log(" API's base URL : " + "localhost:"+sqlConfig.portNumber);
console.log(' ');
console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ');
console.log(" Xmysql took : %d seconds", dataHelp.round(t2, 1));
console.log(" API's base URL : " + "localhost:" + sqlConfig.portNumber);
console.log(' ');
console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ');
});
/**************** END : setup Xapi ****************/
}
function start(sqlConfig) {
//handle cmd line arguments
cmdargs.handle(sqlConfig);
if (cluster.isMaster && sqlConfig.useCpuCores > 1) {
console.log(`Master ${process.pid} is running`);
for (let i = 0; i < numCPUs && i < sqlConfig.useCpuCores; i++) {
console.log(`Forking process number ${i}...`);
cluster.fork();
}
cluster.on('exit', function(worker, code, signal) {
console.log('Worker ' + worker.process.pid + ' died with code: '
+ code + ', and signal: ' + signal);
console.log('Starting a new worker');
cluster.fork();
});
} else {
startXmysql(sqlConfig);
}
}
start(sqlConfig);
})
/**************** END : setup Xapi ****************/

78
examples/aws-lambda/index.js
Unescape View File

@ -31,51 +31,51 @@ var morgan = require('morgan');
var app = express();
var onXapiInitialized = new Promise(function (resolve, reject) {
try {
// /**************** START : setup express ****************/
app.use(morgan('tiny'));
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
// /**************** END : setup express ****************/
try {
// /**************** START : setup express ****************/
app.use(morgan('tiny'));
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
// /**************** END : setup express ****************/
app.use(function (req, res, next) {
// You can add authentication here
console.log('Received request for: ' + req.url, req);
next();
});
app.use(function (req, res, next) {
// You can add authentication here
console.log('Received request for: ' + req.url, req);
next();
});
var mysqlConfig = {
host: config.mysql.host,
port: 3306,
database: config.mysql.database,
user: config.mysql.user,
password: config.mysql.password,
apiPrefix: '/',
ipAddress: 'localhost',
portNumber: 3000,
ignoreTables: [],
storageFolder: __dirname
};
var mysqlConfig = {
host: config.mysql.host,
port: 3306,
database: config.mysql.database,
user: config.mysql.user,
password: config.mysql.password,
apiPrefix: '/',
ipAddress: 'localhost',
portNumber: 3000,
ignoreTables: [],
storageFolder: __dirname
};
var mysqlPool = mysql.createPool(mysqlConfig);
var xapi = new Xapi(mysqlConfig, mysqlPool, app);
xapi.init(function (err, results) {
app.listen(3000);
resolve();
});
}
catch (err) {
reject(err);
}
var mysqlPool = mysql.createPool(mysqlConfig);
var xapi = new Xapi(mysqlConfig, mysqlPool, app);
xapi.init(function (err, results) {
app.listen(3000);
resolve();
});
}
catch (err) {
reject(err);
}
});
function handler(event, context, callback) {
onXapiInitialized.then(function () {
serverless(app)(event, context, callback);
});
onXapiInitialized.then(function () {
serverless(app)(event, context, callback);
});
}
exports.handler = handler;

101
index.js
Unescape View File

@ -7,51 +7,88 @@ const sqlConfig = require('commander');
const mysql = require('mysql');
const cors = require('cors');
const dataHelp = require('./lib/util/data.helper.js');
const Xapi = require('./lib/xapi.js');
const cmdargs = require('./lib/util/cmd.helper.js');
const cluster = require('cluster');
const numCPUs = require('os').cpus().length;
function startXmysql(sqlConfig) {
/**************** START : setup express ****************/
let app = express();
app.use(morgan('tiny'));
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
/**************** END : setup express ****************/
/**************** START : setup mysql ****************/
let mysqlPool = mysql.createPool(sqlConfig);
/**************** END : setup mysql ****************/
/**************** START : setup Xapi ****************/
console.log('');
console.log('');
console.log('');
console.log(' Generating REST APIs at the speed of your thought.. ');
console.log('');
let t = process.hrtime();
let moreApis = new Xapi(sqlConfig, mysqlPool, app);
moreApis.init((err, results) => {
app.listen(sqlConfig.portNumber, sqlConfig.ipAddress);
var t1 = process.hrtime(t);
var t2 = t1[0] + t1[1] / 1000000000;
console.log(" Xmysql took : %d seconds", dataHelp.round(t2, 1));
console.log(" API's base URL : " + "localhost:" + sqlConfig.portNumber);
console.log(' ');
console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ');
});
/**************** END : setup Xapi ****************/
}
function start(sqlConfig) {
cmdargs.handle(sqlConfig);
cmdargs.handle(sqlConfig);
if (cluster.isMaster && sqlConfig.useCpuCores > 1) {
console.log(`Master ${process.pid} is running`);
for (let i = 0; i < numCPUs && i < sqlConfig.useCpuCores; i++) {
console.log(`Forking process number ${i}...`);
cluster.fork();
}
/**************** START : setup express ****************/
let app = express();
app.use(morgan('tiny'));
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
/**************** END : setup express ****************/
cluster.on('exit', function(worker, code, signal) {
console.log('Worker ' + worker.process.pid + ' died with code: '
+ code + ', and signal: ' + signal);
console.log('Starting a new worker');
cluster.fork();
});
/**************** START : setup mysql ****************/
let mysqlPool = mysql.createPool(sqlConfig);
/**************** END : setup mysql ****************/
} else {
startXmysql(sqlConfig);
/**************** START : setup Xapi ****************/
console.log('');
console.log('');
console.log('');
console.log(' Generating REST APIs at the speed of your thought.. ');
console.log('');
}
}
let t = process.hrtime();
let moreApis = new Xapi(sqlConfig,mysqlPool,app);
moreApis.init((err, results) => {
start(sqlConfig);
app.listen(sqlConfig.portNumber,sqlConfig.ipAddress);
var t1 = process.hrtime(t);
var t2 = t1[0]+t1[1]/1000000000;
console.log(" Xmysql took : %d seconds",dataHelp.round(t2,1));
console.log(" API's base URL : " + "localhost:"+sqlConfig.portNumber);
console.log(' ');
console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ');
});
/**************** END : setup Xapi ****************/

7
lib/util/cmd.helper.js
Unescape View File

@ -2,6 +2,7 @@
const program = require('commander');
const colors = require('colors');
const pkginfo = require('pkginfo')(module);
const maxCpus = require('os').cpus().length;
program.on('--help', () => {
@ -25,6 +26,7 @@ program
.option('-i, --ignoreTables <n>', 'comma separated table names to ignore')
.option('-a, --apiPrefix <n>', 'api url prefix / "/api/" by default')
.option('-y, --readOnly', 'readonly apis / false by default')
.option('-c, --useCpuCores <n>', 'use number of CPU cores (using cluster) / 1 by default')
.parse(process.argv)
@ -63,6 +65,11 @@ exports.handle = program => {
program.storageFolder = program.storageFolder || process.cwd()
program.apiPrefix = program.apiPrefix || '/api/'
program.readOnly = program.readOnly || false;
program.useCpuCores = program.useCpuCores || 1;
if(program.useCpuCores === '0') {
program.useCpuCores = maxCpus;
}
if (program.ignoreTables) {
let ignoreTables = program.ignoreTables.split(',')

2
lib/xsql.js
Unescape View File

@ -255,7 +255,7 @@ class Xsql {
/**************** START : prepare value object in prepared statement ****************/
// iterate over sent object array
// iterate over sent object array
let arrOfArr = []
for (var i = 0; i < objectArray.length; ++i) {
let arrValues = []

14
package-lock.json generated
Unescape View File

@ -126,6 +126,15 @@
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
"integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
},
"cluster": {
"version": "0.7.7",
"resolved": "https://registry.npmjs.org/cluster/-/cluster-0.7.7.tgz",
"integrity": "sha1-5JfiZ8yVa9CwUTrbSqOTNX0Ahe8=",
"requires": {
"log": "1.4.0",
"mkdirp": "0.5.1"
}
},
"colors": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz",
@ -472,6 +481,11 @@
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"log": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/log/-/log-1.4.0.tgz",
"integrity": "sha1-S6HYkP3iSbAx3KA7w36q8yVlbxw="
},
"media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",

1
package.json
Unescape View File

@ -26,6 +26,7 @@
"dependencies": {
"assert": "^1.4.1",
"body-parser": "^1.18.2",
"cluster": "^0.7.7",
"colors": "^1.1.2",
"commander": "^2.11.0",
"cors": "^2.8.4",

Write Preview
Loading…
Cancel
Save