github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Feature : option to support multiple cpu cores (using cluster)

pull/39/head
o1lab 7 years ago
parent
ad42700863
commit
a13f26cf60
8 changed files with 202 additions and 104 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      README.md
  2. 100
      bin/index.js
  3. 78
      examples/aws-lambda/index.js
  4. 101
      index.js
  5. 7
      lib/util/cmd.helper.js
  6. 2
      lib/xsql.js
  7. 14
      package-lock.json
  8. 1
      package.json

3
README.md
Unescape View File

@ -88,9 +88,11 @@ Powered by popular node packages : ([express](https://github.com/expressjs/expre
* Upload multiple files * Upload multiple files
* Download file * Download file
* Health and version apis * Health and version apis
* Use more than one CPU Cores
* [Docker support](#docker) and [Nginx reverse proxy config](#nginx-reverse-proxy-config-with-docker) :fire::fire::fire: - Thanks to [@markuman](https://github.com/markuman) * [Docker support](#docker) and [Nginx reverse proxy config](#nginx-reverse-proxy-config-with-docker) :fire::fire::fire: - Thanks to [@markuman](https://github.com/markuman)
* AWS Lambda Example - Thanks to [@bertyhell](https://github.com/bertyhell) :fire::fire::fire: * AWS Lambda Example - Thanks to [@bertyhell](https://github.com/bertyhell) :fire::fire::fire:
Use HTTP clients like [Postman](https://www.getpostman.com/) or [similar tools](https://chrome.google.com/webstore/search/http%20client?_category=apps) to invoke REST API calls Use HTTP clients like [Postman](https://www.getpostman.com/) or [similar tools](https://chrome.google.com/webstore/search/http%20client?_category=apps) to invoke REST API calls
____ ____
@ -939,6 +941,7 @@ http://localhost:3000/_version
-a, --apiPrefix <n> Api url prefix -> /api/ by default -a, --apiPrefix <n> Api url prefix -> /api/ by default
-s, --storageFolder <n> Storage folder -> current working dir by default (available only with local) -s, --storageFolder <n> Storage folder -> current working dir by default (available only with local)
-i, --ignoreTables <n> Comma separated table names to ignore -i, --ignoreTables <n> Comma separated table names to ignore
-c, --useCpuCores <n> Specify number of cpu cores to use / 1 by default / 0 to use max
-y, --readOnly readonly apis -> false by default -y, --readOnly readonly apis -> false by default
-h, --help Output usage information -h, --help Output usage information

100
bin/index.js
Unescape View File

@ -9,50 +9,86 @@ const cors = require('cors');
const dataHelp = require('../lib/util/data.helper.js'); const dataHelp = require('../lib/util/data.helper.js');
const Xapi = require('../lib/xapi.js'); const Xapi = require('../lib/xapi.js');
const cmdargs = require('../lib/util/cmd.helper.js'); const cmdargs = require('../lib/util/cmd.helper.js');
const cluster = require('cluster');
const numCPUs = require('os').cpus().length;
cmdargs.handle(sqlConfig)
function startXmysql(sqlConfig) {
/**************** START : setup express ****************/
let app = express();
app.use(morgan('tiny'));
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
/**************** END : setup express ****************/
/**************** START : setup express ****************/
let app = express();
app.use(morgan('tiny'))
app.use(cors())
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({
extended: true
}))
/**************** END : setup express ****************/
/**************** START : setup mysql ****************/
let mysqlPool = mysql.createPool(sqlConfig);
/**************** END : setup mysql ****************/
/**************** START : setup mysql ****************/
let mysqlPool = mysql.createPool(sqlConfig);
/**************** END : setup mysql ****************/
/**************** START : setup Xapi ****************/
console.log('');
console.log('');
console.log('');
console.log(' Generating REST APIs at the speed of your thought.. ');
console.log('');
/**************** START : setup Xapi ****************/ let t = process.hrtime();
console.log(''); let moreApis = new Xapi(sqlConfig, mysqlPool, app);
console.log('');
console.log('');
console.log(' Generating REST APIs at the speed of your thought.. ');
console.log('');
let t = process.hrtime(); moreApis.init((err, results) => {
let moreApis = new Xapi(sqlConfig,mysqlPool,app);
moreApis.init((err, results) => { app.listen(sqlConfig.portNumber, sqlConfig.ipAddress);
var t1 = process.hrtime(t);
var t2 = t1[0] + t1[1] / 1000000000;
app.listen(sqlConfig.portNumber,sqlConfig.ipAddress)
var t1 = process.hrtime(t);
var t2 = t1[0]+t1[1]/1000000000;
console.log(' '); console.log(" Xmysql took : %d seconds", dataHelp.round(t2, 1));
console.log(" Xmysql took : %d seconds",dataHelp.round(t2,1)); console.log(" API's base URL : " + "localhost:" + sqlConfig.portNumber);
console.log(" API's base URL : " + "localhost:"+sqlConfig.portNumber); console.log(' ');
console.log(' '); console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ');
console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ');
});
/**************** END : setup Xapi ****************/
}
function start(sqlConfig) {
//handle cmd line arguments
cmdargs.handle(sqlConfig);
if (cluster.isMaster && sqlConfig.useCpuCores > 1) {
console.log(`Master ${process.pid} is running`);
for (let i = 0; i < numCPUs && i < sqlConfig.useCpuCores; i++) {
console.log(`Forking process number ${i}...`);
cluster.fork();
}
cluster.on('exit', function(worker, code, signal) {
console.log('Worker ' + worker.process.pid + ' died with code: '
+ code + ', and signal: ' + signal);
console.log('Starting a new worker');
cluster.fork();
});
} else {
startXmysql(sqlConfig);
}
}
start(sqlConfig);
})
/**************** END : setup Xapi ****************/

78
examples/aws-lambda/index.js
Unescape View File

@ -31,51 +31,51 @@ var morgan = require('morgan');
var app = express(); var app = express();
var onXapiInitialized = new Promise(function (resolve, reject) { var onXapiInitialized = new Promise(function (resolve, reject) {
try { try {
// /**************** START : setup express ****************/ // /**************** START : setup express ****************/
app.use(morgan('tiny')); app.use(morgan('tiny'));
app.use(cors()); app.use(cors());
app.use(bodyParser.json()); app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ app.use(bodyParser.urlencoded({
extended: true extended: true
})); }));
// /**************** END : setup express ****************/ // /**************** END : setup express ****************/
app.use(function (req, res, next) { app.use(function (req, res, next) {
// You can add authentication here // You can add authentication here
console.log('Received request for: ' + req.url, req); console.log('Received request for: ' + req.url, req);
next(); next();
}); });
var mysqlConfig = { var mysqlConfig = {
host: config.mysql.host, host: config.mysql.host,
port: 3306, port: 3306,
database: config.mysql.database, database: config.mysql.database,
user: config.mysql.user, user: config.mysql.user,
password: config.mysql.password, password: config.mysql.password,
apiPrefix: '/', apiPrefix: '/',
ipAddress: 'localhost', ipAddress: 'localhost',
portNumber: 3000, portNumber: 3000,
ignoreTables: [], ignoreTables: [],
storageFolder: __dirname storageFolder: __dirname
}; };
var mysqlPool = mysql.createPool(mysqlConfig); var mysqlPool = mysql.createPool(mysqlConfig);
var xapi = new Xapi(mysqlConfig, mysqlPool, app); var xapi = new Xapi(mysqlConfig, mysqlPool, app);
xapi.init(function (err, results) { xapi.init(function (err, results) {
app.listen(3000); app.listen(3000);
resolve(); resolve();
}); });
} }
catch (err) { catch (err) {
reject(err); reject(err);
} }
}); });
function handler(event, context, callback) { function handler(event, context, callback) {
onXapiInitialized.then(function () { onXapiInitialized.then(function () {
serverless(app)(event, context, callback); serverless(app)(event, context, callback);
}); });
} }
exports.handler = handler; exports.handler = handler;

101
index.js
Unescape View File

@ -7,51 +7,88 @@ const sqlConfig = require('commander');
const mysql = require('mysql'); const mysql = require('mysql');
const cors = require('cors'); const cors = require('cors');
const dataHelp = require('./lib/util/data.helper.js'); const dataHelp = require('./lib/util/data.helper.js');
const Xapi = require('./lib/xapi.js'); const Xapi = require('./lib/xapi.js');
const cmdargs = require('./lib/util/cmd.helper.js'); const cmdargs = require('./lib/util/cmd.helper.js');
const cluster = require('cluster');
const numCPUs = require('os').cpus().length;
function startXmysql(sqlConfig) {
/**************** START : setup express ****************/
let app = express();
app.use(morgan('tiny'));
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
/**************** END : setup express ****************/
/**************** START : setup mysql ****************/
let mysqlPool = mysql.createPool(sqlConfig);
/**************** END : setup mysql ****************/
/**************** START : setup Xapi ****************/
console.log('');
console.log('');
console.log('');
console.log(' Generating REST APIs at the speed of your thought.. ');
console.log('');
let t = process.hrtime();
let moreApis = new Xapi(sqlConfig, mysqlPool, app);
moreApis.init((err, results) => {
app.listen(sqlConfig.portNumber, sqlConfig.ipAddress);
var t1 = process.hrtime(t);
var t2 = t1[0] + t1[1] / 1000000000;
console.log(" Xmysql took : %d seconds", dataHelp.round(t2, 1));
console.log(" API's base URL : " + "localhost:" + sqlConfig.portNumber);
console.log(' ');
console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ');
});
/**************** END : setup Xapi ****************/
}
function start(sqlConfig) {
cmdargs.handle(sqlConfig);
cmdargs.handle(sqlConfig); if (cluster.isMaster && sqlConfig.useCpuCores > 1) {
console.log(`Master ${process.pid} is running`);
for (let i = 0; i < numCPUs && i < sqlConfig.useCpuCores; i++) {
console.log(`Forking process number ${i}...`);
cluster.fork();
}
/**************** START : setup express ****************/ cluster.on('exit', function(worker, code, signal) {
let app = express(); console.log('Worker ' + worker.process.pid + ' died with code: '
app.use(morgan('tiny')); + code + ', and signal: ' + signal);
app.use(cors()); console.log('Starting a new worker');
app.use(bodyParser.json()); cluster.fork();
app.use(bodyParser.urlencoded({ });
extended: true
}));
/**************** END : setup express ****************/
/**************** START : setup mysql ****************/ } else {
let mysqlPool = mysql.createPool(sqlConfig);
/**************** END : setup mysql ****************/
startXmysql(sqlConfig);
/**************** START : setup Xapi ****************/ }
console.log(''); }
console.log('');
console.log('');
console.log(' Generating REST APIs at the speed of your thought.. ');
console.log('');
let t = process.hrtime();
let moreApis = new Xapi(sqlConfig,mysqlPool,app);
moreApis.init((err, results) => { start(sqlConfig);
app.listen(sqlConfig.portNumber,sqlConfig.ipAddress);
var t1 = process.hrtime(t);
var t2 = t1[0]+t1[1]/1000000000;
console.log(" Xmysql took : %d seconds",dataHelp.round(t2,1));
console.log(" API's base URL : " + "localhost:"+sqlConfig.portNumber);
console.log(' ');
console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ');
});
/**************** END : setup Xapi ****************/

7
lib/util/cmd.helper.js
Unescape View File

@ -2,6 +2,7 @@
const program = require('commander'); const program = require('commander');
const colors = require('colors'); const colors = require('colors');
const pkginfo = require('pkginfo')(module); const pkginfo = require('pkginfo')(module);
const maxCpus = require('os').cpus().length;
program.on('--help', () => { program.on('--help', () => {
@ -25,6 +26,7 @@ program
.option('-i, --ignoreTables <n>', 'comma separated table names to ignore') .option('-i, --ignoreTables <n>', 'comma separated table names to ignore')
.option('-a, --apiPrefix <n>', 'api url prefix / "/api/" by default') .option('-a, --apiPrefix <n>', 'api url prefix / "/api/" by default')
.option('-y, --readOnly', 'readonly apis / false by default') .option('-y, --readOnly', 'readonly apis / false by default')
.option('-c, --useCpuCores <n>', 'use number of CPU cores (using cluster) / 1 by default')
.parse(process.argv) .parse(process.argv)
@ -63,6 +65,11 @@ exports.handle = program => {
program.storageFolder = program.storageFolder || process.cwd() program.storageFolder = program.storageFolder || process.cwd()
program.apiPrefix = program.apiPrefix || '/api/' program.apiPrefix = program.apiPrefix || '/api/'
program.readOnly = program.readOnly || false; program.readOnly = program.readOnly || false;
program.useCpuCores = program.useCpuCores || 1;
if(program.useCpuCores === '0') {
program.useCpuCores = maxCpus;
}
if (program.ignoreTables) { if (program.ignoreTables) {
let ignoreTables = program.ignoreTables.split(',') let ignoreTables = program.ignoreTables.split(',')

2
lib/xsql.js
Unescape View File

@ -255,7 +255,7 @@ class Xsql {
/**************** START : prepare value object in prepared statement ****************/ /**************** START : prepare value object in prepared statement ****************/
// iterate over sent object array // iterate over sent object array
let arrOfArr = [] let arrOfArr = []
for (var i = 0; i < objectArray.length; ++i) { for (var i = 0; i < objectArray.length; ++i) {
let arrValues = [] let arrValues = []

14
package-lock.json generated
Unescape View File

@ -126,6 +126,15 @@
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
"integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
}, },
"cluster": {
"version": "0.7.7",
"resolved": "https://registry.npmjs.org/cluster/-/cluster-0.7.7.tgz",
"integrity": "sha1-5JfiZ8yVa9CwUTrbSqOTNX0Ahe8=",
"requires": {
"log": "1.4.0",
"mkdirp": "0.5.1"
}
},
"colors": { "colors": {
"version": "1.1.2", "version": "1.1.2",
"resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz",
@ -472,6 +481,11 @@
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
}, },
"log": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/log/-/log-1.4.0.tgz",
"integrity": "sha1-S6HYkP3iSbAx3KA7w36q8yVlbxw="
},
"media-typer": { "media-typer": {
"version": "0.3.0", "version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",

1
package.json
Unescape View File

@ -26,6 +26,7 @@
"dependencies": { "dependencies": {
"assert": "^1.4.1", "assert": "^1.4.1",
"body-parser": "^1.18.2", "body-parser": "^1.18.2",
"cluster": "^0.7.7",
"colors": "^1.1.2", "colors": "^1.1.2",
"commander": "^2.11.0", "commander": "^2.11.0",
"cors": "^2.8.4", "cors": "^2.8.4",

Write Preview
Loading…
Cancel
Save