From fb970517531ce431707ca7ea0397251c13ccca23 Mon Sep 17 00:00:00 2001 From: oof1lab Date: Mon, 13 Nov 2017 12:41:00 +0530 Subject: [PATCH] Feature: Bulk insert, delete, read - npm v0.2.5 --- README.md | 49 +++++++------- assets/log.gif | Bin 7162 -> 7754 bytes bin/index.js | 4 +- lib/util/cmd.helper.js | 2 +- lib/xapi.js | 141 +++++++++++++++++++++++++++++------------ lib/xsql.js | 71 ++++++++++++++++++++- package.json | 2 +- tests/tests.js | 109 +++++++++++++++++++++++++++++++ 8 files changed, 311 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 27b8a880fe..83e80f271a 100644 --- a/README.md +++ b/README.md @@ -92,29 +92,32 @@ if you haven't on your system. ## API Overview -|# | HTTP Type | API URL | Comments | -|--|-----------|----------------------------------|--------------------------------------------------------- -|01| GET | / | Gets all REST APIs | -|02| GET | /api/tableName | Lists rows of table | -|03| POST | /api/tableName | Create a new row | -|04| PUT | /api/tableName | Replaces existing row with new row | -|05| GET | /api/tableName/:id | Retrieves a row by primary key | -|06| PATCH | /api/tableName/:id | Updates a row by primary key | -|07| GET | /api/tableName/findOne | Works as list but gets single record matching criteria | -|08| GET | /api/tableName/count | Count number of rows in a table | -|09| GET | /api/tableName/:id/exists | True or false whether a row exists or not | -|10| DELETE | /api/tableName/:id | Delete a row by primary key | -|11| GET | [/api/parentTable/:id/childTable](#relational-tables) | Get list of child table rows with parent table foreign key | -|12| GET :fire:| [/api/tableName/aggregate](#aggregate-functions) | Aggregate results of numeric column(s) | -|13| GET :fire:| [/api/tableName/groupby](#group-by-having-as-api) | Group by results of column(s) | -|14| GET :fire:| [/api/tableName/ugroupby](#union-of-multiple-group-by-statements) | Multiple group by results using one call | -|15| GET :fire:| [/api/tableName/chart](#chart) | Numeric column distribution based on (min,max,step) or(step array) or (automagic)| -|16| GET :fire:| [/dynamic](#run-dynamic-queries) | execute dynamic mysql statements with params | -|17| GET :fire:| [/upload](#upload-single-file) | upload single file | -|18| GET :fire:| [/uploads](#upload-multiple-files) | upload multiple files | -|19| GET :fire:| [/download](#download-file) | download a file | -|20| GET | /api/tableName/describe| describe each table for its columns | -|21| GET | /api/tables| get all tables in database | +| HTTP Type | API URL | Comments | +|-----------|----------------------------------|--------------------------------------------------------- +| GET | / | Gets all REST APIs | +| GET | /api/tableName | Lists rows of table | +| POST | /api/tableName | Create a new row | +| PUT | /api/tableName | Replaces existing row with new row | +| POST :fire:| /api/tableName/bulk | Create multiple rows - send object array in request body| +| GET :fire:| /api/tableName/bulk | Lists multiple rows - /api/tableName/bulk?_ids=1,2,3 | +| DELETE :fire:| /api/tableName/bulk | Deletes multiple rows - /api/tableName/bulk?_ids=1,2,3 | +| GET | /api/tableName/:id | Retrieves a row by primary key | +| PATCH | /api/tableName/:id | Updates row element by primary key | +| DELETE | /api/tableName/:id | Delete a row by primary key | +| GET | /api/tableName/findOne | Works as list but gets single record matching criteria | +| GET | /api/tableName/count | Count number of rows in a table | +| GET | /api/tableName/:id/exists | True or false whether a row exists or not | +| GET | [/api/parentTable/:id/childTable](#relational-tables) | Get list of child table rows with parent table foreign key | +| GET :fire:| [/api/tableName/aggregate](#aggregate-functions) | Aggregate results of numeric column(s) | +| GET :fire:| [/api/tableName/groupby](#group-by-having-as-api) | Group by results of column(s) | +| GET :fire:| [/api/tableName/ugroupby](#union-of-multiple-group-by-statements) | Multiple group by results using one call | +| GET :fire:| [/api/tableName/chart](#chart) | Numeric column distribution based on (min,max,step) or(step array) or (automagic)| +| GET :fire:| [/dynamic](#run-dynamic-queries) | execute dynamic mysql statements with params | +| GET :fire:| [/upload](#upload-single-file) | upload single file | +| GET :fire:| [/uploads](#upload-multiple-files) | upload multiple files | +| GET :fire:| [/download](#download-file) | download a file | +| GET | /api/tableName/describe| describe each table for its columns | +| GET | /api/tables| get all tables in database | diff --git a/assets/log.gif b/assets/log.gif index e3be0cc0b5a2c1c95212e1f6cb325f0b19de8d18..7d011eb992d31679202eec674478bd71953e9f79 100644 GIT binary patch delta 2016 zcmV<62Os$QH_AK;M@dFFIbq-f1_F@^F9hHN1_F^iAb;}gA_$G=`p)(t zeAy1krt=AnN~hGS^@`1gmIaWj!`|iIFfBf>#Pk;UP-;aO(`tQ$w z|Nj3EzyJzxfCMa{0S}141S)WW3~ZnS9|*w+N^pV{te^!ih`|hMkSYZL04x9i0004# z=mad2jRa}46b1MJlh6hplMDytlUxW7lko>@0XUNl2YHjw1`V_F2)zN5pbDQ#00000 z0RRC200II`0RSv00Y(5A01f~E0x*n^8Nkg7k8?53R^fd)ZD3HL_$>%%O3Rjpu3BEN z8*Aq(quyQOx^C)qB&;FvQUH_j5Nng43os1;00001009610s>3{lg$favw95e0R!X# zRFi5BR4DrmkIU!uy8VvN>-YS={|^`_I7nD%c!-#&xX9S(_y`#(IZ0V*d5M#;5;}j~ z15gbRbFGm5VO@RD?QPI44!+GWPHv5$9%BCB?!FU0!EOUTGA~|co{uu$kAa^)ZR-j` z5EuaAz;^T!0JIlSoM9%@n8!eR&hg#Ri=;An6m2#q2W0wGCI$2}4KP+G}{E@4cW z;Y`N7IkRRuo!}gROu0bgPm%F@`mBFvv?9)WG%XrU+H@ULs6mm=3>xyL!E09?F6D_d zE4rjqy>$FWG_A~cA={cn`?lmnxL50b+}d`f1&nO-sy&C5E?&WK0T+(zbMQcpF7>`0 z`dBeymsPT{gKSx{)Tx0FXFjY=YhSrl$J%xKE@wl(lM@4tOn5PB)}&v5mK}dvg5%q; zar-jHM$A)Vo@BMn`=;H^UTQ2Lhrtr(fL0g|~TP@_ftTBtHEoS`r z=*rI~r=F5;d-A`Zvqz6|xAVU0>=D0j4j+KL4Y;3iz?pX-dSUd(pmFl~hhTHtxEG*< z&H;p>g08(sS#G>}SK)vD3AkQfebtpG8G+PM2;vhtg;khnp)p0%H#~_&<9AJIN25@h z*~m{Z^LjDl49})@o<7kr=oE0h+Th5~u-x{Z+ zc0pN5#z{}%B?yCW=n;6}aDPQe;fM&HSXz1^a;Tt+ANJSP7A6AtT#dh_RA5ahC?_0? zDA+^>U$OmG9epGQ_@HYt)fb!!LzXm~kCUhZFn^~v0_@4B0et@HC!hiRDd+-(23qK# zeOhqn3wu`Zs1b=Oia?`;Mj9ukk79a&rUF#@sRN!;8Yrk2jOqfX5vUpjtBQt7Dy6Pw z3hSn@c6w_9v*OAts--eoYpyEn$^fxDxLW?`sK&ObtgyQp+bplrBD$!d!oC{mrMNC? zYk#(aYD;aflWwpqx!3-xfUS5oTP?P^LK|$Xocj6ztI?Lq?z!^bdT*>9l&b-_(^4C* zzH5RDuc@MHD=@kMcl&9->f&o}qy9>4@Uig@tZ>J0CYmtE>BjpkzvFfcai96B?Ci!D zGwiYu^v+uGoD=)&u*vn3tS`+y^E`5|EmjAev&TgfF|*4x%jqq;)6Sc5zeG#ViN0cXrX(pT?mGtZVz8nUeCers{D$+9i&+17&0?b|130Pxqj%`+HdNF{=p{U*z|hoN{i@~EQy2pcQt=hayRb zh}OPtq%5TZ(LhC0ssfd%R3$1{=}If2GM2}HB`u>j%Uez` ym%0>&EqfV^Ujp+N!W?E^cxlXIN-~+hLnbqIQOsxJ0-^x3p%_>Lv%na40RcO>L(Gu? delta 1688 zcmV;J250%oJo+~ZM@dFFIbnbU+yRjaF9d)C+yRk2Ab%r*$o7rr`p)I9Yolb#3VladH)lL-h90XUPN2YHi# z2Mx1?3B3W6Pz#U&D3gE-8k6q~1`vgu`C#nil2>T0HC6xsEHIOR3l|m{Ub;OS7EU9W znOvEiUfQm3xLUQu6_b+;G66u7unQQI?F>hga1u+iG!5+m17ra>lg<-9lVT4Rf7=a+ z#}9Hj-5#jp8}<7>KNkWpF^Ev;b7=9$keD~pXjqn52d7aGSt)9|$%+Y^D$)2#bGwjh*(s<>#@YH?8r#e37s$7#SK7Df{G6L+ z%9SWqLQKn^T^qLK6La3qIX!u861r~Zq)lVFH|Da2*b1vm={GC2aE6oBJcteKo5>G> zQNDa4E0w^QlhFYwLiiRvhlLw=LEP{0kB ze_7tQ?YIaZDX*@*ruX4|)o0)GcOcu}W&Zv0zdsy+NnpoTe6W?|pIr)KrPT!E zS%*_l!gaS`Y>OaIM+AYf8HkX2^%bQdg#GEap?&79Ct!&^fL9=gEY$WPg_^Bew11Y;N|8b3O13!7i%cO(PB%KQ0E*#LLt&5lkE8zhmk>(Ko%;0XP$cQ$!DMD z2molH1_VmzpoIoN=%EWFYN(=!3TgqPF92Err9>oZsi6o+`Y5D#W=d(M2Y@QTrlmSS z>Z7Bcs%fOGin;=<5Xkxit(~rlYpS2VTB@wJs){PFzz$371iH5H{wo8??m(-esUBPB zv#jE(?6ivt3$3V_Hqh*Uq1!fUX{eA&`faDAI?BPa=~`=QwZM8BEw+ww+A6p5Qd_UO z_2MgTuJZnx?Y;Z1dqTMbd)q6#bbh<6w(KT=X~O728?3p*_N%G1?3U}WzxW=}@wFD~ zyXL$PxBIck8J|ony$WbtD9nQTD(t=yU(9k4A-^o~z;j|;EY2ua+v~BrDjy9rzX|iq zbiUKloN^I6OB=PE;hqY$%{dDV@X1{lJ#^AOAG@>CAds9vz-CkY^S=ly+_1nrUybhE z=Uz=Mxepus_T6Yl@HYmWGVE;Kejby)7d?NDd+xgL&U^2^|BecfgE^Q1e7azm^6{;qepNe;ql00ra;p&mHy4Gq{jXdE7)^mF7>FA|~2DPof>uAvj2J@Mna4 zJ;FU!VU6h*_~VS6tWN>g+(<_zDjT!>J?4uw52*^MRa*%{9q#+N9$V4h~ zk)XIlgy;ZCKtz%elXQeZ9LdElMk150!DN$`1ja7U*vnS@GMLC1CNU$p%VQ!ll*-IYGMhOJV?rG>7}A{P iFRE!xXEHOJ+j}NAwF%2_qQslx)I|XRvjH0Y0|7hG)jI { }) program - .version('0.2.4') + .version('0.2.5') .option('-h, --host ', 'hostname') .option('-d, --database ', 'database schema name') .option('-u, --user ', 'username of database / root by default') diff --git a/lib/xapi.js b/lib/xapi.js index 0e2b8bd03d..8c36e00c04 100644 --- a/lib/xapi.js +++ b/lib/xapi.js @@ -149,6 +149,21 @@ class Xapi { .get(this.asyncMiddleware(this.read.bind(this))); break; + case 'bulkInsert': + this.app.route(routes[i]['routeUrl']) + .post(this.asyncMiddleware(this.bulkInsert.bind(this))); + break; + + case 'bulkRead': + this.app.route(routes[i]['routeUrl']) + .get(this.asyncMiddleware(this.bulkRead.bind(this))); + break; + + case 'bulkDelete': + this.app.route(routes[i]['routeUrl']) + .delete(this.asyncMiddleware(this.bulkDelete.bind(this))); + break; + case 'patch': this.app.route(routes[i]['routeUrl']) .patch(this.asyncMiddleware(this.patch.bind(this))); @@ -230,7 +245,7 @@ class Xapi { console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - '); console.log(' '); console.log(' Database : %s', this.config.database); - console.log(' Number of resources : %s', stat.tables); + console.log(' Number of Tables : %s', stat.tables); console.log(' '); console.log(' REST APIs Generated : %s'.green.bold, stat.apis); console.log(' '); @@ -256,7 +271,7 @@ class Xapi { * @param req * @param res * @param queryParamsObj : {query, params} - * @param listType : 0:list, 1:nested, 2:findOne + * @param listType : 0:list, 1:nested, 2:findOne, 3:bulkRead * * Updates query, params for query of type listType */ @@ -297,6 +312,28 @@ class Xapi { this.mysql.getWhereClause(req.query._where, req.app.locals._tableName, queryParamsObj, ' and '); + } else if (listType === 3) { //bulkRead + + // select * from table where pk in (ids) and whereConditions + queryParamsObj.params.push(req.app.locals._tableName); + queryParamsObj.query += ' where ?? in '; + queryParamsObj.params.push(this.mysql.getPrimaryKeyName(req.app.locals._tableName)); + + queryParamsObj.query += '(' + + if (req.query && req.query._ids) { + let ids = req.query._ids.split(',') + for (var i = 0; i < ids.length; ++i) { + if (i) { + queryParamsObj.query += ',' + } + queryParamsObj.query += '?' + queryParamsObj.params.push(ids[i]) + } + } + queryParamsObj.query += ') ' + this.mysql.getWhereClause(req.query._where, req.app.locals._tableName, queryParamsObj, ' and '); + } else { queryParamsObj.params.push(req.app.locals._tableName); @@ -468,44 +505,6 @@ class Xapi { } - - // async update(req, res) { - // - // let query = 'UPDATE ?? SET '; - // let keys = Object.keys(req.body); - // - // // SET clause - // let updateKeys = ''; - // for (let i = 0; i < keys.length; ++i) { - // updateKeys += keys[i] + ' = ? ' - // if (i !== keys.length - 1) - // updateKeys += ', ' - // } - // - // // where clause - // query += updateKeys + ' where ' - // let clause = this.mysql.getPrimaryKeyWhereClause(req.app.locals._tableName, - // req.params.id.split('___')); - // - // if (!clause) { - // return res.status(400).send({ - // error: "Table is made of composite primary keys - all keys were not in input" - // }) - // } - // - // query += clause; - // - // // params - // let params = []; - // params.push(req.app.locals._tableName); - // params = params.concat(Object.values(req.body)); - // - // let results = await this.mysql.exec(query, params); - // res.status(200).json(results); - // - // - // } - async delete(req, res) { let query = 'DELETE FROM ?? WHERE '; @@ -530,6 +529,68 @@ class Xapi { } + async bulkInsert(req, res) { + + let queryParamsObj = {} + queryParamsObj.query = '' + queryParamsObj.params = [] + let results = [] + + //console.log(req.app.locals._tableName, req.body); + + this.mysql.getBulkInsertStatement(req.app.locals._tableName, req.body, queryParamsObj) + + results = await this.mysql.exec(queryParamsObj.query, queryParamsObj.params); + res.status(200).json(results); + + } + + async bulkDelete(req, res) { + + let query = 'delete from ?? where ?? in '; + let params = []; + + params.push(req.app.locals._tableName); + params.push(this.mysql.getPrimaryKeyName(req.app.locals._tableName)); + + query += '(' + + if (req.query && req.query._ids) { + let ids = req.query._ids.split(',') + for (var i = 0; i < ids.length; ++i) { + if (i) { + query += ',' + } + query += '?' + params.push(ids[i]) + } + } + + query += ')' + + //console.log(query, params); + + var results = await this.mysql.exec(query, params); + res.status(200).json(results); + + } + + async bulkRead(req, res) { + + let queryParamsObj = {} + queryParamsObj.query = '' + queryParamsObj.params = [] + + this.prepareListQuery(req, res, queryParamsObj, 3); + + //console.log(queryParamsObj.query, queryParamsObj.params); + + let results = await this.mysql.exec(queryParamsObj.query, queryParamsObj.params); + res.status(200).json(results); + + } + + async count(req, res) { let query = 'select count(1) as no_of_rows from ??'; diff --git a/lib/xsql.js b/lib/xsql.js index f9d3b0a5ca..426944d325 100644 --- a/lib/xsql.js +++ b/lib/xsql.js @@ -173,6 +173,65 @@ class Xsql { } + getBulkInsertStatement(tableName, objectArray, queryParamsObj) { + + if (tableName in this.metaDb.tables && objectArray) { + + let insObj = objectArray[0]; + + // goal => insert into ?? (?,?..?) values ? [tablName, col1,col2...coln,[[ObjValues_1],[ObjValues_2],...[ObjValues_N]] + queryParamsObj.query = ' INSERT INTO ?? ( ' + queryParamsObj.params.push(tableName) + + let cols = []; + let colPresent = false; + + /**************** START : prepare column names to be inserted ****************/ + // iterate over all column in table and have only ones existing in objects to be inserted + for (var i = 0; i < this.metaDb.tables[tableName]['columns'].length; ++i) { + + let colName = this.metaDb.tables[tableName]['columns'][i]['column_name'] + + + if (colName in insObj) { + + if (colPresent) { + queryParamsObj.query += ',' + } + + queryParamsObj.query += colName + + colPresent = true; + + } + + cols.push(colName) + + //console.log('> > ', queryParamsObj.query); + + } + + queryParamsObj.query += ' ) values ?' + /**************** END : prepare column names to be inserted ****************/ + + + /**************** START : prepare value object in prepared statement ****************/ + // iterate over sent object array + let arrOfArr = [] + for (var i = 0; i < objectArray.length; ++i) { + let arrValues = [] + for (var j = 0; j < cols.length; ++j) { + if (cols[j] in objectArray[i]) + arrValues.push(objectArray[i][cols[j]]) + } + arrOfArr.push(arrValues); + } + queryParamsObj.params.push(arrOfArr) + /**************** END : prepare value object in prepared statement ****************/ + } + } + + getGroupByClause(_groupby, tableName, queryParamsObj) { if (_groupby) { @@ -350,6 +409,14 @@ class Xsql { } + getPrimaryKeyName(tableName) { + let pk = null + if (tableName in this.metaDb.tables) { + pk = this.metaDb.tables[tableName].primaryKeys[0]['column_name'] + } + return pk + } + getPrimaryKeyWhereClause(tableName, pksValues) { let whereClause = ''; @@ -462,6 +529,9 @@ class Xsql { routes.push(this.prepareRoute(internal, 'get', apiPrefix, tableName + '/findOne', 'findOne')) routes.push(this.prepareRoute(internal, 'post', apiPrefix, tableName, 'create')) routes.push(this.prepareRoute(internal, 'get', apiPrefix, tableName, 'list')) + routes.push(this.prepareRoute(internal, 'post', apiPrefix, tableName + '/bulk', 'bulkInsert')) + routes.push(this.prepareRoute(internal, 'get', apiPrefix, tableName + '/bulk', 'bulkRead')) + routes.push(this.prepareRoute(internal, 'delete', apiPrefix, tableName + '/bulk', 'bulkDelete')) routes.push(this.prepareRoute(internal, 'put', apiPrefix, tableName, 'update')) routes.push(this.prepareRoute(internal, 'get', apiPrefix, tableName + '/:id', 'read')) routes.push(this.prepareRoute(internal, 'patch', apiPrefix, tableName + '/:id', 'patch')) @@ -570,7 +640,6 @@ class Xsql { } - } diff --git a/package.json b/package.json index 8a4146ef28..65aa081667 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "xmysql", - "version": "0.2.4", + "version": "0.2.5", "description": "One command to generate REST APIs for any MySql database", "main": "index.js", "scripts": { diff --git a/tests/tests.js b/tests/tests.js index 7ba690b52f..02bcfcdf23 100644 --- a/tests/tests.js +++ b/tests/tests.js @@ -457,6 +457,115 @@ describe('xmysql : tests', function () { }); }); + it('POST /api/productlines/bulk should PASS', function (done) { + + var objArray = [] + + var obj = {}; + obj['productLine'] = 'Bulletrain' + obj['textDescription'] = 'Japan' + + var obj1 = {}; + obj1['productLine'] = 'Bulletrain_1' + obj1['textDescription'] = 'China' + + objArray.push(obj) + objArray.push(obj1) + + //post to an url with data + agent.post('/api/productlines/bulk') //enter url + .send(objArray) //postdata + .expect(200)//200 for success 4xx for failure + .end(function (err, res) { + // Handle /api/v error + if (err) { + return done(err); + } + + //validate response + + res.body['affectedRows'].should.be.equals(2) + + return done(); + + }); + }); + + it('POST /api/productlines/bulk should PASS', function (done) { + + var objArray = [] + + var obj = {}; + obj['productLine'] = 'Bulletrain_2' + + var obj1 = {}; + obj1['productLine'] = 'Bulletrain_3' + + + objArray.push(obj) + objArray.push(obj1) + + //post to an url with data + agent.post('/api/productlines/bulk') //enter url + .send(objArray) //postdata + .expect(200)//200 for success 4xx for failure + .end(function (err, res) { + // Handle /api/v error + if (err) { + return done(err); + } + + //validate response + + res.body['affectedRows'].should.be.equals(2) + + return done(); + + }); + }); + + it('GET /api/productlines/bulk should PASS', function (done) { + + //post to an url with data + agent.get('/api/productlines/bulk?_ids=Bulletrain,Bulletrain_1,Bulletrain_2,Bulletrain_3') //enter url + .expect(200)//200 for success 4xx for failure + .end(function (err, res) { + // Handle /api/v error + if (err) { + return done(err); + } + + //validate response + + res.body.length.should.be.equals(4) + + return done(); + + }); + }); + + + it('DELETE /api/productlines/bulk should PASS', function (done) { + + //post to an url with data + agent.del('/api/productlines/bulk?_ids=Bulletrain,Bulletrain_1,Bulletrain_2,Bulletrain_3') //enter url + .expect(200)//200 for success 4xx for failure + .end(function (err, res) { + // Handle /api/v error + if (err) { + return done(err); + } + + //validate response + + res.body['affectedRows'].should.be.equals(4) + + return done(); + + }); + }); + + it('PUT /api/productlines should PASS', function (done) { var obj = {};