Merge pull request #122 from jeznag/jeznag/call-stored-procedures

Revive calling stored procedures + improve groupby + allow aggregations in queries

.gitignore

@@ -75,3 +75,5 @@ mongod
 *.ntvs*
 *.njsproj
 *.sln
+
+.history

docker-compose.yml

@@ -33,6 +33,9 @@ services:
   test:
     <<: *db-api
     command: sh -c "npm test"
+    volumes:
+    - ./tests:/usr/src/app/tests
+    - ./lib:/usr/src/app/lib
 
 volumes:
   db-data:

lib/util/data.helper.js

@@ -146,6 +146,10 @@ exports.getSchemaQuery = function() {
   );
 };
 
+exports.getRoutines = function () {
+  return 'select routine_name from information_schema.routines'
+}
+
 exports.getChartQuery = function() {
   return "select ? as ??, count(*) as _count from ?? where ?? between ? and ? ";
 };

lib/xapi.js

@@ -98,9 +98,10 @@ class Xapi {
   }
 
   setupRoutes() {
-    let stat = {};
+    let stat = {}
-    stat.tables = 0;
+    stat.tables = 0
-    stat.apis = 0;
+    stat.apis = 0
+    stat.routines = 0
 
     // console.log('this.config while setting up routes', this.config);
 
@@ -345,30 +346,29 @@ class Xapi {
     stat.apis += 2;
     /**************** END : health and version ****************/
 
-    let statStr =
+    /**************** START : call stored procedures ****************/
-      "     Generated: " +
+    this.app.get('/_proc', this.asyncMiddleware(this.proc.bind(this)))
-      stat.apis +
+    stat.apis += 1
-      " REST APIs for " +
+    const procResources = this.mysql.getProcList(true, this.config.apiPrefix)
-      stat.tables +
+    this.app.post('/_proc/:proc', this.asyncMiddleware(this.callProc.bind(this)))
-      " tables ";
+    stat.routines += procResources.length
+    stat.apis += procResources.length
+    /**************** END : call stored procedures ****************/
 
-    console.log(
-      " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "
-    );
-    console.log("                                                            ");
-    console.log(
-      "          Database              :    %s",
-      this.config.database
-    );
-    console.log("          Number of Tables      :    %s", stat.tables);
-    console.log("                                                            ");
-    console.log(
-      "          REST APIs Generated   :    %s",
-      stat.apis
-    );
-    console.log("                                                            ");
 
-    return stat;
+    let statStr = '     Generated: ' + stat.apis + ' REST APIs for ' + stat.tables + ' tables '
+
+    console.log(' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ');
+    console.log('                                                            ');
+    console.log('          Database              :    %s', this.config.database);
+    console.log('          Number of Tables      :    %s', stat.tables);
+    console.log('          Number of Routines    :    %s', stat.routines);
+    console.log('                                                            ');
+    console.log('          REST APIs Generated   :    %s'.green.bold, stat.apis);
+    console.log('                                                            ');
+
+    return stat
+
   }
 
   async xjoin(req, res) {
@@ -487,6 +487,21 @@ class Xapi {
   }
 
   /**************** END : health and version ****************/
+
+  async proc(req, res) {
+    let query = 'SELECT routine_name AS resource FROM information_schema.routines WHERE routine_schema = ? ';
+    let params = [this.config.database];
+    let results = await this.mysql.exec(query, params)
+    res.status(200).json(results)
+  }
+
+  async callProc(req, res) {
+    let query = 'CALL ??(?)'
+    let params = [req.params.proc, Object.values(req.body)]
+    let results = await this.mysql.exec(query, params)
+    res.status(200).json(results)
+  }
+
 }
 
 //expose class

lib/xctrl.js

@@ -294,7 +294,7 @@ class xctrl {
       let tableName = req.app.locals._tableName;
       queryParamsObj.params.push(tableName);
 
-      this.mysql.getColumnsForSelectStmt(
+      this.mysql.getColumnsForGroupBy(
         req.app.locals._tableName,
         req.query,
         queryParamsObj

lib/xsql.js

@@ -13,6 +13,7 @@ class Xsql {
     this.pool = {};
     this.metaDb = {};
     this.metaDb.tables = {};
+    this.metaDb.routines = [];
 
     this.sqlConfig = sqlConfig;
     this.pool = pool;
@@ -56,7 +57,14 @@ class Xsql {
 
           // osx mysql server has limitations related to open_tables
           self.pool.query("FLUSH TABLES", [], (err, results) => {
-            cbk(null, null);
+            self.pool.query(dataHelp.getRoutines(), [this.sqlConfig.database], (err, results) => {
+              if (err) {
+                cbk(err, results)
+              } else {
+                self.iterateToCacheRoutines(results)
+                cbk(null, null)
+              }
+            })
           });
         }
       }
@@ -80,6 +88,14 @@ class Xsql {
     }
   }
 
+  iterateToCacheRoutines(routineResults) {
+      for (let i = 0; i < routineResults.length; i++) {
+          const routine = routineResults[i]
+          const routineName = routine.routine_name
+          this.metaDb.routines.push(routineName)
+      }
+  }
+
   iterateToCacheTableColumns(schemaResults) {
     for (let i = 0; i < schemaResults.length; ++i) {
       let schemaRow = schemaResults[i];
@@ -332,13 +348,16 @@ class Xsql {
         if (i) {
           queryParamsObj.query += ", ";
         }
-        if (orderByCols[i][0] === "-") {
+        const aggregationFunction = this.getAggregationFunction(orderByCols[i]);
-          let len = orderByCols[i].length;
+        const columnName = this.getColumnNameWithoutAggregationFunctions(orderByCols[i]);
-          queryParamsObj.query += " ?? DESC";
+        const orderByDirection = orderByCols[i][0] === "-" ? 'DESC' : 'ASC';
-          queryParamsObj.params.push(orderByCols[i].substring(1, len));
+
+        if (aggregationFunction) {
+          queryParamsObj.query += `${aggregationFunction}(??) ${orderByDirection}`;
+          queryParamsObj.params.push(columnName);
         } else {
-          queryParamsObj.query += " ?? ASC";
+          queryParamsObj.query += `?? ${orderByDirection}`;
-          queryParamsObj.params.push(orderByCols[i]);
+          queryParamsObj.params.push(columnName);
         }
       }
     }
@@ -358,6 +377,16 @@ class Xsql {
     queryParamsObj.query += ",count(1) as _count ";
   }
 
+  getColumnsForGroupBy(tableName, reqQueryParams, queryParamsObj) {
+    const updatedQueryParams = Object.assign({}, reqQueryParams);
+    if ('_groupbyfields' in updatedQueryParams) {
+      // allows you to group by different fields than you have in the select
+      updatedQueryParams['_fields'] = updatedQueryParams['_groupbyfields'];
+    }
+
+    return this.getColumnsForSelectStmt(tableName, updatedQueryParams, queryParamsObj)
+  }
+
   getColumnsForSelectStmt(tableName, reqQueryParams, queryParamsObj) {
     let table = this.metaDb.tables[tableName];
     let cols = [];
@@ -398,13 +427,41 @@ class Xsql {
       if (i) {
         queryParamsObj.query += ",";
       }
+      const aggregationFunction = this.getAggregationFunction(cols[i]);
+
+      if (aggregationFunction) {
+        queryParamsObj.query += `${aggregationFunction}(??)`;
+        const columnName = this.getColumnNameWithoutAggregationFunctions(cols[i]);
+        queryParamsObj.params.push(columnName);
+      } else {
         queryParamsObj.query += "??";
         queryParamsObj.params.push(cols[i]);
       }
+    }
 
     return cols.join(",");
   }
 
+  getAggregationFunction(rawColumnName) {
+    const AGGREGATION_FUNCTION_REGEX = /^[-]?(AVG|BIT_AND|BIT_OR|BIT_XOR|COUNT|COUNTDISTINCT|GROUP_CONCAT|JSON_ARRAYAGG|JSON_OBJECTAGG|MAX|MIN|STD|STDDEV|STDDEV_POP|STDDEV_SAMP|SUM|VAR_POP|VAR_SAMP|VARIANCE)\((.*)\)$/i;
+    const aggFuncMatch = rawColumnName.match(AGGREGATION_FUNCTION_REGEX);
+    if (aggFuncMatch && aggFuncMatch.length === 3) {
+      // match will look like (3) ["AVG(timestamp)", "AVG", "timestamp", index: 0, input: "AVG(timestamp)", groups: undefined]
+      return aggFuncMatch[1];
+    }
+    return null;
+  }
+
+  getColumnNameWithoutAggregationFunctions(rawColumnName) {
+    const AGGREGATION_FUNCTION_REGEX = /^[-]?(AVG|BIT_AND|BIT_OR|BIT_XOR|COUNT|COUNTDISTINCT|GROUP_CONCAT|JSON_ARRAYAGG|JSON_OBJECTAGG|MAX|MIN|STD|STDDEV|STDDEV_POP|STDDEV_SAMP|SUM|VAR_POP|VAR_SAMP|VARIANCE)\((.*)\)$/i;
+    const aggFuncMatch = rawColumnName.match(AGGREGATION_FUNCTION_REGEX);
+    if (aggFuncMatch && aggFuncMatch.length === 3) {
+      // match will look like (3) ["AVG(timestamp)", "AVG", "timestamp", index: 0, input: "AVG(timestamp)", groups: undefined]
+      return aggFuncMatch[2];
+    }
+    return rawColumnName.replace(/-/, '');
+  }
+
   removeUnknownColumns(inputColumns, tableName) {
     let cols = inputColumns;
     let unknown_cols_in_input = [];
@@ -414,9 +471,11 @@ class Xsql {
     // find unknown fields if any
     for (var j = 0; j < cols.length; ++j) {
       let found = 0;
+      // Used to allow aggregation functions like AVG(timestamp)
+      let columnNameWithoutAggregationClauses = this.getColumnNameWithoutAggregationFunctions(cols[j]);
 
       for (var i = 0; i < tableColumns.length; ++i) {
-        if (tableColumns[i]["column_name"] === cols[j]) {
+        if (tableColumns[i]["column_name"] === columnNameWithoutAggregationClauses) {
           found = 1;
           break;
         }
@@ -732,7 +791,12 @@ class Xsql {
           }
         }
 
-        tableObj["routes"] = routes;
+        var procList = this.getProcList()
+        for (var j = 0; j < procList.length; j++) {
+          routes.push(this.prepareRoute(internal, 'post', apiPrefix, '_proc/' + procList[j]))
+        }
+
+        tableObj['routes'] = routes;
 
         schemaRoutes.push(tableObj);
       }
@@ -741,6 +805,14 @@ class Xsql {
     return schemaRoutes;
   }
 
+  getProcList() {
+    let procRoutes = []
+    for (let procName in this.metaDb.routines) {
+      procRoutes.push(this.metaDb.routines[procName])
+    }
+    return procRoutes
+  }
+
   getJoinType(joinInQueryParams) {
     //console.log('joinInQueryParams',joinInQueryParams);
 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "xmysql",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "xmysql",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "One command to generate REST APIs for any MySql database",
   "main": "index.js",
   "scripts": {

tests/tests.js

@@ -1178,6 +1178,27 @@ describe("xmysql : tests", function() {
     }
   );
 
+  it(`GET ${apiPrefix}customers/groupby?_fields=city&_sort=city&_groupbyfields=country should PASS`,
+    function(done) {
+      agent
+        .get(apiPrefix + "customers/groupby?_fields=avg(creditLimit),country,city&_sort=-avg(creditLimit),city&_groupbyfields=country,city")
+        .expect(200)
+        .end(function(err, res) {
+          if (err) {
+            return done(err);
+          }
+
+          res.body[0]['avg(`creditLimit`)'].should.be.equals(210500);
+          res.body[0]["country"].should.be.equals("USA");
+          res.body[0]["city"].should.be.equals("San Rafael");
+          res.body[0]["_count"].should.be.equals(1);
+          res.body.length.should.be.equals(95);
+
+          return done();
+        });
+    }
+  );
+
   it(
     "GET " + apiPrefix + "offices/ugroupby?_fields=country should PASS",
     function(done) {
@@ -2303,4 +2324,32 @@ describe("xmysql : tests", function() {
 
     done();
   });
+
+  it('calls procedure set_credit_limit', function (done) {
+      const obj = {
+          customeNumber: 103,
+          creditLimit: 23134.00
+      }
+      agent.post('/_proc/set_credit_limit')
+          .send(obj)
+          .expect(200)
+          .end(function (err, res) {
+              if (err) {
+                  return done(err);
+              }
+
+              res.body.should.be.deepEqual({
+                fieldCount: 0,
+                affectedRows: 0,
+                insertId: 0,
+                serverStatus: 2,
+                warningCount: 0,
+                message: '',
+                protocol41: true,
+                changedRows: 0
+              })
+
+              return done();
+          });
+  });
 });