Allow aggregation columns in queries

docker-compose.yml

@@ -33,6 +33,9 @@ services:
   test:
     <<: *db-api
     command: sh -c "npm test"
+    volumes:
+    - ./tests:/usr/src/app/tests
+    - ./lib:/usr/src/app/lib
 
 volumes:
   db-data:

lib/xctrl.js

@@ -294,7 +294,7 @@ class xctrl {
       let tableName = req.app.locals._tableName;
       queryParamsObj.params.push(tableName);
 
-      this.mysql.getColumnsForSelectStmt(
+      this.mysql.getColumnsForGroupBy(
         req.app.locals._tableName,
         req.query,
         queryParamsObj

lib/xsql.js

@@ -50,23 +50,23 @@ class Xsql {
             }
           }
 
-        self.iterateToCacheTables(results)
-        self.iterateToCacheTablePks(results)
-        self.iterateToCacheTableColumns(results)
-        self.iterateToCacheTableFks(results)
-        self.iterateToCacheRoutines(results)
-
-        // osx mysql server has limitations related to open_tables
-        self.pool.query('FLUSH TABLES', [], (err, results) => {
-          self.pool.query(dataHelp.getRoutines(), [this.sqlConfig.database], (err, results) => {
-            if (err) {
-              cbk(err, results)
-            } else {
-              self.iterateToCacheRoutines(results)
-              cbk(null, null)
-            }
-          })
-        })
+          self.iterateToCacheTables(results);
+          self.iterateToCacheTablePks(results);
+          self.iterateToCacheTableColumns(results);
+          self.iterateToCacheTableFks(results);
+
+          // osx mysql server has limitations related to open_tables
+          self.pool.query("FLUSH TABLES", [], (err, results) => {
+            self.pool.query(dataHelp.getRoutines(), [this.sqlConfig.database], (err, results) => {
+              if (err) {
+                cbk(err, results)
+              } else {
+                self.iterateToCacheRoutines(results)
+                cbk(null, null)
+              }
+            })
+          });
+        }
       }
     );
   }
@@ -374,6 +374,16 @@ class Xsql {
     queryParamsObj.query += ",count(1) as _count ";
   }
 
+  getColumnsForGroupBy(tableName, reqQueryParams, queryParamsObj) {
+    const updatedQueryParams = Object.assign({}, reqQueryParams);
+    if ("_groupbyfields" in updatedQueryParams) {
+      // allows you to group by different fields than you have in the select
+      updatedQueryParams['_fields'] = updatedQueryParams['_groupbyfields'];
+    }
+
+    return this.getColumnsForSelectStmt(tableName, updatedQueryParams, queryParamsObj)
+  }
+
   getColumnsForSelectStmt(tableName, reqQueryParams, queryParamsObj) {
     let table = this.metaDb.tables[tableName];
     let cols = [];
@@ -421,6 +431,16 @@ class Xsql {
     return cols.join(",");
   }
 
+  getColumnNameWithoutAggregationFunctions(rawColumnName) {
+    const AGGREGATION_FUNCTION_REGEX = /^(AVG|BIT_AND|BIT_OR|BIT_XOR|COUNT|COUNTDISTINCT|GROUP_CONCAT|JSON_ARRAYAGG|JSON_OBJECTAGG|MAX|MIN|STD|STDDEV|STDDEV_POP|STDDEV_SAMP|SUM|VAR_POP|VAR_SAMP|VARIANCE)\((.*)\)$/;
+    const aggFuncMatch = rawColumnName.match(AGGREGATION_FUNCTION_REGEX);
+    if (aggFuncMatch && aggFuncMatch.length === 3) {
+      // match will look like (3) ["AVG(timestamp)", "AVG", "timestamp", index: 0, input: "AVG(timestamp)", groups: undefined]
+      return aggFuncMatch[2];
+    }
+    return rawColumnName;
+  }
+
   removeUnknownColumns(inputColumns, tableName) {
     let cols = inputColumns;
     let unknown_cols_in_input = [];
@@ -430,9 +450,11 @@ class Xsql {
     // find unknown fields if any
     for (var j = 0; j < cols.length; ++j) {
       let found = 0;
+      // Used to allow aggregation functions like AVG(timestamp)
+      let columnNameWithoutAggregationClauses = this.getColumnNameWithoutAggregationFunctions(cols[j]);
 
       for (var i = 0; i < tableColumns.length; ++i) {
-        if (tableColumns[i]["column_name"] === cols[j]) {
+        if (tableColumns[i]["column_name"] === columnNameWithoutAggregationClauses) {
           found = 1;
           break;
         }
@@ -764,7 +786,7 @@ class Xsql {
 
   getProcList() {
     let procRoutes = []
-    for (var procName in this.metaDb.routines) {
+    for (let procName in this.metaDb.routines) {
       procRoutes.push(this.metaDb.routines[procName])
     }
     return procRoutes

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "xmysql",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

tests/tests.js

@@ -1178,6 +1178,30 @@ describe("xmysql : tests", function() {
     }
   );
 
+  it(
+    "GET " +
+      apiPrefix +
+      "customers/groupby?_fields=city&_sort=city&_groupbyfields=country should PASS",
+    function(done) {
+      //post to an url with data
+      agent
+        .get(apiPrefix + "customers/groupby?_fields=avg(creditLimit),country,city&_sort=city&_groupbyfields=country,city") //enter url
+        .expect(200) //200 for success 4xx for failure
+        .end(function(err, res) {
+          // Handle /api/v error
+          if (err) {
+            return done(err);
+          }
+
+          //validate response
+          res.body[0]["city"].should.be.equals("Aachen");
+          res.body.length.should.be.equals(95);
+
+          return done();
+        });
+    }
+  );
+
   it(
     "GET " + apiPrefix + "offices/ugroupby?_fields=country should PASS",
     function(done) {