From 4ca418cbeaef8a5c9781cd7cbdd8bb7acb579465 Mon Sep 17 00:00:00 2001 From: Pranav C <61551451+pranavxc@users.noreply.github.com> Date: Mon, 2 Aug 2021 15:29:45 +0530 Subject: [PATCH] feat: Basic formula support Signed-off-by: Pranav C <61551451+pranavxc@users.noreply.github.com> --- .../spreadsheet/components/editColumn.vue | 31 ++++++-- .../components/editColumn/formulaOptions.vue | 26 +++++++ packages/nocodb/package-lock.json | 5 ++ packages/nocodb/package.json | 1 + packages/nocodb/src/__tests__/formula.test.ts | 77 +++++++++++++++++++ .../lib/dataMapper/lib/sql/BaseModelSql.ts | 23 ++++-- .../dataMapper/lib/sql/formulaQueryBuilder.ts | 70 +++++++++++++++++ packages/nocodb/src/plugins/mino/Minio.ts | 2 - 8 files changed, 222 insertions(+), 13 deletions(-) create mode 100644 packages/nc-gui/components/project/spreadsheet/components/editColumn/formulaOptions.vue create mode 100644 packages/nocodb/src/__tests__/formula.test.ts create mode 100644 packages/nocodb/src/lib/dataMapper/lib/sql/formulaQueryBuilder.ts diff --git a/packages/nc-gui/components/project/spreadsheet/components/editColumn.vue b/packages/nc-gui/components/project/spreadsheet/components/editColumn.vue index 3074f832db..c53bba7605 100644 --- a/packages/nc-gui/components/project/spreadsheet/components/editColumn.vue +++ b/packages/nc-gui/components/project/spreadsheet/components/editColumn.vue @@ -328,7 +328,18 @@ @@ -368,6 +379,7 @@ + + diff --git a/packages/nocodb/package-lock.json b/packages/nocodb/package-lock.json index efd8ca6607..8564b3b165 100644 --- a/packages/nocodb/package-lock.json +++ b/packages/nocodb/package-lock.json @@ -10025,6 +10025,11 @@ "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" }, + "jsep": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/jsep/-/jsep-0.4.0.tgz", + "integrity": "sha512-UDkrzhJK8hmgXeGK8WIiecc/cuW4Vnx5nnrRma7yaxK0WXlvZ4VerGrcxPzifd/CA6QdcI1hpXqr22tHKXpcQA==" + }, "jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", diff --git a/packages/nocodb/package.json b/packages/nocodb/package.json index 21ab9388ba..9524e69c2b 100644 --- a/packages/nocodb/package.json +++ b/packages/nocodb/package.json @@ -128,6 +128,7 @@ "inflection": "^1.12.0", "is-docker": "^2.2.1", "js-beautify": "^1.11.0", + "jsep": "^0.4.0", "json2csv": "^5.0.6", "jsonfile": "^6.1.0", "jsonwebtoken": "^8.5.1", diff --git a/packages/nocodb/src/__tests__/formula.test.ts b/packages/nocodb/src/__tests__/formula.test.ts new file mode 100644 index 0000000000..73922a2ba8 --- /dev/null +++ b/packages/nocodb/src/__tests__/formula.test.ts @@ -0,0 +1,77 @@ +import {expect} from 'chai'; +import 'mocha'; +import knex from '../lib/dataMapper/lib/sql/CustomKnex'; +import formulaQueryBuilder from "../lib/dataMapper/lib/sql/formulaQueryBuilder"; + +process.env.TEST = 'test'; + +describe('{Auth, CRUD, HasMany, Belongs} Tests', () => { + + let knexMysqlRef; + let knexPgRef; + let knexMssqlRef; + let knexSqliteRef; + + + // Called once before any of the tests in this block begin. + before(function (done) { + knexMysqlRef = knex({client:'mysql2'}) + knexMssqlRef = knex({client:'mssql'}) + knexPgRef = knex({client:'pg'}) + knexSqliteRef = knex({client:'sqlite3'}) + done() + }); + + + after((done) => { + done(); + }); + + + describe('Formulas', function () { + + it('Simple formula', function (done) { + expect(formulaQueryBuilder("concat(city, ' _ ',city_id+country_id)", 'a',knexMysqlRef).toQuery()).eq('concat(`city`,\' _ \',`city_id` + `country_id`) as a') + expect(formulaQueryBuilder("concat(city, ' _ ',city_id+country_id)", 'a',knexPgRef).toQuery()).eq('concat("city",\' _ \',"city_id" + "country_id") as a') + expect(formulaQueryBuilder("concat(city, ' _ ',city_id+country_id)", 'a',knexMssqlRef).toQuery()).eq('concat([city],\' _ \',[city_id] + [country_id]) as a') + expect(formulaQueryBuilder("concat(city, ' _ ',city_id+country_id)", 'a',knexSqliteRef).toQuery()).eq('`city` || (\' _ \' || (`city_id` + `country_id`)) as a') + done() + }); + it('Addition', function (done) { + expect(formulaQueryBuilder("ADD(city_id,country_id,2,3,4,5,4)", 'a',knexMysqlRef).toQuery()).eq('`city_id` + (`country_id` + (2 + (3 + (4 + (5 + 4))))) as a') + expect(formulaQueryBuilder("ADD(city_id,country_id,2,3,4,5,4)", 'a',knexPgRef).toQuery()).eq('"city_id" + ("country_id" + (2 + (3 + (4 + (5 + 4))))) as a') + expect(formulaQueryBuilder("ADD(city_id,country_id,2,3,4,5,4)", 'a',knexMssqlRef).toQuery()).eq('[city_id] + ([country_id] + (2 + (3 + (4 + (5 + 4))))) as a') + expect(formulaQueryBuilder("ADD(city_id,country_id,2,3,4,5,4)", 'a',knexSqliteRef).toQuery()).eq('`city_id` + (`country_id` + (2 + (3 + (4 + (5 + 4))))) as a') + done() + }); + it('Average', function (done) { + expect(formulaQueryBuilder("AVG(city_id,country_id,2,3,4,5,4)", 'a',knexMysqlRef).toQuery()).eq('(`city_id` + (`country_id` + (2 + (3 + (4 + (5 + 4)))))) / 7 as a') + expect(formulaQueryBuilder("AVG(city_id,country_id,2,3,4,5,4)", 'a',knexPgRef).toQuery()).eq('("city_id" + ("country_id" + (2 + (3 + (4 + (5 + 4)))))) / 7 as a') + expect(formulaQueryBuilder("AVG(city_id,country_id,2,3,4,5,4)", 'a',knexMssqlRef).toQuery()).eq('([city_id] + ([country_id] + (2 + (3 + (4 + (5 + 4)))))) / 7 as a') + expect(formulaQueryBuilder("AVG(city_id,country_id,2,3,4,5,4)", 'a',knexSqliteRef).toQuery()).eq('(`city_id` + (`country_id` + (2 + (3 + (4 + (5 + 4)))))) / 7 as a') + done() + }); + }); + +});/** + * @copyright Copyright (c) 2021, Xgene Cloud Ltd + * + * @author Naveen MR + * @author Pranav C Balan + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + *"concat(city, ' _ ',city_id+country_id)", 'a' + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ diff --git a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts index 92ded9ebd7..edcc0dd22c 100644 --- a/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts +++ b/packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts @@ -3,6 +3,7 @@ import _ from 'lodash'; import Validator from 'validator'; import BaseModel, {XcFilter, XcFilterWithAlias} from '../BaseModel'; +import formulaQueryBuilder from "./formulaQueryBuilder"; /** @@ -661,6 +662,7 @@ class BaseModelSql extends BaseModel { try { return await this._run( this.$db.select(this.selectQuery('*')) + .select(...this.selectFormulas) .conditionGraph(args?.conditionGraph) .where(this._wherePk(id)).first() ) || {}; @@ -715,13 +717,10 @@ class BaseModelSql extends BaseModel { const {fields, where, limit, offset, sort, condition, conditionGraph = null} = this._getListArgs(args); - // if (fields === '*') { - // fields = `${this.tn}.*`; - // } - const query = this.$db // .select(...fields.split(',')) .select(this.selectQuery(fields)) + .select(...this.selectFormulas) .xwhere(where, this.selectQuery('')) .condition(condition, this.selectQuery('')) .conditionGraph(conditionGraph); @@ -1742,10 +1741,11 @@ class BaseModelSql extends BaseModel { : '*'; } + // @ts-ignore public selectQuery(fields) { const fieldsArr = fields.split(','); - return this.columns?.reduce((selectObj, col) => { + const selectObj = this.columns?.reduce((selectObj, col) => { if ( !fields || fieldsArr.includes('*') @@ -1757,6 +1757,10 @@ class BaseModelSql extends BaseModel { } return selectObj; }, {}) || '*'; + + + return selectObj; + } // @ts-ignore @@ -1857,6 +1861,15 @@ class BaseModelSql extends BaseModel { } } + protected get selectFormulas() { + return (this.virtualColumns || [])?.reduce((arr, v) => { + if (v.formula?.value) { + arr.push(formulaQueryBuilder(v.formula?.value, v._cn, this.dbDriver)) + } + return arr; + }, []) + } + } diff --git a/packages/nocodb/src/lib/dataMapper/lib/sql/formulaQueryBuilder.ts b/packages/nocodb/src/lib/dataMapper/lib/sql/formulaQueryBuilder.ts new file mode 100644 index 0000000000..0fa0308db2 --- /dev/null +++ b/packages/nocodb/src/lib/dataMapper/lib/sql/formulaQueryBuilder.ts @@ -0,0 +1,70 @@ +import jsep from 'jsep'; + + +// todo: switch function based on database + +export default function formulaQueryBuilder(str, alias, knex) { + const fn = (pt, a?, nestedBinary?) => { + const colAlias = a ? ` as ${a}` : ''; + if (pt.type === 'CallExpression') { + switch (pt.callee.name) { + case 'ADD': + case 'SUM': + if (pt.arguments.length > 1) { + return fn({ + type: 'BinaryExpression', + operator: '+', + left: pt.arguments[0], + right: {...pt, arguments: pt.arguments.slice(1)} + }, a, nestedBinary) + } else { + return fn(pt.arguments[0], a, nestedBinary) + } + break; + case 'AVG': + if (pt.arguments.length > 1) { + return fn({ + type: 'BinaryExpression', + operator: '/', + left: {...pt, callee: {name: 'SUM'}}, + right: {type: 'Literal', value: pt.arguments.length} + }, a, nestedBinary) + } else { + return fn(pt.arguments[0], a, nestedBinary) + } + break; + case 'concat': + case 'CONCAT': + if (knex.clientType() === 'sqlite3') { + if (pt.arguments.length > 1) { + return fn({ + type: 'BinaryExpression', + operator: '||', + left: pt.arguments[0], + right: {...pt, arguments: pt.arguments.slice(1)} + }, a, nestedBinary) + } else { + return fn(pt.arguments[0], a, nestedBinary) + } + } + break; + } + + return knex.raw(`${pt.callee.name}(${pt.arguments.map(arg => fn(arg).toQuery()).join()})${colAlias}`) + } else if (pt.type === 'Literal') { + return knex.raw(`?${colAlias}`, [pt.value]); + } else if (pt.type === 'Identifier') { + return knex.raw(`??${colAlias}`, [pt.name]); + } else if (pt.type === 'BinaryExpression') { + const query = knex.raw(`${fn(pt.left, null, true).toQuery()} ${pt.operator} ${fn(pt.right, null, true).toQuery()}${colAlias}`) + if (nestedBinary) { + query.wrap('(', ')') + } + return query; + } + }; + return fn(jsep(str), alias) +} + + + diff --git a/packages/nocodb/src/plugins/mino/Minio.ts b/packages/nocodb/src/plugins/mino/Minio.ts index de142e6bcf..687a6fc50f 100644 --- a/packages/nocodb/src/plugins/mino/Minio.ts +++ b/packages/nocodb/src/plugins/mino/Minio.ts @@ -58,8 +58,6 @@ export default class Minio implements IStorageAdapter { } public async init(): Promise { - - // todo: update in ui(checkbox and number field) this.input.port = +this.input.port || 9000; this.input.useSSL = this.input.useSSL ==='true';