Browse Source

feat: Basic formula support

Signed-off-by: Pranav C <61551451+pranavxc@users.noreply.github.com>
pull/448/head
Pranav C 3 years ago committed by Pranav C
parent
commit
4ca418cbea
  1. 31
      packages/nc-gui/components/project/spreadsheet/components/editColumn.vue
  2. 26
      packages/nc-gui/components/project/spreadsheet/components/editColumn/formulaOptions.vue
  3. 5
      packages/nocodb/package-lock.json
  4. 1
      packages/nocodb/package.json
  5. 77
      packages/nocodb/src/__tests__/formula.test.ts
  6. 23
      packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts
  7. 70
      packages/nocodb/src/lib/dataMapper/lib/sql/formulaQueryBuilder.ts
  8. 2
      packages/nocodb/src/plugins/mino/Minio.ts

31
packages/nc-gui/components/project/spreadsheet/components/editColumn.vue

@ -328,7 +328,18 @@
</template> </template>
<template v-else> <template v-else>
<v-col cols12> <v-col cols12>
<v-autocomplete <formula-options
ref="formula"
:column="newColumn"
:nodes="nodes"
:meta="meta"
:is-s-q-lite="isSQLite"
:alias="newColumn.cn"
:is-m-s-s-q-l="isMSSQL"
v-on="$listeners"
/>
<!-- <v-autocomplete
label="Formula" label="Formula"
hide-details hide-details
class="caption formula-type" class="caption formula-type"
@ -337,9 +348,9 @@
:items="formulas" :items="formulas"
> >
<template #item="{item}"> <template #item="{item}">
<span class="green--text text--darken-2 caption font-weight-regular">{{ item }}</span> <span class="green&#45;&#45;text text&#45;&#45;darken-2 caption font-weight-regular">{{ item }}</span>
</template> </template>
</v-autocomplete> </v-autocomplete>-->
</v-col> </v-col>
</template> </template>
</template> </template>
@ -368,6 +379,7 @@
</template> </template>
<script> <script>
import FormulaOptions from '@/components/project/spreadsheet/components/editColumn/formulaOptions'
import LookupOptions from '@/components/project/spreadsheet/components/editColumn/lookupOptions' import LookupOptions from '@/components/project/spreadsheet/components/editColumn/lookupOptions'
import { uiTypes } from '@/components/project/spreadsheet/helpers/uiTypes' import { uiTypes } from '@/components/project/spreadsheet/helpers/uiTypes'
import CustomSelectOptions from '@/components/project/spreadsheet/components/editColumn/customSelectOptions' import CustomSelectOptions from '@/components/project/spreadsheet/components/editColumn/customSelectOptions'
@ -379,7 +391,14 @@ import { MssqlUi } from '@/helpers/MssqlUi'
export default { export default {
name: 'EditColumn', name: 'EditColumn',
components: { LookupOptions, LinkedToAnotherOptions, DlgLabelSubmitCancel, RelationOptions, CustomSelectOptions }, components: {
FormulaOptions,
LookupOptions,
LinkedToAnotherOptions,
DlgLabelSubmitCancel,
RelationOptions,
CustomSelectOptions
},
props: { props: {
nodes: Object, nodes: Object,
sqlUi: [Object, Function], sqlUi: [Object, Function],
@ -393,9 +412,9 @@ export default {
valid: false, valid: false,
relationDeleteDlg: false, relationDeleteDlg: false,
newColumn: {}, newColumn: {},
uiTypes, uiTypes
// dataTypes: [], // dataTypes: [],
formulas: ['AVERAGE()', 'COUNT()', 'COUNTA()', 'COUNTALL()', 'SUM()', 'MIN()', 'MAX()', 'AND()', 'OR()', 'TRUE()', 'FALSE()', 'NOT()', 'XOR()', 'ISERROR()', 'IF()', 'LEN()', 'MID()', 'LEFT()', 'RIGHT()', 'FIND()', 'CONCATENATE()', 'T()', 'VALUE()', 'ARRAYJOIN()', 'ARRAYUNIQUE()', 'ARRAYCOMPACT()', 'ARRAYFLATTEN()', 'ROUND()', 'ROUNDUP()', 'ROUNDDOWN()', 'INT()', 'EVEN()', 'ODD()', 'MOD()', 'LOG()', 'EXP()', 'POWER()', 'SQRT()', 'CEILING()', 'FLOOR()', 'ABS()', 'RECORD_ID()', 'CREATED_TIME()', 'ERROR()', 'BLANK()', 'YEAR()', 'MONTH()', 'DAY()', 'HOUR()', 'MINUTE()', 'SECOND()', 'TODAY()', 'NOW()', 'WORKDAY()', 'DATETIME_PARSE()', 'DATETIME_FORMAT()', 'SET_LOCALE()', 'SET_TIMEZONE()', 'DATESTR()', 'TIMESTR()', 'TONOW()', 'FROMNOW()', 'DATEADD()', 'WEEKDAY()', 'WEEKNUM()', 'DATETIME_DIFF()', 'WORKDAY_DIFF()', 'IS_BEFORE()', 'IS_SAME()', 'IS_AFTER()', 'REPLACE()', 'REPT()', 'LOWER()', 'UPPER()', 'TRIM()', 'SUBSTITUTE()', 'SEARCH()', 'SWITCH()', 'LAST_MODIFIED_TIME()', 'ENCODE_URL_COMPONENT()', 'REGEX_EXTRACT()', 'REGEX_MATCH()', 'REGEX_REPLACE()'] // formulas: ['AVERAGE()', 'COUNT()', 'COUNTA()', 'COUNTALL()', 'SUM()', 'MIN()', 'MAX()', 'AND()', 'OR()', 'TRUE()', 'FALSE()', 'NOT()', 'XOR()', 'ISERROR()', 'IF()', 'LEN()', 'MID()', 'LEFT()', 'RIGHT()', 'FIND()', 'CONCATENATE()', 'T()', 'VALUE()', 'ARRAYJOIN()', 'ARRAYUNIQUE()', 'ARRAYCOMPACT()', 'ARRAYFLATTEN()', 'ROUND()', 'ROUNDUP()', 'ROUNDDOWN()', 'INT()', 'EVEN()', 'ODD()', 'MOD()', 'LOG()', 'EXP()', 'POWER()', 'SQRT()', 'CEILING()', 'FLOOR()', 'ABS()', 'RECORD_ID()', 'CREATED_TIME()', 'ERROR()', 'BLANK()', 'YEAR()', 'MONTH()', 'DAY()', 'HOUR()', 'MINUTE()', 'SECOND()', 'TODAY()', 'NOW()', 'WORKDAY()', 'DATETIME_PARSE()', 'DATETIME_FORMAT()', 'SET_LOCALE()', 'SET_TIMEZONE()', 'DATESTR()', 'TIMESTR()', 'TONOW()', 'FROMNOW()', 'DATEADD()', 'WEEKDAY()', 'WEEKNUM()', 'DATETIME_DIFF()', 'WORKDAY_DIFF()', 'IS_BEFORE()', 'IS_SAME()', 'IS_AFTER()', 'REPLACE()', 'REPT()', 'LOWER()', 'UPPER()', 'TRIM()', 'SUBSTITUTE()', 'SEARCH()', 'SWITCH()', 'LAST_MODIFIED_TIME()', 'ENCODE_URL_COMPONENT()', 'REGEX_EXTRACT()', 'REGEX_MATCH()', 'REGEX_REPLACE()']
}), }),
computed: { computed: {
isEditDisabled() { isEditDisabled() {

26
packages/nc-gui/components/project/spreadsheet/components/editColumn/formulaOptions.vue

@ -0,0 +1,26 @@
<template>
<v-text-field
v-model="formula.value"
outlined
class="caption"
hide-details="auto"
label="Formula"
persistent-hint
hint="Available formulas are ADD, AVG, CONCAT, +, -, /"
/>
</template>
<script>
export default {
name: 'FormulaOptions',
props: ['nodes', 'column', 'meta', 'isSQLite', 'alias'],
data: () => ({
formula: {},
formulas: ['AVERAGE()', 'COUNT()', 'COUNTA()', 'COUNTALL()', 'SUM()', 'MIN()', 'MAX()', 'AND()', 'OR()', 'TRUE()', 'FALSE()', 'NOT()', 'XOR()', 'ISERROR()', 'IF()', 'LEN()', 'MID()', 'LEFT()', 'RIGHT()', 'FIND()', 'CONCATENATE()', 'T()', 'VALUE()', 'ARRAYJOIN()', 'ARRAYUNIQUE()', 'ARRAYCOMPACT()', 'ARRAYFLATTEN()', 'ROUND()', 'ROUNDUP()', 'ROUNDDOWN()', 'INT()', 'EVEN()', 'ODD()', 'MOD()', 'LOG()', 'EXP()', 'POWER()', 'SQRT()', 'CEILING()', 'FLOOR()', 'ABS()', 'RECORD_ID()', 'CREATED_TIME()', 'ERROR()', 'BLANK()', 'YEAR()', 'MONTH()', 'DAY()', 'HOUR()', 'MINUTE()', 'SECOND()', 'TODAY()', 'NOW()', 'WORKDAY()', 'DATETIME_PARSE()', 'DATETIME_FORMAT()', 'SET_LOCALE()', 'SET_TIMEZONE()', 'DATESTR()', 'TIMESTR()', 'TONOW()', 'FROMNOW()', 'DATEADD()', 'WEEKDAY()', 'WEEKNUM()', 'DATETIME_DIFF()', 'WORKDAY_DIFF()', 'IS_BEFORE()', 'IS_SAME()', 'IS_AFTER()', 'REPLACE()', 'REPT()', 'LOWER()', 'UPPER()', 'TRIM()', 'SUBSTITUTE()', 'SEARCH()', 'SWITCH()', 'LAST_MODIFIED_TIME()', 'ENCODE_URL_COMPONENT()', 'REGEX_EXTRACT()', 'REGEX_MATCH()', 'REGEX_REPLACE()']
})
}
</script>
<style scoped>
</style>

5
packages/nocodb/package-lock.json generated

@ -10025,6 +10025,11 @@
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
}, },
"jsep": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/jsep/-/jsep-0.4.0.tgz",
"integrity": "sha512-UDkrzhJK8hmgXeGK8WIiecc/cuW4Vnx5nnrRma7yaxK0WXlvZ4VerGrcxPzifd/CA6QdcI1hpXqr22tHKXpcQA=="
},
"jsesc": { "jsesc": {
"version": "2.5.2", "version": "2.5.2",
"resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz",

1
packages/nocodb/package.json

@ -128,6 +128,7 @@
"inflection": "^1.12.0", "inflection": "^1.12.0",
"is-docker": "^2.2.1", "is-docker": "^2.2.1",
"js-beautify": "^1.11.0", "js-beautify": "^1.11.0",
"jsep": "^0.4.0",
"json2csv": "^5.0.6", "json2csv": "^5.0.6",
"jsonfile": "^6.1.0", "jsonfile": "^6.1.0",
"jsonwebtoken": "^8.5.1", "jsonwebtoken": "^8.5.1",

77
packages/nocodb/src/__tests__/formula.test.ts

@ -0,0 +1,77 @@
import {expect} from 'chai';
import 'mocha';
import knex from '../lib/dataMapper/lib/sql/CustomKnex';
import formulaQueryBuilder from "../lib/dataMapper/lib/sql/formulaQueryBuilder";
process.env.TEST = 'test';
describe('{Auth, CRUD, HasMany, Belongs} Tests', () => {
let knexMysqlRef;
let knexPgRef;
let knexMssqlRef;
let knexSqliteRef;
// Called once before any of the tests in this block begin.
before(function (done) {
knexMysqlRef = knex({client:'mysql2'})
knexMssqlRef = knex({client:'mssql'})
knexPgRef = knex({client:'pg'})
knexSqliteRef = knex({client:'sqlite3'})
done()
});
after((done) => {
done();
});
describe('Formulas', function () {
it('Simple formula', function (done) {
expect(formulaQueryBuilder("concat(city, ' _ ',city_id+country_id)", 'a',knexMysqlRef).toQuery()).eq('concat(`city`,\' _ \',`city_id` + `country_id`) as a')
expect(formulaQueryBuilder("concat(city, ' _ ',city_id+country_id)", 'a',knexPgRef).toQuery()).eq('concat("city",\' _ \',"city_id" + "country_id") as a')
expect(formulaQueryBuilder("concat(city, ' _ ',city_id+country_id)", 'a',knexMssqlRef).toQuery()).eq('concat([city],\' _ \',[city_id] + [country_id]) as a')
expect(formulaQueryBuilder("concat(city, ' _ ',city_id+country_id)", 'a',knexSqliteRef).toQuery()).eq('`city` || (\' _ \' || (`city_id` + `country_id`)) as a')
done()
});
it('Addition', function (done) {
expect(formulaQueryBuilder("ADD(city_id,country_id,2,3,4,5,4)", 'a',knexMysqlRef).toQuery()).eq('`city_id` + (`country_id` + (2 + (3 + (4 + (5 + 4))))) as a')
expect(formulaQueryBuilder("ADD(city_id,country_id,2,3,4,5,4)", 'a',knexPgRef).toQuery()).eq('"city_id" + ("country_id" + (2 + (3 + (4 + (5 + 4))))) as a')
expect(formulaQueryBuilder("ADD(city_id,country_id,2,3,4,5,4)", 'a',knexMssqlRef).toQuery()).eq('[city_id] + ([country_id] + (2 + (3 + (4 + (5 + 4))))) as a')
expect(formulaQueryBuilder("ADD(city_id,country_id,2,3,4,5,4)", 'a',knexSqliteRef).toQuery()).eq('`city_id` + (`country_id` + (2 + (3 + (4 + (5 + 4))))) as a')
done()
});
it('Average', function (done) {
expect(formulaQueryBuilder("AVG(city_id,country_id,2,3,4,5,4)", 'a',knexMysqlRef).toQuery()).eq('(`city_id` + (`country_id` + (2 + (3 + (4 + (5 + 4)))))) / 7 as a')
expect(formulaQueryBuilder("AVG(city_id,country_id,2,3,4,5,4)", 'a',knexPgRef).toQuery()).eq('("city_id" + ("country_id" + (2 + (3 + (4 + (5 + 4)))))) / 7 as a')
expect(formulaQueryBuilder("AVG(city_id,country_id,2,3,4,5,4)", 'a',knexMssqlRef).toQuery()).eq('([city_id] + ([country_id] + (2 + (3 + (4 + (5 + 4)))))) / 7 as a')
expect(formulaQueryBuilder("AVG(city_id,country_id,2,3,4,5,4)", 'a',knexSqliteRef).toQuery()).eq('(`city_id` + (`country_id` + (2 + (3 + (4 + (5 + 4)))))) / 7 as a')
done()
});
});
});/**
* @copyright Copyright (c) 2021, Xgene Cloud Ltd
*
* @author Naveen MR <oof1lab@gmail.com>
* @author Pranav C Balan <pranavxc@gmail.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*"concat(city, ' _ ',city_id+country_id)", 'a'
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/

23
packages/nocodb/src/lib/dataMapper/lib/sql/BaseModelSql.ts

@ -3,6 +3,7 @@ import _ from 'lodash';
import Validator from 'validator'; import Validator from 'validator';
import BaseModel, {XcFilter, XcFilterWithAlias} from '../BaseModel'; import BaseModel, {XcFilter, XcFilterWithAlias} from '../BaseModel';
import formulaQueryBuilder from "./formulaQueryBuilder";
/** /**
@ -661,6 +662,7 @@ class BaseModelSql extends BaseModel {
try { try {
return await this._run( return await this._run(
this.$db.select(this.selectQuery('*')) this.$db.select(this.selectQuery('*'))
.select(...this.selectFormulas)
.conditionGraph(args?.conditionGraph) .conditionGraph(args?.conditionGraph)
.where(this._wherePk(id)).first() .where(this._wherePk(id)).first()
) || {}; ) || {};
@ -715,13 +717,10 @@ class BaseModelSql extends BaseModel {
const {fields, where, limit, offset, sort, condition, conditionGraph = null} = this._getListArgs(args); const {fields, where, limit, offset, sort, condition, conditionGraph = null} = this._getListArgs(args);
// if (fields === '*') {
// fields = `${this.tn}.*`;
// }
const query = this.$db const query = this.$db
// .select(...fields.split(',')) // .select(...fields.split(','))
.select(this.selectQuery(fields)) .select(this.selectQuery(fields))
.select(...this.selectFormulas)
.xwhere(where, this.selectQuery('')) .xwhere(where, this.selectQuery(''))
.condition(condition, this.selectQuery('')) .condition(condition, this.selectQuery(''))
.conditionGraph(conditionGraph); .conditionGraph(conditionGraph);
@ -1742,10 +1741,11 @@ class BaseModelSql extends BaseModel {
: '*'; : '*';
} }
// @ts-ignore // @ts-ignore
public selectQuery(fields) { public selectQuery(fields) {
const fieldsArr = fields.split(','); const fieldsArr = fields.split(',');
return this.columns?.reduce((selectObj, col) => { const selectObj = this.columns?.reduce((selectObj, col) => {
if ( if (
!fields !fields
|| fieldsArr.includes('*') || fieldsArr.includes('*')
@ -1757,6 +1757,10 @@ class BaseModelSql extends BaseModel {
} }
return selectObj; return selectObj;
}, {}) || '*'; }, {}) || '*';
return selectObj;
} }
// @ts-ignore // @ts-ignore
@ -1857,6 +1861,15 @@ class BaseModelSql extends BaseModel {
} }
} }
protected get selectFormulas() {
return (this.virtualColumns || [])?.reduce((arr, v) => {
if (v.formula?.value) {
arr.push(formulaQueryBuilder(v.formula?.value, v._cn, this.dbDriver))
}
return arr;
}, [])
}
} }

70
packages/nocodb/src/lib/dataMapper/lib/sql/formulaQueryBuilder.ts

@ -0,0 +1,70 @@
import jsep from 'jsep';
// todo: switch function based on database
export default function formulaQueryBuilder(str, alias, knex) {
const fn = (pt, a?, nestedBinary?) => {
const colAlias = a ? ` as ${a}` : '';
if (pt.type === 'CallExpression') {
switch (pt.callee.name) {
case 'ADD':
case 'SUM':
if (pt.arguments.length > 1) {
return fn({
type: 'BinaryExpression',
operator: '+',
left: pt.arguments[0],
right: {...pt, arguments: pt.arguments.slice(1)}
}, a, nestedBinary)
} else {
return fn(pt.arguments[0], a, nestedBinary)
}
break;
case 'AVG':
if (pt.arguments.length > 1) {
return fn({
type: 'BinaryExpression',
operator: '/',
left: {...pt, callee: {name: 'SUM'}},
right: {type: 'Literal', value: pt.arguments.length}
}, a, nestedBinary)
} else {
return fn(pt.arguments[0], a, nestedBinary)
}
break;
case 'concat':
case 'CONCAT':
if (knex.clientType() === 'sqlite3') {
if (pt.arguments.length > 1) {
return fn({
type: 'BinaryExpression',
operator: '||',
left: pt.arguments[0],
right: {...pt, arguments: pt.arguments.slice(1)}
}, a, nestedBinary)
} else {
return fn(pt.arguments[0], a, nestedBinary)
}
}
break;
}
return knex.raw(`${pt.callee.name}(${pt.arguments.map(arg => fn(arg).toQuery()).join()})${colAlias}`)
} else if (pt.type === 'Literal') {
return knex.raw(`?${colAlias}`, [pt.value]);
} else if (pt.type === 'Identifier') {
return knex.raw(`??${colAlias}`, [pt.name]);
} else if (pt.type === 'BinaryExpression') {
const query = knex.raw(`${fn(pt.left, null, true).toQuery()} ${pt.operator} ${fn(pt.right, null, true).toQuery()}${colAlias}`)
if (nestedBinary) {
query.wrap('(', ')')
}
return query;
}
};
return fn(jsep(str), alias)
}

2
packages/nocodb/src/plugins/mino/Minio.ts

@ -58,8 +58,6 @@ export default class Minio implements IStorageAdapter {
} }
public async init(): Promise<any> { public async init(): Promise<any> {
// todo: update in ui(checkbox and number field) // todo: update in ui(checkbox and number field)
this.input.port = +this.input.port || 9000; this.input.port = +this.input.port || 9000;
this.input.useSSL = this.input.useSSL ==='true'; this.input.useSSL = this.input.useSSL ==='true';

Loading…
Cancel
Save