From a8bf1878ebe566ca35b84900485e67125d70a2ac Mon Sep 17 00:00:00 2001
From: Ramesh Mane <101566080+rameshmane7218@users.noreply.github.com>
Date: Thu, 21 Dec 2023 13:51:29 +0530
Subject: [PATCH 001/262] fix: checkbox alignment issue in grid cell
---
packages/nc-gui/components/cell/Checkbox.vue | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/packages/nc-gui/components/cell/Checkbox.vue b/packages/nc-gui/components/cell/Checkbox.vue
index be70d8d314..5bf86c4058 100644
--- a/packages/nc-gui/components/cell/Checkbox.vue
+++ b/packages/nc-gui/components/cell/Checkbox.vue
@@ -42,6 +42,8 @@ const readOnly = inject(ReadonlyInj)
const isExpandedFormOpen = inject(IsExpandedFormOpenInj, ref(false))
+const rowHeight = inject(RowHeightInj, ref())
+
const checkboxMeta = computed(() => {
return {
icon: {
@@ -89,11 +91,18 @@ useSelectedCellKeyupListener(active, (e) => {
'nc-cell-hover-show': !vModel && !readOnly,
'opacity-0': readOnly && !vModel,
}"
+ :style="{
+ height: isForm || isExpandedFormOpen || isGallery ? undefined : `max(${(rowHeight || 1) * 1.8}rem, 41px)`,
+ }"
@click="onClick(false, $event)"
>
From 262ef725bccb2d985ebf618dc29a88ed4aceb942 Mon Sep 17 00:00:00 2001
From: Ramesh Mane <101566080+rameshmane7218@users.noreply.github.com>
Date: Thu, 21 Dec 2023 13:54:47 +0530
Subject: [PATCH 002/262] fix: barcode cell alignment issue #7253
---
.../components/virtual-cell/barcode/Barcode.vue | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/packages/nc-gui/components/virtual-cell/barcode/Barcode.vue b/packages/nc-gui/components/virtual-cell/barcode/Barcode.vue
index 2c27509ecf..52ddbdac27 100644
--- a/packages/nc-gui/components/virtual-cell/barcode/Barcode.vue
+++ b/packages/nc-gui/components/virtual-cell/barcode/Barcode.vue
@@ -55,7 +55,7 @@ const rowHeight = inject(RowHeightInj, ref(undefined))
@@ -98,3 +98,11 @@ const rowHeight = inject(RowHeightInj, ref(undefined))
{{ $t('msg.warning.nonEditableFields.barcodeFieldsCannotBeDirectlyChanged') }}
+
+
From e8a316a9cd21d22e2384cce6f7c5621a0615a479 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:57 +0000
Subject: [PATCH 003/262] refactor: add proper typing
---
packages/nc-gui/utils/formulaUtils.ts | 30 ++++++++++++++++++++-------
1 file changed, 23 insertions(+), 7 deletions(-)
diff --git a/packages/nc-gui/utils/formulaUtils.ts b/packages/nc-gui/utils/formulaUtils.ts
index 8449d27da7..a6df066f23 100644
--- a/packages/nc-gui/utils/formulaUtils.ts
+++ b/packages/nc-gui/utils/formulaUtils.ts
@@ -1,14 +1,30 @@
import type { Input as AntInput } from 'ant-design-vue'
-const formulaTypes = {
- NUMERIC: 'numeric',
- STRING: 'string',
- DATE: 'date',
- LOGICAL: 'logical',
- COND_EXP: 'conditional_expression',
+enum formulaTypes {
+ NUMERIC = 'numeric',
+ STRING = 'string',
+ DATE = 'date',
+ LOGICAL = 'logical',
+ COND_EXP = 'conditional_expression',
}
-const formulas: Record = {
+interface FormulaMeta {
+ type?: string
+ validation?: {
+ args?: {
+ min?: number
+ max?: number
+ rqd?: number
+ }
+ }
+ description?: string
+ syntax?: string
+ examples?: string[]
+ returnType?: ((args: any[]) => formulaTypes) | formulaTypes
+}
+
+const formulas: Record = {
+
AVG: {
type: formulaTypes.NUMERIC,
validation: {
From 77941ebef119a2a65528c0597e52a46dfc9b6720 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:57 +0000
Subject: [PATCH 004/262] feat: define function return types - WIP
---
packages/nc-gui/utils/formulaUtils.ts | 60 +++++++++++++++++++++++++++
1 file changed, 60 insertions(+)
diff --git a/packages/nc-gui/utils/formulaUtils.ts b/packages/nc-gui/utils/formulaUtils.ts
index a6df066f23..30a8625795 100644
--- a/packages/nc-gui/utils/formulaUtils.ts
+++ b/packages/nc-gui/utils/formulaUtils.ts
@@ -6,6 +6,8 @@ enum formulaTypes {
DATE = 'date',
LOGICAL = 'logical',
COND_EXP = 'conditional_expression',
+ NULL = 'null',
+ BOOLEAN = 'boolean',
}
interface FormulaMeta {
@@ -35,6 +37,7 @@ const formulas: Record = {
description: 'Average of input parameters',
syntax: 'AVG(value1, [value2, ...])',
examples: ['AVG(10, 5) => 7.5', 'AVG({column1}, {column2})', 'AVG({column1}, {column2}, {column3})'],
+ returnType: formulaTypes.NUMERIC,
},
ADD: {
type: formulaTypes.NUMERIC,
@@ -46,6 +49,7 @@ const formulas: Record = {
description: 'Sum of input parameters',
syntax: 'ADD(value1, [value2, ...])',
examples: ['ADD(5, 5) => 10', 'ADD({column1}, {column2})', 'ADD({column1}, {column2}, {column3})'],
+ returnType: formulaTypes.NUMERIC,
},
DATEADD: {
type: formulaTypes.DATE,
@@ -66,6 +70,7 @@ const formulas: Record = {
'DATEADD({column1}, 2, "year")',
'DATEADD({column1}, -2, "year")',
],
+ returnType: formulaTypes.DATE,
},
DATETIME_DIFF: {
type: formulaTypes.DATE,
@@ -89,6 +94,7 @@ const formulas: Record = {
'DATEDIFF({column1}, {column2}, "days")',
'DATEDIFF({column1}, {column2}, "d")',
],
+ returnType: formulaTypes.NUMERIC,
},
AND: {
type: formulaTypes.COND_EXP,
@@ -100,6 +106,7 @@ const formulas: Record = {
description: 'TRUE if all expr evaluate to TRUE',
syntax: 'AND(expr1, [expr2, ...])',
examples: ['AND(5 > 2, 5 < 10) => 1', 'AND({column1} > 2, {column2} < 10)'],
+ returnType: formulaTypes.COND_EXP,
},
OR: {
type: formulaTypes.COND_EXP,
@@ -111,6 +118,7 @@ const formulas: Record = {
description: 'TRUE if at least one expr evaluates to TRUE',
syntax: 'OR(expr1, [expr2, ...])',
examples: ['OR(5 > 2, 5 < 10) => 1', 'OR({column1} > 2, {column2} < 10)'],
+ returnType: formulaTypes.COND_EXP,
},
CONCAT: {
type: formulaTypes.STRING,
@@ -122,6 +130,7 @@ const formulas: Record = {
description: 'Concatenated string of input parameters',
syntax: 'CONCAT(str1, [str2, ...])',
examples: ['CONCAT("AA", "BB", "CC") => "AABBCC"', 'CONCAT({column1}, {column2}, {column3})'],
+ returnType: formulaTypes.STRING,
},
TRIM: {
type: formulaTypes.STRING,
@@ -133,6 +142,7 @@ const formulas: Record = {
description: 'Remove trailing and leading whitespaces from input parameter',
syntax: 'TRIM(str)',
examples: ['TRIM(" HELLO WORLD ") => "HELLO WORLD"', 'TRIM({column1})'],
+ returnType: formulaTypes.STRING,
},
UPPER: {
type: formulaTypes.STRING,
@@ -144,6 +154,7 @@ const formulas: Record = {
description: 'Upper case converted string of input parameter',
syntax: 'UPPER(str)',
examples: ['UPPER("nocodb") => "NOCODB"', 'UPPER({column1})'],
+ returnType: formulaTypes.STRING,
},
LOWER: {
type: formulaTypes.STRING,
@@ -155,6 +166,7 @@ const formulas: Record = {
description: 'Lower case converted string of input parameter',
syntax: 'LOWER(str)',
examples: ['LOWER("NOCODB") => "nocodb"', 'LOWER({column1})'],
+ returnType: formulaTypes.STRING,
},
LEN: {
type: formulaTypes.STRING,
@@ -166,6 +178,7 @@ const formulas: Record = {
description: 'Input parameter character length',
syntax: 'LEN(value)',
examples: ['LEN("NocoDB") => 6', 'LEN({column1})'],
+ returnType: formulaTypes.NUMERIC,
},
MIN: {
type: formulaTypes.NUMERIC,
@@ -177,6 +190,7 @@ const formulas: Record = {
description: 'Minimum value amongst input parameters',
syntax: 'MIN(value1, [value2, ...])',
examples: ['MIN(1000, 2000) => 1000', 'MIN({column1}, {column2})'],
+ returnType: formulaTypes.NUMERIC,
},
MAX: {
type: formulaTypes.NUMERIC,
@@ -188,6 +202,7 @@ const formulas: Record = {
description: 'Maximum value amongst input parameters',
syntax: 'MAX(value1, [value2, ...])',
examples: ['MAX(1000, 2000) => 2000', 'MAX({column1}, {column2})'],
+ returnType: formulaTypes.NUMERIC,
},
CEILING: {
type: formulaTypes.NUMERIC,
@@ -199,6 +214,7 @@ const formulas: Record = {
description: 'Rounded next largest integer value of input parameter',
syntax: 'CEILING(value)',
examples: ['CEILING(1.01) => 2', 'CEILING({column1})'],
+ returnType: formulaTypes.NUMERIC,
},
FLOOR: {
type: formulaTypes.NUMERIC,
@@ -210,6 +226,7 @@ const formulas: Record = {
description: 'Rounded largest integer less than or equal to input parameter',
syntax: 'FLOOR(value)',
examples: ['FLOOR(3.1415) => 3', 'FLOOR({column1})'],
+ returnType: formulaTypes.NUMERIC,
},
ROUND: {
type: formulaTypes.NUMERIC,
@@ -222,6 +239,7 @@ const formulas: Record = {
description: 'Rounded number to a specified number of decimal places or the nearest integer if not specified',
syntax: 'ROUND(value, precision), ROUND(value)',
examples: ['ROUND(3.1415) => 3', 'ROUND(3.1415, 2) => 3.14', 'ROUND({column1}, 3)'],
+ returnType: formulaTypes.NUMERIC,
},
MOD: {
type: formulaTypes.NUMERIC,
@@ -233,6 +251,7 @@ const formulas: Record = {
description: 'Remainder after integer division of input parameters',
syntax: 'MOD(value1, value2)',
examples: ['MOD(1024, 1000) => 24', 'MOD({column}, 2)'],
+ returnType: formulaTypes.NUMERIC,
},
REPEAT: {
type: formulaTypes.STRING,
@@ -244,6 +263,7 @@ const formulas: Record = {
description: 'Specified copies of the input parameter string concatenated together',
syntax: 'REPEAT(str, count)',
examples: ['REPEAT("A", 5) => "AAAAA"', 'REPEAT({column}, 5)'],
+ returnType: formulaTypes.STRING,
},
LOG: {
type: formulaTypes.NUMERIC,
@@ -251,6 +271,7 @@ const formulas: Record = {
description: 'Logarithm of input parameter to the base (default = e) specified',
syntax: 'LOG([base], value)',
examples: ['LOG(2, 1024) => 10', 'LOG(2, {column1})'],
+ returnType: formulaTypes.NUMERIC,
},
EXP: {
type: formulaTypes.NUMERIC,
@@ -258,6 +279,7 @@ const formulas: Record = {
description: 'Exponential value of input parameter (e ^ power)',
syntax: 'EXP(power)',
examples: ['EXP(1) => 2.718281828459045', 'EXP({column1})'],
+ returnType: formulaTypes.NUMERIC,
},
POWER: {
type: formulaTypes.NUMERIC,
@@ -269,6 +291,7 @@ const formulas: Record = {
description: 'base to the exponent power, as in base ^ exponent',
syntax: 'POWER(base, exponent)',
examples: ['POWER(2, 10) => 1024', 'POWER({column1}, 10)'],
+ returnType: formulaTypes.NUMERIC,
},
SQRT: {
type: formulaTypes.NUMERIC,
@@ -280,6 +303,7 @@ const formulas: Record = {
description: 'Square root of the input parameter',
syntax: 'SQRT(value)',
examples: ['SQRT(100) => 10', 'SQRT({column1})'],
+ returnType: formulaTypes.NUMERIC,
},
ABS: {
type: formulaTypes.NUMERIC,
@@ -291,6 +315,7 @@ const formulas: Record = {
description: 'Absolute value of the input parameter',
syntax: 'ABS(value)',
examples: ['ABS({column1})'],
+ returnType: formulaTypes.NUMERIC,
},
NOW: {
type: formulaTypes.DATE,
@@ -302,6 +327,7 @@ const formulas: Record = {
description: 'Returns the current time and day',
syntax: 'NOW()',
examples: ['NOW() => 2022-05-19 17:20:43'],
+ returnType: formulaTypes.DATE,
},
REPLACE: {
type: formulaTypes.STRING,
@@ -313,6 +339,7 @@ const formulas: Record = {
description: 'String, after replacing all occurrences of srchStr with rplcStr',
syntax: 'REPLACE(str, srchStr, rplcStr)',
examples: ['REPLACE("AABBCC", "AA", "BB") => "BBBBCC"', 'REPLACE({column1}, {column2}, {column3})'],
+ returnType: formulaTypes.STRING,
},
SEARCH: {
type: formulaTypes.STRING,
@@ -324,6 +351,7 @@ const formulas: Record = {
description: 'Index of srchStr specified if found, 0 otherwise',
syntax: 'SEARCH(str, srchStr)',
examples: ['SEARCH("HELLO WORLD", "WORLD") => 7', 'SEARCH({column1}, "abc")'],
+ returnType: formulaTypes.NUMERIC,
},
INT: {
type: formulaTypes.NUMERIC,
@@ -335,6 +363,7 @@ const formulas: Record = {
description: 'Integer value of input parameter',
syntax: 'INT(value)',
examples: ['INT(3.1415) => 3', 'INT({column1})'],
+ returnType: formulaTypes.NUMERIC,
},
RIGHT: {
type: formulaTypes.STRING,
@@ -346,6 +375,7 @@ const formulas: Record = {
description: 'n characters from the end of input parameter',
syntax: 'RIGHT(str, n)',
examples: ['RIGHT("HELLO WORLD", 5) => WORLD', 'RIGHT({column1}, 3)'],
+ returnType: formulaTypes.STRING,
},
LEFT: {
type: formulaTypes.STRING,
@@ -357,6 +387,7 @@ const formulas: Record = {
description: 'n characters from the beginning of input parameter',
syntax: 'LEFT(str, n)',
examples: ['LEFT({column1}, 2)', 'LEFT("ABCD", 2) => "AB"'],
+ returnType: formulaTypes.STRING,
},
SUBSTR: {
type: formulaTypes.STRING,
@@ -369,6 +400,7 @@ const formulas: Record = {
description: 'Substring of length n of input string from the postition specified',
syntax: ' SUBTR(str, position, [n])',
examples: ['SUBSTR("HELLO WORLD", 7) => WORLD', 'SUBSTR("HELLO WORLD", 7, 3) => WOR', 'SUBSTR({column1}, 7, 5)'],
+ returnType: formulaTypes.STRING,
},
MID: {
type: formulaTypes.STRING,
@@ -380,6 +412,7 @@ const formulas: Record = {
description: 'Alias for SUBSTR',
syntax: 'MID(str, position, [count])',
examples: ['MID("NocoDB", 3, 2) => "co"', 'MID({column1}, 3, 2)'],
+ returnType: formulaTypes.STRING,
},
IF: {
type: formulaTypes.COND_EXP,
@@ -392,6 +425,7 @@ const formulas: Record = {
description: 'SuccessCase if expr evaluates to TRUE, elseCase otherwise',
syntax: 'IF(expr, successCase, elseCase)',
examples: ['IF(5 > 1, "YES", "NO") => "YES"', 'IF({column} > 1, "YES", "NO")'],
+ returnType: formulaTypes.STRING,
},
SWITCH: {
type: formulaTypes.COND_EXP,
@@ -408,6 +442,10 @@ const formulas: Record = {
'SWITCH(3, 1, "One", 2, "Two", "N/A") => "N/A"',
'SWITCH({column1}, 1, "One", 2, "Two", "N/A")',
],
+ // todo: resolve return type based on the args
+ returnType: () => {
+ return formulaTypes.STRING;
+ },
},
URL: {
type: formulaTypes.STRING,
@@ -419,6 +457,7 @@ const formulas: Record = {
description: 'Convert to a hyperlink if it is a valid URL',
syntax: 'URL(str)',
examples: ['URL("https://github.com/nocodb/nocodb")', 'URL({column1})'],
+ returnType: formulaTypes.STRING,
},
WEEKDAY: {
type: formulaTypes.NUMERIC,
@@ -431,6 +470,7 @@ const formulas: Record = {
description: 'Returns the day of the week as an integer between 0 and 6 inclusive starting from Monday by default',
syntax: 'WEEKDAY(date, [startDayOfWeek])',
examples: ['WEEKDAY("2021-06-09")', 'WEEKDAY(NOW(), "sunday")'],
+ returnType: formulaTypes.NUMERIC,
},
TRUE: {
@@ -443,6 +483,7 @@ const formulas: Record = {
description: 'Returns 1',
syntax: 'TRUE()',
examples: ['TRUE()'],
+ returnType: formulaTypes.NUMERIC,
},
FALSE: {
@@ -455,6 +496,7 @@ const formulas: Record = {
description: 'Returns 0',
syntax: 'FALSE()',
examples: ['FALSE()'],
+ returnType: formulaTypes.NUMERIC,
},
REGEX_MATCH: {
@@ -467,6 +509,7 @@ const formulas: Record = {
description: 'Returns 1 if the input text matches a regular expression or 0 if it does not.',
syntax: 'REGEX_MATCH(string, regex)',
examples: ['REGEX_MATCH({title}, "abc.*")'],
+ returnType: formulaTypes.NUMERIC,
},
REGEX_EXTRACT: {
@@ -479,6 +522,7 @@ const formulas: Record = {
description: 'Returns the first match of a regular expression in a string.',
syntax: 'REGEX_EXTRACT(string, regex)',
examples: ['REGEX_EXTRACT({title}, "abc.*")'],
+ returnType: formulaTypes.STRING,
},
REGEX_REPLACE: {
type: formulaTypes.STRING,
@@ -490,6 +534,7 @@ const formulas: Record = {
description: 'Replaces all matches of a regular expression in a string with a replacement string',
syntax: 'REGEX_MATCH(string, regex, replacement)',
examples: ['REGEX_EXTRACT({title}, "abc.*", "abcd")'],
+ returnType: formulaTypes.STRING,
},
BLANK: {
type: formulaTypes.STRING,
@@ -501,6 +546,7 @@ const formulas: Record = {
description: 'Returns a blank value(null)',
syntax: 'BLANK()',
examples: ['BLANK()'],
+ returnType: formulaTypes.NULL,
},
XOR: {
type: formulaTypes.NUMERIC,
@@ -512,6 +558,7 @@ const formulas: Record = {
description: 'Returns true if an odd number of arguments are true, and false otherwise.',
syntax: 'XOR(expression, [exp2, ...])',
examples: ['XOR(TRUE(), FALSE(), TRUE())'],
+ returnType: formulaTypes.BOOLEAN,
},
EVEN: {
type: formulaTypes.NUMERIC,
@@ -523,6 +570,7 @@ const formulas: Record = {
description: 'Returns the nearest even integer that is greater than or equal to the specified value',
syntax: 'EVEN(value)',
examples: ['EVEN({column})'],
+ returnType: formulaTypes.NUMERIC,
},
ODD: {
type: formulaTypes.NUMERIC,
@@ -534,6 +582,7 @@ const formulas: Record = {
description: 'Returns the nearest odd integer that is greater than or equal to the specified value',
syntax: 'ODD(value)',
examples: ['ODD({column})'],
+ returnType: formulaTypes.NUMERIC,
},
RECORD_ID: {
validation: {
@@ -544,6 +593,11 @@ const formulas: Record = {
description: 'Returns the record id of the current record',
syntax: 'RECORD_ID()',
examples: ['RECORD_ID()'],
+
+ // todo: resolve return type based on the args
+ returnType: () => {
+ return formulaTypes.STRING;
+ },
},
COUNTA: {
validation: {
@@ -554,6 +608,7 @@ const formulas: Record = {
description: 'Counts the number of non-empty arguments',
syntax: 'COUNTA(value1, [value2, ...])',
examples: ['COUNTA({field1}, {field2})'],
+ returnType: formulaTypes.NUMERIC,
},
COUNT: {
validation: {
@@ -564,6 +619,7 @@ const formulas: Record = {
description: 'Count the number of arguments that are numbers',
syntax: 'COUNT(value1, [value2, ...])',
examples: ['COUNT({field1}, {field2})'],
+ returnType: formulaTypes.NUMERIC,
},
COUNTALL: {
validation: {
@@ -574,6 +630,7 @@ const formulas: Record = {
description: 'Counts the number of arguments',
syntax: 'COUNTALL(value1, [value2, ...])',
examples: ['COUNTALL({field1}, {field2})'],
+ returnType: formulaTypes.NUMERIC,
},
ROUNDDOWN: {
type: formulaTypes.NUMERIC,
@@ -587,6 +644,7 @@ const formulas: Record = {
'Round down the value after the decimal point to the number of decimal places given by "precision"(default is 0)',
syntax: 'ROUNDDOWN(value, [precision])',
examples: ['ROUNDDOWN({field1})', 'ROUNDDOWN({field1}, 2)'],
+ returnType: formulaTypes.NUMERIC,
},
ROUNDUP: {
type: formulaTypes.NUMERIC,
@@ -599,6 +657,7 @@ const formulas: Record = {
description: 'Round up the value after the decimal point to the number of decimal places given by "precision"(default is 0)',
syntax: 'ROUNDUP(value, [precision])',
examples: ['ROUNDUP({field1})', 'ROUNDUP({field1}, 2)'],
+ returnType: formulaTypes.NUMERIC,
},
VALUE: {
validation: {
@@ -610,6 +669,7 @@ const formulas: Record = {
'Extract the numeric value from a string, if `%` or `-` is present, it will handle it accordingly and return the numeric value',
syntax: 'VALUE(value)',
examples: ['VALUE({field})', 'VALUE("abc10000%")', 'VALUE("$10000")'],
+ returnType: formulaTypes.NUMERIC,
},
// Disabling these functions for now; these act as alias for CreatedAt & UpdatedAt fields;
// Issue: Error noticed if CreatedAt & UpdatedAt fields are removed from the table after creating these formulas
From e2e2f74183797acc4b57928314e3c93f8e3d9f96 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:57 +0000
Subject: [PATCH 005/262] feat: define function return types - WIP
---
packages/nc-gui/utils/formulaUtils.ts | 19 ++++++++++++++-----
1 file changed, 14 insertions(+), 5 deletions(-)
diff --git a/packages/nc-gui/utils/formulaUtils.ts b/packages/nc-gui/utils/formulaUtils.ts
index 30a8625795..e0291f80ea 100644
--- a/packages/nc-gui/utils/formulaUtils.ts
+++ b/packages/nc-gui/utils/formulaUtils.ts
@@ -26,7 +26,6 @@ interface FormulaMeta {
}
const formulas: Record = {
-
AVG: {
type: formulaTypes.NUMERIC,
validation: {
@@ -425,7 +424,17 @@ const formulas: Record = {
description: 'SuccessCase if expr evaluates to TRUE, elseCase otherwise',
syntax: 'IF(expr, successCase, elseCase)',
examples: ['IF(5 > 1, "YES", "NO") => "YES"', 'IF({column} > 1, "YES", "NO")'],
- returnType: formulaTypes.STRING,
+ returnType: (argsTypes: formulaTypes[]) => {
+ if (argsTypes.slice(1).includes(formulaTypes.STRING)) {
+ return formulaTypes.STRING
+ } else if (argsTypes.slice(1).includes(formulaTypes.NUMERIC)) {
+ return formulaTypes.NUMERIC
+ } else if (argsTypes.slice(1).includes(formulaTypes.BOOLEAN)) {
+ return formulaTypes.BOOLEAN
+ }
+
+ return argsTypes[1]
+ },
},
SWITCH: {
type: formulaTypes.COND_EXP,
@@ -443,8 +452,8 @@ const formulas: Record = {
'SWITCH({column1}, 1, "One", 2, "Two", "N/A")',
],
// todo: resolve return type based on the args
- returnType: () => {
- return formulaTypes.STRING;
+ returnType: (argTypes: formulaTypes[]) => {
+ return formulaTypes.STRING
},
},
URL: {
@@ -596,7 +605,7 @@ const formulas: Record = {
// todo: resolve return type based on the args
returnType: () => {
- return formulaTypes.STRING;
+ return formulaTypes.STRING
},
},
COUNTA: {
From 814b1eed1ecf7bf9b49d2a3dcac15a846efda3cd Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:57 +0000
Subject: [PATCH 006/262] feat: define function return types - WIP
---
.../smartsheet/column/FormulaOptions.vue | 4 +++-
packages/nc-gui/utils/formulaUtils.ts | 23 ++++++++++++++++++-
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
index f87585f490..c656659de3 100644
--- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
+++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
@@ -190,6 +190,7 @@ function parseAndValidateFormula(formula: string) {
}
function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = new Set()) {
+ let type: formulaTypes;
if (parsedTree.type === JSEPNode.CALL_EXP) {
const calleeName = parsedTree.callee.name.toUpperCase()
// validate function name
@@ -207,6 +208,7 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
errors.add(t('msg.formula.maxRequiredArgumentsFormula', { maxRequiredArguments: validation.args.max, calleeName }))
}
}
+
parsedTree.arguments.map((arg: Record) => validateAgainstMeta(arg, errors))
// validate data type
@@ -445,7 +447,7 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
} else {
errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'))
}
- return errors
+ return {errors, type}
}
function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set()) {
diff --git a/packages/nc-gui/utils/formulaUtils.ts b/packages/nc-gui/utils/formulaUtils.ts
index e0291f80ea..c4f4fef23b 100644
--- a/packages/nc-gui/utils/formulaUtils.ts
+++ b/packages/nc-gui/utils/formulaUtils.ts
@@ -435,6 +435,17 @@ const formulas: Record = {
return argsTypes[1]
},
+ returnType: (argTypes: formulaTypes[]) => {
+ if (argTypes.slice(1).includes(formulaTypes.STRING)) {
+ return formulaTypes.STRING
+ } else if (argTypes.slice(1).includes(formulaTypes.NUMERIC)) {
+ return formulaTypes.NUMERIC
+ } else if (argTypes.slice(1).includes(formulaTypes.BOOLEAN)) {
+ return formulaTypes.BOOLEAN
+ }
+
+ return argTypes[1]
+ },
},
SWITCH: {
type: formulaTypes.COND_EXP,
@@ -453,7 +464,17 @@ const formulas: Record = {
],
// todo: resolve return type based on the args
returnType: (argTypes: formulaTypes[]) => {
- return formulaTypes.STRING
+ const returnArgTypes = argTypes.slice(2).filter((_, i) => i % 2 === 1)
+
+ if (returnArgTypes.slice(1).includes(formulaTypes.STRING)) {
+ return formulaTypes.STRING
+ } else if (returnArgTypes.slice(1).includes(formulaTypes.NUMERIC)) {
+ return formulaTypes.NUMERIC
+ } else if (returnArgTypes.slice(1).includes(formulaTypes.BOOLEAN)) {
+ return formulaTypes.BOOLEAN
+ }
+
+ return returnArgTypes[1]
},
},
URL: {
From 8cbe49cd4d3fedd4b18394046afae4adde2d2445 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:57 +0000
Subject: [PATCH 007/262] fix: add formula validation - WIP
---
.../smartsheet/column/FormulaOptions.vue | 65 +++++++++++++++++--
1 file changed, 58 insertions(+), 7 deletions(-)
diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
index c656659de3..6e5039b0f7 100644
--- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
+++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
@@ -190,7 +190,7 @@ function parseAndValidateFormula(formula: string) {
}
function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = new Set()) {
- let type: formulaTypes;
+ let returnType: formulaTypes
if (parsedTree.type === JSEPNode.CALL_EXP) {
const calleeName = parsedTree.callee.name.toUpperCase()
// validate function name
@@ -203,17 +203,41 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
if (validation.args.rqd !== undefined && validation.args.rqd !== parsedTree.arguments.length) {
errors.add(t('msg.formula.requiredArgumentsFormula', { requiredArguments: validation.args.rqd, calleeName }))
} else if (validation.args.min !== undefined && validation.args.min > parsedTree.arguments.length) {
- errors.add(t('msg.formula.minRequiredArgumentsFormula', { minRequiredArguments: validation.args.min, calleeName }))
+ errors.add(
+ t('msg.formula.minRequiredArgumentsFormula', {
+ minRequiredArguments: validation.args.min,
+ calleeName,
+ }),
+ )
} else if (validation.args.max !== undefined && validation.args.max < parsedTree.arguments.length) {
- errors.add(t('msg.formula.maxRequiredArgumentsFormula', { maxRequiredArguments: validation.args.max, calleeName }))
+ errors.add(
+ t('msg.formula.maxRequiredArgumentsFormula', {
+ maxRequiredArguments: validation.args.max,
+ calleeName,
+ }),
+ )
}
}
parsedTree.arguments.map((arg: Record) => validateAgainstMeta(arg, errors))
+ // get args type and validate
+ const validateResult = parsedTree.arguments.map((arg) => {
+ return validateAgainstMeta(arg, errors, typeErrors)
+ })
+
+ const argsTypes = validateResult.map((v: any) => v.returnType);
+
+ if (typeof validateResult[0].returnType === 'function') {
+ returnType = formulas[calleeName].returnType(argsTypes)
+ } else if (validateResult[0]) {
+ returnType = formulas[calleeName].returnType
+ }
+
// validate data type
if (parsedTree.callee.type === JSEPNode.IDENTIFIER) {
const expectedType = formulas[calleeName.toUpperCase()].type
+
if (expectedType === formulaTypes.NUMERIC) {
if (calleeName === 'WEEKDAY') {
// parsedTree.arguments[0] = date
@@ -242,7 +266,7 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
typeErrors,
)
} else {
- parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors))
+ parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors, argsTypes))
}
} else if (expectedType === formulaTypes.DATE) {
if (calleeName === 'DATEADD') {
@@ -438,7 +462,21 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
}
validateAgainstMeta(parsedTree.left, errors)
validateAgainstMeta(parsedTree.right, errors)
+
+ // todo: type extraction for binary exps
+ returnType = formulaTypes.NUMERIC
} else if (parsedTree.type === JSEPNode.LITERAL || parsedTree.type === JSEPNode.UNARY_EXP) {
+ if (parsedTree.type === JSEPNode.LITERAL) {
+ if (typeof parsedTree.value === 'number') {
+ returnType = formulaTypes.NUMERIC
+ } else if (typeof parsedTree.value === 'string') {
+ returnType = formulaTypes.STRING
+ } else if (typeof parsedTree.value === 'boolean') {
+ returnType = formulaTypes.BOOLEAN
+ } else {
+ returnType = formulaTypes.STRING
+ }
+ }
// do nothing
} else if (parsedTree.type === JSEPNode.COMPOUND) {
if (parsedTree.body.length) {
@@ -447,10 +485,11 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
} else {
errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'))
}
- return {errors, type}
+ return { errors, returnType }
}
-function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set()) {
+function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set(), argTypes: formulaTypes = []) {
+ let type
if (parsedTree === false || typeof parsedTree === 'undefined') {
return typeErrors
}
@@ -460,10 +499,14 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t
} else if (expectedType === formulaTypes.NUMERIC) {
if (typeof parsedTree.value !== 'number') {
typeErrors.add(t('msg.formula.numericTypeIsExpected'))
+ } else {
+ type = formulaTypes.NUMERIC
}
} else if (expectedType === formulaTypes.STRING) {
if (typeof parsedTree.value !== 'string') {
typeErrors.add(t('msg.formula.stringTypeIsExpected'))
+ } else {
+ type = formulaTypes.STRING
}
}
} else if (parsedTree.type === JSEPNode.IDENTIFIER) {
@@ -475,6 +518,7 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t
if (col.uidt === UITypes.Formula) {
const foundType = getRootDataType(jsep(col.colOptions?.formula_raw))
+ type = foundType
if (foundType === 'N/A') {
typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: col.title }))
} else if (expectedType !== foundType) {
@@ -504,6 +548,7 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t
}),
)
}
+ type = formulaTypes.STRING
break
// numeric
@@ -523,6 +568,7 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t
}),
)
}
+ type = formulaTypes.NUMERIC
break
// date
@@ -539,6 +585,7 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t
}),
)
}
+ type = formulaTypes.DATE
break
case UITypes.Rollup: {
@@ -616,6 +663,8 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t
}),
)
}
+
+ type = formulaTypes.NUMERIC
} else if (parsedTree.type === JSEPNode.CALL_EXP) {
const calleeName = parsedTree.callee.name.toUpperCase()
if (formulas[calleeName]?.type && expectedType !== formulas[calleeName].type) {
@@ -626,8 +675,10 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t
}),
)
}
+ // todo: derive type from returnType
+ type = formulas[calleeName]?.type
}
- return typeErrors
+ return { type, typeErrors }
}
function getRootDataType(parsedTree: any): any {
From 3ce53b879167e8fe62c50535d5082697a89de04d Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:58 +0000
Subject: [PATCH 008/262] chore: add jest unit test in nocodb-sdk
---
packages/nocodb-sdk/jest.config.js | 5 +
.../nocodb-sdk/src/lib/formulaHelpers.spec.ts | 6 +
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 528 ++++++++++++++++++
packages/nocodb-sdk/tsconfig.json | 2 +-
4 files changed, 540 insertions(+), 1 deletion(-)
create mode 100644 packages/nocodb-sdk/jest.config.js
create mode 100644 packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts
diff --git a/packages/nocodb-sdk/jest.config.js b/packages/nocodb-sdk/jest.config.js
new file mode 100644
index 0000000000..b413e106db
--- /dev/null
+++ b/packages/nocodb-sdk/jest.config.js
@@ -0,0 +1,5 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+ preset: 'ts-jest',
+ testEnvironment: 'node',
+};
\ No newline at end of file
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts
new file mode 100644
index 0000000000..6fce60edb8
--- /dev/null
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts
@@ -0,0 +1,6 @@
+describe('auth', () => {
+ it('Formula parsing and type validation', async () => {
+ const response = {userId: 'fakeUserId'};
+ expect(response).toEqual({ userId: 'fakeUserId' });
+ });
+});
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 5a28417d84..8d5cbcbd15 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1,6 +1,7 @@
import jsep from 'jsep';
import { ColumnType } from './Api';
+import {UITypes} from "../../build/main";
export const jsepCurlyHook = {
name: 'curly',
@@ -189,3 +190,530 @@ function escapeLiteral(v: string) {
.replace(/'/g, `\\'`)
);
}
+
+
+
+
+function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = new Set()) {
+ let returnType: formulaTypes
+ if (parsedTree.type === JSEPNode.CALL_EXP) {
+ const calleeName = parsedTree.callee.name.toUpperCase()
+ // validate function name
+ if (!availableFunctions.includes(calleeName)) {
+ errors.add(t('msg.formula.functionNotAvailable', { function: calleeName }))
+ }
+ // validate arguments
+ const validation = formulas[calleeName] && formulas[calleeName].validation
+ if (validation && validation.args) {
+ if (validation.args.rqd !== undefined && validation.args.rqd !== parsedTree.arguments.length) {
+ errors.add(t('msg.formula.requiredArgumentsFormula', { requiredArguments: validation.args.rqd, calleeName }))
+ } else if (validation.args.min !== undefined && validation.args.min > parsedTree.arguments.length) {
+ errors.add(
+ t('msg.formula.minRequiredArgumentsFormula', {
+ minRequiredArguments: validation.args.min,
+ calleeName,
+ }),
+ )
+ } else if (validation.args.max !== undefined && validation.args.max < parsedTree.arguments.length) {
+ errors.add(
+ t('msg.formula.maxRequiredArgumentsFormula', {
+ maxRequiredArguments: validation.args.max,
+ calleeName,
+ }),
+ )
+ }
+ }
+
+ parsedTree.arguments.map((arg: Record) => validateAgainstMeta(arg, errors))
+
+ // get args type and validate
+ const validateResult = parsedTree.arguments.map((arg) => {
+ return validateAgainstMeta(arg, errors, typeErrors)
+ })
+
+ const argsTypes = validateResult.map((v: any) => v.returnType);
+
+ if (typeof validateResult[0].returnType === 'function') {
+ returnType = formulas[calleeName].returnType(argsTypes)
+ } else if (validateResult[0]) {
+ returnType = formulas[calleeName].returnType
+ }
+
+ // validate data type
+ if (parsedTree.callee.type === JSEPNode.IDENTIFIER) {
+ const expectedType = formulas[calleeName.toUpperCase()].type
+
+ if (expectedType === formulaTypes.NUMERIC) {
+ if (calleeName === 'WEEKDAY') {
+ // parsedTree.arguments[0] = date
+ validateAgainstType(
+ parsedTree.arguments[0],
+ formulaTypes.DATE,
+ (v: any) => {
+ if (!validateDateWithUnknownFormat(v)) {
+ typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate'))
+ }
+ },
+ typeErrors,
+ )
+ // parsedTree.arguments[1] = startDayOfWeek (optional)
+ validateAgainstType(
+ parsedTree.arguments[1],
+ formulaTypes.STRING,
+ (v: any) => {
+ if (
+ typeof v !== 'string' ||
+ !['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'].includes(v.toLowerCase())
+ ) {
+ typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate'))
+ }
+ },
+ typeErrors,
+ )
+ } else {
+ parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors, argsTypes))
+ }
+ } else if (expectedType === formulaTypes.DATE) {
+ if (calleeName === 'DATEADD') {
+ // parsedTree.arguments[0] = date
+ validateAgainstType(
+ parsedTree.arguments[0],
+ formulaTypes.DATE,
+ (v: any) => {
+ if (!validateDateWithUnknownFormat(v)) {
+ typeErrors.add(t('msg.formula.firstParamDateAddHaveDate'))
+ }
+ },
+ typeErrors,
+ )
+ // parsedTree.arguments[1] = numeric
+ validateAgainstType(
+ parsedTree.arguments[1],
+ formulaTypes.NUMERIC,
+ (v: any) => {
+ if (typeof v !== 'number') {
+ typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber'))
+ }
+ },
+ typeErrors,
+ )
+ // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"]
+ validateAgainstType(
+ parsedTree.arguments[2],
+ formulaTypes.STRING,
+ (v: any) => {
+ if (!['day', 'week', 'month', 'year'].includes(v)) {
+ typeErrors.add(typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate')))
+ }
+ },
+ typeErrors,
+ )
+ } else if (calleeName === 'DATETIME_DIFF') {
+ // parsedTree.arguments[0] = date
+ validateAgainstType(
+ parsedTree.arguments[0],
+ formulaTypes.DATE,
+ (v: any) => {
+ if (!validateDateWithUnknownFormat(v)) {
+ typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate'))
+ }
+ },
+ typeErrors,
+ )
+ // parsedTree.arguments[1] = date
+ validateAgainstType(
+ parsedTree.arguments[1],
+ formulaTypes.DATE,
+ (v: any) => {
+ if (!validateDateWithUnknownFormat(v)) {
+ typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate'))
+ }
+ },
+ typeErrors,
+ )
+ // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"]
+ validateAgainstType(
+ parsedTree.arguments[2],
+ formulaTypes.STRING,
+ (v: any) => {
+ if (
+ ![
+ 'milliseconds',
+ 'ms',
+ 'seconds',
+ 's',
+ 'minutes',
+ 'm',
+ 'hours',
+ 'h',
+ 'days',
+ 'd',
+ 'weeks',
+ 'w',
+ 'months',
+ 'M',
+ 'quarters',
+ 'Q',
+ 'years',
+ 'y',
+ ].includes(v)
+ ) {
+ typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate'))
+ }
+ },
+ typeErrors,
+ )
+ }
+ }
+ }
+
+ errors = new Set([...errors, ...typeErrors])
+ } else if (parsedTree.type === JSEPNode.IDENTIFIER) {
+ if (supportedColumns.value.filter((c) => !column || column.value?.id !== c.id).every((c) => c.title !== parsedTree.name)) {
+ errors.add(
+ t('msg.formula.columnNotAvailable', {
+ columnName: parsedTree.name,
+ }),
+ )
+ }
+
+ // check circular reference
+ // e.g. formula1 -> formula2 -> formula1 should return circular reference error
+
+ // get all formula columns excluding itself
+ const formulaPaths = supportedColumns.value
+ .filter((c) => c.id !== column.value?.id && c.uidt === UITypes.Formula)
+ .reduce((res: Record[], c: Record) => {
+ // in `formula`, get all the (unique) target neighbours
+ // i.e. all column id (e.g. cl_xxxxxxxxxxxxxx) with formula type
+ const neighbours = [
+ ...new Set(
+ (c.colOptions.formula.match(/cl_\w{14}/g) || []).filter(
+ (colId: string) =>
+ supportedColumns.value.filter((col: ColumnType) => col.id === colId && col.uidt === UITypes.Formula).length,
+ ),
+ ),
+ ]
+ if (neighbours.length > 0) {
+ // e.g. formula column 1 -> [formula column 2, formula column3]
+ res.push({ [c.id]: neighbours })
+ }
+ return res
+ }, [])
+ // include target formula column (i.e. the one to be saved if applicable)
+ const targetFormulaCol = supportedColumns.value.find(
+ (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula,
+ )
+
+ if (targetFormulaCol && column.value?.id) {
+ formulaPaths.push({
+ [column.value?.id as string]: [targetFormulaCol.id],
+ })
+ }
+ const vertices = formulaPaths.length
+ if (vertices > 0) {
+ // perform kahn's algo for cycle detection
+ const adj = new Map()
+ const inDegrees = new Map()
+ // init adjacency list & indegree
+
+ for (const [_, v] of Object.entries(formulaPaths)) {
+ const src = Object.keys(v)[0]
+ const neighbours = v[src]
+ inDegrees.set(src, inDegrees.get(src) || 0)
+ for (const neighbour of neighbours) {
+ adj.set(src, (adj.get(src) || new Set()).add(neighbour))
+ inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1)
+ }
+ }
+ const queue: string[] = []
+ // put all vertices with in-degree = 0 (i.e. no incoming edges) to queue
+ inDegrees.forEach((inDegree, col) => {
+ if (inDegree === 0) {
+ // in-degree = 0 means we start traversing from this node
+ queue.push(col)
+ }
+ })
+ // init count of visited vertices
+ let visited = 0
+ // BFS
+ while (queue.length !== 0) {
+ // remove a vertex from the queue
+ const src = queue.shift()
+ // if this node has neighbours, increase visited by 1
+ const neighbours = adj.get(src) || new Set()
+ if (neighbours.size > 0) {
+ visited += 1
+ }
+ // iterate each neighbouring nodes
+ neighbours.forEach((neighbour: string) => {
+ // decrease in-degree of its neighbours by 1
+ inDegrees.set(neighbour, inDegrees.get(neighbour) - 1)
+ // if in-degree becomes 0
+ if (inDegrees.get(neighbour) === 0) {
+ // then put the neighboring node to the queue
+ queue.push(neighbour)
+ }
+ })
+ }
+ // vertices not same as visited = cycle found
+ if (vertices !== visited) {
+ errors.add(t('msg.formula.cantSaveCircularReference'))
+ }
+ }
+ } else if (parsedTree.type === JSEPNode.BINARY_EXP) {
+ if (!availableBinOps.includes(parsedTree.operator)) {
+ errors.add(t('msg.formula.operationNotAvailable', { operation: parsedTree.operator }))
+ }
+ validateAgainstMeta(parsedTree.left, errors)
+ validateAgainstMeta(parsedTree.right, errors)
+
+ // todo: type extraction for binary exps
+ returnType = formulaTypes.NUMERIC
+ } else if (parsedTree.type === JSEPNode.LITERAL || parsedTree.type === JSEPNode.UNARY_EXP) {
+ if (parsedTree.type === JSEPNode.LITERAL) {
+ if (typeof parsedTree.value === 'number') {
+ returnType = formulaTypes.NUMERIC
+ } else if (typeof parsedTree.value === 'string') {
+ returnType = formulaTypes.STRING
+ } else if (typeof parsedTree.value === 'boolean') {
+ returnType = formulaTypes.BOOLEAN
+ } else {
+ returnType = formulaTypes.STRING
+ }
+ }
+ // do nothing
+ } else if (parsedTree.type === JSEPNode.COMPOUND) {
+ if (parsedTree.body.length) {
+ errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'))
+ }
+ } else {
+ errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'))
+ }
+ return { errors, returnType }
+}
+
+function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set(), argTypes: formulaTypes = []) {
+ let type
+ if (parsedTree === false || typeof parsedTree === 'undefined') {
+ return typeErrors
+ }
+ if (parsedTree.type === JSEPNode.LITERAL) {
+ if (typeof func === 'function') {
+ func(parsedTree.value)
+ } else if (expectedType === formulaTypes.NUMERIC) {
+ if (typeof parsedTree.value !== 'number') {
+ typeErrors.add(t('msg.formula.numericTypeIsExpected'))
+ } else {
+ type = formulaTypes.NUMERIC
+ }
+ } else if (expectedType === formulaTypes.STRING) {
+ if (typeof parsedTree.value !== 'string') {
+ typeErrors.add(t('msg.formula.stringTypeIsExpected'))
+ } else {
+ type = formulaTypes.STRING
+ }
+ }
+ } else if (parsedTree.type === JSEPNode.IDENTIFIER) {
+ const col = supportedColumns.value.find((c) => c.title === parsedTree.name)
+
+ if (col === undefined) {
+ return
+ }
+
+ if (col.uidt === UITypes.Formula) {
+ const foundType = getRootDataType(jsep(col.colOptions?.formula_raw))
+ type = foundType
+ if (foundType === 'N/A') {
+ typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: col.title }))
+ } else if (expectedType !== foundType) {
+ typeErrors.add(
+ t('msg.formula.typeIsExpectedButFound', {
+ type: expectedType,
+ found: foundType,
+ }),
+ )
+ }
+ } else {
+ switch (col.uidt) {
+ // string
+ case UITypes.SingleLineText:
+ case UITypes.LongText:
+ case UITypes.MultiSelect:
+ case UITypes.SingleSelect:
+ case UITypes.PhoneNumber:
+ case UITypes.Email:
+ case UITypes.URL:
+ if (expectedType !== formulaTypes.STRING) {
+ typeErrors.add(
+ t('msg.formula.columnWithTypeFoundButExpected', {
+ columnName: parsedTree.name,
+ columnType: formulaTypes.STRING,
+ expectedType,
+ }),
+ )
+ }
+ type = formulaTypes.STRING
+ break
+
+ // numeric
+ case UITypes.Year:
+ case UITypes.Number:
+ case UITypes.Decimal:
+ case UITypes.Rating:
+ case UITypes.Count:
+ case UITypes.AutoNumber:
+ case UITypes.Currency:
+ if (expectedType !== formulaTypes.NUMERIC) {
+ typeErrors.add(
+ t('msg.formula.columnWithTypeFoundButExpected', {
+ columnName: parsedTree.name,
+ columnType: formulaTypes.NUMERIC,
+ expectedType,
+ }),
+ )
+ }
+ type = formulaTypes.NUMERIC
+ break
+
+ // date
+ case UITypes.Date:
+ case UITypes.DateTime:
+ case UITypes.CreateTime:
+ case UITypes.LastModifiedTime:
+ if (expectedType !== formulaTypes.DATE) {
+ typeErrors.add(
+ t('msg.formula.columnWithTypeFoundButExpected', {
+ columnName: parsedTree.name,
+ columnType: formulaTypes.DATE,
+ expectedType,
+ }),
+ )
+ }
+ type = formulaTypes.DATE
+ break
+
+ // not supported
+ case UITypes.ForeignKey:
+ case UITypes.Attachment:
+ case UITypes.ID:
+ case UITypes.Time:
+ case UITypes.Percent:
+ case UITypes.Duration:
+ case UITypes.Rollup:
+ case UITypes.Lookup:
+ case UITypes.Barcode:
+ case UITypes.Button:
+ case UITypes.Checkbox:
+ case UITypes.Collaborator:
+ case UITypes.QrCode:
+ default:
+ typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: parsedTree.name }))
+ break
+ }
+ }
+ } else if (parsedTree.type === JSEPNode.UNARY_EXP || parsedTree.type === JSEPNode.BINARY_EXP) {
+ if (expectedType !== formulaTypes.NUMERIC) {
+ // parsedTree.name won't be available here
+ typeErrors.add(
+ t('msg.formula.typeIsExpectedButFound', {
+ type: formulaTypes.NUMERIC,
+ found: expectedType,
+ }),
+ )
+ }
+
+ type = formulaTypes.NUMERIC
+ } else if (parsedTree.type === JSEPNode.CALL_EXP) {
+ const calleeName = parsedTree.callee.name.toUpperCase()
+ if (formulas[calleeName]?.type && expectedType !== formulas[calleeName].type) {
+ typeErrors.add(
+ t('msg.formula.typeIsExpectedButFound', {
+ type: expectedType,
+ found: formulas[calleeName].type,
+ }),
+ )
+ }
+ // todo: derive type from returnType
+ type = formulas[calleeName]?.type
+ }
+ return { type, typeErrors }
+}
+
+function getRootDataType(parsedTree: any): any {
+ // given a parse tree, return the data type of it
+ if (parsedTree.type === JSEPNode.CALL_EXP) {
+ return formulas[parsedTree.callee.name.toUpperCase()].type
+ } else if (parsedTree.type === JSEPNode.IDENTIFIER) {
+ const col = supportedColumns.value.find((c) => c.title === parsedTree.name) as Record
+ if (col?.uidt === UITypes.Formula) {
+ return getRootDataType(jsep(col?.formula_raw))
+ } else {
+ switch (col?.uidt) {
+ // string
+ case UITypes.SingleLineText:
+ case UITypes.LongText:
+ case UITypes.MultiSelect:
+ case UITypes.SingleSelect:
+ case UITypes.PhoneNumber:
+ case UITypes.Email:
+ case UITypes.URL:
+ return formulaTypes.STRING
+
+ // numeric
+ case UITypes.Year:
+ case UITypes.Number:
+ case UITypes.Decimal:
+ case UITypes.Rating:
+ case UITypes.Count:
+ case UITypes.AutoNumber:
+ return formulaTypes.NUMERIC
+
+ // date
+ case UITypes.Date:
+ case UITypes.DateTime:
+ case UITypes.CreateTime:
+ case UITypes.LastModifiedTime:
+ return formulaTypes.DATE
+
+ // not supported
+ case UITypes.ForeignKey:
+ case UITypes.Attachment:
+ case UITypes.ID:
+ case UITypes.Time:
+ case UITypes.Currency:
+ case UITypes.Percent:
+ case UITypes.Duration:
+ case UITypes.Rollup:
+ case UITypes.Lookup:
+ case UITypes.Barcode:
+ case UITypes.Button:
+ case UITypes.Checkbox:
+ case UITypes.Collaborator:
+ case UITypes.QrCode:
+ default:
+ return 'N/A'
+ }
+ }
+ } else if (parsedTree.type === JSEPNode.BINARY_EXP || parsedTree.type === JSEPNode.UNARY_EXP) {
+ return formulaTypes.NUMERIC
+ } else if (parsedTree.type === JSEPNode.LITERAL) {
+ return typeof parsedTree.value
+ } else {
+ return 'N/A'
+ }
+}
+
+function isCurlyBracketBalanced() {
+ // count number of opening curly brackets and closing curly brackets
+ const cntCurlyBrackets = (formulaRef.value.$el.value.match(/\{|}/g) || []).reduce(
+ (acc: Record, cur: number) => {
+ acc[cur] = (acc[cur] || 0) + 1
+ return acc
+ },
+ {},
+ )
+ return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0)
+}
+
+
diff --git a/packages/nocodb-sdk/tsconfig.json b/packages/nocodb-sdk/tsconfig.json
index 8af501e5e0..0bd395cc39 100644
--- a/packages/nocodb-sdk/tsconfig.json
+++ b/packages/nocodb-sdk/tsconfig.json
@@ -38,7 +38,7 @@
// "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */,
"lib": ["es2017","dom"],
- "types": [],
+ "types": ["jest", "node"],
"typeRoots": ["node_modules/@types", "src/types"],
"baseUrl": "./src",
"paths": {
From a6e0bedc6fe1f8b30355f2b6a871a563b0b8337f Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:58 +0000
Subject: [PATCH 009/262] feat: add method to validate and extract type
---
.../nocodb-sdk/src/lib/formulaHelpers.spec.ts | 67 +-
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 1398 +++++++++++++++--
2 files changed, 1293 insertions(+), 172 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts
index 6fce60edb8..071733e6fb 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts
@@ -1,6 +1,65 @@
-describe('auth', () => {
- it('Formula parsing and type validation', async () => {
- const response = {userId: 'fakeUserId'};
- expect(response).toEqual({ userId: 'fakeUserId' });
+import {
+ FormulaDataTypes,
+ validateFormulaAndExtractTreeWithType,
+} from './formulaHelpers';
+import UITypes from './UITypes';
+
+describe('Formula parsing and type validation', () => {
+ it('Simple formula', async () => {
+ const result = validateFormulaAndExtractTreeWithType('1 + 2', []);
+
+ expect(result.dataType).toEqual(FormulaDataTypes.NUMERIC);
+ });
+
+ it('Formula with IF condition', async () => {
+ const result = validateFormulaAndExtractTreeWithType(
+ 'IF({column}, "Found", BLANK())',
+ [
+ {
+ id: 'cid',
+ title: 'column',
+ uidt: UITypes.Number,
+ },
+ ]
+ );
+
+ expect(result.dataType).toEqual(FormulaDataTypes.STRING);
+ });
+ it('Complex formula', async () => {
+ const result = validateFormulaAndExtractTreeWithType(
+ 'SWITCH({column2},"value1",IF({column1}, "Found", BLANK()),"value2", 2)',
+ [
+ {
+ id: 'id1',
+ title: 'column1',
+ uidt: UITypes.Number,
+ },
+ {
+ id: 'id2',
+ title: 'column2',
+ uidt: UITypes.SingleLineText,
+ },
+ ]
+ );
+
+ expect(result.dataType).toEqual(FormulaDataTypes.STRING);
+
+ const result1 = validateFormulaAndExtractTreeWithType(
+ 'SWITCH({column2},"value1",IF({column1}, 1, 2),"value2", 2)',
+ [
+ {
+ id: 'id1',
+ title: 'column1',
+ uidt: UITypes.Number,
+ },
+ {
+ id: 'id2',
+ title: 'column2',
+ uidt: UITypes.SingleLineText,
+ },
+ ]
+ );
+
+ expect(result1.dataType).toEqual(FormulaDataTypes.NUMERIC);
});
});
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 8d5cbcbd15..c1c19b2562 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1,7 +1,7 @@
import jsep from 'jsep';
import { ColumnType } from './Api';
-import {UITypes} from "../../build/main";
+import UITypes from './UITypes';
export const jsepCurlyHook = {
name: 'curly',
@@ -191,150 +191,955 @@ function escapeLiteral(v: string) {
);
}
+export enum FormulaDataTypes {
+ NUMERIC = 'numeric',
+ STRING = 'string',
+ DATE = 'date',
+ LOGICAL = 'logical',
+ COND_EXP = 'conditional_expression',
+ NULL = 'null',
+ BOOLEAN = 'boolean',
+}
+
+export enum JSEPNode {
+ COMPOUND = 'Compound',
+ IDENTIFIER = 'Identifier',
+ MEMBER_EXP = 'MemberExpression',
+ LITERAL = 'Literal',
+ THIS_EXP = 'ThisExpression',
+ CALL_EXP = 'CallExpression',
+ UNARY_EXP = 'UnaryExpression',
+ BINARY_EXP = 'BinaryExpression',
+ ARRAY_EXP = 'ArrayExpression',
+}
+
+interface FormulaMeta {
+ type?: string;
+ validation?: {
+ args?: {
+ min?: number;
+ max?: number;
+ rqd?: number;
+ };
+ };
+ description?: string;
+ syntax?: string;
+ examples?: string[];
+ returnType?: ((args: any[]) => FormulaDataTypes) | FormulaDataTypes;
+}
+
+const formulas: Record = {
+ AVG: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description: 'Average of input parameters',
+ syntax: 'AVG(value1, [value2, ...])',
+ examples: [
+ 'AVG(10, 5) => 7.5',
+ 'AVG({column1}, {column2})',
+ 'AVG({column1}, {column2}, {column3})',
+ ],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ ADD: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description: 'Sum of input parameters',
+ syntax: 'ADD(value1, [value2, ...])',
+ examples: [
+ 'ADD(5, 5) => 10',
+ 'ADD({column1}, {column2})',
+ 'ADD({column1}, {column2}, {column3})',
+ ],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ DATEADD: {
+ type: FormulaDataTypes.DATE,
+ validation: {
+ args: {
+ rqd: 3,
+ },
+ },
+ description: 'Adds a "count" units to Datetime.',
+ syntax:
+ 'DATEADD(date | datetime, value, ["day" | "week" | "month" | "year"])',
+ examples: [
+ 'DATEADD({column1}, 2, "day")',
+ 'DATEADD({column1}, -2, "day")',
+ 'DATEADD({column1}, 2, "week")',
+ 'DATEADD({column1}, -2, "week")',
+ 'DATEADD({column1}, 2, "month")',
+ 'DATEADD({column1}, -2, "month")',
+ 'DATEADD({column1}, 2, "year")',
+ 'DATEADD({column1}, -2, "year")',
+ ],
+ returnType: FormulaDataTypes.DATE,
+ },
+ DATETIME_DIFF: {
+ type: FormulaDataTypes.DATE,
+ validation: {
+ args: {
+ min: 2,
+ max: 3,
+ },
+ },
+ description:
+ 'Calculate the difference of two given date / datetime in specified units.',
+ syntax:
+ 'DATETIME_DIFF(date | datetime, date | datetime, ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"])',
+ examples: [
+ 'DATEDIFF({column1}, {column2})',
+ 'DATEDIFF({column1}, {column2}, "seconds")',
+ 'DATEDIFF({column1}, {column2}, "s")',
+ 'DATEDIFF({column1}, {column2}, "years")',
+ 'DATEDIFF({column1}, {column2}, "y")',
+ 'DATEDIFF({column1}, {column2}, "minutes")',
+ 'DATEDIFF({column1}, {column2}, "m")',
+ 'DATEDIFF({column1}, {column2}, "days")',
+ 'DATEDIFF({column1}, {column2}, "d")',
+ ],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ AND: {
+ type: FormulaDataTypes.COND_EXP,
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description: 'TRUE if all expr evaluate to TRUE',
+ syntax: 'AND(expr1, [expr2, ...])',
+ examples: ['AND(5 > 2, 5 < 10) => 1', 'AND({column1} > 2, {column2} < 10)'],
+ returnType: FormulaDataTypes.COND_EXP,
+ },
+ OR: {
+ type: FormulaDataTypes.COND_EXP,
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description: 'TRUE if at least one expr evaluates to TRUE',
+ syntax: 'OR(expr1, [expr2, ...])',
+ examples: ['OR(5 > 2, 5 < 10) => 1', 'OR({column1} > 2, {column2} < 10)'],
+ returnType: FormulaDataTypes.COND_EXP,
+ },
+ CONCAT: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description: 'Concatenated string of input parameters',
+ syntax: 'CONCAT(str1, [str2, ...])',
+ examples: [
+ 'CONCAT("AA", "BB", "CC") => "AABBCC"',
+ 'CONCAT({column1}, {column2}, {column3})',
+ ],
+ returnType: FormulaDataTypes.STRING,
+ },
+ TRIM: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description: 'Remove trailing and leading whitespaces from input parameter',
+ syntax: 'TRIM(str)',
+ examples: [
+ 'TRIM(" HELLO WORLD ") => "HELLO WORLD"',
+ 'TRIM({column1})',
+ ],
+ returnType: FormulaDataTypes.STRING,
+ },
+ UPPER: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description: 'Upper case converted string of input parameter',
+ syntax: 'UPPER(str)',
+ examples: ['UPPER("nocodb") => "NOCODB"', 'UPPER({column1})'],
+ returnType: FormulaDataTypes.STRING,
+ },
+ LOWER: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description: 'Lower case converted string of input parameter',
+ syntax: 'LOWER(str)',
+ examples: ['LOWER("NOCODB") => "nocodb"', 'LOWER({column1})'],
+ returnType: FormulaDataTypes.STRING,
+ },
+ LEN: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description: 'Input parameter character length',
+ syntax: 'LEN(value)',
+ examples: ['LEN("NocoDB") => 6', 'LEN({column1})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ MIN: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description: 'Minimum value amongst input parameters',
+ syntax: 'MIN(value1, [value2, ...])',
+ examples: ['MIN(1000, 2000) => 1000', 'MIN({column1}, {column2})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ MAX: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description: 'Maximum value amongst input parameters',
+ syntax: 'MAX(value1, [value2, ...])',
+ examples: ['MAX(1000, 2000) => 2000', 'MAX({column1}, {column2})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ CEILING: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description: 'Rounded next largest integer value of input parameter',
+ syntax: 'CEILING(value)',
+ examples: ['CEILING(1.01) => 2', 'CEILING({column1})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ FLOOR: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description:
+ 'Rounded largest integer less than or equal to input parameter',
+ syntax: 'FLOOR(value)',
+ examples: ['FLOOR(3.1415) => 3', 'FLOOR({column1})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ ROUND: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ min: 1,
+ max: 2,
+ },
+ },
+ description:
+ 'Rounded number to a specified number of decimal places or the nearest integer if not specified',
+ syntax: 'ROUND(value, precision), ROUND(value)',
+ examples: [
+ 'ROUND(3.1415) => 3',
+ 'ROUND(3.1415, 2) => 3.14',
+ 'ROUND({column1}, 3)',
+ ],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ MOD: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ rqd: 2,
+ },
+ },
+ description: 'Remainder after integer division of input parameters',
+ syntax: 'MOD(value1, value2)',
+ examples: ['MOD(1024, 1000) => 24', 'MOD({column}, 2)'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ REPEAT: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 2,
+ },
+ },
+ description:
+ 'Specified copies of the input parameter string concatenated together',
+ syntax: 'REPEAT(str, count)',
+ examples: ['REPEAT("A", 5) => "AAAAA"', 'REPEAT({column}, 5)'],
+ returnType: FormulaDataTypes.STRING,
+ },
+ LOG: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {},
+ description:
+ 'Logarithm of input parameter to the base (default = e) specified',
+ syntax: 'LOG([base], value)',
+ examples: ['LOG(2, 1024) => 10', 'LOG(2, {column1})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ EXP: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {},
+ description: 'Exponential value of input parameter (e ^ power)',
+ syntax: 'EXP(power)',
+ examples: ['EXP(1) => 2.718281828459045', 'EXP({column1})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ POWER: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ rqd: 2,
+ },
+ },
+ description: 'base to the exponent power, as in base ^ exponent',
+ syntax: 'POWER(base, exponent)',
+ examples: ['POWER(2, 10) => 1024', 'POWER({column1}, 10)'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ SQRT: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description: 'Square root of the input parameter',
+ syntax: 'SQRT(value)',
+ examples: ['SQRT(100) => 10', 'SQRT({column1})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ ABS: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description: 'Absolute value of the input parameter',
+ syntax: 'ABS(value)',
+ examples: ['ABS({column1})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ NOW: {
+ type: FormulaDataTypes.DATE,
+ validation: {
+ args: {
+ rqd: 0,
+ },
+ },
+ description: 'Returns the current time and day',
+ syntax: 'NOW()',
+ examples: ['NOW() => 2022-05-19 17:20:43'],
+ returnType: FormulaDataTypes.DATE,
+ },
+ REPLACE: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 3,
+ },
+ },
+ description:
+ 'String, after replacing all occurrences of srchStr with rplcStr',
+ syntax: 'REPLACE(str, srchStr, rplcStr)',
+ examples: [
+ 'REPLACE("AABBCC", "AA", "BB") => "BBBBCC"',
+ 'REPLACE({column1}, {column2}, {column3})',
+ ],
+ returnType: FormulaDataTypes.STRING,
+ },
+ SEARCH: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 2,
+ },
+ },
+ description: 'Index of srchStr specified if found, 0 otherwise',
+ syntax: 'SEARCH(str, srchStr)',
+ examples: [
+ 'SEARCH("HELLO WORLD", "WORLD") => 7',
+ 'SEARCH({column1}, "abc")',
+ ],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ INT: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description: 'Integer value of input parameter',
+ syntax: 'INT(value)',
+ examples: ['INT(3.1415) => 3', 'INT({column1})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ RIGHT: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 2,
+ },
+ },
+ description: 'n characters from the end of input parameter',
+ syntax: 'RIGHT(str, n)',
+ examples: ['RIGHT("HELLO WORLD", 5) => WORLD', 'RIGHT({column1}, 3)'],
+ returnType: FormulaDataTypes.STRING,
+ },
+ LEFT: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 2,
+ },
+ },
+ description: 'n characters from the beginning of input parameter',
+ syntax: 'LEFT(str, n)',
+ examples: ['LEFT({column1}, 2)', 'LEFT("ABCD", 2) => "AB"'],
+ returnType: FormulaDataTypes.STRING,
+ },
+ SUBSTR: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ min: 2,
+ max: 3,
+ },
+ },
+ description:
+ 'Substring of length n of input string from the postition specified',
+ syntax: ' SUBTR(str, position, [n])',
+ examples: [
+ 'SUBSTR("HELLO WORLD", 7) => WORLD',
+ 'SUBSTR("HELLO WORLD", 7, 3) => WOR',
+ 'SUBSTR({column1}, 7, 5)',
+ ],
+ returnType: FormulaDataTypes.STRING,
+ },
+ MID: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 3,
+ },
+ },
+ description: 'Alias for SUBSTR',
+ syntax: 'MID(str, position, [count])',
+ examples: ['MID("NocoDB", 3, 2) => "co"', 'MID({column1}, 3, 2)'],
+ returnType: FormulaDataTypes.STRING,
+ },
+ IF: {
+ type: FormulaDataTypes.COND_EXP,
+ validation: {
+ args: {
+ min: 2,
+ max: 3,
+ },
+ },
+ description: 'SuccessCase if expr evaluates to TRUE, elseCase otherwise',
+ syntax: 'IF(expr, successCase, elseCase)',
+ examples: [
+ 'IF(5 > 1, "YES", "NO") => "YES"',
+ 'IF({column} > 1, "YES", "NO")',
+ ],
+ returnType: (argsTypes: FormulaDataTypes[]) => {
+ if (argsTypes.slice(1).includes(FormulaDataTypes.STRING)) {
+ return FormulaDataTypes.STRING;
+ } else if (argsTypes.slice(1).includes(FormulaDataTypes.NUMERIC)) {
+ return FormulaDataTypes.NUMERIC;
+ } else if (argsTypes.slice(1).includes(FormulaDataTypes.BOOLEAN)) {
+ return FormulaDataTypes.BOOLEAN;
+ }
+
+ return argsTypes[1];
+ },
+ },
+ SWITCH: {
+ type: FormulaDataTypes.COND_EXP,
+ validation: {
+ args: {
+ min: 3,
+ },
+ },
+ description: 'Switch case value based on expr output',
+ syntax: 'SWITCH(expr, [pattern, value, ..., default])',
+ examples: [
+ 'SWITCH(1, 1, "One", 2, "Two", "N/A") => "One""',
+ 'SWITCH(2, 1, "One", 2, "Two", "N/A") => "Two"',
+ 'SWITCH(3, 1, "One", 2, "Two", "N/A") => "N/A"',
+ 'SWITCH({column1}, 1, "One", 2, "Two", "N/A")',
+ ],
+ // todo: resolve return type based on the args
+ returnType: (argTypes: FormulaDataTypes[]) => {
+ const returnArgTypes = argTypes.slice(2).filter((_, i) => i % 2 === 0);
+
+ if (returnArgTypes.includes(FormulaDataTypes.STRING)) {
+ return FormulaDataTypes.STRING;
+ } else if (returnArgTypes.includes(FormulaDataTypes.NUMERIC)) {
+ return FormulaDataTypes.NUMERIC;
+ } else if (returnArgTypes.includes(FormulaDataTypes.BOOLEAN)) {
+ return FormulaDataTypes.BOOLEAN;
+ }
+
+ return returnArgTypes[0];
+ },
+ },
+ URL: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description: 'Convert to a hyperlink if it is a valid URL',
+ syntax: 'URL(str)',
+ examples: ['URL("https://github.com/nocodb/nocodb")', 'URL({column1})'],
+ returnType: FormulaDataTypes.STRING,
+ },
+ WEEKDAY: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ min: 1,
+ max: 2,
+ },
+ },
+ description:
+ 'Returns the day of the week as an integer between 0 and 6 inclusive starting from Monday by default',
+ syntax: 'WEEKDAY(date, [startDayOfWeek])',
+ examples: ['WEEKDAY("2021-06-09")', 'WEEKDAY(NOW(), "sunday")'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+
+ TRUE: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ max: 0,
+ },
+ },
+ description: 'Returns 1',
+ syntax: 'TRUE()',
+ examples: ['TRUE()'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+
+ FALSE: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ max: 0,
+ },
+ },
+ description: 'Returns 0',
+ syntax: 'FALSE()',
+ examples: ['FALSE()'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+
+ REGEX_MATCH: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 2,
+ },
+ },
+ description:
+ 'Returns 1 if the input text matches a regular expression or 0 if it does not.',
+ syntax: 'REGEX_MATCH(string, regex)',
+ examples: ['REGEX_MATCH({title}, "abc.*")'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ REGEX_EXTRACT: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 2,
+ },
+ },
+ description: 'Returns the first match of a regular expression in a string.',
+ syntax: 'REGEX_EXTRACT(string, regex)',
+ examples: ['REGEX_EXTRACT({title}, "abc.*")'],
+ returnType: FormulaDataTypes.STRING,
+ },
+ REGEX_REPLACE: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 3,
+ },
+ },
+ description:
+ 'Replaces all matches of a regular expression in a string with a replacement string',
+ syntax: 'REGEX_MATCH(string, regex, replacement)',
+ examples: ['REGEX_EXTRACT({title}, "abc.*", "abcd")'],
+ returnType: FormulaDataTypes.STRING,
+ },
+ BLANK: {
+ type: FormulaDataTypes.STRING,
+ validation: {
+ args: {
+ rqd: 0,
+ },
+ },
+ description: 'Returns a blank value(null)',
+ syntax: 'BLANK()',
+ examples: ['BLANK()'],
+ returnType: FormulaDataTypes.NULL,
+ },
+ XOR: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description:
+ 'Returns true if an odd number of arguments are true, and false otherwise.',
+ syntax: 'XOR(expression, [exp2, ...])',
+ examples: ['XOR(TRUE(), FALSE(), TRUE())'],
+ returnType: FormulaDataTypes.BOOLEAN,
+ },
+ EVEN: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description:
+ 'Returns the nearest even integer that is greater than or equal to the specified value',
+ syntax: 'EVEN(value)',
+ examples: ['EVEN({column})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ ODD: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description:
+ 'Returns the nearest odd integer that is greater than or equal to the specified value',
+ syntax: 'ODD(value)',
+ examples: ['ODD({column})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ RECORD_ID: {
+ validation: {
+ args: {
+ rqd: 0,
+ },
+ },
+ description: 'Returns the record id of the current record',
+ syntax: 'RECORD_ID()',
+ examples: ['RECORD_ID()'],
+ // todo: resolve return type based on the args
+ returnType: () => {
+ return FormulaDataTypes.STRING;
+ },
+ },
+ COUNTA: {
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description: 'Counts the number of non-empty arguments',
+ syntax: 'COUNTA(value1, [value2, ...])',
+ examples: ['COUNTA({field1}, {field2})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ COUNT: {
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description: 'Count the number of arguments that are numbers',
+ syntax: 'COUNT(value1, [value2, ...])',
+ examples: ['COUNT({field1}, {field2})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ COUNTALL: {
+ validation: {
+ args: {
+ min: 1,
+ },
+ },
+ description: 'Counts the number of arguments',
+ syntax: 'COUNTALL(value1, [value2, ...])',
+ examples: ['COUNTALL({field1}, {field2})'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ ROUNDDOWN: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ min: 1,
+ max: 2,
+ },
+ },
+ description:
+ 'Round down the value after the decimal point to the number of decimal places given by "precision"(default is 0)',
+ syntax: 'ROUNDDOWN(value, [precision])',
+ examples: ['ROUNDDOWN({field1})', 'ROUNDDOWN({field1}, 2)'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ ROUNDUP: {
+ type: FormulaDataTypes.NUMERIC,
+ validation: {
+ args: {
+ min: 1,
+ max: 2,
+ },
+ },
+ description:
+ 'Round up the value after the decimal point to the number of decimal places given by "precision"(default is 0)',
+ syntax: 'ROUNDUP(value, [precision])',
+ examples: ['ROUNDUP({field1})', 'ROUNDUP({field1}, 2)'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ VALUE: {
+ validation: {
+ args: {
+ rqd: 1,
+ },
+ },
+ description:
+ 'Extract the numeric value from a string, if `%` or `-` is present, it will handle it accordingly and return the numeric value',
+ syntax: 'VALUE(value)',
+ examples: ['VALUE({field})', 'VALUE("abc10000%")', 'VALUE("$10000")'],
+ returnType: FormulaDataTypes.NUMERIC,
+ },
+ // Disabling these functions for now; these act as alias for CreatedAt & UpdatedAt fields;
+ // Issue: Error noticed if CreatedAt & UpdatedAt fields are removed from the table after creating these formulas
+ //
+ // CREATED_TIME: {
+ // validation: {
+ // args: {
+ // rqd: 0,
+ // },
+ // },
+ // description: 'Returns the created time of the current record if it exists',
+ // syntax: 'CREATED_TIME()',
+ // examples: ['CREATED_TIME()'],
+ // },
+ // LAST_MODIFIED_TIME: {
+ // validation: {
+ // args: {
+ // rqd: 0,
+ // },
+ // },
+ // description: 'Returns the last modified time of the current record if it exists',
+ // syntax: ' LAST_MODIFIED_TIME()',
+ // examples: [' LAST_MODIFIED_TIME()'],
+ // },
+};
-function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = new Set()) {
- let returnType: formulaTypes
+/*
+function validateAgainstMeta(
+ parsedTree: any,
+ errors = new Set(),
+ typeErrors = new Set()
+) {
+ let returnType: FormulaDataTypes;
if (parsedTree.type === JSEPNode.CALL_EXP) {
- const calleeName = parsedTree.callee.name.toUpperCase()
+ const calleeName = parsedTree.callee.name.toUpperCase();
// validate function name
if (!availableFunctions.includes(calleeName)) {
- errors.add(t('msg.formula.functionNotAvailable', { function: calleeName }))
+ errors.add(
+ t('msg.formula.functionNotAvailable', { function: calleeName })
+ );
}
// validate arguments
- const validation = formulas[calleeName] && formulas[calleeName].validation
+ const validation = formulas[calleeName] && formulas[calleeName].validation;
if (validation && validation.args) {
- if (validation.args.rqd !== undefined && validation.args.rqd !== parsedTree.arguments.length) {
- errors.add(t('msg.formula.requiredArgumentsFormula', { requiredArguments: validation.args.rqd, calleeName }))
- } else if (validation.args.min !== undefined && validation.args.min > parsedTree.arguments.length) {
+ if (
+ validation.args.rqd !== undefined &&
+ validation.args.rqd !== parsedTree.arguments.length
+ ) {
+ errors.add(
+ t('msg.formula.requiredArgumentsFormula', {
+ requiredArguments: validation.args.rqd,
+ calleeName,
+ })
+ );
+ } else if (
+ validation.args.min !== undefined &&
+ validation.args.min > parsedTree.arguments.length
+ ) {
errors.add(
t('msg.formula.minRequiredArgumentsFormula', {
minRequiredArguments: validation.args.min,
calleeName,
- }),
- )
- } else if (validation.args.max !== undefined && validation.args.max < parsedTree.arguments.length) {
+ })
+ );
+ } else if (
+ validation.args.max !== undefined &&
+ validation.args.max < parsedTree.arguments.length
+ ) {
errors.add(
t('msg.formula.maxRequiredArgumentsFormula', {
maxRequiredArguments: validation.args.max,
calleeName,
- }),
- )
+ })
+ );
}
}
- parsedTree.arguments.map((arg: Record) => validateAgainstMeta(arg, errors))
+ parsedTree.arguments.map((arg: Record) =>
+ validateAgainstMeta(arg, errors)
+ );
// get args type and validate
const validateResult = parsedTree.arguments.map((arg) => {
- return validateAgainstMeta(arg, errors, typeErrors)
- })
+ return validateAgainstMeta(arg, errors, typeErrors);
+ });
const argsTypes = validateResult.map((v: any) => v.returnType);
if (typeof validateResult[0].returnType === 'function') {
- returnType = formulas[calleeName].returnType(argsTypes)
+ returnType = formulas[calleeName].returnType(argsTypes);
} else if (validateResult[0]) {
- returnType = formulas[calleeName].returnType
+ returnType = formulas[calleeName].returnType;
}
// validate data type
if (parsedTree.callee.type === JSEPNode.IDENTIFIER) {
- const expectedType = formulas[calleeName.toUpperCase()].type
+ const expectedType = formulas[calleeName.toUpperCase()].type;
- if (expectedType === formulaTypes.NUMERIC) {
+ if (expectedType === FormulaDataTypes.NUMERIC) {
if (calleeName === 'WEEKDAY') {
// parsedTree.arguments[0] = date
validateAgainstType(
parsedTree.arguments[0],
- formulaTypes.DATE,
+ FormulaDataTypes.DATE,
(v: any) => {
if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate'))
+ typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate'));
}
},
- typeErrors,
- )
+ typeErrors
+ );
// parsedTree.arguments[1] = startDayOfWeek (optional)
validateAgainstType(
parsedTree.arguments[1],
- formulaTypes.STRING,
+ FormulaDataTypes.STRING,
(v: any) => {
if (
typeof v !== 'string' ||
- !['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'].includes(v.toLowerCase())
+ ![
+ 'sunday',
+ 'monday',
+ 'tuesday',
+ 'wednesday',
+ 'thursday',
+ 'friday',
+ 'saturday',
+ ].includes(v.toLowerCase())
) {
- typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate'))
+ typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate'));
}
},
- typeErrors,
- )
+ typeErrors
+ );
} else {
- parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors, argsTypes))
+ parsedTree.arguments.map((arg: Record) =>
+ validateAgainstType(arg, expectedType, null, typeErrors, argsTypes)
+ );
}
- } else if (expectedType === formulaTypes.DATE) {
+ } else if (expectedType === FormulaDataTypes.DATE) {
if (calleeName === 'DATEADD') {
// parsedTree.arguments[0] = date
validateAgainstType(
parsedTree.arguments[0],
- formulaTypes.DATE,
+ FormulaDataTypes.DATE,
(v: any) => {
if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.firstParamDateAddHaveDate'))
+ typeErrors.add(t('msg.formula.firstParamDateAddHaveDate'));
}
},
- typeErrors,
- )
+ typeErrors
+ );
// parsedTree.arguments[1] = numeric
validateAgainstType(
parsedTree.arguments[1],
- formulaTypes.NUMERIC,
+ FormulaDataTypes.NUMERIC,
(v: any) => {
if (typeof v !== 'number') {
- typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber'))
+ typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber'));
}
},
- typeErrors,
- )
+ typeErrors
+ );
// parsedTree.arguments[2] = ["day" | "week" | "month" | "year"]
validateAgainstType(
parsedTree.arguments[2],
- formulaTypes.STRING,
+ FormulaDataTypes.STRING,
(v: any) => {
if (!['day', 'week', 'month', 'year'].includes(v)) {
- typeErrors.add(typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate')))
+ typeErrors.add(
+ typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate'))
+ );
}
},
- typeErrors,
- )
+ typeErrors
+ );
} else if (calleeName === 'DATETIME_DIFF') {
// parsedTree.arguments[0] = date
validateAgainstType(
parsedTree.arguments[0],
- formulaTypes.DATE,
+ FormulaDataTypes.DATE,
(v: any) => {
if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate'))
+ typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate'));
}
},
- typeErrors,
- )
+ typeErrors
+ );
// parsedTree.arguments[1] = date
validateAgainstType(
parsedTree.arguments[1],
- formulaTypes.DATE,
+ FormulaDataTypes.DATE,
(v: any) => {
if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate'))
+ typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate'));
}
},
- typeErrors,
- )
+ typeErrors
+ );
// parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"]
validateAgainstType(
parsedTree.arguments[2],
- formulaTypes.STRING,
+ FormulaDataTypes.STRING,
(v: any) => {
if (
![
@@ -358,23 +1163,27 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
'y',
].includes(v)
) {
- typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate'))
+ typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate'));
}
},
- typeErrors,
- )
+ typeErrors
+ );
}
}
}
- errors = new Set([...errors, ...typeErrors])
+ errors = new Set([...errors, ...typeErrors]);
} else if (parsedTree.type === JSEPNode.IDENTIFIER) {
- if (supportedColumns.value.filter((c) => !column || column.value?.id !== c.id).every((c) => c.title !== parsedTree.name)) {
+ if (
+ supportedColumns.value
+ .filter((c) => !column || column.value?.id !== c.id)
+ .every((c) => c.title !== parsedTree.name)
+ ) {
errors.add(
t('msg.formula.columnNotAvailable', {
columnName: parsedTree.name,
- }),
- )
+ })
+ );
}
// check circular reference
@@ -390,149 +1199,170 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
...new Set(
(c.colOptions.formula.match(/cl_\w{14}/g) || []).filter(
(colId: string) =>
- supportedColumns.value.filter((col: ColumnType) => col.id === colId && col.uidt === UITypes.Formula).length,
- ),
+ supportedColumns.value.filter(
+ (col: ColumnType) =>
+ col.id === colId && col.uidt === UITypes.Formula
+ ).length
+ )
),
- ]
+ ];
if (neighbours.length > 0) {
// e.g. formula column 1 -> [formula column 2, formula column3]
- res.push({ [c.id]: neighbours })
+ res.push({ [c.id]: neighbours });
}
- return res
- }, [])
+ return res;
+ }, []);
// include target formula column (i.e. the one to be saved if applicable)
const targetFormulaCol = supportedColumns.value.find(
- (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula,
- )
+ (c: ColumnType) =>
+ c.title === parsedTree.name && c.uidt === UITypes.Formula
+ );
if (targetFormulaCol && column.value?.id) {
formulaPaths.push({
[column.value?.id as string]: [targetFormulaCol.id],
- })
+ });
}
- const vertices = formulaPaths.length
+ const vertices = formulaPaths.length;
if (vertices > 0) {
// perform kahn's algo for cycle detection
- const adj = new Map()
- const inDegrees = new Map()
+ const adj = new Map();
+ const inDegrees = new Map();
// init adjacency list & indegree
for (const [_, v] of Object.entries(formulaPaths)) {
- const src = Object.keys(v)[0]
- const neighbours = v[src]
- inDegrees.set(src, inDegrees.get(src) || 0)
+ const src = Object.keys(v)[0];
+ const neighbours = v[src];
+ inDegrees.set(src, inDegrees.get(src) || 0);
for (const neighbour of neighbours) {
- adj.set(src, (adj.get(src) || new Set()).add(neighbour))
- inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1)
+ adj.set(src, (adj.get(src) || new Set()).add(neighbour));
+ inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1);
}
}
- const queue: string[] = []
+ const queue: string[] = [];
// put all vertices with in-degree = 0 (i.e. no incoming edges) to queue
inDegrees.forEach((inDegree, col) => {
if (inDegree === 0) {
// in-degree = 0 means we start traversing from this node
- queue.push(col)
+ queue.push(col);
}
- })
+ });
// init count of visited vertices
- let visited = 0
+ let visited = 0;
// BFS
while (queue.length !== 0) {
// remove a vertex from the queue
- const src = queue.shift()
+ const src = queue.shift();
// if this node has neighbours, increase visited by 1
- const neighbours = adj.get(src) || new Set()
+ const neighbours = adj.get(src) || new Set();
if (neighbours.size > 0) {
- visited += 1
+ visited += 1;
}
// iterate each neighbouring nodes
neighbours.forEach((neighbour: string) => {
// decrease in-degree of its neighbours by 1
- inDegrees.set(neighbour, inDegrees.get(neighbour) - 1)
+ inDegrees.set(neighbour, inDegrees.get(neighbour) - 1);
// if in-degree becomes 0
if (inDegrees.get(neighbour) === 0) {
// then put the neighboring node to the queue
- queue.push(neighbour)
+ queue.push(neighbour);
}
- })
+ });
}
// vertices not same as visited = cycle found
if (vertices !== visited) {
- errors.add(t('msg.formula.cantSaveCircularReference'))
+ errors.add(t('msg.formula.cantSaveCircularReference'));
}
}
} else if (parsedTree.type === JSEPNode.BINARY_EXP) {
if (!availableBinOps.includes(parsedTree.operator)) {
- errors.add(t('msg.formula.operationNotAvailable', { operation: parsedTree.operator }))
+ errors.add(
+ t('msg.formula.operationNotAvailable', {
+ operation: parsedTree.operator,
+ })
+ );
}
- validateAgainstMeta(parsedTree.left, errors)
- validateAgainstMeta(parsedTree.right, errors)
+ validateAgainstMeta(parsedTree.left, errors);
+ validateAgainstMeta(parsedTree.right, errors);
// todo: type extraction for binary exps
- returnType = formulaTypes.NUMERIC
- } else if (parsedTree.type === JSEPNode.LITERAL || parsedTree.type === JSEPNode.UNARY_EXP) {
+ returnType = FormulaDataTypes.NUMERIC;
+ } else if (
+ parsedTree.type === JSEPNode.LITERAL ||
+ parsedTree.type === JSEPNode.UNARY_EXP
+ ) {
if (parsedTree.type === JSEPNode.LITERAL) {
if (typeof parsedTree.value === 'number') {
- returnType = formulaTypes.NUMERIC
+ returnType = FormulaDataTypes.NUMERIC;
} else if (typeof parsedTree.value === 'string') {
- returnType = formulaTypes.STRING
+ returnType = FormulaDataTypes.STRING;
} else if (typeof parsedTree.value === 'boolean') {
- returnType = formulaTypes.BOOLEAN
+ returnType = FormulaDataTypes.BOOLEAN;
} else {
- returnType = formulaTypes.STRING
+ returnType = FormulaDataTypes.STRING;
}
}
// do nothing
} else if (parsedTree.type === JSEPNode.COMPOUND) {
if (parsedTree.body.length) {
- errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'))
+ errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'));
}
} else {
- errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'))
+ errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'));
}
- return { errors, returnType }
+ return { errors, returnType };
}
-function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set(), argTypes: formulaTypes = []) {
- let type
+function validateAgainstType(
+ parsedTree: any,
+ expectedType: string,
+ func: any,
+ typeErrors = new Set(),
+ argTypes: FormulaDataTypes = []
+) {
+ let type;
if (parsedTree === false || typeof parsedTree === 'undefined') {
- return typeErrors
+ return typeErrors;
}
if (parsedTree.type === JSEPNode.LITERAL) {
if (typeof func === 'function') {
- func(parsedTree.value)
- } else if (expectedType === formulaTypes.NUMERIC) {
+ func(parsedTree.value);
+ } else if (expectedType === FormulaDataTypes.NUMERIC) {
if (typeof parsedTree.value !== 'number') {
- typeErrors.add(t('msg.formula.numericTypeIsExpected'))
+ typeErrors.add(t('msg.formula.numericTypeIsExpected'));
} else {
- type = formulaTypes.NUMERIC
+ type = FormulaDataTypes.NUMERIC;
}
- } else if (expectedType === formulaTypes.STRING) {
+ } else if (expectedType === FormulaDataTypes.STRING) {
if (typeof parsedTree.value !== 'string') {
- typeErrors.add(t('msg.formula.stringTypeIsExpected'))
+ typeErrors.add(t('msg.formula.stringTypeIsExpected'));
} else {
- type = formulaTypes.STRING
+ type = FormulaDataTypes.STRING;
}
}
} else if (parsedTree.type === JSEPNode.IDENTIFIER) {
- const col = supportedColumns.value.find((c) => c.title === parsedTree.name)
+ const col = supportedColumns.value.find((c) => c.title === parsedTree.name);
if (col === undefined) {
- return
+ return;
}
if (col.uidt === UITypes.Formula) {
- const foundType = getRootDataType(jsep(col.colOptions?.formula_raw))
- type = foundType
+ const foundType = getRootDataType(jsep(col.colOptions?.formula_raw));
+ type = foundType;
if (foundType === 'N/A') {
- typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: col.title }))
+ typeErrors.add(
+ t('msg.formula.notSupportedToReferenceColumn', {
+ columnName: col.title,
+ })
+ );
} else if (expectedType !== foundType) {
typeErrors.add(
t('msg.formula.typeIsExpectedButFound', {
type: expectedType,
found: foundType,
- }),
- )
+ })
+ );
}
} else {
switch (col.uidt) {
@@ -544,17 +1374,17 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t
case UITypes.PhoneNumber:
case UITypes.Email:
case UITypes.URL:
- if (expectedType !== formulaTypes.STRING) {
+ if (expectedType !== FormulaDataTypes.STRING) {
typeErrors.add(
t('msg.formula.columnWithTypeFoundButExpected', {
columnName: parsedTree.name,
- columnType: formulaTypes.STRING,
+ columnType: FormulaDataTypes.STRING,
expectedType,
- }),
- )
+ })
+ );
}
- type = formulaTypes.STRING
- break
+ type = FormulaDataTypes.STRING;
+ break;
// numeric
case UITypes.Year:
@@ -564,34 +1394,34 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t
case UITypes.Count:
case UITypes.AutoNumber:
case UITypes.Currency:
- if (expectedType !== formulaTypes.NUMERIC) {
+ if (expectedType !== FormulaDataTypes.NUMERIC) {
typeErrors.add(
t('msg.formula.columnWithTypeFoundButExpected', {
columnName: parsedTree.name,
- columnType: formulaTypes.NUMERIC,
+ columnType: FormulaDataTypes.NUMERIC,
expectedType,
- }),
- )
+ })
+ );
}
- type = formulaTypes.NUMERIC
- break
+ type = FormulaDataTypes.NUMERIC;
+ break;
// date
case UITypes.Date:
case UITypes.DateTime:
case UITypes.CreateTime:
case UITypes.LastModifiedTime:
- if (expectedType !== formulaTypes.DATE) {
+ if (expectedType !== FormulaDataTypes.DATE) {
typeErrors.add(
t('msg.formula.columnWithTypeFoundButExpected', {
columnName: parsedTree.name,
- columnType: formulaTypes.DATE,
+ columnType: FormulaDataTypes.DATE,
expectedType,
- }),
- )
+ })
+ );
}
- type = formulaTypes.DATE
- break
+ type = FormulaDataTypes.DATE;
+ break;
// not supported
case UITypes.ForeignKey:
@@ -608,46 +1438,58 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t
case UITypes.Collaborator:
case UITypes.QrCode:
default:
- typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: parsedTree.name }))
- break
+ typeErrors.add(
+ t('msg.formula.notSupportedToReferenceColumn', {
+ columnName: parsedTree.name,
+ })
+ );
+ break;
}
}
- } else if (parsedTree.type === JSEPNode.UNARY_EXP || parsedTree.type === JSEPNode.BINARY_EXP) {
- if (expectedType !== formulaTypes.NUMERIC) {
+ } else if (
+ parsedTree.type === JSEPNode.UNARY_EXP ||
+ parsedTree.type === JSEPNode.BINARY_EXP
+ ) {
+ if (expectedType !== FormulaDataTypes.NUMERIC) {
// parsedTree.name won't be available here
typeErrors.add(
t('msg.formula.typeIsExpectedButFound', {
- type: formulaTypes.NUMERIC,
+ type: FormulaDataTypes.NUMERIC,
found: expectedType,
- }),
- )
+ })
+ );
}
- type = formulaTypes.NUMERIC
+ type = FormulaDataTypes.NUMERIC;
} else if (parsedTree.type === JSEPNode.CALL_EXP) {
- const calleeName = parsedTree.callee.name.toUpperCase()
- if (formulas[calleeName]?.type && expectedType !== formulas[calleeName].type) {
+ const calleeName = parsedTree.callee.name.toUpperCase();
+ if (
+ formulas[calleeName]?.type &&
+ expectedType !== formulas[calleeName].type
+ ) {
typeErrors.add(
t('msg.formula.typeIsExpectedButFound', {
type: expectedType,
found: formulas[calleeName].type,
- }),
- )
+ })
+ );
}
// todo: derive type from returnType
- type = formulas[calleeName]?.type
+ type = formulas[calleeName]?.type;
}
- return { type, typeErrors }
+ return { type, typeErrors };
}
function getRootDataType(parsedTree: any): any {
// given a parse tree, return the data type of it
if (parsedTree.type === JSEPNode.CALL_EXP) {
- return formulas[parsedTree.callee.name.toUpperCase()].type
+ return formulas[parsedTree.callee.name.toUpperCase()].type;
} else if (parsedTree.type === JSEPNode.IDENTIFIER) {
- const col = supportedColumns.value.find((c) => c.title === parsedTree.name) as Record
+ const col = supportedColumns.value.find(
+ (c) => c.title === parsedTree.name
+ ) as Record;
if (col?.uidt === UITypes.Formula) {
- return getRootDataType(jsep(col?.formula_raw))
+ return getRootDataType(jsep(col?.formula_raw));
} else {
switch (col?.uidt) {
// string
@@ -658,7 +1500,7 @@ function getRootDataType(parsedTree: any): any {
case UITypes.PhoneNumber:
case UITypes.Email:
case UITypes.URL:
- return formulaTypes.STRING
+ return FormulaDataTypes.STRING;
// numeric
case UITypes.Year:
@@ -667,14 +1509,14 @@ function getRootDataType(parsedTree: any): any {
case UITypes.Rating:
case UITypes.Count:
case UITypes.AutoNumber:
- return formulaTypes.NUMERIC
+ return FormulaDataTypes.NUMERIC;
// date
case UITypes.Date:
case UITypes.DateTime:
case UITypes.CreateTime:
case UITypes.LastModifiedTime:
- return formulaTypes.DATE
+ return FormulaDataTypes.DATE;
// not supported
case UITypes.ForeignKey:
@@ -692,28 +1534,248 @@ function getRootDataType(parsedTree: any): any {
case UITypes.Collaborator:
case UITypes.QrCode:
default:
- return 'N/A'
+ return 'N/A';
}
}
- } else if (parsedTree.type === JSEPNode.BINARY_EXP || parsedTree.type === JSEPNode.UNARY_EXP) {
- return formulaTypes.NUMERIC
+ } else if (
+ parsedTree.type === JSEPNode.BINARY_EXP ||
+ parsedTree.type === JSEPNode.UNARY_EXP
+ ) {
+ return FormulaDataTypes.NUMERIC;
} else if (parsedTree.type === JSEPNode.LITERAL) {
- return typeof parsedTree.value
+ return typeof parsedTree.value;
} else {
- return 'N/A'
+ return 'N/A';
}
}
function isCurlyBracketBalanced() {
// count number of opening curly brackets and closing curly brackets
- const cntCurlyBrackets = (formulaRef.value.$el.value.match(/\{|}/g) || []).reduce(
- (acc: Record, cur: number) => {
- acc[cur] = (acc[cur] || 0) + 1
- return acc
- },
- {},
- )
- return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0)
+ const cntCurlyBrackets = (
+ formulaRef.value.$el.value.match(/\{|}/g) || []
+ ).reduce((acc: Record, cur: number) => {
+ acc[cur] = (acc[cur] || 0) + 1;
+ return acc;
+ }, {});
+ return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0);
+}
+*/
+
+enum FormulaErrorType {
+ NOT_AVAILABLE = 'NOT_AVAILABLE',
+ NOT_SUPPORTED = 'NOT_SUPPORTED',
+ 'MIN_ARG' = 'MIN_ARG',
+ 'MAX_ARG' = 'MAX_ARG',
+ 'TYPE_MISMATCH' = 'TYPE_MISMATCH',
+ 'INVALID_ARG' = 'INVALID_ARG',
+ 'INVALID_ARG_TYPE' = 'INVALID_ARG_TYPE',
+ 'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE',
+ 'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT',
}
+class FormulaError extends Error {
+ public name: string;
+ public type: FormulaErrorType;
+ constructor(
+ type: FormulaErrorType,
+ name: string,
+ message: string = 'Formula Error'
+ ) {
+ super(message);
+ this.name = name;
+ this.type = type;
+ }
+}
+
+export function validateFormulaAndExtractTreeWithType(
+ formula,
+ columns: ColumnType[]
+) {
+ const colAliasToColMap = {};
+ const colIdToColMap = {};
+
+ for (const col of columns) {
+ colAliasToColMap[col.title] = col;
+ colIdToColMap[col.id] = col;
+ }
+
+ const validateAndExtract = (parsedTree: any) => {
+ const res: {
+ dataType?: FormulaDataTypes;
+ errors?: Set;
+ [key: string]: any;
+ } = { ...parsedTree };
+
+ if (parsedTree.type === JSEPNode.CALL_EXP) {
+ const calleeName = parsedTree.callee.name.toUpperCase();
+ // validate function name
+ if (!formulas[calleeName]) {
+ throw new FormulaError(
+ FormulaErrorType.INVALID_ARG_TYPE,
+ 'Function not available'
+ );
+ //t('msg.formula.functionNotAvailable', { function: calleeName })
+ }
+ // validate arguments
+ const validation =
+ formulas[calleeName] && formulas[calleeName].validation;
+ if (validation && validation.args) {
+ if (
+ validation.args.rqd !== undefined &&
+ validation.args.rqd !== parsedTree.arguments.length
+ ) {
+ throw new FormulaError(
+ FormulaErrorType.INVALID_ARG,
+ calleeName,
+ 'Required arguments missing'
+ );
+
+ // errors.add(
+ // t('msg.formula.requiredArgumentsFormula', {
+ // requiredArguments: validation.args.rqd,
+ // calleeName,
+ // })
+ // );
+ } else if (
+ validation.args.min !== undefined &&
+ validation.args.min > parsedTree.arguments.length
+ ) {
+ throw new FormulaError(
+ FormulaErrorType.MIN_ARG,
+ calleeName,
+ 'Minimum arguments required'
+ );
+
+ // errors.add(
+ // t('msg.formula.minRequiredArgumentsFormula', {
+ // minRequiredArguments: validation.args.min,
+ // calleeName,
+ // })
+ // );
+ } else if (
+ validation.args.max !== undefined &&
+ validation.args.max < parsedTree.arguments.length
+ ) {
+ throw new FormulaError(
+ FormulaErrorType.MAX_ARG,
+ calleeName,
+ 'Maximum arguments required'
+ );
+
+ // errors.add(
+ // t('msg.formula.maxRequiredArgumentsFormula', {
+ // maxRequiredArguments: validation.args.max,
+ // calleeName,
+ // })
+ // );
+ }
+ }
+ // get args type and validate
+ const validateResult = res.arguments = parsedTree.arguments.map((arg) => {
+ return validateAndExtract(arg);
+ });
+
+ const argsTypes = validateResult.map((v: any) => v.dataType);
+
+ if (typeof formulas[calleeName].returnType === 'function') {
+ res.dataType = (formulas[calleeName].returnType as any)?.(
+ argsTypes
+ ) as FormulaDataTypes;
+ } else if (formulas[calleeName].returnType) {
+ res.dataType = formulas[calleeName].returnType as FormulaDataTypes;
+ }
+ } else if (parsedTree.type === JSEPNode.IDENTIFIER) {
+ const col = columns.find[parsedTree.name] as Record<
+ string,
+ any
+ >;
+ res.name = col.id;
+
+ if (col?.uidt === UITypes.Formula) {
+ // todo: check for circular reference
+
+ // todo: extract the type and return
+ const formulaRes = validateFormulaAndExtractTreeWithType(
+ col.colOptions.formula,
+ columns
+ );
+
+ res.dataType = formulaRes as any;
+ } else {
+
+ switch (col?.uidt) {
+ // string
+ case UITypes.SingleLineText:
+ case UITypes.LongText:
+ case UITypes.MultiSelect:
+ case UITypes.SingleSelect:
+ case UITypes.PhoneNumber:
+ case UITypes.Email:
+ case UITypes.URL:
+ res.dataType = FormulaDataTypes.STRING;
+ break;
+ // numeric
+ case UITypes.Year:
+ case UITypes.Number:
+ case UITypes.Decimal:
+ case UITypes.Rating:
+ case UITypes.Count:
+ case UITypes.AutoNumber:
+ res.dataType = FormulaDataTypes.NUMERIC;
+ break;
+ // date
+ case UITypes.Date:
+ case UITypes.DateTime:
+ case UITypes.CreateTime:
+ case UITypes.LastModifiedTime:
+ res.dataType = FormulaDataTypes.DATE;
+ break;
+ // not supported
+ case UITypes.ForeignKey:
+ case UITypes.Attachment:
+ case UITypes.ID:
+ case UITypes.Time:
+ case UITypes.Currency:
+ case UITypes.Percent:
+ case UITypes.Duration:
+ case UITypes.Rollup:
+ case UITypes.Lookup:
+ case UITypes.Barcode:
+ case UITypes.Button:
+ case UITypes.Checkbox:
+ case UITypes.Collaborator:
+ case UITypes.QrCode:
+ default:
+ throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, '');
+ }
+ }
+ } else if (parsedTree.type === JSEPNode.LITERAL) {
+ if (typeof parsedTree.value === 'number') {
+ res.dataType = FormulaDataTypes.NUMERIC;
+ } else if (typeof parsedTree.value === 'string') {
+ res.dataType = FormulaDataTypes.STRING;
+ } else if (typeof parsedTree.value === 'boolean') {
+ res.dataType = FormulaDataTypes.BOOLEAN;
+ } else {
+ res.dataType = FormulaDataTypes.STRING;
+ }
+ } else if (
+ parsedTree.type === JSEPNode.BINARY_EXP ||
+ parsedTree.type === JSEPNode.UNARY_EXP
+ ) {
+ res.left = validateAndExtract(parsedTree.left);
+ res.right = validateAndExtract(parsedTree.right);
+ res.dataType = FormulaDataTypes.NUMERIC;
+ } else {
+ // res.type= 'N/A';
+ }
+
+ return res;
+ };
+ // register jsep curly hook
+ jsep.plugins.register(jsepCurlyHook);
+ const parsedFormula = jsep(formula);
+ const result = validateAndExtract(parsedFormula);
+ return result;
+}
From f4944a2dd72d93df4f9df7b21c43ea2494f7ae20 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:58 +0000
Subject: [PATCH 010/262] feat: support non conditional_expression in if
formula
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 2 +-
packages/nocodb-sdk/tsconfig.json | 2 +-
.../src/db/formulav2/formulaQueryBuilderv2.ts | 15 ++++++++++---
.../src/db/functionMappings/commonFns.ts | 21 ++++++++++++++++---
4 files changed, 32 insertions(+), 8 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index c1c19b2562..5d0dba7305 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1685,7 +1685,7 @@ export function validateFormulaAndExtractTreeWithType(
res.dataType = formulas[calleeName].returnType as FormulaDataTypes;
}
} else if (parsedTree.type === JSEPNode.IDENTIFIER) {
- const col = columns.find[parsedTree.name] as Record<
+ const col = (colIdToColMap[parsedTree.name] || colAliasToColMap[parsedTree.name]) as Record<
string,
any
>;
diff --git a/packages/nocodb-sdk/tsconfig.json b/packages/nocodb-sdk/tsconfig.json
index 0bd395cc39..86fdaa2404 100644
--- a/packages/nocodb-sdk/tsconfig.json
+++ b/packages/nocodb-sdk/tsconfig.json
@@ -38,7 +38,7 @@
// "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */,
"lib": ["es2017","dom"],
- "types": ["jest", "node"],
+ "types": ["jest"],
"typeRoots": ["node_modules/@types", "src/types"],
"baseUrl": "./src",
"paths": {
diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
index 390c733025..e8bd658f97 100644
--- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
+++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
@@ -2,6 +2,7 @@ import jsep from 'jsep';
import {
jsepCurlyHook,
UITypes,
+ validateFormulaAndExtractTreeWithType,
validateDateWithUnknownFormat,
} from 'nocodb-sdk';
import mapFunctionName from '../mapFunctionName';
@@ -14,7 +15,10 @@ import type LookupColumn from '~/models/LookupColumn';
import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
import NocoCache from '~/cache/NocoCache';
import { CacheGetType, CacheScope } from '~/utils/globals';
-import { convertDateFormatForConcat } from '~/helpers/formulaFnHelper';
+import {
+ convertDateFormatForConcat,
+ validateDateWithUnknownFormat,
+} from '~/helpers/formulaFnHelper';
import FormulaColumn from '~/models/FormulaColumn';
// todo: switch function based on database
@@ -62,14 +66,19 @@ async function _formulaQueryBuilder(
) {
const knex = baseModelSqlv2.dbDriver;
+ const columns = await model.getColumns();
// formula may include double curly brackets in previous version
// convert to single curly bracket here for compatibility
- const tree = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}'));
+ // const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}'));
+ const tree = validateFormulaAndExtractTreeWithType(
+ _tree.replaceAll('{{', '{').replaceAll('}}', '}'),
+ columns,
+ );
const columnIdToUidt = {};
// todo: improve - implement a common solution for filter, sort, formula, etc
- for (const col of await model.getColumns()) {
+ for (const col of columns) {
columnIdToUidt[col.id] = col.uidt;
if (col.id in aliasToColumn) continue;
switch (col.uidt) {
diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts
index 547f65796e..6f87f5f84d 100644
--- a/packages/nocodb/src/db/functionMappings/commonFns.ts
+++ b/packages/nocodb/src/db/functionMappings/commonFns.ts
@@ -1,3 +1,4 @@
+import { FormulaDataTypes } from 'nocodb-sdk';
import type { MapFnArgs } from '../mapFunctionName';
import { NcError } from '~/helpers/catchError';
@@ -36,11 +37,25 @@ export default {
};
},
IF: async (args: MapFnArgs) => {
+ const condArg = (await args.fn(args.pt.arguments[0])).builder.toQuery();
+
+ let cond = condArg;
+
+ switch (args.pt.arguments[0].dataType as FormulaDataTypes) {
+ case FormulaDataTypes.NUMERIC:
+ cond = `(${condArg}) IS NOT NULL OR (${condArg}) != 0`;
+ break;
+ case FormulaDataTypes.STRING:
+ cond = `(${condArg}) IS NOT NULL OR (${condArg}) != ''`;
+ break;
+ case FormulaDataTypes.BOOLEAN:
+ cond = `(${condArg}) IS NOT NULL OR (${condArg}) != false`;
+ break;
+ }
+
let query = args.knex
.raw(
- `\n\tWHEN ${(
- await args.fn(args.pt.arguments[0])
- ).builder.toQuery()} THEN ${(
+ `\n\tWHEN ${cond} THEN ${(
await args.fn(args.pt.arguments[1])
).builder.toQuery()}`,
)
From 019e5b0919536b18157fa42385f4f964f8a98251 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:58 +0000
Subject: [PATCH 011/262] fix: correction in IF formula
---
packages/nocodb/src/db/functionMappings/commonFns.ts | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts
index 6f87f5f84d..8e39d1f4de 100644
--- a/packages/nocodb/src/db/functionMappings/commonFns.ts
+++ b/packages/nocodb/src/db/functionMappings/commonFns.ts
@@ -43,13 +43,13 @@ export default {
switch (args.pt.arguments[0].dataType as FormulaDataTypes) {
case FormulaDataTypes.NUMERIC:
- cond = `(${condArg}) IS NOT NULL OR (${condArg}) != 0`;
+ cond = `(${condArg}) IS NOT NULL AND (${condArg}) != 0`;
break;
case FormulaDataTypes.STRING:
- cond = `(${condArg}) IS NOT NULL OR (${condArg}) != ''`;
+ cond = `(${condArg}) IS NOT NULL AND (${condArg}) != ''`;
break;
case FormulaDataTypes.BOOLEAN:
- cond = `(${condArg}) IS NOT NULL OR (${condArg}) != false`;
+ cond = `(${condArg}) IS NOT NULL AND (${condArg}) != false`;
break;
}
From 82e8bcd52368ccfc845dce4f73902259878fc702 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:58 +0000
Subject: [PATCH 012/262] fix: DATE value with if
---
packages/nocodb/src/db/functionMappings/commonFns.ts | 3 +++
1 file changed, 3 insertions(+)
diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts
index 8e39d1f4de..d5d987c866 100644
--- a/packages/nocodb/src/db/functionMappings/commonFns.ts
+++ b/packages/nocodb/src/db/functionMappings/commonFns.ts
@@ -51,6 +51,9 @@ export default {
case FormulaDataTypes.BOOLEAN:
cond = `(${condArg}) IS NOT NULL AND (${condArg}) != false`;
break;
+ case FormulaDataTypes.DATE:
+ cond = `(${condArg}) IS NOT NULL`;
+ break;
}
let query = args.knex
From 36277d55093890b318eca2796b8c416f8baa676f Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:58 +0000
Subject: [PATCH 013/262] feat: add migration and hide parsed tree data from
api response
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 713 +++---------------
.../src/db/formulav2/formulaQueryBuilderv2.ts | 24 +-
.../v2/nc_038_formula_parsed_tree_column.ts | 16 +
packages/nocodb/src/models/FormulaColumn.ts | 24 +-
.../nocodb/src/services/columns.service.ts | 6 +-
5 files changed, 162 insertions(+), 621 deletions(-)
create mode 100644 packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 5d0dba7305..94b3dcc62d 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -2,6 +2,7 @@ import jsep from 'jsep';
import { ColumnType } from './Api';
import UITypes from './UITypes';
+import { formulaTypes } from '../../../nc-gui/utils';
export const jsepCurlyHook = {
name: 'curly',
@@ -220,6 +221,7 @@ interface FormulaMeta {
min?: number;
max?: number;
rqd?: number;
+ validator?: (args: formulaTypes[]) => boolean;
};
};
description?: string;
@@ -230,7 +232,6 @@ interface FormulaMeta {
const formulas: Record = {
AVG: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
min: 1,
@@ -309,7 +310,6 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
AND: {
- type: FormulaDataTypes.COND_EXP,
validation: {
args: {
min: 1,
@@ -321,7 +321,6 @@ const formulas: Record = {
returnType: FormulaDataTypes.COND_EXP,
},
OR: {
- type: FormulaDataTypes.COND_EXP,
validation: {
args: {
min: 1,
@@ -964,603 +963,6 @@ const formulas: Record = {
// },
};
-/*
-function validateAgainstMeta(
- parsedTree: any,
- errors = new Set(),
- typeErrors = new Set()
-) {
- let returnType: FormulaDataTypes;
- if (parsedTree.type === JSEPNode.CALL_EXP) {
- const calleeName = parsedTree.callee.name.toUpperCase();
- // validate function name
- if (!availableFunctions.includes(calleeName)) {
- errors.add(
- t('msg.formula.functionNotAvailable', { function: calleeName })
- );
- }
- // validate arguments
- const validation = formulas[calleeName] && formulas[calleeName].validation;
- if (validation && validation.args) {
- if (
- validation.args.rqd !== undefined &&
- validation.args.rqd !== parsedTree.arguments.length
- ) {
- errors.add(
- t('msg.formula.requiredArgumentsFormula', {
- requiredArguments: validation.args.rqd,
- calleeName,
- })
- );
- } else if (
- validation.args.min !== undefined &&
- validation.args.min > parsedTree.arguments.length
- ) {
- errors.add(
- t('msg.formula.minRequiredArgumentsFormula', {
- minRequiredArguments: validation.args.min,
- calleeName,
- })
- );
- } else if (
- validation.args.max !== undefined &&
- validation.args.max < parsedTree.arguments.length
- ) {
- errors.add(
- t('msg.formula.maxRequiredArgumentsFormula', {
- maxRequiredArguments: validation.args.max,
- calleeName,
- })
- );
- }
- }
-
- parsedTree.arguments.map((arg: Record) =>
- validateAgainstMeta(arg, errors)
- );
-
- // get args type and validate
- const validateResult = parsedTree.arguments.map((arg) => {
- return validateAgainstMeta(arg, errors, typeErrors);
- });
-
- const argsTypes = validateResult.map((v: any) => v.returnType);
-
- if (typeof validateResult[0].returnType === 'function') {
- returnType = formulas[calleeName].returnType(argsTypes);
- } else if (validateResult[0]) {
- returnType = formulas[calleeName].returnType;
- }
-
- // validate data type
- if (parsedTree.callee.type === JSEPNode.IDENTIFIER) {
- const expectedType = formulas[calleeName.toUpperCase()].type;
-
- if (expectedType === FormulaDataTypes.NUMERIC) {
- if (calleeName === 'WEEKDAY') {
- // parsedTree.arguments[0] = date
- validateAgainstType(
- parsedTree.arguments[0],
- FormulaDataTypes.DATE,
- (v: any) => {
- if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate'));
- }
- },
- typeErrors
- );
- // parsedTree.arguments[1] = startDayOfWeek (optional)
- validateAgainstType(
- parsedTree.arguments[1],
- FormulaDataTypes.STRING,
- (v: any) => {
- if (
- typeof v !== 'string' ||
- ![
- 'sunday',
- 'monday',
- 'tuesday',
- 'wednesday',
- 'thursday',
- 'friday',
- 'saturday',
- ].includes(v.toLowerCase())
- ) {
- typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate'));
- }
- },
- typeErrors
- );
- } else {
- parsedTree.arguments.map((arg: Record) =>
- validateAgainstType(arg, expectedType, null, typeErrors, argsTypes)
- );
- }
- } else if (expectedType === FormulaDataTypes.DATE) {
- if (calleeName === 'DATEADD') {
- // parsedTree.arguments[0] = date
- validateAgainstType(
- parsedTree.arguments[0],
- FormulaDataTypes.DATE,
- (v: any) => {
- if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.firstParamDateAddHaveDate'));
- }
- },
- typeErrors
- );
- // parsedTree.arguments[1] = numeric
- validateAgainstType(
- parsedTree.arguments[1],
- FormulaDataTypes.NUMERIC,
- (v: any) => {
- if (typeof v !== 'number') {
- typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber'));
- }
- },
- typeErrors
- );
- // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"]
- validateAgainstType(
- parsedTree.arguments[2],
- FormulaDataTypes.STRING,
- (v: any) => {
- if (!['day', 'week', 'month', 'year'].includes(v)) {
- typeErrors.add(
- typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate'))
- );
- }
- },
- typeErrors
- );
- } else if (calleeName === 'DATETIME_DIFF') {
- // parsedTree.arguments[0] = date
- validateAgainstType(
- parsedTree.arguments[0],
- FormulaDataTypes.DATE,
- (v: any) => {
- if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate'));
- }
- },
- typeErrors
- );
- // parsedTree.arguments[1] = date
- validateAgainstType(
- parsedTree.arguments[1],
- FormulaDataTypes.DATE,
- (v: any) => {
- if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate'));
- }
- },
- typeErrors
- );
- // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"]
- validateAgainstType(
- parsedTree.arguments[2],
- FormulaDataTypes.STRING,
- (v: any) => {
- if (
- ![
- 'milliseconds',
- 'ms',
- 'seconds',
- 's',
- 'minutes',
- 'm',
- 'hours',
- 'h',
- 'days',
- 'd',
- 'weeks',
- 'w',
- 'months',
- 'M',
- 'quarters',
- 'Q',
- 'years',
- 'y',
- ].includes(v)
- ) {
- typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate'));
- }
- },
- typeErrors
- );
- }
- }
- }
-
- errors = new Set([...errors, ...typeErrors]);
- } else if (parsedTree.type === JSEPNode.IDENTIFIER) {
- if (
- supportedColumns.value
- .filter((c) => !column || column.value?.id !== c.id)
- .every((c) => c.title !== parsedTree.name)
- ) {
- errors.add(
- t('msg.formula.columnNotAvailable', {
- columnName: parsedTree.name,
- })
- );
- }
-
- // check circular reference
- // e.g. formula1 -> formula2 -> formula1 should return circular reference error
-
- // get all formula columns excluding itself
- const formulaPaths = supportedColumns.value
- .filter((c) => c.id !== column.value?.id && c.uidt === UITypes.Formula)
- .reduce((res: Record[], c: Record) => {
- // in `formula`, get all the (unique) target neighbours
- // i.e. all column id (e.g. cl_xxxxxxxxxxxxxx) with formula type
- const neighbours = [
- ...new Set(
- (c.colOptions.formula.match(/cl_\w{14}/g) || []).filter(
- (colId: string) =>
- supportedColumns.value.filter(
- (col: ColumnType) =>
- col.id === colId && col.uidt === UITypes.Formula
- ).length
- )
- ),
- ];
- if (neighbours.length > 0) {
- // e.g. formula column 1 -> [formula column 2, formula column3]
- res.push({ [c.id]: neighbours });
- }
- return res;
- }, []);
- // include target formula column (i.e. the one to be saved if applicable)
- const targetFormulaCol = supportedColumns.value.find(
- (c: ColumnType) =>
- c.title === parsedTree.name && c.uidt === UITypes.Formula
- );
-
- if (targetFormulaCol && column.value?.id) {
- formulaPaths.push({
- [column.value?.id as string]: [targetFormulaCol.id],
- });
- }
- const vertices = formulaPaths.length;
- if (vertices > 0) {
- // perform kahn's algo for cycle detection
- const adj = new Map();
- const inDegrees = new Map();
- // init adjacency list & indegree
-
- for (const [_, v] of Object.entries(formulaPaths)) {
- const src = Object.keys(v)[0];
- const neighbours = v[src];
- inDegrees.set(src, inDegrees.get(src) || 0);
- for (const neighbour of neighbours) {
- adj.set(src, (adj.get(src) || new Set()).add(neighbour));
- inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1);
- }
- }
- const queue: string[] = [];
- // put all vertices with in-degree = 0 (i.e. no incoming edges) to queue
- inDegrees.forEach((inDegree, col) => {
- if (inDegree === 0) {
- // in-degree = 0 means we start traversing from this node
- queue.push(col);
- }
- });
- // init count of visited vertices
- let visited = 0;
- // BFS
- while (queue.length !== 0) {
- // remove a vertex from the queue
- const src = queue.shift();
- // if this node has neighbours, increase visited by 1
- const neighbours = adj.get(src) || new Set();
- if (neighbours.size > 0) {
- visited += 1;
- }
- // iterate each neighbouring nodes
- neighbours.forEach((neighbour: string) => {
- // decrease in-degree of its neighbours by 1
- inDegrees.set(neighbour, inDegrees.get(neighbour) - 1);
- // if in-degree becomes 0
- if (inDegrees.get(neighbour) === 0) {
- // then put the neighboring node to the queue
- queue.push(neighbour);
- }
- });
- }
- // vertices not same as visited = cycle found
- if (vertices !== visited) {
- errors.add(t('msg.formula.cantSaveCircularReference'));
- }
- }
- } else if (parsedTree.type === JSEPNode.BINARY_EXP) {
- if (!availableBinOps.includes(parsedTree.operator)) {
- errors.add(
- t('msg.formula.operationNotAvailable', {
- operation: parsedTree.operator,
- })
- );
- }
- validateAgainstMeta(parsedTree.left, errors);
- validateAgainstMeta(parsedTree.right, errors);
-
- // todo: type extraction for binary exps
- returnType = FormulaDataTypes.NUMERIC;
- } else if (
- parsedTree.type === JSEPNode.LITERAL ||
- parsedTree.type === JSEPNode.UNARY_EXP
- ) {
- if (parsedTree.type === JSEPNode.LITERAL) {
- if (typeof parsedTree.value === 'number') {
- returnType = FormulaDataTypes.NUMERIC;
- } else if (typeof parsedTree.value === 'string') {
- returnType = FormulaDataTypes.STRING;
- } else if (typeof parsedTree.value === 'boolean') {
- returnType = FormulaDataTypes.BOOLEAN;
- } else {
- returnType = FormulaDataTypes.STRING;
- }
- }
- // do nothing
- } else if (parsedTree.type === JSEPNode.COMPOUND) {
- if (parsedTree.body.length) {
- errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'));
- }
- } else {
- errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'));
- }
- return { errors, returnType };
-}
-
-function validateAgainstType(
- parsedTree: any,
- expectedType: string,
- func: any,
- typeErrors = new Set(),
- argTypes: FormulaDataTypes = []
-) {
- let type;
- if (parsedTree === false || typeof parsedTree === 'undefined') {
- return typeErrors;
- }
- if (parsedTree.type === JSEPNode.LITERAL) {
- if (typeof func === 'function') {
- func(parsedTree.value);
- } else if (expectedType === FormulaDataTypes.NUMERIC) {
- if (typeof parsedTree.value !== 'number') {
- typeErrors.add(t('msg.formula.numericTypeIsExpected'));
- } else {
- type = FormulaDataTypes.NUMERIC;
- }
- } else if (expectedType === FormulaDataTypes.STRING) {
- if (typeof parsedTree.value !== 'string') {
- typeErrors.add(t('msg.formula.stringTypeIsExpected'));
- } else {
- type = FormulaDataTypes.STRING;
- }
- }
- } else if (parsedTree.type === JSEPNode.IDENTIFIER) {
- const col = supportedColumns.value.find((c) => c.title === parsedTree.name);
-
- if (col === undefined) {
- return;
- }
-
- if (col.uidt === UITypes.Formula) {
- const foundType = getRootDataType(jsep(col.colOptions?.formula_raw));
- type = foundType;
- if (foundType === 'N/A') {
- typeErrors.add(
- t('msg.formula.notSupportedToReferenceColumn', {
- columnName: col.title,
- })
- );
- } else if (expectedType !== foundType) {
- typeErrors.add(
- t('msg.formula.typeIsExpectedButFound', {
- type: expectedType,
- found: foundType,
- })
- );
- }
- } else {
- switch (col.uidt) {
- // string
- case UITypes.SingleLineText:
- case UITypes.LongText:
- case UITypes.MultiSelect:
- case UITypes.SingleSelect:
- case UITypes.PhoneNumber:
- case UITypes.Email:
- case UITypes.URL:
- if (expectedType !== FormulaDataTypes.STRING) {
- typeErrors.add(
- t('msg.formula.columnWithTypeFoundButExpected', {
- columnName: parsedTree.name,
- columnType: FormulaDataTypes.STRING,
- expectedType,
- })
- );
- }
- type = FormulaDataTypes.STRING;
- break;
-
- // numeric
- case UITypes.Year:
- case UITypes.Number:
- case UITypes.Decimal:
- case UITypes.Rating:
- case UITypes.Count:
- case UITypes.AutoNumber:
- case UITypes.Currency:
- if (expectedType !== FormulaDataTypes.NUMERIC) {
- typeErrors.add(
- t('msg.formula.columnWithTypeFoundButExpected', {
- columnName: parsedTree.name,
- columnType: FormulaDataTypes.NUMERIC,
- expectedType,
- })
- );
- }
- type = FormulaDataTypes.NUMERIC;
- break;
-
- // date
- case UITypes.Date:
- case UITypes.DateTime:
- case UITypes.CreateTime:
- case UITypes.LastModifiedTime:
- if (expectedType !== FormulaDataTypes.DATE) {
- typeErrors.add(
- t('msg.formula.columnWithTypeFoundButExpected', {
- columnName: parsedTree.name,
- columnType: FormulaDataTypes.DATE,
- expectedType,
- })
- );
- }
- type = FormulaDataTypes.DATE;
- break;
-
- // not supported
- case UITypes.ForeignKey:
- case UITypes.Attachment:
- case UITypes.ID:
- case UITypes.Time:
- case UITypes.Percent:
- case UITypes.Duration:
- case UITypes.Rollup:
- case UITypes.Lookup:
- case UITypes.Barcode:
- case UITypes.Button:
- case UITypes.Checkbox:
- case UITypes.Collaborator:
- case UITypes.QrCode:
- default:
- typeErrors.add(
- t('msg.formula.notSupportedToReferenceColumn', {
- columnName: parsedTree.name,
- })
- );
- break;
- }
- }
- } else if (
- parsedTree.type === JSEPNode.UNARY_EXP ||
- parsedTree.type === JSEPNode.BINARY_EXP
- ) {
- if (expectedType !== FormulaDataTypes.NUMERIC) {
- // parsedTree.name won't be available here
- typeErrors.add(
- t('msg.formula.typeIsExpectedButFound', {
- type: FormulaDataTypes.NUMERIC,
- found: expectedType,
- })
- );
- }
-
- type = FormulaDataTypes.NUMERIC;
- } else if (parsedTree.type === JSEPNode.CALL_EXP) {
- const calleeName = parsedTree.callee.name.toUpperCase();
- if (
- formulas[calleeName]?.type &&
- expectedType !== formulas[calleeName].type
- ) {
- typeErrors.add(
- t('msg.formula.typeIsExpectedButFound', {
- type: expectedType,
- found: formulas[calleeName].type,
- })
- );
- }
- // todo: derive type from returnType
- type = formulas[calleeName]?.type;
- }
- return { type, typeErrors };
-}
-
-function getRootDataType(parsedTree: any): any {
- // given a parse tree, return the data type of it
- if (parsedTree.type === JSEPNode.CALL_EXP) {
- return formulas[parsedTree.callee.name.toUpperCase()].type;
- } else if (parsedTree.type === JSEPNode.IDENTIFIER) {
- const col = supportedColumns.value.find(
- (c) => c.title === parsedTree.name
- ) as Record;
- if (col?.uidt === UITypes.Formula) {
- return getRootDataType(jsep(col?.formula_raw));
- } else {
- switch (col?.uidt) {
- // string
- case UITypes.SingleLineText:
- case UITypes.LongText:
- case UITypes.MultiSelect:
- case UITypes.SingleSelect:
- case UITypes.PhoneNumber:
- case UITypes.Email:
- case UITypes.URL:
- return FormulaDataTypes.STRING;
-
- // numeric
- case UITypes.Year:
- case UITypes.Number:
- case UITypes.Decimal:
- case UITypes.Rating:
- case UITypes.Count:
- case UITypes.AutoNumber:
- return FormulaDataTypes.NUMERIC;
-
- // date
- case UITypes.Date:
- case UITypes.DateTime:
- case UITypes.CreateTime:
- case UITypes.LastModifiedTime:
- return FormulaDataTypes.DATE;
-
- // not supported
- case UITypes.ForeignKey:
- case UITypes.Attachment:
- case UITypes.ID:
- case UITypes.Time:
- case UITypes.Currency:
- case UITypes.Percent:
- case UITypes.Duration:
- case UITypes.Rollup:
- case UITypes.Lookup:
- case UITypes.Barcode:
- case UITypes.Button:
- case UITypes.Checkbox:
- case UITypes.Collaborator:
- case UITypes.QrCode:
- default:
- return 'N/A';
- }
- }
- } else if (
- parsedTree.type === JSEPNode.BINARY_EXP ||
- parsedTree.type === JSEPNode.UNARY_EXP
- ) {
- return FormulaDataTypes.NUMERIC;
- } else if (parsedTree.type === JSEPNode.LITERAL) {
- return typeof parsedTree.value;
- } else {
- return 'N/A';
- }
-}
-
-function isCurlyBracketBalanced() {
- // count number of opening curly brackets and closing curly brackets
- const cntCurlyBrackets = (
- formulaRef.value.$el.value.match(/\{|}/g) || []
- ).reduce((acc: Record, cur: number) => {
- acc[cur] = (acc[cur] || 0) + 1;
- return acc;
- }, {});
- return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0);
-}
-*/
-
enum FormulaErrorType {
NOT_AVAILABLE = 'NOT_AVAILABLE',
NOT_SUPPORTED = 'NOT_SUPPORTED',
@@ -1571,11 +973,13 @@ enum FormulaErrorType {
'INVALID_ARG_TYPE' = 'INVALID_ARG_TYPE',
'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE',
'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT',
+ CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE',
}
class FormulaError extends Error {
public name: string;
public type: FormulaErrorType;
+
constructor(
type: FormulaErrorType,
name: string,
@@ -1671,9 +1075,11 @@ export function validateFormulaAndExtractTreeWithType(
}
}
// get args type and validate
- const validateResult = res.arguments = parsedTree.arguments.map((arg) => {
- return validateAndExtract(arg);
- });
+ const validateResult = (res.arguments = parsedTree.arguments.map(
+ (arg) => {
+ return validateAndExtract(arg);
+ }
+ ));
const argsTypes = validateResult.map((v: any) => v.dataType);
@@ -1685,10 +1091,8 @@ export function validateFormulaAndExtractTreeWithType(
res.dataType = formulas[calleeName].returnType as FormulaDataTypes;
}
} else if (parsedTree.type === JSEPNode.IDENTIFIER) {
- const col = (colIdToColMap[parsedTree.name] || colAliasToColMap[parsedTree.name]) as Record<
- string,
- any
- >;
+ const col = (colIdToColMap[parsedTree.name] ||
+ colAliasToColMap[parsedTree.name]) as Record;
res.name = col.id;
if (col?.uidt === UITypes.Formula) {
@@ -1702,7 +1106,6 @@ export function validateFormulaAndExtractTreeWithType(
res.dataType = formulaRes as any;
} else {
-
switch (col?.uidt) {
// string
case UITypes.SingleLineText:
@@ -1766,8 +1169,6 @@ export function validateFormulaAndExtractTreeWithType(
res.left = validateAndExtract(parsedTree.left);
res.right = validateAndExtract(parsedTree.right);
res.dataType = FormulaDataTypes.NUMERIC;
- } else {
- // res.type= 'N/A';
}
return res;
@@ -1779,3 +1180,95 @@ export function validateFormulaAndExtractTreeWithType(
const result = validateAndExtract(parsedFormula);
return result;
}
+
+function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
+ // check circular reference
+ // e.g. formula1 -> formula2 -> formula1 should return circular reference error
+
+ // get all formula columns excluding itself
+ const formulaPaths = columns.value
+ .filter((c) => c.id !== formulaCol.value?.id && c.uidt === UITypes.Formula)
+ .reduce((res: Record[], c: Record) => {
+ // in `formula`, get all the (unique) target neighbours
+ // i.e. all column id (e.g. cxxxxxxxxxxxxxx) with formula type
+ const neighbours = [
+ ...new Set(
+ (c.colOptions.formula.match(/c\w{15}/g) || []).filter(
+ (colId: string) =>
+ columns.value.filter(
+ (col: ColumnType) =>
+ col.id === colId && col.uidt === UITypes.Formula
+ ).length
+ )
+ ),
+ ];
+ if (neighbours.length > 0) {
+ // e.g. formula column 1 -> [formula column 2, formula column3]
+ res.push({ [c.id]: neighbours });
+ }
+ return res;
+ }, []);
+
+ // include target formula column (i.e. the one to be saved if applicable)
+ const targetFormulaCol = columns.value.find(
+ (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula
+ );
+
+ if (targetFormulaCol && formulaCol.value?.id) {
+ formulaPaths.push({
+ [formulaCol.value?.id as string]: [targetFormulaCol.id],
+ });
+ }
+ const vertices = formulaPaths.length;
+ if (vertices > 0) {
+ // perform kahn's algo for cycle detection
+ const adj = new Map();
+ const inDegrees = new Map();
+ // init adjacency list & indegree
+
+ for (const [_, v] of Object.entries(formulaPaths)) {
+ const src = Object.keys(v)[0];
+ const neighbours = v[src];
+ inDegrees.set(src, inDegrees.get(src) || 0);
+ for (const neighbour of neighbours) {
+ adj.set(src, (adj.get(src) || new Set()).add(neighbour));
+ inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1);
+ }
+ }
+ const queue: string[] = [];
+ // put all vertices with in-degree = 0 (i.e. no incoming edges) to queue
+ inDegrees.forEach((inDegree, col) => {
+ if (inDegree === 0) {
+ // in-degree = 0 means we start traversing from this node
+ queue.push(col);
+ }
+ });
+ // init count of visited vertices
+ let visited = 0;
+ // BFS
+ while (queue.length !== 0) {
+ // remove a vertex from the queue
+ const src = queue.shift();
+ // if this node has neighbours, increase visited by 1
+ const neighbours = adj.get(src) || new Set();
+ if (neighbours.size > 0) {
+ visited += 1;
+ }
+ // iterate each neighbouring nodes
+ neighbours.forEach((neighbour: string) => {
+ // decrease in-degree of its neighbours by 1
+ inDegrees.set(neighbour, inDegrees.get(neighbour) - 1);
+ // if in-degree becomes 0
+ if (inDegrees.get(neighbour) === 0) {
+ // then put the neighboring node to the queue
+ queue.push(neighbour);
+ }
+ });
+ }
+ // vertices not same as visited = cycle found
+ if (vertices !== visited) {
+ // errors.add(t('msg.formula.cantSaveCircularReference'))
+ throw new FormulaError(FormulaErrorType.CIRCULAR_REFERENCE, '');
+ }
+ }
+}
diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
index e8bd658f97..f1ac991183 100644
--- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
+++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
@@ -63,17 +63,22 @@ async function _formulaQueryBuilder(
model: Model,
aliasToColumn: Record Promise<{ builder: any }>> = {},
tableAlias?: string,
+ parsedTree?: any,
) {
const knex = baseModelSqlv2.dbDriver;
const columns = await model.getColumns();
- // formula may include double curly brackets in previous version
- // convert to single curly bracket here for compatibility
- // const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}'));
- const tree = validateFormulaAndExtractTreeWithType(
- _tree.replaceAll('{{', '{').replaceAll('}}', '}'),
- columns,
- );
+
+ let tree = parsedTree;
+ if (!tree) {
+ // formula may include double curly brackets in previous version
+ // convert to single curly bracket here for compatibility
+ // const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}'));
+ tree = validateFormulaAndExtractTreeWithType(
+ _tree.replaceAll('{{', '{').replaceAll('}}', '}'),
+ columns,
+ );
+ }
const columnIdToUidt = {};
@@ -93,6 +98,7 @@ async function _formulaQueryBuilder(
model,
{ ...aliasToColumn, [col.id]: null },
tableAlias,
+ formulOption.getParsedTree(),
);
builder.sql = '(' + builder.sql + ')';
return {
@@ -413,6 +419,7 @@ async function _formulaQueryBuilder(
'',
lookupModel,
aliasToColumn,
+ formulaOption.getParsedTree()
);
if (isMany) {
const qb = selectQb;
@@ -968,6 +975,9 @@ export default async function formulaQueryBuilderv2(
model,
aliasToColumn,
tableAlias,
+ await column
+ ?.getColOptions()
+ .then((formula) => formula?.getParsedTree()),
);
if (!validateFormula) return qb;
diff --git a/packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts b/packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts
new file mode 100644
index 0000000000..5e50af058c
--- /dev/null
+++ b/packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts
@@ -0,0 +1,16 @@
+import type { Knex } from 'knex';
+import { MetaTable } from '~/utils/globals';
+
+const up = async (knex: Knex) => {
+ await knex.schema.alterTable(MetaTable.COL_FORMULA, (table) => {
+ table.text('parsed_tree');
+ });
+};
+
+const down = async (knex: Knex) => {
+ await knex.schema.alterTable(MetaTable.COL_FORMULA, (table) => {
+ table.dropColumn('parsed_tree');
+ });
+};
+
+export { up, down };
diff --git a/packages/nocodb/src/models/FormulaColumn.ts b/packages/nocodb/src/models/FormulaColumn.ts
index 955d480b11..d3f1234a44 100644
--- a/packages/nocodb/src/models/FormulaColumn.ts
+++ b/packages/nocodb/src/models/FormulaColumn.ts
@@ -2,19 +2,21 @@ import Noco from '~/Noco';
import NocoCache from '~/cache/NocoCache';
import { extractProps } from '~/helpers/extractProps';
import { CacheGetType, CacheScope, MetaTable } from '~/utils/globals';
+import { parseMetaProp, stringifyMetaProp } from '~/utils/modelUtils';
export default class FormulaColumn {
formula: string;
formula_raw: string;
fk_column_id: string;
error: string;
+ private parsed_tree?: any;
constructor(data: Partial) {
Object.assign(this, data);
}
public static async insert(
- formulaColumn: Partial,
+ formulaColumn: Partial,
ncMeta = Noco.ncMeta,
) {
const insertObj = extractProps(formulaColumn, [
@@ -22,11 +24,16 @@ export default class FormulaColumn {
'formula_raw',
'formula',
'error',
+ 'parsed_tree',
]);
+
+ insertObj.parsed_tree = stringifyMetaProp(insertObj, 'parsed_tree');
+
await ncMeta.metaInsert2(null, null, MetaTable.COL_FORMULA, insertObj);
return this.read(formulaColumn.fk_column_id, ncMeta);
}
+
public static async read(columnId: string, ncMeta = Noco.ncMeta) {
let column =
columnId &&
@@ -41,7 +48,10 @@ export default class FormulaColumn {
MetaTable.COL_FORMULA,
{ fk_column_id: columnId },
);
- await NocoCache.set(`${CacheScope.COL_FORMULA}:${columnId}`, column);
+ if (column) {
+ column.parsed_tree = parseMetaProp(column, 'parsed_tree');
+ await NocoCache.set(`${CacheScope.COL_FORMULA}:${columnId}`, column);
+ }
}
return column ? new FormulaColumn(column) : null;
@@ -51,7 +61,7 @@ export default class FormulaColumn {
static async update(
id: string,
- formula: Partial,
+ formula: Partial & { parsed_tree?: any },
ncMeta = Noco.ncMeta,
) {
const updateObj = extractProps(formula, [
@@ -59,7 +69,11 @@ export default class FormulaColumn {
'formula_raw',
'fk_column_id',
'error',
+ 'parsed_tree',
]);
+
+ updateObj.parsed_tree = stringifyMetaProp(insertObj, 'parsed_tree');
+
// get existing cache
const key = `${CacheScope.COL_FORMULA}:${id}`;
let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT);
@@ -71,4 +85,8 @@ export default class FormulaColumn {
// set meta
await ncMeta.metaUpdate(null, null, MetaTable.COL_FORMULA, updateObj, id);
}
+
+ public getParsedTree() {
+ return this.parsed_tree;
+ }
}
diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts
index 09cecbd6e6..48c2f30464 100644
--- a/packages/nocodb/src/services/columns.service.ts
+++ b/packages/nocodb/src/services/columns.service.ts
@@ -5,7 +5,7 @@ import {
isVirtualCol,
substituteColumnAliasWithIdInFormula,
substituteColumnIdWithAliasInFormula,
- UITypes,
+ UITypes, validateFormulaAndExtractTreeWithType,
} from 'nocodb-sdk';
import { pluralize, singularize } from 'inflection';
import hash from 'object-hash';
@@ -1197,6 +1197,10 @@ export class ColumnsService {
colBody.formula_raw || colBody.formula,
table.columns,
);
+ colBody.parsed_tree = validateFormulaAndExtractTreeWithType(
+ colBody.formula_raw || colBody.formula,
+ table.columns,
+ );
try {
const baseModel = await reuseOrSave('baseModel', reuse, async () =>
From 106858fc61da4f62644a147ca4dce41e8366507f Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:58 +0000
Subject: [PATCH 014/262] fix: corrections
---
packages/nocodb/src/models/Column.ts | 6 +++++-
packages/nocodb/src/models/FormulaColumn.ts | 4 ++--
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts
index 484ae6579b..f9996aea0a 100644
--- a/packages/nocodb/src/models/Column.ts
+++ b/packages/nocodb/src/models/Column.ts
@@ -713,7 +713,11 @@ export default class Column implements ColumnType {
title: col?.title,
})
)
- await FormulaColumn.update(formulaCol.id, formula, ncMeta);
+ await FormulaColumn.update(
+ formulaCol.id,
+ formula as FormulaColumn & { parsed_tree?: any },
+ ncMeta,
+ );
}
}
diff --git a/packages/nocodb/src/models/FormulaColumn.ts b/packages/nocodb/src/models/FormulaColumn.ts
index d3f1234a44..4c786125e9 100644
--- a/packages/nocodb/src/models/FormulaColumn.ts
+++ b/packages/nocodb/src/models/FormulaColumn.ts
@@ -16,7 +16,7 @@ export default class FormulaColumn {
}
public static async insert(
- formulaColumn: Partial,
+ formulaColumn: Partial & { parsed_tree?: any },
ncMeta = Noco.ncMeta,
) {
const insertObj = extractProps(formulaColumn, [
@@ -72,7 +72,7 @@ export default class FormulaColumn {
'parsed_tree',
]);
- updateObj.parsed_tree = stringifyMetaProp(insertObj, 'parsed_tree');
+ updateObj.parsed_tree = stringifyMetaProp(updateObj, 'parsed_tree');
// get existing cache
const key = `${CacheScope.COL_FORMULA}:${id}`;
From 60983af77cacab8aece2ff0873fcc2367665a82b Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:58 +0000
Subject: [PATCH 015/262] refactor: exclude parsed tree from response
---
.../src/db/formulav2/formulaQueryBuilderv2.ts | 10 ++++++----
.../src/meta/migrations/XcMigrationSourcev2.ts | 4 ++++
packages/nocodb/src/models/Column.ts | 1 +
packages/nocodb/src/models/FormulaColumn.ts | 6 ++++--
packages/nocodb/src/services/columns.service.ts | 14 +++++++++++++-
5 files changed, 28 insertions(+), 7 deletions(-)
diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
index f1ac991183..3a6dcb8775 100644
--- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
+++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
@@ -419,7 +419,7 @@ async function _formulaQueryBuilder(
'',
lookupModel,
aliasToColumn,
- formulaOption.getParsedTree()
+ formulaOption.getParsedTree(),
);
if (isMany) {
const qb = selectQb;
@@ -963,6 +963,7 @@ export default async function formulaQueryBuilderv2(
aliasToColumn = {},
tableAlias?: string,
validateFormula = false,
+ parsedTree?: any,
) {
const knex = baseModelSqlv2.dbDriver;
// register jsep curly hook once only
@@ -975,9 +976,10 @@ export default async function formulaQueryBuilderv2(
model,
aliasToColumn,
tableAlias,
- await column
- ?.getColOptions()
- .then((formula) => formula?.getParsedTree()),
+ parsedTree ??
+ (await column
+ ?.getColOptions()
+ .then((formula) => formula?.getParsedTree())),
);
if (!validateFormula) return qb;
diff --git a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts
index 6dedcf2843..2df5d90b5a 100644
--- a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts
+++ b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts
@@ -24,6 +24,7 @@ import * as nc_034_erd_filter_and_notification from '~/meta/migrations/v2/nc_034
import * as nc_035_add_username_to_users from '~/meta/migrations/v2/nc_035_add_username_to_users';
import * as nc_036_base_deleted from '~/meta/migrations/v2/nc_036_base_deleted';
import * as nc_037_rename_project_and_base from '~/meta/migrations/v2/nc_037_rename_project_and_base';
+import * as nc_038_formula_parsed_tree_column from '~/meta/migrations/v2/nc_038_formula_parsed_tree_column';
// Create a custom migration source class
export default class XcMigrationSourcev2 {
@@ -59,6 +60,7 @@ export default class XcMigrationSourcev2 {
'nc_035_add_username_to_users',
'nc_036_base_deleted',
'nc_037_rename_project_and_base',
+ 'nc_038_formula_parsed_tree_column'
]);
}
@@ -120,6 +122,8 @@ export default class XcMigrationSourcev2 {
return nc_036_base_deleted;
case 'nc_037_rename_project_and_base':
return nc_037_rename_project_and_base;
+ case 'nc_038_formula_parsed_tree_column':
+ return nc_038_formula_parsed_tree_column;
}
}
}
diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts
index f9996aea0a..2d76ed8860 100644
--- a/packages/nocodb/src/models/Column.ts
+++ b/packages/nocodb/src/models/Column.ts
@@ -308,6 +308,7 @@ export default class Column implements ColumnType {
fk_column_id: colId,
formula: column.formula,
formula_raw: column.formula_raw,
+ parsed_tree: column.parsed_tree,
},
ncMeta,
);
diff --git a/packages/nocodb/src/models/FormulaColumn.ts b/packages/nocodb/src/models/FormulaColumn.ts
index 4c786125e9..f807737286 100644
--- a/packages/nocodb/src/models/FormulaColumn.ts
+++ b/packages/nocodb/src/models/FormulaColumn.ts
@@ -11,8 +11,10 @@ export default class FormulaColumn {
error: string;
private parsed_tree?: any;
- constructor(data: Partial) {
- Object.assign(this, data);
+ constructor(data: Partial & { parsed_tree?: any }) {
+ const { parsed_tree, ...rest } = data;
+ this.parsed_tree = parsed_tree;
+ Object.assign(this, rest);
}
public static async insert(
diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts
index 48c2f30464..edace7a76d 100644
--- a/packages/nocodb/src/services/columns.service.ts
+++ b/packages/nocodb/src/services/columns.service.ts
@@ -5,7 +5,8 @@ import {
isVirtualCol,
substituteColumnAliasWithIdInFormula,
substituteColumnIdWithAliasInFormula,
- UITypes, validateFormulaAndExtractTreeWithType,
+ UITypes,
+ validateFormulaAndExtractTreeWithType,
} from 'nocodb-sdk';
import { pluralize, singularize } from 'inflection';
import hash from 'object-hash';
@@ -184,6 +185,7 @@ export class ColumnsService {
let colBody = { ...param.column } as Column & {
formula?: string;
formula_raw?: string;
+ parsed_tree?: any;
};
if (
[
@@ -208,6 +210,10 @@ export class ColumnsService {
colBody.formula_raw || colBody.formula,
table.columns,
);
+ colBody.parsed_tree = validateFormulaAndExtractTreeWithType(
+ colBody.formula_raw || colBody.formula,
+ table.columns,
+ );
try {
const baseModel = await reuseOrSave('baseModel', reuse, async () =>
@@ -227,6 +233,7 @@ export class ColumnsService {
{},
null,
true,
+ colBody.parsed_tree
);
} catch (e) {
console.error(e);
@@ -931,6 +938,7 @@ export class ColumnsService {
]);
await FormulaColumn.update(c.id, {
formula_raw: new_formula_raw,
+ parsed_tree: validateFormulaAndExtractTreeWithType(new_formula_raw, table.columns)
});
}
}
@@ -992,6 +1000,10 @@ export class ColumnsService {
]);
await FormulaColumn.update(c.id, {
formula_raw: new_formula_raw,
+ parsed_tree: validateFormulaAndExtractTreeWithType(
+ new_formula_raw,
+ table.columns,
+ ),
});
}
}
From 24ee8745e158f0339ff344ac9abd85dfb246fef7 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:59 +0000
Subject: [PATCH 016/262] feat:populate and save parsed tree if missing
---
packages/nc-gui/utils/formulaUtils.ts | 11 -----------
.../src/db/formulav2/formulaQueryBuilderv2.ts | 17 +++++++++++++++--
2 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/packages/nc-gui/utils/formulaUtils.ts b/packages/nc-gui/utils/formulaUtils.ts
index c4f4fef23b..13788c1a07 100644
--- a/packages/nc-gui/utils/formulaUtils.ts
+++ b/packages/nc-gui/utils/formulaUtils.ts
@@ -435,17 +435,6 @@ const formulas: Record = {
return argsTypes[1]
},
- returnType: (argTypes: formulaTypes[]) => {
- if (argTypes.slice(1).includes(formulaTypes.STRING)) {
- return formulaTypes.STRING
- } else if (argTypes.slice(1).includes(formulaTypes.NUMERIC)) {
- return formulaTypes.NUMERIC
- } else if (argTypes.slice(1).includes(formulaTypes.BOOLEAN)) {
- return formulaTypes.BOOLEAN
- }
-
- return argTypes[1]
- },
},
SWITCH: {
type: formulaTypes.COND_EXP,
diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
index 3a6dcb8775..f6826bb8fb 100644
--- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
+++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
@@ -7,12 +7,12 @@ import {
} from 'nocodb-sdk';
import mapFunctionName from '../mapFunctionName';
import genRollupSelectv2 from '../genRollupSelectv2';
-import type Column from '~/models/Column';
import type Model from '~/models/Model';
import type RollupColumn from '~/models/RollupColumn';
import type LinkToAnotherRecordColumn from '~/models/LinkToAnotherRecordColumn';
import type LookupColumn from '~/models/LookupColumn';
import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
+import type Column from '~/models/Column';
import NocoCache from '~/cache/NocoCache';
import { CacheGetType, CacheScope } from '~/utils/globals';
import {
@@ -21,7 +21,7 @@ import {
} from '~/helpers/formulaFnHelper';
import FormulaColumn from '~/models/FormulaColumn';
-// todo: switch function based on database
+const logger = new Logger('FormulaQueryBuilderv2');
// @ts-ignore
const getAggregateFn: (fnName: string) => (args: { qb; knex?; cn }) => any = (
@@ -64,6 +64,7 @@ async function _formulaQueryBuilder(
aliasToColumn: Record Promise<{ builder: any }>> = {},
tableAlias?: string,
parsedTree?: any,
+ column: Column = null,
) {
const knex = baseModelSqlv2.dbDriver;
@@ -78,6 +79,18 @@ async function _formulaQueryBuilder(
_tree.replaceAll('{{', '{').replaceAll('}}', '}'),
columns,
);
+
+ // populate and save parsedTree to column if not exist
+ if (column) {
+ FormulaColumn.update(column.id, { parsed_tree: tree }).then(
+ () => {
+ // ignore
+ },
+ (err) => {
+ logger.error(err);
+ },
+ );
+ }
}
const columnIdToUidt = {};
From c64cf83bba57733dafd559dbf00789407b859c78 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:59 +0000
Subject: [PATCH 017/262] fix: corrections
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 94b3dcc62d..d14bc44c18 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -2,7 +2,6 @@ import jsep from 'jsep';
import { ColumnType } from './Api';
import UITypes from './UITypes';
-import { formulaTypes } from '../../../nc-gui/utils';
export const jsepCurlyHook = {
name: 'curly',
@@ -221,7 +220,7 @@ interface FormulaMeta {
min?: number;
max?: number;
rqd?: number;
- validator?: (args: formulaTypes[]) => boolean;
+ validator?: (args: FormulaDataTypes[]) => boolean;
};
};
description?: string;
@@ -1180,6 +1179,7 @@ export function validateFormulaAndExtractTreeWithType(
const result = validateAndExtract(parsedFormula);
return result;
}
+/*
function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
// check circular reference
@@ -1272,3 +1272,4 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
}
}
}
+*/
From 3412a0e54af3ce272191272a80e4f2dc846aa8bd Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:59 +0000
Subject: [PATCH 018/262] fix: missing import statement
---
packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
index f6826bb8fb..f6688082a3 100644
--- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
+++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
@@ -5,6 +5,7 @@ import {
validateFormulaAndExtractTreeWithType,
validateDateWithUnknownFormat,
} from 'nocodb-sdk';
+import { Logger } from '@nestjs/common';
import mapFunctionName from '../mapFunctionName';
import genRollupSelectv2 from '../genRollupSelectv2';
import type Model from '~/models/Model';
From 0457968fc2e549cf9e237eef2147167efaa92d74 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:59 +0000
Subject: [PATCH 019/262] fix: replace double curly brace
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 7 ++++---
packages/nocodb/src/services/columns.service.ts | 4 +++-
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index d14bc44c18..0085b617d2 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1097,9 +1097,10 @@ export function validateFormulaAndExtractTreeWithType(
if (col?.uidt === UITypes.Formula) {
// todo: check for circular reference
- // todo: extract the type and return
- const formulaRes = validateFormulaAndExtractTreeWithType(
- col.colOptions.formula,
+ const formulaRes = col.colOptions?.parsed_tree || validateFormulaAndExtractTreeWithType(
+ // formula may include double curly brackets in previous version
+ // convert to single curly bracket here for compatibility
+ col.colOptions.formula.replaceAll('{{', '{').replaceAll('}}', '}'),
columns
);
diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts
index edace7a76d..f9731fc115 100644
--- a/packages/nocodb/src/services/columns.service.ts
+++ b/packages/nocodb/src/services/columns.service.ts
@@ -1210,7 +1210,9 @@ export class ColumnsService {
table.columns,
);
colBody.parsed_tree = validateFormulaAndExtractTreeWithType(
- colBody.formula_raw || colBody.formula,
+ // formula may include double curly brackets in previous version
+ // convert to single curly bracket here for compatibility
+ colBody.formula_raw || colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'),
table.columns,
);
From 7b80cabe4b806615b415a6ce5c53ad6bd0451cd2 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:59 +0000
Subject: [PATCH 020/262] fix: circular reference validation
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 0085b617d2..5285d8eebb 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1095,7 +1095,8 @@ export function validateFormulaAndExtractTreeWithType(
res.name = col.id;
if (col?.uidt === UITypes.Formula) {
- // todo: check for circular reference
+ // check for circular reference
+ checkForCircularFormulaRef(col, parsedTree, columns);
const formulaRes = col.colOptions?.parsed_tree || validateFormulaAndExtractTreeWithType(
// formula may include double curly brackets in previous version
@@ -1180,7 +1181,6 @@ export function validateFormulaAndExtractTreeWithType(
const result = validateAndExtract(parsedFormula);
return result;
}
-/*
function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
// check circular reference
@@ -1188,15 +1188,15 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
// get all formula columns excluding itself
const formulaPaths = columns.value
- .filter((c) => c.id !== formulaCol.value?.id && c.uidt === UITypes.Formula)
+ .filter((c) => c.id !== formulaCol?.id && c.uidt === UITypes.Formula)
.reduce((res: Record[], c: Record) => {
// in `formula`, get all the (unique) target neighbours
// i.e. all column id (e.g. cxxxxxxxxxxxxxx) with formula type
const neighbours = [
...new Set(
- (c.colOptions.formula.match(/c\w{15}/g) || []).filter(
+ (c.colOptions.formula.match(/c_?\w{14,15}/g) || []).filter(
(colId: string) =>
- columns.value.filter(
+ columns.filter(
(col: ColumnType) =>
col.id === colId && col.uidt === UITypes.Formula
).length
@@ -1211,13 +1211,13 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
}, []);
// include target formula column (i.e. the one to be saved if applicable)
- const targetFormulaCol = columns.value.find(
+ const targetFormulaCol = columns.find(
(c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula
);
- if (targetFormulaCol && formulaCol.value?.id) {
+ if (targetFormulaCol && formulaCol?.id) {
formulaPaths.push({
- [formulaCol.value?.id as string]: [targetFormulaCol.id],
+ [formulaCol?.id as string]: [targetFormulaCol.id],
});
}
const vertices = formulaPaths.length;
@@ -1273,4 +1273,4 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
}
}
}
-*/
+
From c7b265e429a15af9f3a6fd64bf4f475400a967f9 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:59 +0000
Subject: [PATCH 021/262] fix: formula errors
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 75 +++++++++----------
1 file changed, 37 insertions(+), 38 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 5285d8eebb..95b98d8b0d 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -976,17 +976,19 @@ enum FormulaErrorType {
}
class FormulaError extends Error {
- public name: string;
public type: FormulaErrorType;
+ public extra: Record;
constructor(
type: FormulaErrorType,
- name: string,
+ extra: {
+ [key: string]: any;
+ },
message: string = 'Formula Error'
) {
super(message);
- this.name = name;
this.type = type;
+ this.extra = extra;
}
}
@@ -1029,48 +1031,39 @@ export function validateFormulaAndExtractTreeWithType(
) {
throw new FormulaError(
FormulaErrorType.INVALID_ARG,
- calleeName,
+ {
+ key: 'msg.formula.requiredArgumentsFormula',
+ requiredArguments: validation.args.rqd,
+ calleeName,
+ },
'Required arguments missing'
);
-
- // errors.add(
- // t('msg.formula.requiredArgumentsFormula', {
- // requiredArguments: validation.args.rqd,
- // calleeName,
- // })
- // );
} else if (
validation.args.min !== undefined &&
validation.args.min > parsedTree.arguments.length
) {
throw new FormulaError(
FormulaErrorType.MIN_ARG,
- calleeName,
+ {
+ key: 'msg.formula.minRequiredArgumentsFormula',
+ minRequiredArguments: validation.args.min,
+ calleeName,
+ },
'Minimum arguments required'
);
-
- // errors.add(
- // t('msg.formula.minRequiredArgumentsFormula', {
- // minRequiredArguments: validation.args.min,
- // calleeName,
- // })
- // );
} else if (
validation.args.max !== undefined &&
validation.args.max < parsedTree.arguments.length
) {
throw new FormulaError(
- FormulaErrorType.MAX_ARG,
- calleeName,
- 'Maximum arguments required'
+ FormulaErrorType.INVALID_ARG,
+ {
+ key: 'msg.formula.maxRequiredArgumentsFormula',
+ maxRequiredArguments: validation.args.max,
+ calleeName,
+ },
+ 'Maximum arguments missing'
);
-
- // errors.add(
- // t('msg.formula.maxRequiredArgumentsFormula', {
- // maxRequiredArguments: validation.args.max,
- // calleeName,
- // })
- // );
}
}
// get args type and validate
@@ -1098,12 +1091,14 @@ export function validateFormulaAndExtractTreeWithType(
// check for circular reference
checkForCircularFormulaRef(col, parsedTree, columns);
- const formulaRes = col.colOptions?.parsed_tree || validateFormulaAndExtractTreeWithType(
- // formula may include double curly brackets in previous version
- // convert to single curly bracket here for compatibility
- col.colOptions.formula.replaceAll('{{', '{').replaceAll('}}', '}'),
- columns
- );
+ const formulaRes =
+ col.colOptions?.parsed_tree ||
+ validateFormulaAndExtractTreeWithType(
+ // formula may include double curly brackets in previous version
+ // convert to single curly bracket here for compatibility
+ col.colOptions.formula.replaceAll('{{', '{').replaceAll('}}', '}'),
+ columns
+ );
res.dataType = formulaRes as any;
} else {
@@ -1268,9 +1263,13 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
}
// vertices not same as visited = cycle found
if (vertices !== visited) {
- // errors.add(t('msg.formula.cantSaveCircularReference'))
- throw new FormulaError(FormulaErrorType.CIRCULAR_REFERENCE, '');
+ throw new FormulaError(
+ FormulaErrorType.CIRCULAR_REFERENCE,
+ {
+ key: 'msg.formula.cantSaveCircularReference',
+ },
+ 'Circular reference detected'
+ );
}
}
}
-
From 1a632051bd401a0f855427aae0d2604cf405fc66 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:59 +0000
Subject: [PATCH 022/262] fix: typo
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 95b98d8b0d..f2281c89f1 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -973,6 +973,7 @@ enum FormulaErrorType {
'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE',
'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT',
CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE',
+ INVALID_FUNCTION_NAME = 'INVALID_FUNCTION_NAME',
}
class FormulaError extends Error {
@@ -1016,7 +1017,8 @@ export function validateFormulaAndExtractTreeWithType(
// validate function name
if (!formulas[calleeName]) {
throw new FormulaError(
- FormulaErrorType.INVALID_ARG_TYPE,
+ FormulaErrorType.INVALID_FUNCTION_NAME,
+ {},
'Function not available'
);
//t('msg.formula.functionNotAvailable', { function: calleeName })
From 2f26553ccd324bb780593a4003062d37837cefec Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:16:59 +0000
Subject: [PATCH 023/262] refactor: SWITCH and IF type extraction correction
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 64 +++++++++++++------
1 file changed, 43 insertions(+), 21 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index f2281c89f1..6bdabf0dcc 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -664,16 +664,28 @@ const formulas: Record = {
'IF(5 > 1, "YES", "NO") => "YES"',
'IF({column} > 1, "YES", "NO")',
],
- returnType: (argsTypes: FormulaDataTypes[]) => {
- if (argsTypes.slice(1).includes(FormulaDataTypes.STRING)) {
+ returnType: (argTypes: FormulaDataTypes[]) => {
+ // extract all return types except NULL, since null can be returned by any type
+ const returnValueTypes = new Set(
+ argTypes.slice(1).filter((type) => type !== FormulaDataTypes.NULL)
+ );
+ // if there are more than one return types or if there is a string return type
+ // return type as string else return the type
+ if (
+ returnValueTypes.size > 1 ||
+ returnValueTypes.has(FormulaDataTypes.STRING)
+ ) {
return FormulaDataTypes.STRING;
- } else if (argsTypes.slice(1).includes(FormulaDataTypes.NUMERIC)) {
+ } else if (returnValueTypes.has(FormulaDataTypes.NUMERIC)) {
return FormulaDataTypes.NUMERIC;
- } else if (argsTypes.slice(1).includes(FormulaDataTypes.BOOLEAN)) {
+ } else if (returnValueTypes.has(FormulaDataTypes.BOOLEAN)) {
return FormulaDataTypes.BOOLEAN;
+ } else if (returnValueTypes.has(FormulaDataTypes.DATE)) {
+ return FormulaDataTypes.DATE;
}
- return argsTypes[1];
+ // if none of the above conditions are met, return the first return argument type
+ return argTypes[1];
},
},
SWITCH: {
@@ -681,7 +693,7 @@ const formulas: Record = {
validation: {
args: {
min: 3,
- },
+ }
},
description: 'Switch case value based on expr output',
syntax: 'SWITCH(expr, [pattern, value, ..., default])',
@@ -691,19 +703,29 @@ const formulas: Record = {
'SWITCH(3, 1, "One", 2, "Two", "N/A") => "N/A"',
'SWITCH({column1}, 1, "One", 2, "Two", "N/A")',
],
- // todo: resolve return type based on the args
returnType: (argTypes: FormulaDataTypes[]) => {
- const returnArgTypes = argTypes.slice(2).filter((_, i) => i % 2 === 0);
+ // extract all return types except NULL, since null can be returned by any type
+ const returnValueTypes = new Set(
+ argTypes.slice(2).filter((_, i) => i % 2 === 0)
+ );
- if (returnArgTypes.includes(FormulaDataTypes.STRING)) {
+ // if there are more than one return types or if there is a string return type
+ // return type as string else return the type
+ if (
+ returnValueTypes.size > 1 ||
+ returnValueTypes.has(FormulaDataTypes.STRING)
+ ) {
return FormulaDataTypes.STRING;
- } else if (returnArgTypes.includes(FormulaDataTypes.NUMERIC)) {
+ } else if (returnValueTypes.has(FormulaDataTypes.NUMERIC)) {
return FormulaDataTypes.NUMERIC;
- } else if (returnArgTypes.includes(FormulaDataTypes.BOOLEAN)) {
+ } else if (returnValueTypes.has(FormulaDataTypes.BOOLEAN)) {
return FormulaDataTypes.BOOLEAN;
+ } else if (returnValueTypes.has(FormulaDataTypes.DATE)) {
+ return FormulaDataTypes.DATE;
}
- return returnArgTypes[0];
+ // if none of the above conditions are met, return the first return argument type
+ return argTypes[1];
},
},
URL: {
@@ -965,13 +987,13 @@ const formulas: Record = {
enum FormulaErrorType {
NOT_AVAILABLE = 'NOT_AVAILABLE',
NOT_SUPPORTED = 'NOT_SUPPORTED',
- 'MIN_ARG' = 'MIN_ARG',
- 'MAX_ARG' = 'MAX_ARG',
- 'TYPE_MISMATCH' = 'TYPE_MISMATCH',
- 'INVALID_ARG' = 'INVALID_ARG',
- 'INVALID_ARG_TYPE' = 'INVALID_ARG_TYPE',
- 'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE',
- 'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT',
+ MIN_ARG = 'MIN_ARG',
+ MAX_ARG = 'MAX_ARG',
+ TYPE_MISMATCH = 'TYPE_MISMATCH',
+ INVALID_ARG = 'INVALID_ARG',
+ INVALID_ARG_TYPE = 'INVALID_ARG_TYPE',
+ INVALID_ARG_VALUE = 'INVALID_ARG_VALUE',
+ INVALID_ARG_COUNT = 'INVALID_ARG_COUNT',
CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE',
INVALID_FUNCTION_NAME = 'INVALID_FUNCTION_NAME',
}
@@ -1075,11 +1097,11 @@ export function validateFormulaAndExtractTreeWithType(
}
));
- const argsTypes = validateResult.map((v: any) => v.dataType);
+ const argTypes = validateResult.map((v: any) => v.dataType);
if (typeof formulas[calleeName].returnType === 'function') {
res.dataType = (formulas[calleeName].returnType as any)?.(
- argsTypes
+ argTypes
) as FormulaDataTypes;
} else if (formulas[calleeName].returnType) {
res.dataType = formulas[calleeName].returnType as FormulaDataTypes;
From 9e8871cf406abd1e2b5ec5c11d2b304c479c9ad1 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:00 +0000
Subject: [PATCH 024/262] fix: treat values in AND/OR based on type
---
.../src/db/functionMappings/commonFns.ts | 48 +++++++++++--------
1 file changed, 28 insertions(+), 20 deletions(-)
diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts
index d5d987c866..d8f1929ac5 100644
--- a/packages/nocodb/src/db/functionMappings/commonFns.ts
+++ b/packages/nocodb/src/db/functionMappings/commonFns.ts
@@ -2,6 +2,31 @@ import { FormulaDataTypes } from 'nocodb-sdk';
import type { MapFnArgs } from '../mapFunctionName';
import { NcError } from '~/helpers/catchError';
+async function treatArgAsConditionalExp(
+ args: MapFnArgs,
+ argument = args.pt?.arguments?.[0],
+) {
+ const condArg = (await args.fn(argument)).builder.toQuery();
+
+ let cond = condArg;
+
+ switch (argument.dataType as FormulaDataTypes) {
+ case FormulaDataTypes.NUMERIC:
+ cond = `(${condArg}) IS NOT NULL AND (${condArg}) != 0`;
+ break;
+ case FormulaDataTypes.STRING:
+ cond = `(${condArg}) IS NOT NULL AND (${condArg}) != ''`;
+ break;
+ case FormulaDataTypes.BOOLEAN:
+ cond = `(${condArg}) IS NOT NULL AND (${condArg}) != false`;
+ break;
+ case FormulaDataTypes.DATE:
+ cond = `(${condArg}) IS NOT NULL`;
+ break;
+ }
+ return { builder: args.knex.raw(cond) };
+}
+
export default {
// todo: handle default case
SWITCH: async (args: MapFnArgs) => {
@@ -37,24 +62,7 @@ export default {
};
},
IF: async (args: MapFnArgs) => {
- const condArg = (await args.fn(args.pt.arguments[0])).builder.toQuery();
-
- let cond = condArg;
-
- switch (args.pt.arguments[0].dataType as FormulaDataTypes) {
- case FormulaDataTypes.NUMERIC:
- cond = `(${condArg}) IS NOT NULL AND (${condArg}) != 0`;
- break;
- case FormulaDataTypes.STRING:
- cond = `(${condArg}) IS NOT NULL AND (${condArg}) != ''`;
- break;
- case FormulaDataTypes.BOOLEAN:
- cond = `(${condArg}) IS NOT NULL AND (${condArg}) != false`;
- break;
- case FormulaDataTypes.DATE:
- cond = `(${condArg}) IS NOT NULL`;
- break;
- }
+ const cond = await treatArgAsConditionalExp(args);
let query = args.knex
.raw(
@@ -80,7 +88,7 @@ export default {
`${(
await Promise.all(
args.pt.arguments.map(async (ar) =>
- (await args.fn(ar)).builder.toQuery(),
+ (await treatArgAsConditionalExp(args, ar)).builder.toQuery(),
),
)
).join(' AND ')}`,
@@ -98,7 +106,7 @@ export default {
`${(
await Promise.all(
args.pt.arguments.map(async (ar) =>
- (await args.fn(ar)).builder.toQuery(),
+ (await treatArgAsConditionalExp(args, ar)).builder.toQuery(),
),
)
).join(' OR ')}`,
From ef36baa15e26bf6a0d6890a6b1ec325e88eb5057 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:00 +0000
Subject: [PATCH 025/262] fix: treat values in AND/OR based on type
---
packages/nocodb/src/db/functionMappings/commonFns.ts | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts
index d8f1929ac5..205bb2f339 100644
--- a/packages/nocodb/src/db/functionMappings/commonFns.ts
+++ b/packages/nocodb/src/db/functionMappings/commonFns.ts
@@ -10,6 +10,11 @@ async function treatArgAsConditionalExp(
let cond = condArg;
+ // based on the data type of the argument, we need to handle the condition
+ // if string - value is not null and not empty then true
+ // if number - value is not null and not 0 then true
+ // if boolean - value is not null and not false then true
+ // if date - value is not null then true
switch (argument.dataType as FormulaDataTypes) {
case FormulaDataTypes.NUMERIC:
cond = `(${condArg}) IS NOT NULL AND (${condArg}) != 0`;
From 19be58a420570762b19137eebddd4e925c6f0ee2 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:00 +0000
Subject: [PATCH 026/262] feat: cast values to string if there are different
types values in IF
---
.../src/db/functionMappings/commonFns.ts | 54 ++++++++++++++-----
.../nocodb/src/db/functionMappings/mysql.ts | 9 ++++
packages/nocodb/src/db/functionMappings/pg.ts | 9 ++++
3 files changed, 59 insertions(+), 13 deletions(-)
diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts
index 205bb2f339..e6382a3a07 100644
--- a/packages/nocodb/src/db/functionMappings/commonFns.ts
+++ b/packages/nocodb/src/db/functionMappings/commonFns.ts
@@ -67,24 +67,52 @@ export default {
};
},
IF: async (args: MapFnArgs) => {
- const cond = await treatArgAsConditionalExp(args);
+ const cond = (await treatArgAsConditionalExp(args)).builder;
+ let thenArg;
+ let elseArg;
+ const returnArgsType = new Set(
+ [args.pt.arguments[1].dataType, args.pt.arguments[2].dataType].filter(
+ (type) => type !== FormulaDataTypes.NULL,
+ ),
+ );
+ // cast to string if the return value types are different
+ if (returnArgsType.size > 1) {
+ thenArg = (
+ await args.fn({
+ type: 'CallExpression',
+ arguments: [args.pt.arguments[1]],
+ callee: {
+ type: 'Identifier',
+ name: 'STRING',
+ },
+ } as any)
+ ).builder;
+ elseArg = (
+ await args.fn({
+ type: 'CallExpression',
+ arguments: [args.pt.arguments[2]],
+ callee: {
+ type: 'Identifier',
+ name: 'STRING',
+ },
+ } as any)
+ ).builder;
+ } else {
+ thenArg = (await args.fn(args.pt.arguments[1])).builder.toQuery();
+ elseArg = (await args.fn(args.pt.arguments[1])).builder.toQuery();
+ }
- let query = args.knex
- .raw(
- `\n\tWHEN ${cond} THEN ${(
- await args.fn(args.pt.arguments[1])
- ).builder.toQuery()}`,
- )
- .toQuery();
+ let query = args.knex.raw(`\n\tWHEN ${cond} THEN ${thenArg}`).toQuery();
if (args.pt.arguments[2]) {
- query += args.knex
- .raw(
- `\n\tELSE ${(await args.fn(args.pt.arguments[2])).builder.toQuery()}`,
- )
- .toQuery();
+ query += args.knex.raw(`\n\tELSE ${elseArg}`).toQuery();
}
return { builder: args.knex.raw(`CASE ${query}\n END${args.colAlias}`) };
},
+ // used only for casting to string internally, this one is dummy function
+ // and will work as fallback for dbs which don't support/implemented CAST
+ STRING(args: MapFnArgs) {
+ return args.fn(args.pt?.arguments?.[0]);
+ },
AND: async (args: MapFnArgs) => {
return {
builder: args.knex.raw(
diff --git a/packages/nocodb/src/db/functionMappings/mysql.ts b/packages/nocodb/src/db/functionMappings/mysql.ts
index 8026806876..588f51aa3a 100644
--- a/packages/nocodb/src/db/functionMappings/mysql.ts
+++ b/packages/nocodb/src/db/functionMappings/mysql.ts
@@ -160,6 +160,15 @@ END) ${colAlias}`,
),
};
},
+ STRING: async (args: MapFnArgs) => {
+ return {
+ builder: args.knex.raw(
+ `CAST(${(await args.fn(args.pt.arguments[0])).builder} AS CHAR) ${
+ args.colAlias
+ }`,
+ ),
+ };
+ },
};
export default mysql2;
diff --git a/packages/nocodb/src/db/functionMappings/pg.ts b/packages/nocodb/src/db/functionMappings/pg.ts
index a26180ab34..9ef7dff338 100644
--- a/packages/nocodb/src/db/functionMappings/pg.ts
+++ b/packages/nocodb/src/db/functionMappings/pg.ts
@@ -299,6 +299,15 @@ END) ${colAlias}`,
),
};
},
+ STRING: async (args: MapFnArgs) => {
+ return {
+ builder: args.knex.raw(
+ `(${(await args.fn(args.pt.arguments[0])).builder})::text ${
+ args.colAlias
+ }`,
+ ),
+ };
+ },
};
export default pg;
From 23270c9078095021c0f8168509128f21020a78cb Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:00 +0000
Subject: [PATCH 027/262] feat: cast values to string if there are different
types values in SWITCH
---
.../src/db/functionMappings/commonFns.ts | 63 ++++++++++++++++---
1 file changed, 53 insertions(+), 10 deletions(-)
diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts
index e6382a3a07..d8648b954e 100644
--- a/packages/nocodb/src/db/functionMappings/commonFns.ts
+++ b/packages/nocodb/src/db/functionMappings/commonFns.ts
@@ -38,27 +38,70 @@ export default {
const count = Math.floor((args.pt.arguments.length - 1) / 2);
let query = '';
+ const returnArgsType = new Set(
+ args.pt.arguments
+ .filter(
+ (type, i) => i > 1 && i % 2 === 0 && type !== FormulaDataTypes.NULL,
+ )
+ .map((type) => type.dataType),
+ );
+
+ // if else case present then push that to types
+ if (args.pt.arguments.length % 2 === 0) {
+ returnArgsType.add(
+ args.pt.arguments[args.pt.arguments.length - 1].dataType,
+ );
+ }
+
const switchVal = (await args.fn(args.pt.arguments[0])).builder.toQuery();
for (let i = 0; i < count; i++) {
+ let val;
+ // cast to string if the return value types are different
+ if (returnArgsType.size > 1) {
+ val = (
+ await args.fn({
+ type: 'CallExpression',
+ arguments: [args.pt.arguments[i * 2 + 2]],
+ callee: {
+ type: 'Identifier',
+ name: 'STRING',
+ },
+ } as any)
+ ).builder.toQuery();
+ } else {
+ val = (await args.fn(args.pt.arguments[i * 2 + 2])).builder.toQuery();
+ }
+
query += args.knex
.raw(
`\n\tWHEN ${(
await args.fn(args.pt.arguments[i * 2 + 1])
- ).builder.toQuery()} THEN ${(
- await args.fn(args.pt.arguments[i * 2 + 2])
- ).builder.toQuery()}`,
+ ).builder.toQuery()} THEN ${val}`,
)
.toQuery();
}
if (args.pt.arguments.length % 2 === 0) {
- query += args.knex
- .raw(
- `\n\tELSE ${(
- await args.fn(args.pt.arguments[args.pt.arguments.length - 1])
- ).builder.toQuery()}`,
- )
- .toQuery();
+ let val;
+ // cast to string if the return value types are different
+ if (returnArgsType.size > 1) {
+ val = (
+ await args.fn({
+ type: 'CallExpression',
+ arguments: [args.pt.arguments[args.pt.arguments.length - 1]],
+ callee: {
+ type: 'Identifier',
+ name: 'STRING',
+ },
+ } as any)
+ ).builder.toQuery();
+ } else {
+ val = (
+ await args.fn(args.pt.arguments[args.pt.arguments.length - 1])
+ ).builder.toQuery();
+ }
+
+ query += `\n\tELSE ${val}`;
}
return {
builder: args.knex.raw(
From a4df91819f7bb67e71595199f4fe63da9a7ed569 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:00 +0000
Subject: [PATCH 028/262] fix: formula if condition validation correction
---
.../smartsheet/column/FormulaOptions.vue | 27 ++++++++++++++-----
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 9 ++++---
.../src/db/functionMappings/commonFns.ts | 2 +-
3 files changed, 27 insertions(+), 11 deletions(-)
diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
index 6e5039b0f7..45624cf039 100644
--- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
+++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
@@ -10,7 +10,7 @@ import {
isSystemColumn,
jsepCurlyHook,
substituteColumnIdWithAliasInFormula,
- validateDateWithUnknownFormat,
+ validateFormulaAndExtractTreeWithType
} from 'nocodb-sdk'
import {
MetaInj,
@@ -99,10 +99,15 @@ const validators = {
validator: (_: any, formula: any) => {
return new Promise((resolve, reject) => {
if (!formula?.trim()) return reject(new Error('Required'))
- const res = parseAndValidateFormula(formula)
- if (res !== true) {
- return reject(new Error(res))
+
+ try {
+ validateFormulaAndExtractTreeWithType(formula, supportedColumns.value)
+ } catch (e: any) {
+ return reject(new Error(e.message))
}
+ // if (res !== true) {
+ // return reject(new Error(res))
+ // }
resolve()
})
},
@@ -226,7 +231,7 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
return validateAgainstMeta(arg, errors, typeErrors)
})
- const argsTypes = validateResult.map((v: any) => v.returnType);
+ const argsTypes = validateResult.map((v: any) => v.returnType)
if (typeof validateResult[0].returnType === 'function') {
returnType = formulas[calleeName].returnType(argsTypes)
@@ -266,7 +271,9 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
typeErrors,
)
} else {
- parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors, argsTypes))
+ parsedTree.arguments.map((arg: Record) =>
+ validateAgainstType(arg, expectedType, null, typeErrors, argsTypes),
+ )
}
} else if (expectedType === formulaTypes.DATE) {
if (calleeName === 'DATEADD') {
@@ -488,7 +495,13 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n
return { errors, returnType }
}
-function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set(), argTypes: formulaTypes = []) {
+function validateAgainstType(
+ parsedTree: any,
+ expectedType: string,
+ func: any,
+ typeErrors = new Set(),
+ argTypes: formulaTypes = [],
+) {
let type
if (parsedTree === false || typeof parsedTree === 'undefined') {
return typeErrors
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 6bdabf0dcc..c93a564bb5 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -693,7 +693,7 @@ const formulas: Record = {
validation: {
args: {
min: 3,
- }
+ },
},
description: 'Switch case value based on expr output',
syntax: 'SWITCH(expr, [pattern, value, ..., default])',
@@ -1169,7 +1169,7 @@ export function validateFormulaAndExtractTreeWithType(
case UITypes.Collaborator:
case UITypes.QrCode:
default:
- throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, '');
+ throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, {});
}
}
} else if (parsedTree.type === JSEPNode.LITERAL) {
@@ -1188,7 +1188,10 @@ export function validateFormulaAndExtractTreeWithType(
) {
res.left = validateAndExtract(parsedTree.left);
res.right = validateAndExtract(parsedTree.right);
- res.dataType = FormulaDataTypes.NUMERIC;
+
+ if (['==', '<', '>', '<=', '>=', '!='].includes(parsedTree.operator)) {
+ res.dataType = FormulaDataTypes.COND_EXP;
+ } else res.dataType = FormulaDataTypes.NUMERIC;
}
return res;
diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts
index d8648b954e..ef3f8b5db3 100644
--- a/packages/nocodb/src/db/functionMappings/commonFns.ts
+++ b/packages/nocodb/src/db/functionMappings/commonFns.ts
@@ -142,7 +142,7 @@ export default {
).builder;
} else {
thenArg = (await args.fn(args.pt.arguments[1])).builder.toQuery();
- elseArg = (await args.fn(args.pt.arguments[1])).builder.toQuery();
+ elseArg = (await args.fn(args.pt.arguments[2])).builder.toQuery();
}
let query = args.knex.raw(`\n\tWHEN ${cond} THEN ${thenArg}`).toQuery();
From aa7bc17c1c8c195c6fa0b6a258b7aaf1daa0df4f Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:00 +0000
Subject: [PATCH 029/262] refactor: WEEKDAY validation
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 207 ++++++++++++++++--
1 file changed, 192 insertions(+), 15 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index c93a564bb5..63b970b327 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -2,6 +2,7 @@ import jsep from 'jsep';
import { ColumnType } from './Api';
import UITypes from './UITypes';
+import { validateDateWithUnknownFormat } from '../../../nc-gui/utils';
export const jsepCurlyHook = {
name: 'curly',
@@ -73,6 +74,20 @@ export async function substituteColumnAliasWithIdInFormula(
return jsepTreeToFormula(parsedFormula);
}
+enum FormulaErrorType {
+ NOT_AVAILABLE = 'NOT_AVAILABLE',
+ NOT_SUPPORTED = 'NOT_SUPPORTED',
+ MIN_ARG = 'MIN_ARG',
+ MAX_ARG = 'MAX_ARG',
+ TYPE_MISMATCH = 'TYPE_MISMATCH',
+ INVALID_ARG = 'INVALID_ARG',
+ INVALID_ARG_TYPE = 'INVALID_ARG_TYPE',
+ INVALID_ARG_VALUE = 'INVALID_ARG_VALUE',
+ INVALID_ARG_COUNT = 'INVALID_ARG_COUNT',
+ CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE',
+ INVALID_FUNCTION_NAME = 'INVALID_FUNCTION_NAME',
+}
+
export function substituteColumnIdWithAliasInFormula(
formula,
columns: ColumnType[],
@@ -747,6 +762,40 @@ const formulas: Record = {
min: 1,
max: 2,
},
+ validator(argTypes: FormulaDataTypes[], parsedTree: any) {
+ if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
+ if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
+ throw new FormulaError(
+ FormulaErrorType.TYPE_MISMATCH,
+ { key: 'msg.formula.firstParamWeekDayHaveDate' },
+ 'First parameter of WEEKDY should be a date'
+ );
+ }
+ }
+
+ if (parsedTree.arguments[1].type === JSEPNode.LITERAL) {
+ const value = parsedTree.arguments[0].value;
+ if (
+ typeof value !== 'string' ||
+ ![
+ 'sunday',
+ 'monday',
+ 'tuesday',
+ 'wednesday',
+ 'thursday',
+ 'friday',
+ 'saturday',
+ ].includes(value.toLowerCase())
+ ) {
+ typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate'));
+ throw new FormulaError(
+ FormulaErrorType.TYPE_MISMATCH,
+ { key: 'msg.formula.secondParamWeekDayHaveDate' },
+ 'Second parameter of WEEKDY should be day of week string'
+ );
+ }
+ }
+ },
},
description:
'Returns the day of the week as an integer between 0 and 6 inclusive starting from Monday by default',
@@ -984,20 +1033,6 @@ const formulas: Record = {
// },
};
-enum FormulaErrorType {
- NOT_AVAILABLE = 'NOT_AVAILABLE',
- NOT_SUPPORTED = 'NOT_SUPPORTED',
- MIN_ARG = 'MIN_ARG',
- MAX_ARG = 'MAX_ARG',
- TYPE_MISMATCH = 'TYPE_MISMATCH',
- INVALID_ARG = 'INVALID_ARG',
- INVALID_ARG_TYPE = 'INVALID_ARG_TYPE',
- INVALID_ARG_VALUE = 'INVALID_ARG_VALUE',
- INVALID_ARG_COUNT = 'INVALID_ARG_COUNT',
- CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE',
- INVALID_FUNCTION_NAME = 'INVALID_FUNCTION_NAME',
-}
-
class FormulaError extends Error {
public type: FormulaErrorType;
public extra: Record;
@@ -1043,7 +1078,149 @@ export function validateFormulaAndExtractTreeWithType(
{},
'Function not available'
);
- //t('msg.formula.functionNotAvailable', { function: calleeName })
+
+ // validate data type
+ if (parsedTree.callee.type === JSEPNode.IDENTIFIER) {
+ const expectedType = formulas[calleeName.toUpperCase()].type;
+ if (expectedType === formulaTypes.NUMERIC) {
+ if (calleeName === 'WEEKDAY') {
+ // parsedTree.arguments[0] = date
+ validateAgainstType(
+ parsedTree.arguments[0],
+ formulaTypes.DATE,
+ (v: any) => {
+ if (!validateDateWithUnknownFormat(v)) {
+ typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate'));
+ }
+ },
+ typeErrors
+ );
+ // parsedTree.arguments[1] = startDayOfWeek (optional)
+ validateAgainstType(
+ parsedTree.arguments[1],
+ formulaTypes.STRING,
+ (v: any) => {
+ if (
+ typeof v !== 'string' ||
+ ![
+ 'sunday',
+ 'monday',
+ 'tuesday',
+ 'wednesday',
+ 'thursday',
+ 'friday',
+ 'saturday',
+ ].includes(v.toLowerCase())
+ ) {
+ typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate'));
+ }
+ },
+ typeErrors
+ );
+ } else {
+ parsedTree.arguments.map((arg: Record) =>
+ validateAgainstType(arg, expectedType, null, typeErrors)
+ );
+ }
+ } else if (expectedType === formulaTypes.DATE) {
+ if (calleeName === 'DATEADD') {
+ // parsedTree.arguments[0] = date
+ validateAgainstType(
+ parsedTree.arguments[0],
+ formulaTypes.DATE,
+ (v: any) => {
+ if (!validateDateWithUnknownFormat(v)) {
+ typeErrors.add(t('msg.formula.firstParamDateAddHaveDate'));
+ }
+ },
+ typeErrors
+ );
+ // parsedTree.arguments[1] = numeric
+ validateAgainstType(
+ parsedTree.arguments[1],
+ formulaTypes.NUMERIC,
+ (v: any) => {
+ if (typeof v !== 'number') {
+ typeErrors.add(
+ t('msg.formula.secondParamDateAddHaveNumber')
+ );
+ }
+ },
+ typeErrors
+ );
+ // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"]
+ validateAgainstType(
+ parsedTree.arguments[2],
+ formulaTypes.STRING,
+ (v: any) => {
+ if (!['day', 'week', 'month', 'year'].includes(v)) {
+ typeErrors.add(
+ typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate'))
+ );
+ }
+ },
+ typeErrors
+ );
+ } else if (calleeName === 'DATETIME_DIFF') {
+ // parsedTree.arguments[0] = date
+ validateAgainstType(
+ parsedTree.arguments[0],
+ formulaTypes.DATE,
+ (v: any) => {
+ if (!validateDateWithUnknownFormat(v)) {
+ typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate'));
+ }
+ },
+ typeErrors
+ );
+ // parsedTree.arguments[1] = date
+ validateAgainstType(
+ parsedTree.arguments[1],
+ formulaTypes.DATE,
+ (v: any) => {
+ if (!validateDateWithUnknownFormat(v)) {
+ typeErrors.add(
+ t('msg.formula.secondParamDateDiffHaveDate')
+ );
+ }
+ },
+ typeErrors
+ );
+ // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"]
+ validateAgainstType(
+ parsedTree.arguments[2],
+ formulaTypes.STRING,
+ (v: any) => {
+ if (
+ ![
+ 'milliseconds',
+ 'ms',
+ 'seconds',
+ 's',
+ 'minutes',
+ 'm',
+ 'hours',
+ 'h',
+ 'days',
+ 'd',
+ 'weeks',
+ 'w',
+ 'months',
+ 'M',
+ 'quarters',
+ 'Q',
+ 'years',
+ 'y',
+ ].includes(v)
+ ) {
+ typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate'));
+ }
+ },
+ typeErrors
+ );
+ }
+ }
+ }
}
// validate arguments
const validation =
From c3142e00a9329b9c92b7ffcd0b1f2cf13ca7b3e3 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:00 +0000
Subject: [PATCH 030/262] refactor: DATEADD validation
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 39 +++++++++++++++++--
1 file changed, 36 insertions(+), 3 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 63b970b327..01c7619831 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -282,6 +282,40 @@ const formulas: Record = {
args: {
rqd: 3,
},
+ validator: (args: FormulaDataTypes[], parsedTree: any) => {
+ if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
+ if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
+ throw new FormulaError(
+ FormulaErrorType.TYPE_MISMATCH,
+ { key: 'msg.formula.firstParamDateAddHaveDate' },
+ 'First parameter of DATEADD should be a date'
+ );
+ }
+ }
+
+ if (parsedTree.arguments[1].type === JSEPNode.LITERAL) {
+ if (typeof parsedTree.arguments[1].value !== 'number') {
+ throw new FormulaError(
+ FormulaErrorType.TYPE_MISMATCH,
+ { key: 'msg.formula.secondParamDateAddHaveNumber' },
+ 'Second parameter of DATEADD should be a number'
+ );
+ }
+ }
+ if (parsedTree.arguments[2].type === JSEPNode.LITERAL) {
+ if (
+ !['day', 'week', 'month', 'year'].includes(
+ parsedTree.arguments[2].value
+ )
+ ) {
+ throw new FormulaError(
+ FormulaErrorType.TYPE_MISMATCH,
+ { key: 'msg.formula.thirdParamDateAddHaveDate' },
+ "Third parameter of DATEADD should be one of 'day', 'week', 'month', 'year'"
+ );
+ }
+ }
+ },
},
description: 'Adds a "count" units to Datetime.',
syntax:
@@ -768,7 +802,7 @@ const formulas: Record = {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
{ key: 'msg.formula.firstParamWeekDayHaveDate' },
- 'First parameter of WEEKDY should be a date'
+ 'First parameter of WEEKDAY should be a date'
);
}
}
@@ -787,11 +821,10 @@ const formulas: Record = {
'saturday',
].includes(value.toLowerCase())
) {
- typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate'));
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
{ key: 'msg.formula.secondParamWeekDayHaveDate' },
- 'Second parameter of WEEKDY should be day of week string'
+ 'Second parameter of WEEKDAY should be day of week string'
);
}
}
From 7f02ac7c2dc606bca2793976fefe189101b37122 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:00 +0000
Subject: [PATCH 031/262] refactor: DATETIME_DIFF validation
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 50 +++++++++++++++++++
1 file changed, 50 insertions(+)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 01c7619831..2180052c72 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -339,6 +339,56 @@ const formulas: Record = {
min: 2,
max: 3,
},
+ validator: (args: FormulaDataTypes[], parsedTree: any) => {
+
+ if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
+ if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
+ throw new FormulaError(
+ FormulaErrorType.TYPE_MISMATCH,
+ { key: 'msg.formula.firstParamDateDiffHaveDate' },
+ 'First parameter of DATETIME_DIFF should be a date'
+ );
+ }
+ }
+
+ if (parsedTree.arguments[1].type === JSEPNode.LITERAL) {
+ if (!validateDateWithUnknownFormat(parsedTree.arguments[1].value)) {
+ throw new FormulaError(
+ FormulaErrorType.TYPE_MISMATCH,
+ { key: 'msg.formula.secondParamDateDiffHaveDate' },
+ 'Second parameter of DATETIME_DIFF should be a date'
+ );
+ }
+ }
+ if (parsedTree.arguments[2].type === JSEPNode.LITERAL) {
+ if (![
+ 'milliseconds',
+ 'ms',
+ 'seconds',
+ 's',
+ 'minutes',
+ 'm',
+ 'hours',
+ 'h',
+ 'days',
+ 'd',
+ 'weeks',
+ 'w',
+ 'months',
+ 'M',
+ 'quarters',
+ 'Q',
+ 'years',
+ 'y',
+ ].includes(parsedTree.arguments[0].value)) {
+ throw new FormulaError(
+ FormulaErrorType.TYPE_MISMATCH,
+ { key: 'msg.formula.thirdParamDateDiffHaveDate' },
+ 'Third parameter of DATETIME_DIFF should be one of \'milliseconds\', \'ms\', \'seconds\', \'s\', \'minutes\', \'m\', \'hours\', \'h\', \'days\', \'d\', \'weeks\', \'w\', \'months\', \'M\', \'quarters\', \'Q\', \'years\', \'y\'''
+ );
+ }
+ }
+ }
},
description:
'Calculate the difference of two given date / datetime in specified units.',
From 248155ec0d8205e8ea3eebf221fe3f5200d7efb8 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:00 +0000
Subject: [PATCH 032/262] refactor: move custom validation to formula meta
object
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 189 +++---------------
1 file changed, 26 insertions(+), 163 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 2180052c72..1fd1d78e62 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1,8 +1,8 @@
import jsep from 'jsep';
-import { ColumnType } from './Api';
+import {ColumnType} from './Api';
import UITypes from './UITypes';
-import { validateDateWithUnknownFormat } from '../../../nc-gui/utils';
+import {validateDateWithUnknownFormat} from '../../../nc-gui/utils';
export const jsepCurlyHook = {
name: 'curly',
@@ -11,7 +11,7 @@ export const jsepCurlyHook = {
const OCURLY_CODE = 123; // {
const CCURLY_CODE = 125; // }
let start = -1;
- const { context } = env;
+ const {context} = env;
if (
!jsep.isIdentifierStart(context.code) &&
context.code === OCURLY_CODE
@@ -28,10 +28,10 @@ export const jsepCurlyHook = {
name: /{{(.*?)}}/.test(context.expr)
? // start would be the position of the first curly bracket
// add 2 to point to the first character for expressions like {{col1}}
- context.expr.slice(start + 2, context.index - 1)
+ context.expr.slice(start + 2, context.index - 1)
: // start would be the position of the first curly bracket
// add 1 to point to the first character for expressions like {col1}
- context.expr.slice(start + 1, context.index - 1),
+ context.expr.slice(start + 1, context.index - 1),
};
return env.node;
} else {
@@ -235,8 +235,8 @@ interface FormulaMeta {
min?: number;
max?: number;
rqd?: number;
- validator?: (args: FormulaDataTypes[]) => boolean;
};
+ custom?: (args: FormulaDataTypes[], parseTree: any) => void;
};
description?: string;
syntax?: string;
@@ -282,12 +282,12 @@ const formulas: Record = {
args: {
rqd: 3,
},
- validator: (args: FormulaDataTypes[], parsedTree: any) => {
+ custom: (args: FormulaDataTypes[], parsedTree: any) => {
if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- { key: 'msg.formula.firstParamDateAddHaveDate' },
+ {key: 'msg.formula.firstParamDateAddHaveDate'},
'First parameter of DATEADD should be a date'
);
}
@@ -297,7 +297,7 @@ const formulas: Record = {
if (typeof parsedTree.arguments[1].value !== 'number') {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- { key: 'msg.formula.secondParamDateAddHaveNumber' },
+ {key: 'msg.formula.secondParamDateAddHaveNumber'},
'Second parameter of DATEADD should be a number'
);
}
@@ -310,7 +310,7 @@ const formulas: Record = {
) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- { key: 'msg.formula.thirdParamDateAddHaveDate' },
+ {key: 'msg.formula.thirdParamDateAddHaveDate'},
"Third parameter of DATEADD should be one of 'day', 'week', 'month', 'year'"
);
}
@@ -339,13 +339,13 @@ const formulas: Record = {
min: 2,
max: 3,
},
- validator: (args: FormulaDataTypes[], parsedTree: any) => {
+ custom: (args: FormulaDataTypes[], parsedTree: any) => {
if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- { key: 'msg.formula.firstParamDateDiffHaveDate' },
+ {key: 'msg.formula.firstParamDateDiffHaveDate'},
'First parameter of DATETIME_DIFF should be a date'
);
}
@@ -355,7 +355,7 @@ const formulas: Record = {
if (!validateDateWithUnknownFormat(parsedTree.arguments[1].value)) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- { key: 'msg.formula.secondParamDateDiffHaveDate' },
+ {key: 'msg.formula.secondParamDateDiffHaveDate'},
'Second parameter of DATETIME_DIFF should be a date'
);
}
@@ -383,8 +383,8 @@ const formulas: Record = {
].includes(parsedTree.arguments[0].value)) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- { key: 'msg.formula.thirdParamDateDiffHaveDate' },
- 'Third parameter of DATETIME_DIFF should be one of \'milliseconds\', \'ms\', \'seconds\', \'s\', \'minutes\', \'m\', \'hours\', \'h\', \'days\', \'d\', \'weeks\', \'w\', \'months\', \'M\', \'quarters\', \'Q\', \'years\', \'y\'''
+ {key: 'msg.formula.thirdParamDateDiffHaveDate'},
+ 'Third parameter of DATETIME_DIFF should be one of \'milliseconds\', \'ms\', \'seconds\', \'s\', \'minutes\', \'m\', \'hours\', \'h\', \'days\', \'d\', \'weeks\', \'w\', \'months\', \'M\', \'quarters\', \'Q\', \'years\', \'y\''
);
}
}
@@ -846,12 +846,12 @@ const formulas: Record = {
min: 1,
max: 2,
},
- validator(argTypes: FormulaDataTypes[], parsedTree: any) {
+ custom(argTypes: FormulaDataTypes[], parsedTree: any) {
if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- { key: 'msg.formula.firstParamWeekDayHaveDate' },
+ {key: 'msg.formula.firstParamWeekDayHaveDate'},
'First parameter of WEEKDAY should be a date'
);
}
@@ -873,7 +873,7 @@ const formulas: Record = {
) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- { key: 'msg.formula.secondParamWeekDayHaveDate' },
+ {key: 'msg.formula.secondParamWeekDayHaveDate'},
'Second parameter of WEEKDAY should be day of week string'
);
}
@@ -1150,7 +1150,7 @@ export function validateFormulaAndExtractTreeWithType(
dataType?: FormulaDataTypes;
errors?: Set;
[key: string]: any;
- } = { ...parsedTree };
+ } = {...parsedTree};
if (parsedTree.type === JSEPNode.CALL_EXP) {
const calleeName = parsedTree.callee.name.toUpperCase();
@@ -1161,150 +1161,8 @@ export function validateFormulaAndExtractTreeWithType(
{},
'Function not available'
);
-
- // validate data type
- if (parsedTree.callee.type === JSEPNode.IDENTIFIER) {
- const expectedType = formulas[calleeName.toUpperCase()].type;
- if (expectedType === formulaTypes.NUMERIC) {
- if (calleeName === 'WEEKDAY') {
- // parsedTree.arguments[0] = date
- validateAgainstType(
- parsedTree.arguments[0],
- formulaTypes.DATE,
- (v: any) => {
- if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate'));
- }
- },
- typeErrors
- );
- // parsedTree.arguments[1] = startDayOfWeek (optional)
- validateAgainstType(
- parsedTree.arguments[1],
- formulaTypes.STRING,
- (v: any) => {
- if (
- typeof v !== 'string' ||
- ![
- 'sunday',
- 'monday',
- 'tuesday',
- 'wednesday',
- 'thursday',
- 'friday',
- 'saturday',
- ].includes(v.toLowerCase())
- ) {
- typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate'));
- }
- },
- typeErrors
- );
- } else {
- parsedTree.arguments.map((arg: Record) =>
- validateAgainstType(arg, expectedType, null, typeErrors)
- );
- }
- } else if (expectedType === formulaTypes.DATE) {
- if (calleeName === 'DATEADD') {
- // parsedTree.arguments[0] = date
- validateAgainstType(
- parsedTree.arguments[0],
- formulaTypes.DATE,
- (v: any) => {
- if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.firstParamDateAddHaveDate'));
- }
- },
- typeErrors
- );
- // parsedTree.arguments[1] = numeric
- validateAgainstType(
- parsedTree.arguments[1],
- formulaTypes.NUMERIC,
- (v: any) => {
- if (typeof v !== 'number') {
- typeErrors.add(
- t('msg.formula.secondParamDateAddHaveNumber')
- );
- }
- },
- typeErrors
- );
- // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"]
- validateAgainstType(
- parsedTree.arguments[2],
- formulaTypes.STRING,
- (v: any) => {
- if (!['day', 'week', 'month', 'year'].includes(v)) {
- typeErrors.add(
- typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate'))
- );
- }
- },
- typeErrors
- );
- } else if (calleeName === 'DATETIME_DIFF') {
- // parsedTree.arguments[0] = date
- validateAgainstType(
- parsedTree.arguments[0],
- formulaTypes.DATE,
- (v: any) => {
- if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate'));
- }
- },
- typeErrors
- );
- // parsedTree.arguments[1] = date
- validateAgainstType(
- parsedTree.arguments[1],
- formulaTypes.DATE,
- (v: any) => {
- if (!validateDateWithUnknownFormat(v)) {
- typeErrors.add(
- t('msg.formula.secondParamDateDiffHaveDate')
- );
- }
- },
- typeErrors
- );
- // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"]
- validateAgainstType(
- parsedTree.arguments[2],
- formulaTypes.STRING,
- (v: any) => {
- if (
- ![
- 'milliseconds',
- 'ms',
- 'seconds',
- 's',
- 'minutes',
- 'm',
- 'hours',
- 'h',
- 'days',
- 'd',
- 'weeks',
- 'w',
- 'months',
- 'M',
- 'quarters',
- 'Q',
- 'years',
- 'y',
- ].includes(v)
- ) {
- typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate'));
- }
- },
- typeErrors
- );
- }
- }
- }
}
+
// validate arguments
const validation =
formulas[calleeName] && formulas[calleeName].validation;
@@ -1359,6 +1217,11 @@ export function validateFormulaAndExtractTreeWithType(
const argTypes = validateResult.map((v: any) => v.dataType);
+ // if validation function is present, call it
+ if (formulas[calleeName].validation?.custom) {
+ formulas[calleeName].validation?.custom(argTypes, parsedTree);
+ }
+
if (typeof formulas[calleeName].returnType === 'function') {
res.dataType = (formulas[calleeName].returnType as any)?.(
argTypes
@@ -1487,7 +1350,7 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
];
if (neighbours.length > 0) {
// e.g. formula column 1 -> [formula column 2, formula column3]
- res.push({ [c.id]: neighbours });
+ res.push({[c.id]: neighbours});
}
return res;
}, []);
From f7b53e0d45fdcf1d4c3731c86dd54c045de467c1 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:00 +0000
Subject: [PATCH 033/262] fix: import statement corrections
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 104 +++++++++++-------
1 file changed, 66 insertions(+), 38 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 1fd1d78e62..efb3031713 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1,8 +1,35 @@
import jsep from 'jsep';
-import {ColumnType} from './Api';
+import { ColumnType } from './Api';
import UITypes from './UITypes';
-import {validateDateWithUnknownFormat} from '../../../nc-gui/utils';
+import dayjs from 'dayjs';
+
+// todo: move to date utils and export, remove duplicate from gui
+
+export const dateFormats = [
+ 'YYYY-MM-DD',
+ 'YYYY/MM/DD',
+ 'DD-MM-YYYY',
+ 'MM-DD-YYYY',
+ 'DD/MM/YYYY',
+ 'MM/DD/YYYY',
+ 'DD MM YYYY',
+ 'MM DD YYYY',
+ 'YYYY MM DD',
+];
+function validateDateWithUnknownFormat(v: string) {
+ for (const format of dateFormats) {
+ if (dayjs(v, format, true).isValid() as any) {
+ return true;
+ }
+ for (const timeFormat of ['HH:mm', 'HH:mm:ss', 'HH:mm:ss.SSS']) {
+ if (dayjs(v, `${format} ${timeFormat}`, true).isValid() as any) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
export const jsepCurlyHook = {
name: 'curly',
@@ -11,7 +38,7 @@ export const jsepCurlyHook = {
const OCURLY_CODE = 123; // {
const CCURLY_CODE = 125; // }
let start = -1;
- const {context} = env;
+ const { context } = env;
if (
!jsep.isIdentifierStart(context.code) &&
context.code === OCURLY_CODE
@@ -28,10 +55,10 @@ export const jsepCurlyHook = {
name: /{{(.*?)}}/.test(context.expr)
? // start would be the position of the first curly bracket
// add 2 to point to the first character for expressions like {{col1}}
- context.expr.slice(start + 2, context.index - 1)
+ context.expr.slice(start + 2, context.index - 1)
: // start would be the position of the first curly bracket
// add 1 to point to the first character for expressions like {col1}
- context.expr.slice(start + 1, context.index - 1),
+ context.expr.slice(start + 1, context.index - 1),
};
return env.node;
} else {
@@ -287,7 +314,7 @@ const formulas: Record = {
if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- {key: 'msg.formula.firstParamDateAddHaveDate'},
+ { key: 'msg.formula.firstParamDateAddHaveDate' },
'First parameter of DATEADD should be a date'
);
}
@@ -297,7 +324,7 @@ const formulas: Record = {
if (typeof parsedTree.arguments[1].value !== 'number') {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- {key: 'msg.formula.secondParamDateAddHaveNumber'},
+ { key: 'msg.formula.secondParamDateAddHaveNumber' },
'Second parameter of DATEADD should be a number'
);
}
@@ -310,7 +337,7 @@ const formulas: Record = {
) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- {key: 'msg.formula.thirdParamDateAddHaveDate'},
+ { key: 'msg.formula.thirdParamDateAddHaveDate' },
"Third parameter of DATEADD should be one of 'day', 'week', 'month', 'year'"
);
}
@@ -340,12 +367,11 @@ const formulas: Record = {
max: 3,
},
custom: (args: FormulaDataTypes[], parsedTree: any) => {
-
if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- {key: 'msg.formula.firstParamDateDiffHaveDate'},
+ { key: 'msg.formula.firstParamDateDiffHaveDate' },
'First parameter of DATETIME_DIFF should be a date'
);
}
@@ -355,40 +381,42 @@ const formulas: Record = {
if (!validateDateWithUnknownFormat(parsedTree.arguments[1].value)) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- {key: 'msg.formula.secondParamDateDiffHaveDate'},
+ { key: 'msg.formula.secondParamDateDiffHaveDate' },
'Second parameter of DATETIME_DIFF should be a date'
);
}
}
if (parsedTree.arguments[2].type === JSEPNode.LITERAL) {
- if (![
- 'milliseconds',
- 'ms',
- 'seconds',
- 's',
- 'minutes',
- 'm',
- 'hours',
- 'h',
- 'days',
- 'd',
- 'weeks',
- 'w',
- 'months',
- 'M',
- 'quarters',
- 'Q',
- 'years',
- 'y',
- ].includes(parsedTree.arguments[0].value)) {
+ if (
+ ![
+ 'milliseconds',
+ 'ms',
+ 'seconds',
+ 's',
+ 'minutes',
+ 'm',
+ 'hours',
+ 'h',
+ 'days',
+ 'd',
+ 'weeks',
+ 'w',
+ 'months',
+ 'M',
+ 'quarters',
+ 'Q',
+ 'years',
+ 'y',
+ ].includes(parsedTree.arguments[0].value)
+ ) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- {key: 'msg.formula.thirdParamDateDiffHaveDate'},
- 'Third parameter of DATETIME_DIFF should be one of \'milliseconds\', \'ms\', \'seconds\', \'s\', \'minutes\', \'m\', \'hours\', \'h\', \'days\', \'d\', \'weeks\', \'w\', \'months\', \'M\', \'quarters\', \'Q\', \'years\', \'y\''
+ { key: 'msg.formula.thirdParamDateDiffHaveDate' },
+ "Third parameter of DATETIME_DIFF should be one of 'milliseconds', 'ms', 'seconds', 's', 'minutes', 'm', 'hours', 'h', 'days', 'd', 'weeks', 'w', 'months', 'M', 'quarters', 'Q', 'years', 'y'"
);
}
}
- }
+ },
},
description:
'Calculate the difference of two given date / datetime in specified units.',
@@ -851,7 +879,7 @@ const formulas: Record = {
if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- {key: 'msg.formula.firstParamWeekDayHaveDate'},
+ { key: 'msg.formula.firstParamWeekDayHaveDate' },
'First parameter of WEEKDAY should be a date'
);
}
@@ -873,7 +901,7 @@ const formulas: Record = {
) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
- {key: 'msg.formula.secondParamWeekDayHaveDate'},
+ { key: 'msg.formula.secondParamWeekDayHaveDate' },
'Second parameter of WEEKDAY should be day of week string'
);
}
@@ -1150,7 +1178,7 @@ export function validateFormulaAndExtractTreeWithType(
dataType?: FormulaDataTypes;
errors?: Set;
[key: string]: any;
- } = {...parsedTree};
+ } = { ...parsedTree };
if (parsedTree.type === JSEPNode.CALL_EXP) {
const calleeName = parsedTree.callee.name.toUpperCase();
@@ -1350,7 +1378,7 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
];
if (neighbours.length > 0) {
// e.g. formula column 1 -> [formula column 2, formula column3]
- res.push({[c.id]: neighbours});
+ res.push({ [c.id]: neighbours });
}
return res;
}, []);
From 27529b39d82e01317988b2b2b712c57f574431c0 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:01 +0000
Subject: [PATCH 034/262] fix: prefix unused vars with _
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index efb3031713..1a95a477d9 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -309,7 +309,7 @@ const formulas: Record = {
args: {
rqd: 3,
},
- custom: (args: FormulaDataTypes[], parsedTree: any) => {
+ custom: (_argTypes: FormulaDataTypes[], parsedTree: any) => {
if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
throw new FormulaError(
@@ -366,7 +366,7 @@ const formulas: Record = {
min: 2,
max: 3,
},
- custom: (args: FormulaDataTypes[], parsedTree: any) => {
+ custom: (_argTypes: FormulaDataTypes[], parsedTree: any) => {
if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
throw new FormulaError(
@@ -874,7 +874,7 @@ const formulas: Record = {
min: 1,
max: 2,
},
- custom(argTypes: FormulaDataTypes[], parsedTree: any) {
+ custom(_argTypes: FormulaDataTypes[], parsedTree: any) {
if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) {
throw new FormulaError(
From ad4ac116a0c5f2bf52033ed7091b7daab1775fca Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:01 +0000
Subject: [PATCH 035/262] refactor: move arg type
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 99 ++++++++++---------
1 file changed, 53 insertions(+), 46 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 1a95a477d9..64628670fb 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -256,12 +256,13 @@ export enum JSEPNode {
}
interface FormulaMeta {
- type?: string;
validation?: {
args?: {
min?: number;
max?: number;
rqd?: number;
+
+ type?: FormulaDataTypes;
};
custom?: (args: FormulaDataTypes[], parseTree: any) => void;
};
@@ -276,6 +277,7 @@ const formulas: Record = {
validation: {
args: {
min: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description: 'Average of input parameters',
@@ -288,10 +290,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
ADD: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
min: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description: 'Sum of input parameters',
@@ -304,10 +306,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
DATEADD: {
- type: FormulaDataTypes.DATE,
validation: {
args: {
rqd: 3,
+ type: FormulaDataTypes.DATE,
},
custom: (_argTypes: FormulaDataTypes[], parsedTree: any) => {
if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
@@ -360,11 +362,11 @@ const formulas: Record = {
returnType: FormulaDataTypes.DATE,
},
DATETIME_DIFF: {
- type: FormulaDataTypes.DATE,
validation: {
args: {
min: 2,
max: 3,
+ type: FormulaDataTypes.DATE,
},
custom: (_argTypes: FormulaDataTypes[], parsedTree: any) => {
if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
@@ -458,10 +460,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.COND_EXP,
},
CONCAT: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
min: 1,
+ type: FormulaDataTypes.STRING,
},
},
description: 'Concatenated string of input parameters',
@@ -473,10 +475,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
TRIM: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.STRING,
},
},
description: 'Remove trailing and leading whitespaces from input parameter',
@@ -488,10 +490,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
UPPER: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.STRING,
},
},
description: 'Upper case converted string of input parameter',
@@ -500,10 +502,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
LOWER: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.STRING,
},
},
description: 'Lower case converted string of input parameter',
@@ -512,10 +514,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
LEN: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.STRING,
},
},
description: 'Input parameter character length',
@@ -524,10 +526,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
MIN: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
min: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description: 'Minimum value amongst input parameters',
@@ -536,10 +538,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
MAX: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
min: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description: 'Maximum value amongst input parameters',
@@ -548,10 +550,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
CEILING: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description: 'Rounded next largest integer value of input parameter',
@@ -560,10 +562,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
FLOOR: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description:
@@ -573,11 +575,11 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
ROUND: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
min: 1,
max: 2,
+ type: FormulaDataTypes.NUMERIC,
},
},
description:
@@ -591,10 +593,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
MOD: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
rqd: 2,
+ type: FormulaDataTypes.NUMERIC,
},
},
description: 'Remainder after integer division of input parameters',
@@ -603,10 +605,11 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
REPEAT: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 2,
+
+ type: FormulaDataTypes.STRING,
},
},
description:
@@ -616,8 +619,11 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
LOG: {
- type: FormulaDataTypes.NUMERIC,
- validation: {},
+ validation: {
+ args: {
+ type: FormulaDataTypes.NUMERIC,
+ },
+ },
description:
'Logarithm of input parameter to the base (default = e) specified',
syntax: 'LOG([base], value)',
@@ -625,18 +631,21 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
EXP: {
- type: FormulaDataTypes.NUMERIC,
- validation: {},
+ validation: {
+ args: {
+ type: FormulaDataTypes.NUMERIC,
+ },
+ },
description: 'Exponential value of input parameter (e ^ power)',
syntax: 'EXP(power)',
examples: ['EXP(1) => 2.718281828459045', 'EXP({column1})'],
returnType: FormulaDataTypes.NUMERIC,
},
POWER: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
rqd: 2,
+ type: FormulaDataTypes.NUMERIC,
},
},
description: 'base to the exponent power, as in base ^ exponent',
@@ -645,10 +654,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
SQRT: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description: 'Square root of the input parameter',
@@ -657,10 +666,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
ABS: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description: 'Absolute value of the input parameter',
@@ -669,10 +678,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
NOW: {
- type: FormulaDataTypes.DATE,
validation: {
args: {
rqd: 0,
+ type: FormulaDataTypes.DATE,
},
},
description: 'Returns the current time and day',
@@ -681,10 +690,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.DATE,
},
REPLACE: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 3,
+ type: FormulaDataTypes.STRING,
},
},
description:
@@ -697,10 +706,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
SEARCH: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 2,
+ type: FormulaDataTypes.STRING,
},
},
description: 'Index of srchStr specified if found, 0 otherwise',
@@ -712,10 +721,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
INT: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description: 'Integer value of input parameter',
@@ -724,10 +733,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
RIGHT: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 2,
+ type: FormulaDataTypes.STRING,
},
},
description: 'n characters from the end of input parameter',
@@ -736,10 +745,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
LEFT: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 2,
+ type: FormulaDataTypes.STRING,
},
},
description: 'n characters from the beginning of input parameter',
@@ -748,11 +757,11 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
SUBSTR: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
min: 2,
max: 3,
+ type: FormulaDataTypes.STRING,
},
},
description:
@@ -766,10 +775,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
MID: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 3,
+ type: FormulaDataTypes.STRING,
},
},
description: 'Alias for SUBSTR',
@@ -778,7 +787,6 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
IF: {
- type: FormulaDataTypes.COND_EXP,
validation: {
args: {
min: 2,
@@ -816,11 +824,13 @@ const formulas: Record = {
},
},
SWITCH: {
- type: FormulaDataTypes.COND_EXP,
validation: {
args: {
min: 3,
},
+ custom: (_argTypes: any[], _parseTree) => {
+ // Todo: Add validation for switch
+ },
},
description: 'Switch case value based on expr output',
syntax: 'SWITCH(expr, [pattern, value, ..., default])',
@@ -856,10 +866,10 @@ const formulas: Record = {
},
},
URL: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.STRING,
},
},
description: 'Convert to a hyperlink if it is a valid URL',
@@ -868,11 +878,11 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
WEEKDAY: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
min: 1,
max: 2,
+ type: FormulaDataTypes.NUMERIC,
},
custom(_argTypes: FormulaDataTypes[], parsedTree: any) {
if (parsedTree.arguments[0].type === JSEPNode.LITERAL) {
@@ -916,7 +926,6 @@ const formulas: Record = {
},
TRUE: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
max: 0,
@@ -929,7 +938,6 @@ const formulas: Record = {
},
FALSE: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
max: 0,
@@ -942,10 +950,10 @@ const formulas: Record = {
},
REGEX_MATCH: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 2,
+ type: FormulaDataTypes.STRING,
},
},
description:
@@ -956,10 +964,10 @@ const formulas: Record = {
},
REGEX_EXTRACT: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 2,
+ type: FormulaDataTypes.STRING,
},
},
description: 'Returns the first match of a regular expression in a string.',
@@ -968,10 +976,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
REGEX_REPLACE: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 3,
+ type: FormulaDataTypes.STRING,
},
},
description:
@@ -981,7 +989,6 @@ const formulas: Record = {
returnType: FormulaDataTypes.STRING,
},
BLANK: {
- type: FormulaDataTypes.STRING,
validation: {
args: {
rqd: 0,
@@ -993,11 +1000,11 @@ const formulas: Record = {
returnType: FormulaDataTypes.NULL,
},
XOR: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
min: 1,
},
+ // todo: validation for boolean
},
description:
'Returns true if an odd number of arguments are true, and false otherwise.',
@@ -1006,10 +1013,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.BOOLEAN,
},
EVEN: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description:
@@ -1019,10 +1026,10 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
ODD: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
rqd: 1,
+ type: FormulaDataTypes.NUMERIC,
},
},
description:
@@ -1080,11 +1087,11 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
ROUNDDOWN: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
min: 1,
max: 2,
+ type: FormulaDataTypes.NUMERIC,
},
},
description:
@@ -1094,11 +1101,11 @@ const formulas: Record = {
returnType: FormulaDataTypes.NUMERIC,
},
ROUNDUP: {
- type: FormulaDataTypes.NUMERIC,
validation: {
args: {
min: 1,
max: 2,
+ type: FormulaDataTypes.NUMERIC,
},
},
description:
From c3b5db43df382236fe8066c5233309b762721274 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:01 +0000
Subject: [PATCH 036/262] refactor: arg validation logic
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 64628670fb..44fb1e2a09 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -17,6 +17,7 @@ export const dateFormats = [
'MM DD YYYY',
'YYYY MM DD',
];
+
function validateDateWithUnknownFormat(v: string) {
for (const format of dateFormats) {
if (dayjs(v, format, true).isValid() as any) {
@@ -1256,6 +1257,24 @@ export function validateFormulaAndExtractTreeWithType(
if (formulas[calleeName].validation?.custom) {
formulas[calleeName].validation?.custom(argTypes, parsedTree);
}
+ // validate against expected arg types if present
+ else if (formulas[calleeName].validation?.args?.type) {
+ const expectedArgType = formulas[calleeName].validation.args.type;
+ if (
+ argTypes.some(
+ (argType) =>
+ argType !== expectedArgType && argType !== FormulaDataTypes.NULL
+ )
+ )
+ throw new FormulaError(
+ FormulaErrorType.INVALID_ARG,
+ {
+ key: 'msg.formula.invalidArgumentType',
+ calleeName,
+ },
+ 'Invalid argument type'
+ );
+ }
if (typeof formulas[calleeName].returnType === 'function') {
res.dataType = (formulas[calleeName].returnType as any)?.(
From 83933e920c9b1b6adbccacb20378cc6608ed599f Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:01 +0000
Subject: [PATCH 037/262] fix: typo correction - remove unnecessary `.value`
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 44fb1e2a09..2299641d74 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1386,7 +1386,7 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
// e.g. formula1 -> formula2 -> formula1 should return circular reference error
// get all formula columns excluding itself
- const formulaPaths = columns.value
+ const formulaPaths = columns
.filter((c) => c.id !== formulaCol?.id && c.uidt === UITypes.Formula)
.reduce((res: Record[], c: Record) => {
// in `formula`, get all the (unique) target neighbours
From d3a84fd5d04f887b65d7f592da5066d808a4ab3a Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:01 +0000
Subject: [PATCH 038/262] fix: extract datatype from nested formula properly
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 2299641d74..5f365fbab2 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1301,7 +1301,7 @@ export function validateFormulaAndExtractTreeWithType(
columns
);
- res.dataType = formulaRes as any;
+ res.dataType = (formulaRes as any)?.dataType;
} else {
switch (col?.uidt) {
// string
From cb3c3d7b006c315804a7b53bfed117cb5b0ccbca Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:01 +0000
Subject: [PATCH 039/262] fix: error message correction
---
.../smartsheet/column/FormulaOptions.vue | 16 ++++++++++----
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 21 +++++++++++++++----
.../nocodb/src/services/columns.service.ts | 14 ++++++++++---
3 files changed, 40 insertions(+), 11 deletions(-)
diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
index 45624cf039..1cd4dfcb3e 100644
--- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
+++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
@@ -2,6 +2,14 @@
import type { Ref } from 'vue'
import type { ListItem as AntListItem } from 'ant-design-vue'
import jsep from 'jsep'
+import type { ColumnType, FormulaType } from 'nocodb-sdk'
+import {
+ FormulaError,
+ UITypes,
+ jsepCurlyHook,
+ substituteColumnIdWithAliasInFormula,
+ validateFormulaAndExtractTreeWithType,
+} from 'nocodb-sdk'
import type { ColumnType, FormulaType, LinkToAnotherRecordType, TableType } from 'nocodb-sdk'
import {
UITypes,
@@ -52,10 +60,6 @@ const { setAdditionalValidations, validateInfos, sqlUi, column } = useColumnCrea
const { t } = useI18n()
-const baseStore = useBase()
-
-const { tables } = storeToRefs(baseStore)
-
const { predictFunction: _predictFunction } = useNocoEe()
enum JSEPNode {
@@ -103,6 +107,10 @@ const validators = {
try {
validateFormulaAndExtractTreeWithType(formula, supportedColumns.value)
} catch (e: any) {
+ if (e instanceof FormulaError && e.extra?.key) {
+ return reject(new Error(t(e.extra.key, e.extra)))
+ }
+
return reject(new Error(e.message))
}
// if (res !== true) {
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 5f365fbab2..7198412eec 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -102,7 +102,7 @@ export async function substituteColumnAliasWithIdInFormula(
return jsepTreeToFormula(parsedFormula);
}
-enum FormulaErrorType {
+export enum FormulaErrorType {
NOT_AVAILABLE = 'NOT_AVAILABLE',
NOT_SUPPORTED = 'NOT_SUPPORTED',
MIN_ARG = 'MIN_ARG',
@@ -1152,7 +1152,7 @@ const formulas: Record = {
// },
};
-class FormulaError extends Error {
+export class FormulaError extends Error {
public type: FormulaErrorType;
public extra: Record;
@@ -1265,15 +1265,28 @@ export function validateFormulaAndExtractTreeWithType(
(argType) =>
argType !== expectedArgType && argType !== FormulaDataTypes.NULL
)
- )
+ ) {
+ let key = '';
+
+ if (expectedArgType === FormulaDataTypes.NUMERIC) {
+ key = 'msg.formula.numericTypeIsExpected';
+ } else if (expectedArgType === FormulaDataTypes.STRING) {
+ key = 'msg.formula.stringTypeIsExpected';
+ } else if (expectedArgType === FormulaDataTypes.BOOLEAN) {
+ key = 'msg.formula.booleanTypeIsExpected';
+ } else if (expectedArgType === FormulaDataTypes.DATE) {
+ key = 'msg.formula.dateTypeIsExpected';
+ }
+
throw new FormulaError(
FormulaErrorType.INVALID_ARG,
{
- key: 'msg.formula.invalidArgumentType',
+ key,
calleeName,
},
'Invalid argument type'
);
+ }
}
if (typeof formulas[calleeName].returnType === 'function') {
diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts
index f9731fc115..81c261f421 100644
--- a/packages/nocodb/src/services/columns.service.ts
+++ b/packages/nocodb/src/services/columns.service.ts
@@ -233,7 +233,7 @@ export class ColumnsService {
{},
null,
true,
- colBody.parsed_tree
+ colBody.parsed_tree,
);
} catch (e) {
console.error(e);
@@ -938,7 +938,10 @@ export class ColumnsService {
]);
await FormulaColumn.update(c.id, {
formula_raw: new_formula_raw,
- parsed_tree: validateFormulaAndExtractTreeWithType(new_formula_raw, table.columns)
+ parsed_tree: validateFormulaAndExtractTreeWithType(
+ new_formula_raw,
+ table.columns,
+ ),
});
}
}
@@ -1209,10 +1212,15 @@ export class ColumnsService {
colBody.formula_raw || colBody.formula,
table.columns,
);
+ console.log(
+ colBody.formula_raw ||
+ colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'),
+ );
colBody.parsed_tree = validateFormulaAndExtractTreeWithType(
// formula may include double curly brackets in previous version
// convert to single curly bracket here for compatibility
- colBody.formula_raw || colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'),
+ colBody.formula_raw ||
+ colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'),
table.columns,
);
From bc9f106b44c7a56c727ba72c7b4561642e8cd522 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:01 +0000
Subject: [PATCH 040/262] fix: validation corrections
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 43 ++++++++++++++-----
1 file changed, 32 insertions(+), 11 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 7198412eec..8ed974081c 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -389,7 +389,10 @@ const formulas: Record = {
);
}
}
- if (parsedTree.arguments[2].type === JSEPNode.LITERAL) {
+ if (
+ parsedTree.arguments[2] &&
+ parsedTree.arguments[2].type === JSEPNode.LITERAL
+ ) {
if (
![
'milliseconds',
@@ -410,7 +413,7 @@ const formulas: Record = {
'Q',
'years',
'y',
- ].includes(parsedTree.arguments[0].value)
+ ].includes(parsedTree.arguments[2].value)
) {
throw new FormulaError(
FormulaErrorType.TYPE_MISMATCH,
@@ -896,8 +899,12 @@ const formulas: Record = {
}
}
- if (parsedTree.arguments[1].type === JSEPNode.LITERAL) {
- const value = parsedTree.arguments[0].value;
+ // if second argument is present and literal then validate it
+ if (
+ parsedTree.arguments[1] &&
+ parsedTree.arguments[1].type === JSEPNode.LITERAL
+ ) {
+ const value = parsedTree.arguments[1].value;
if (
typeof value !== 'string' ||
![
@@ -1343,23 +1350,37 @@ export function validateFormulaAndExtractTreeWithType(
case UITypes.LastModifiedTime:
res.dataType = FormulaDataTypes.DATE;
break;
- // not supported
- case UITypes.ForeignKey:
- case UITypes.Attachment:
- case UITypes.ID:
- case UITypes.Time:
+
case UITypes.Currency:
case UITypes.Percent:
case UITypes.Duration:
+ case UITypes.Links:
case UITypes.Rollup:
+ res.dataType = FormulaDataTypes.NUMERIC;
+ break;
+
+ case UITypes.Attachment:
+ res.dataType = FormulaDataTypes.STRING;
+ break;
+ case UITypes.Checkbox:
+ res.dataType = FormulaDataTypes.NUMERIC;
+ break;
+ case UITypes.ID:
+ case UITypes.ForeignKey:
+ {
+ res.dataType = FormulaDataTypes.NUMERIC;
+ }
+ break;
+ // not supported
+ case UITypes.Time:
case UITypes.Lookup:
case UITypes.Barcode:
case UITypes.Button:
- case UITypes.Checkbox:
case UITypes.Collaborator:
case UITypes.QrCode:
default:
- throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, {});
+ break;
+ // throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, {});
}
}
} else if (parsedTree.type === JSEPNode.LITERAL) {
From 0ad6b17298e8941ac7da2e506ffc7eece868a34c Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:01 +0000
Subject: [PATCH 041/262] fix: extract type of id/foreign key using sqlui
---
.../smartsheet/column/FormulaOptions.vue | 3 +-
.../nocodb-sdk/src/lib/formulaHelpers.spec.ts | 40 ++++++-----
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 72 ++++++++++++++++---
.../nocodb/src/services/columns.service.ts | 37 +++++-----
4 files changed, 111 insertions(+), 41 deletions(-)
diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
index 1cd4dfcb3e..049d5dab30 100644
--- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
+++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
@@ -105,8 +105,9 @@ const validators = {
if (!formula?.trim()) return reject(new Error('Required'))
try {
- validateFormulaAndExtractTreeWithType(formula, supportedColumns.value)
+ validateFormulaAndExtractTreeWithType({ formula, columns: supportedColumns.value, clientOrSqlUi: sqlUi.value })
} catch (e: any) {
+ console.log(e)
if (e instanceof FormulaError && e.extra?.key) {
return reject(new Error(t(e.extra.key, e.extra)))
}
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts
index 071733e6fb..f0fcb1f697 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts
@@ -6,29 +6,35 @@ import UITypes from './UITypes';
describe('Formula parsing and type validation', () => {
it('Simple formula', async () => {
- const result = validateFormulaAndExtractTreeWithType('1 + 2', []);
+ const result = validateFormulaAndExtractTreeWithType({
+ formula: '1 + 2',
+ columns: [],
+ clientOrSqlUi: 'mysql2',
+ });
expect(result.dataType).toEqual(FormulaDataTypes.NUMERIC);
});
it('Formula with IF condition', async () => {
- const result = validateFormulaAndExtractTreeWithType(
- 'IF({column}, "Found", BLANK())',
- [
+ const result = validateFormulaAndExtractTreeWithType({
+ formula: 'IF({column}, "Found", BLANK())',
+ columns: [
{
id: 'cid',
title: 'column',
uidt: UITypes.Number,
},
- ]
- );
+ ],
+ clientOrSqlUi: 'mysql2',
+ });
expect(result.dataType).toEqual(FormulaDataTypes.STRING);
});
it('Complex formula', async () => {
- const result = validateFormulaAndExtractTreeWithType(
- 'SWITCH({column2},"value1",IF({column1}, "Found", BLANK()),"value2", 2)',
- [
+ const result = validateFormulaAndExtractTreeWithType({
+ formula:
+ 'SWITCH({column2},"value1",IF({column1}, "Found", BLANK()),"value2", 2)',
+ columns: [
{
id: 'id1',
title: 'column1',
@@ -39,14 +45,15 @@ describe('Formula parsing and type validation', () => {
title: 'column2',
uidt: UITypes.SingleLineText,
},
- ]
- );
+ ],
+ clientOrSqlUi: 'mysql2',
+ });
expect(result.dataType).toEqual(FormulaDataTypes.STRING);
- const result1 = validateFormulaAndExtractTreeWithType(
- 'SWITCH({column2},"value1",IF({column1}, 1, 2),"value2", 2)',
- [
+ const result1 = validateFormulaAndExtractTreeWithType({
+ formula: 'SWITCH({column2},"value1",IF({column1}, 1, 2),"value2", 2)',
+ columns: [
{
id: 'id1',
title: 'column1',
@@ -57,8 +64,9 @@ describe('Formula parsing and type validation', () => {
title: 'column2',
uidt: UITypes.SingleLineText,
},
- ]
- );
+ ],
+ clientOrSqlUi: 'mysql2',
+ });
expect(result1.dataType).toEqual(FormulaDataTypes.NUMERIC);
});
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 8ed974081c..fe8db215bf 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -3,6 +3,14 @@ import jsep from 'jsep';
import { ColumnType } from './Api';
import UITypes from './UITypes';
import dayjs from 'dayjs';
+import {
+ MssqlUi,
+ MysqlUi,
+ OracleUi,
+ PgUi,
+ SnowflakeUi,
+ SqlUiFactory,
+} from './sqlUi';
// todo: move to date utils and export, remove duplicate from gui
@@ -242,6 +250,7 @@ export enum FormulaDataTypes {
COND_EXP = 'conditional_expression',
NULL = 'null',
BOOLEAN = 'boolean',
+ UNKNOWN = 'unknown',
}
export enum JSEPNode {
@@ -1176,10 +1185,30 @@ export class FormulaError extends Error {
}
}
-export function validateFormulaAndExtractTreeWithType(
+export function validateFormulaAndExtractTreeWithType({
formula,
- columns: ColumnType[]
-) {
+ columns,
+ clientOrSqlUi,
+ getMeta,
+}: {
+ formula: string;
+ columns: ColumnType[];
+ clientOrSqlUi:
+ | 'mysql'
+ | 'pg'
+ | 'sqlite3'
+ | 'mssql'
+ | 'mysql2'
+ | 'oracledb'
+ | 'mariadb'
+ | 'sqlite'
+ | MysqlUi
+ | MssqlUi
+ | SnowflakeUi
+ | PgUi
+ | OracleUi;
+ getMeta?: (tableId: string) => Promise;
+}) {
const colAliasToColMap = {};
const colIdToColMap = {};
@@ -1270,7 +1299,9 @@ export function validateFormulaAndExtractTreeWithType(
if (
argTypes.some(
(argType) =>
- argType !== expectedArgType && argType !== FormulaDataTypes.NULL
+ argType !== expectedArgType &&
+ argType !== FormulaDataTypes.NULL &&
+ argType !== FormulaDataTypes.UNKNOWN
)
) {
let key = '';
@@ -1317,8 +1348,14 @@ export function validateFormulaAndExtractTreeWithType(
validateFormulaAndExtractTreeWithType(
// formula may include double curly brackets in previous version
// convert to single curly bracket here for compatibility
- col.colOptions.formula.replaceAll('{{', '{').replaceAll('}}', '}'),
- columns
+ {
+ formula: col.colOptions.formula
+ .replaceAll('{{', '{')
+ .replaceAll('}}', '}'),
+ columns,
+ clientOrSqlUi,
+ getMeta,
+ }
);
res.dataType = (formulaRes as any)?.dataType;
@@ -1368,7 +1405,26 @@ export function validateFormulaAndExtractTreeWithType(
case UITypes.ID:
case UITypes.ForeignKey:
{
- res.dataType = FormulaDataTypes.NUMERIC;
+ const sqlUI =
+ typeof clientOrSqlUi === 'string'
+ ? SqlUiFactory.create(clientOrSqlUi)
+ : clientOrSqlUi;
+ if (sqlUI) {
+ const abstractType = sqlUI.getAbstractType(col);
+ if (['integer', 'float', 'decimal'].includes(abstractType)) {
+ res.dataType = FormulaDataTypes.NUMERIC;
+ } else if (['boolean'].includes(abstractType)) {
+ res.dataType = FormulaDataTypes.BOOLEAN;
+ } else if (
+ ['date', 'datetime', 'time', 'year'].includes(abstractType)
+ ) {
+ res.dataType = FormulaDataTypes.DATE;
+ } else {
+ res.dataType = FormulaDataTypes.STRING;
+ }
+ } else {
+ res.dataType = FormulaDataTypes.UNKNOWN;
+ }
}
break;
// not supported
@@ -1379,8 +1435,8 @@ export function validateFormulaAndExtractTreeWithType(
case UITypes.Collaborator:
case UITypes.QrCode:
default:
+ res.dataType = FormulaDataTypes.UNKNOWN;
break;
- // throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, {});
}
}
} else if (parsedTree.type === JSEPNode.LITERAL) {
diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts
index 81c261f421..c39c312fe7 100644
--- a/packages/nocodb/src/services/columns.service.ts
+++ b/packages/nocodb/src/services/columns.service.ts
@@ -210,10 +210,11 @@ export class ColumnsService {
colBody.formula_raw || colBody.formula,
table.columns,
);
- colBody.parsed_tree = validateFormulaAndExtractTreeWithType(
- colBody.formula_raw || colBody.formula,
- table.columns,
- );
+ colBody.parsed_tree = validateFormulaAndExtractTreeWithType({
+ formula: colBody.formula_raw || colBody.formula,
+ columns: table.columns,
+ clientOrSqlUi: source.type,
+ });
try {
const baseModel = await reuseOrSave('baseModel', reuse, async () =>
@@ -938,10 +939,11 @@ export class ColumnsService {
]);
await FormulaColumn.update(c.id, {
formula_raw: new_formula_raw,
- parsed_tree: validateFormulaAndExtractTreeWithType(
- new_formula_raw,
- table.columns,
- ),
+ parsed_tree: validateFormulaAndExtractTreeWithType({
+ formula: new_formula_raw,
+ columns: table.columns,
+ clientOrSqlUi: source.type,
+ }),
});
}
}
@@ -1003,10 +1005,11 @@ export class ColumnsService {
]);
await FormulaColumn.update(c.id, {
formula_raw: new_formula_raw,
- parsed_tree: validateFormulaAndExtractTreeWithType(
- new_formula_raw,
- table.columns,
- ),
+ parsed_tree: validateFormulaAndExtractTreeWithType({
+ formula: new_formula_raw,
+ columns: table.columns,
+ clientOrSqlUi: source.type,
+ }),
});
}
}
@@ -1216,13 +1219,15 @@ export class ColumnsService {
colBody.formula_raw ||
colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'),
);
- colBody.parsed_tree = validateFormulaAndExtractTreeWithType(
+ colBody.parsed_tree = validateFormulaAndExtractTreeWithType({
// formula may include double curly brackets in previous version
// convert to single curly bracket here for compatibility
- colBody.formula_raw ||
+ formula:
+ colBody.formula_raw ||
colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'),
- table.columns,
- );
+ columns: table.columns,
+ clientOrSqlUi: source.type,
+ });
try {
const baseModel = await reuseOrSave('baseModel', reuse, async () =>
From 59e5ba83bda63224e90c29700bce6b0d0922050a Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:01 +0000
Subject: [PATCH 042/262] fix: type definition correction
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index fe8db215bf..b7f96413f6 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1202,11 +1202,11 @@ export function validateFormulaAndExtractTreeWithType({
| 'oracledb'
| 'mariadb'
| 'sqlite'
- | MysqlUi
- | MssqlUi
- | SnowflakeUi
- | PgUi
- | OracleUi;
+ | typeof MysqlUi
+ | typeof MssqlUi
+ | typeof SnowflakeUi
+ | typeof PgUi
+ | typeof OracleUi;
getMeta?: (tableId: string) => Promise;
}) {
const colAliasToColMap = {};
From 20f1c04e7e65ec507c8801ac4ac3177969262a3a Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:02 +0000
Subject: [PATCH 043/262] fix: method usage correction
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 1 +
.../nocodb/src/db/formulav2/formulaQueryBuilderv2.ts | 10 +++++++---
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index b7f96413f6..1775c57a8a 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1202,6 +1202,7 @@ export function validateFormulaAndExtractTreeWithType({
| 'oracledb'
| 'mariadb'
| 'sqlite'
+ | 'snowflake'
| typeof MysqlUi
| typeof MssqlUi
| typeof SnowflakeUi
diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
index f6688082a3..b9294e6591 100644
--- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
+++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
@@ -21,6 +21,7 @@ import {
validateDateWithUnknownFormat,
} from '~/helpers/formulaFnHelper';
import FormulaColumn from '~/models/FormulaColumn';
+import { Source } from '~/models';
const logger = new Logger('FormulaQueryBuilderv2');
@@ -76,10 +77,13 @@ async function _formulaQueryBuilder(
// formula may include double curly brackets in previous version
// convert to single curly bracket here for compatibility
// const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}'));
- tree = validateFormulaAndExtractTreeWithType(
- _tree.replaceAll('{{', '{').replaceAll('}}', '}'),
+ tree = validateFormulaAndExtractTreeWithType({
+ formula: _tree.replaceAll('{{', '{').replaceAll('}}', '}'),
columns,
- );
+ clientOrSqlUi: await Source.get(column.source_id).then(
+ (source) => source.type,
+ ),
+ });
// populate and save parsedTree to column if not exist
if (column) {
From 748322ab669c50934fbbff990c828c6b550fbeae Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:02 +0000
Subject: [PATCH 044/262] fix: extract source id from model/column
---
packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
index b9294e6591..993dedf412 100644
--- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
+++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
@@ -80,9 +80,9 @@ async function _formulaQueryBuilder(
tree = validateFormulaAndExtractTreeWithType({
formula: _tree.replaceAll('{{', '{').replaceAll('}}', '}'),
columns,
- clientOrSqlUi: await Source.get(column.source_id).then(
- (source) => source.type,
- ),
+ clientOrSqlUi: await Source.get(
+ model?.source_id ?? column?.source_id,
+ ).then((source) => source.type),
});
// populate and save parsedTree to column if not exist
From e0435b5804dc62a39f236f86d1a7e6f775d98740 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:02 +0000
Subject: [PATCH 045/262] fix: pass argument as object and extract client from
basemodelSql
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 5 ++++-
.../src/db/formulav2/formulaQueryBuilderv2.ts | 13 ++++++++++---
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 1775c57a8a..fc3858dc47 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -46,6 +46,9 @@ export const jsepCurlyHook = {
jsep.hooks.add('gobble-token', function gobbleCurlyLiteral(env) {
const OCURLY_CODE = 123; // {
const CCURLY_CODE = 125; // }
+ // jsep.addIdentifierChar('.');
+ // jsep.addIdentifierChar('*');
+ // jsep.addIdentifierChar('?');
let start = -1;
const { context } = env;
if (
@@ -1408,7 +1411,7 @@ export function validateFormulaAndExtractTreeWithType({
{
const sqlUI =
typeof clientOrSqlUi === 'string'
- ? SqlUiFactory.create(clientOrSqlUi)
+ ? SqlUiFactory.create({ client: clientOrSqlUi })
: clientOrSqlUi;
if (sqlUI) {
const abstractType = sqlUI.getAbstractType(col);
diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
index 993dedf412..e7b056c5bf 100644
--- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
+++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
@@ -80,9 +80,16 @@ async function _formulaQueryBuilder(
tree = validateFormulaAndExtractTreeWithType({
formula: _tree.replaceAll('{{', '{').replaceAll('}}', '}'),
columns,
- clientOrSqlUi: await Source.get(
- model?.source_id ?? column?.source_id,
- ).then((source) => source.type),
+ clientOrSqlUi: baseModelSqlv2.clientType as
+ | 'mysql'
+ | 'pg'
+ | 'sqlite3'
+ | 'mssql'
+ | 'mysql2'
+ | 'oracledb'
+ | 'mariadb'
+ | 'sqlite'
+ | 'snowflake',
});
// populate and save parsedTree to column if not exist
From 8f3550a8eed25b4935cf3c9224acd7e048705b78 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:02 +0000
Subject: [PATCH 046/262] fix: extract type of specificDBtype column using
sqlui
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index fc3858dc47..1e6ff74aac 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1408,6 +1408,7 @@ export function validateFormulaAndExtractTreeWithType({
break;
case UITypes.ID:
case UITypes.ForeignKey:
+ case UITypes.SpecificDBType:
{
const sqlUI =
typeof clientOrSqlUi === 'string'
From 568eb90b1a6e403e745f1a56cb5373fc75cb05b4 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:02 +0000
Subject: [PATCH 047/262] fix: add support for special chars in identifier if
wrapped within curly bracket
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 66 ++++++++++++++++++-
1 file changed, 63 insertions(+), 3 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 1e6ff74aac..3d59f668a8 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -26,6 +26,54 @@ export const dateFormats = [
'YYYY MM DD',
];
+// opening and closing string code
+const OCURLY_CODE = 123; // '{'
+const CCURLY_CODE = 125; // '}'
+
+export const jsepCurlyHook = {
+ name: 'curly',
+ init(jsep) {
+ // Match identifier in following pattern: {abc-cde}
+ jsep.hooks.add('gobble-token', function escapedIdentifier(env) {
+ // check if the current token is an opening curly bracket
+ if (this.code === OCURLY_CODE) {
+ const patternIndex = this.index;
+ // move to the next character until we find a closing curly bracket
+ while (this.index < this.expr.length) {
+ ++this.index;
+ if (this.code === CCURLY_CODE) {
+ let identifier = this.expr.slice(patternIndex, ++this.index);
+
+ // if starting with double curley brace then check for ending double curley brace
+ // if found include with the identifier
+ if (
+ identifier.startsWith('{{') &&
+ this.expr.slice(patternIndex, this.index + 1).endsWith('}')
+ ) {
+ identifier = this.expr.slice(patternIndex, ++this.index);
+ }
+ env.node = {
+ type: jsep.IDENTIFIER,
+ name: /^{{.*}}$/.test(identifier)
+ ? // start would be the position of the first curly bracket
+ // add 2 to point to the first character for expressions like {{col1}}
+ identifier.slice(2, -2)
+ : // start would be the position of the first curly bracket
+ // add 1 to point to the first character for expressions like {col1}
+ identifier.slice(1, -1),
+ raw: identifier,
+ };
+
+ // env.node = this.gobbleTokenProperty(env.node);
+ return env.node;
+ }
+ }
+ this.throwError('Unclosed }');
+ }
+ });
+ },
+} as jsep.IPlugin;
+
function validateDateWithUnknownFormat(v: string) {
for (const format of dateFormats) {
if (dayjs(v, format, true).isValid() as any) {
@@ -40,15 +88,13 @@ function validateDateWithUnknownFormat(v: string) {
return false;
}
+/*
export const jsepCurlyHook = {
name: 'curly',
init(jsep) {
jsep.hooks.add('gobble-token', function gobbleCurlyLiteral(env) {
const OCURLY_CODE = 123; // {
const CCURLY_CODE = 125; // }
- // jsep.addIdentifierChar('.');
- // jsep.addIdentifierChar('*');
- // jsep.addIdentifierChar('?');
let start = -1;
const { context } = env;
if (
@@ -80,6 +126,7 @@ export const jsepCurlyHook = {
});
},
} as jsep.IPlugin;
+*/
export async function substituteColumnAliasWithIdInFormula(
formula,
@@ -125,6 +172,7 @@ export enum FormulaErrorType {
INVALID_ARG_COUNT = 'INVALID_ARG_COUNT',
CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE',
INVALID_FUNCTION_NAME = 'INVALID_FUNCTION_NAME',
+ INVALID_COLUMN = 'INVALID_COLUMN',
}
export function substituteColumnIdWithAliasInFormula(
@@ -1341,6 +1389,18 @@ export function validateFormulaAndExtractTreeWithType({
} else if (parsedTree.type === JSEPNode.IDENTIFIER) {
const col = (colIdToColMap[parsedTree.name] ||
colAliasToColMap[parsedTree.name]) as Record;
+
+ if (!col) {
+ throw new FormulaError(
+ FormulaErrorType.INVALID_COLUMN,
+ {
+ key: 'msg.formula.invalidColumn',
+ column: parsedTree.name,
+ },
+ `Invalid column name/id ${JSON.stringify(parsedTree.name)} in formula`
+ );
+ }
+
res.name = col.id;
if (col?.uidt === UITypes.Formula) {
From ff172ae5e3f0a236d483651abb1df195f391040f Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:02 +0000
Subject: [PATCH 048/262] fix: avoid passing index as arg
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 8 ++++++--
packages/nocodb/src/services/columns.service.ts | 4 ----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 3d59f668a8..956ac11fe0 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -255,13 +255,17 @@ export function jsepTreeToFormula(node, isCallExpId = false) {
return (
jsepTreeToFormula(node.callee, true) +
'(' +
- node.arguments.map(jsepTreeToFormula).join(', ') +
+ node.arguments.map((argPt) => jsepTreeToFormula(argPt)).join(', ') +
')'
);
}
if (node.type === 'ArrayExpression') {
- return '[' + node.elements.map(jsepTreeToFormula).join(', ') + ']';
+ return (
+ '[' +
+ node.elements.map((elePt) => jsepTreeToFormula(elePt)).join(', ') +
+ ']'
+ );
}
if (node.type === 'Compound') {
diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts
index c39c312fe7..3e3f0c4073 100644
--- a/packages/nocodb/src/services/columns.service.ts
+++ b/packages/nocodb/src/services/columns.service.ts
@@ -1215,10 +1215,6 @@ export class ColumnsService {
colBody.formula_raw || colBody.formula,
table.columns,
);
- console.log(
- colBody.formula_raw ||
- colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'),
- );
colBody.parsed_tree = validateFormulaAndExtractTreeWithType({
// formula may include double curly brackets in previous version
// convert to single curly bracket here for compatibility
From ece0749f549ec233daf3488841779c772211f53f Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:02 +0000
Subject: [PATCH 049/262] chore: cleanup
---
.../smartsheet/column/FormulaOptions.vue | 69 +------------------
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 40 -----------
2 files changed, 1 insertion(+), 108 deletions(-)
diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
index 049d5dab30..319a5319cc 100644
--- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
+++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue
@@ -25,7 +25,6 @@ import {
NcAutocompleteTree,
computed,
formulaList,
- formulaTypes,
formulas,
getUIDTIcon,
getWordUntilCaret,
@@ -62,18 +61,6 @@ const { t } = useI18n()
const { predictFunction: _predictFunction } = useNocoEe()
-enum JSEPNode {
- COMPOUND = 'Compound',
- IDENTIFIER = 'Identifier',
- MEMBER_EXP = 'MemberExpression',
- LITERAL = 'Literal',
- THIS_EXP = 'ThisExpression',
- CALL_EXP = 'CallExpression',
- UNARY_EXP = 'UnaryExpression',
- BINARY_EXP = 'BinaryExpression',
- ARRAY_EXP = 'ArrayExpression',
-}
-
const meta = inject(MetaInj, ref())
const supportedColumns = computed(
@@ -107,16 +94,12 @@ const validators = {
try {
validateFormulaAndExtractTreeWithType({ formula, columns: supportedColumns.value, clientOrSqlUi: sqlUi.value })
} catch (e: any) {
- console.log(e)
if (e instanceof FormulaError && e.extra?.key) {
return reject(new Error(t(e.extra.key, e.extra)))
}
return reject(new Error(e.message))
}
- // if (res !== true) {
- // return reject(new Error(res))
- // }
resolve()
})
},
@@ -610,53 +593,6 @@ function validateAgainstType(
type = formulaTypes.DATE
break
- case UITypes.Rollup: {
- const rollupFunction = col.colOptions.rollup_function
- if (['count', 'avg', 'sum', 'countDistinct', 'sumDistinct', 'avgDistinct'].includes(rollupFunction)) {
- // these functions produce a numeric value, which can be used in numeric functions
- if (expectedType !== formulaTypes.NUMERIC) {
- typeErrors.add(
- t('msg.formula.columnWithTypeFoundButExpected', {
- columnName: parsedTree.name,
- columnType: formulaTypes.NUMERIC,
- expectedType,
- }),
- )
- }
- } else {
- // the value is based on the foreign rollup column type
- const selectedTable = refTables.value.find((t) => t.column.id === col.colOptions.fk_relation_column_id)
- const refTableColumns = metas.value[selectedTable.id].columns.filter(
- (c: ColumnType) =>
- vModel.value.fk_lookup_column_id === c.id ||
- (!isSystemColumn(c) && c.id !== vModel.value.id && c.uidt !== UITypes.Links),
- )
- const childFieldColumn = refTableColumns.find(
- (column: ColumnType) => column.id === col.colOptions.fk_rollup_column_id,
- )
- const abstractType = sqlUi.value.getAbstractType(childFieldColumn)
-
- if (expectedType === formulaTypes.DATE && !isDate(childFieldColumn, sqlUi.value.getAbstractType(childFieldColumn))) {
- typeErrors.add(
- t('msg.formula.columnWithTypeFoundButExpected', {
- columnName: parsedTree.name,
- columnType: abstractType,
- expectedType,
- }),
- )
- } else if (expectedType === formulaTypes.NUMERIC && !isNumericCol(childFieldColumn)) {
- typeErrors.add(
- t('msg.formula.columnWithTypeFoundButExpected', {
- columnName: parsedTree.name,
- columnType: abstractType,
- expectedType,
- }),
- )
- }
- }
- break
- }
-
// not supported
case UITypes.ForeignKey:
case UITypes.Attachment:
@@ -664,6 +600,7 @@ function validateAgainstType(
case UITypes.Time:
case UITypes.Percent:
case UITypes.Duration:
+ case UITypes.Rollup:
case UITypes.Lookup:
case UITypes.Barcode:
case UITypes.Button:
@@ -871,10 +808,6 @@ setAdditionalValidations({
onMounted(() => {
jsep.plugins.register(jsepCurlyHook)
})
-
-// const predictFunction = async () => {
-// await _predictFunction(formState, meta, supportedColumns, suggestionsList, vModel)
-// }
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index 956ac11fe0..f64f1fede0 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -88,46 +88,6 @@ function validateDateWithUnknownFormat(v: string) {
return false;
}
-/*
-export const jsepCurlyHook = {
- name: 'curly',
- init(jsep) {
- jsep.hooks.add('gobble-token', function gobbleCurlyLiteral(env) {
- const OCURLY_CODE = 123; // {
- const CCURLY_CODE = 125; // }
- let start = -1;
- const { context } = env;
- if (
- !jsep.isIdentifierStart(context.code) &&
- context.code === OCURLY_CODE
- ) {
- if (start == -1) {
- start = context.index;
- }
- context.index += 1;
- context.gobbleExpressions(CCURLY_CODE);
- if (context.code === CCURLY_CODE) {
- context.index += 1;
- env.node = {
- type: jsep.IDENTIFIER,
- name: /{{(.*?)}}/.test(context.expr)
- ? // start would be the position of the first curly bracket
- // add 2 to point to the first character for expressions like {{col1}}
- context.expr.slice(start + 2, context.index - 1)
- : // start would be the position of the first curly bracket
- // add 1 to point to the first character for expressions like {col1}
- context.expr.slice(start + 1, context.index - 1),
- };
- return env.node;
- } else {
- context.throwError('Unclosed }');
- }
- }
- });
- },
-} as jsep.IPlugin;
-*/
-
export async function substituteColumnAliasWithIdInFormula(
formula,
columns: ColumnType[]
From 2aead9ef223d803b2508832189d6b75a99639bc3 Mon Sep 17 00:00:00 2001
From: Pranav C
Date: Thu, 21 Dec 2023 09:17:02 +0000
Subject: [PATCH 050/262] refactor: improved/corrected validation messages
---
packages/nocodb-sdk/src/lib/formulaHelpers.ts | 76 ++++++++++++-------
1 file changed, 47 insertions(+), 29 deletions(-)
diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
index f64f1fede0..db6a72668b 100644
--- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts
+++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts
@@ -1312,34 +1312,52 @@ export function validateFormulaAndExtractTreeWithType({
// validate against expected arg types if present
else if (formulas[calleeName].validation?.args?.type) {
const expectedArgType = formulas[calleeName].validation.args.type;
- if (
- argTypes.some(
- (argType) =>
- argType !== expectedArgType &&
- argType !== FormulaDataTypes.NULL &&
- argType !== FormulaDataTypes.UNKNOWN
- )
- ) {
- let key = '';
-
- if (expectedArgType === FormulaDataTypes.NUMERIC) {
- key = 'msg.formula.numericTypeIsExpected';
- } else if (expectedArgType === FormulaDataTypes.STRING) {
- key = 'msg.formula.stringTypeIsExpected';
- } else if (expectedArgType === FormulaDataTypes.BOOLEAN) {
- key = 'msg.formula.booleanTypeIsExpected';
- } else if (expectedArgType === FormulaDataTypes.DATE) {
- key = 'msg.formula.dateTypeIsExpected';
- }
- throw new FormulaError(
- FormulaErrorType.INVALID_ARG,
- {
- key,
- calleeName,
- },
- 'Invalid argument type'
- );
+ for (const argPt of validateResult) {
+ if (
+ argPt.dataType !== expectedArgType &&
+ argPt.dataType !== FormulaDataTypes.NULL &&
+ argPt.dataType !== FormulaDataTypes.UNKNOWN
+ ) {
+ if (argPt.type === JSEPNode.IDENTIFIER) {
+ throw new FormulaError(
+ FormulaErrorType.INVALID_ARG,
+ {
+ key: 'msg.formula.columnWithTypeFoundButExpected',
+ columnName: parsedTree.name,
+ columnType: parsedTree.dataType,
+ expectedType: expectedArgType,
+ },
+ `Field ${parsedTree.name} with ${parsedTree.dataType} type is found but ${expectedArgType} type is expected`
+ );
+ } else {
+ let key = '',
+ message = 'Invalid argument type';
+
+ if (expectedArgType === FormulaDataTypes.NUMERIC) {
+ key = 'msg.formula.numericTypeIsExpected';
+ message = 'Numeric type is expected';
+ } else if (expectedArgType === FormulaDataTypes.STRING) {
+ key = 'msg.formula.stringTypeIsExpected';
+ message = 'String type is expected';
+ } else if (expectedArgType === FormulaDataTypes.BOOLEAN) {
+ key = 'msg.formula.booleanTypeIsExpected';
+ message = 'Boolean type is expected';
+ } else if (expectedArgType === FormulaDataTypes.DATE) {
+ key = 'msg.formula.dateTypeIsExpected';
+ message = 'Date type is expected';
+ }
+
+ throw new FormulaError(
+ FormulaErrorType.INVALID_ARG,
+ {
+ key,
+ calleeName,
+ },
+ message
+ );
+ }
+ }
}
}
@@ -1358,8 +1376,8 @@ export function validateFormulaAndExtractTreeWithType({
throw new FormulaError(
FormulaErrorType.INVALID_COLUMN,
{
- key: 'msg.formula.invalidColumn',
- column: parsedTree.name,
+ key: 'msg.formula.columnNotAvailable',
+ columnName: parsedTree.name,
},
`Invalid column name/id ${JSON.stringify(parsedTree.name)} in formula`
);
From d730d401251bc3e767180467913028089776ca69 Mon Sep 17 00:00:00 2001
From: Pranav C