From e8a316a9cd21d22e2384cce6f7c5621a0615a479 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:57 +0000 Subject: [PATCH 001/188] refactor: add proper typing --- packages/nc-gui/utils/formulaUtils.ts | 30 ++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/nc-gui/utils/formulaUtils.ts b/packages/nc-gui/utils/formulaUtils.ts index 8449d27da7..a6df066f23 100644 --- a/packages/nc-gui/utils/formulaUtils.ts +++ b/packages/nc-gui/utils/formulaUtils.ts @@ -1,14 +1,30 @@ import type { Input as AntInput } from 'ant-design-vue' -const formulaTypes = { - NUMERIC: 'numeric', - STRING: 'string', - DATE: 'date', - LOGICAL: 'logical', - COND_EXP: 'conditional_expression', +enum formulaTypes { + NUMERIC = 'numeric', + STRING = 'string', + DATE = 'date', + LOGICAL = 'logical', + COND_EXP = 'conditional_expression', } -const formulas: Record = { +interface FormulaMeta { + type?: string + validation?: { + args?: { + min?: number + max?: number + rqd?: number + } + } + description?: string + syntax?: string + examples?: string[] + returnType?: ((args: any[]) => formulaTypes) | formulaTypes +} + +const formulas: Record = { + AVG: { type: formulaTypes.NUMERIC, validation: { From 77941ebef119a2a65528c0597e52a46dfc9b6720 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:57 +0000 Subject: [PATCH 002/188] feat: define function return types - WIP --- packages/nc-gui/utils/formulaUtils.ts | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/packages/nc-gui/utils/formulaUtils.ts b/packages/nc-gui/utils/formulaUtils.ts index a6df066f23..30a8625795 100644 --- a/packages/nc-gui/utils/formulaUtils.ts +++ b/packages/nc-gui/utils/formulaUtils.ts @@ -6,6 +6,8 @@ enum formulaTypes { DATE = 'date', LOGICAL = 'logical', COND_EXP = 'conditional_expression', + NULL = 'null', + BOOLEAN = 'boolean', } interface FormulaMeta { @@ -35,6 +37,7 @@ const formulas: Record = { description: 'Average of input parameters', syntax: 'AVG(value1, [value2, ...])', examples: ['AVG(10, 5) => 7.5', 'AVG({column1}, {column2})', 'AVG({column1}, {column2}, {column3})'], + returnType: formulaTypes.NUMERIC, }, ADD: { type: formulaTypes.NUMERIC, @@ -46,6 +49,7 @@ const formulas: Record = { description: 'Sum of input parameters', syntax: 'ADD(value1, [value2, ...])', examples: ['ADD(5, 5) => 10', 'ADD({column1}, {column2})', 'ADD({column1}, {column2}, {column3})'], + returnType: formulaTypes.NUMERIC, }, DATEADD: { type: formulaTypes.DATE, @@ -66,6 +70,7 @@ const formulas: Record = { 'DATEADD({column1}, 2, "year")', 'DATEADD({column1}, -2, "year")', ], + returnType: formulaTypes.DATE, }, DATETIME_DIFF: { type: formulaTypes.DATE, @@ -89,6 +94,7 @@ const formulas: Record = { 'DATEDIFF({column1}, {column2}, "days")', 'DATEDIFF({column1}, {column2}, "d")', ], + returnType: formulaTypes.NUMERIC, }, AND: { type: formulaTypes.COND_EXP, @@ -100,6 +106,7 @@ const formulas: Record = { description: 'TRUE if all expr evaluate to TRUE', syntax: 'AND(expr1, [expr2, ...])', examples: ['AND(5 > 2, 5 < 10) => 1', 'AND({column1} > 2, {column2} < 10)'], + returnType: formulaTypes.COND_EXP, }, OR: { type: formulaTypes.COND_EXP, @@ -111,6 +118,7 @@ const formulas: Record = { description: 'TRUE if at least one expr evaluates to TRUE', syntax: 'OR(expr1, [expr2, ...])', examples: ['OR(5 > 2, 5 < 10) => 1', 'OR({column1} > 2, {column2} < 10)'], + returnType: formulaTypes.COND_EXP, }, CONCAT: { type: formulaTypes.STRING, @@ -122,6 +130,7 @@ const formulas: Record = { description: 'Concatenated string of input parameters', syntax: 'CONCAT(str1, [str2, ...])', examples: ['CONCAT("AA", "BB", "CC") => "AABBCC"', 'CONCAT({column1}, {column2}, {column3})'], + returnType: formulaTypes.STRING, }, TRIM: { type: formulaTypes.STRING, @@ -133,6 +142,7 @@ const formulas: Record = { description: 'Remove trailing and leading whitespaces from input parameter', syntax: 'TRIM(str)', examples: ['TRIM(" HELLO WORLD ") => "HELLO WORLD"', 'TRIM({column1})'], + returnType: formulaTypes.STRING, }, UPPER: { type: formulaTypes.STRING, @@ -144,6 +154,7 @@ const formulas: Record = { description: 'Upper case converted string of input parameter', syntax: 'UPPER(str)', examples: ['UPPER("nocodb") => "NOCODB"', 'UPPER({column1})'], + returnType: formulaTypes.STRING, }, LOWER: { type: formulaTypes.STRING, @@ -155,6 +166,7 @@ const formulas: Record = { description: 'Lower case converted string of input parameter', syntax: 'LOWER(str)', examples: ['LOWER("NOCODB") => "nocodb"', 'LOWER({column1})'], + returnType: formulaTypes.STRING, }, LEN: { type: formulaTypes.STRING, @@ -166,6 +178,7 @@ const formulas: Record = { description: 'Input parameter character length', syntax: 'LEN(value)', examples: ['LEN("NocoDB") => 6', 'LEN({column1})'], + returnType: formulaTypes.NUMERIC, }, MIN: { type: formulaTypes.NUMERIC, @@ -177,6 +190,7 @@ const formulas: Record = { description: 'Minimum value amongst input parameters', syntax: 'MIN(value1, [value2, ...])', examples: ['MIN(1000, 2000) => 1000', 'MIN({column1}, {column2})'], + returnType: formulaTypes.NUMERIC, }, MAX: { type: formulaTypes.NUMERIC, @@ -188,6 +202,7 @@ const formulas: Record = { description: 'Maximum value amongst input parameters', syntax: 'MAX(value1, [value2, ...])', examples: ['MAX(1000, 2000) => 2000', 'MAX({column1}, {column2})'], + returnType: formulaTypes.NUMERIC, }, CEILING: { type: formulaTypes.NUMERIC, @@ -199,6 +214,7 @@ const formulas: Record = { description: 'Rounded next largest integer value of input parameter', syntax: 'CEILING(value)', examples: ['CEILING(1.01) => 2', 'CEILING({column1})'], + returnType: formulaTypes.NUMERIC, }, FLOOR: { type: formulaTypes.NUMERIC, @@ -210,6 +226,7 @@ const formulas: Record = { description: 'Rounded largest integer less than or equal to input parameter', syntax: 'FLOOR(value)', examples: ['FLOOR(3.1415) => 3', 'FLOOR({column1})'], + returnType: formulaTypes.NUMERIC, }, ROUND: { type: formulaTypes.NUMERIC, @@ -222,6 +239,7 @@ const formulas: Record = { description: 'Rounded number to a specified number of decimal places or the nearest integer if not specified', syntax: 'ROUND(value, precision), ROUND(value)', examples: ['ROUND(3.1415) => 3', 'ROUND(3.1415, 2) => 3.14', 'ROUND({column1}, 3)'], + returnType: formulaTypes.NUMERIC, }, MOD: { type: formulaTypes.NUMERIC, @@ -233,6 +251,7 @@ const formulas: Record = { description: 'Remainder after integer division of input parameters', syntax: 'MOD(value1, value2)', examples: ['MOD(1024, 1000) => 24', 'MOD({column}, 2)'], + returnType: formulaTypes.NUMERIC, }, REPEAT: { type: formulaTypes.STRING, @@ -244,6 +263,7 @@ const formulas: Record = { description: 'Specified copies of the input parameter string concatenated together', syntax: 'REPEAT(str, count)', examples: ['REPEAT("A", 5) => "AAAAA"', 'REPEAT({column}, 5)'], + returnType: formulaTypes.STRING, }, LOG: { type: formulaTypes.NUMERIC, @@ -251,6 +271,7 @@ const formulas: Record = { description: 'Logarithm of input parameter to the base (default = e) specified', syntax: 'LOG([base], value)', examples: ['LOG(2, 1024) => 10', 'LOG(2, {column1})'], + returnType: formulaTypes.NUMERIC, }, EXP: { type: formulaTypes.NUMERIC, @@ -258,6 +279,7 @@ const formulas: Record = { description: 'Exponential value of input parameter (e ^ power)', syntax: 'EXP(power)', examples: ['EXP(1) => 2.718281828459045', 'EXP({column1})'], + returnType: formulaTypes.NUMERIC, }, POWER: { type: formulaTypes.NUMERIC, @@ -269,6 +291,7 @@ const formulas: Record = { description: 'base to the exponent power, as in base ^ exponent', syntax: 'POWER(base, exponent)', examples: ['POWER(2, 10) => 1024', 'POWER({column1}, 10)'], + returnType: formulaTypes.NUMERIC, }, SQRT: { type: formulaTypes.NUMERIC, @@ -280,6 +303,7 @@ const formulas: Record = { description: 'Square root of the input parameter', syntax: 'SQRT(value)', examples: ['SQRT(100) => 10', 'SQRT({column1})'], + returnType: formulaTypes.NUMERIC, }, ABS: { type: formulaTypes.NUMERIC, @@ -291,6 +315,7 @@ const formulas: Record = { description: 'Absolute value of the input parameter', syntax: 'ABS(value)', examples: ['ABS({column1})'], + returnType: formulaTypes.NUMERIC, }, NOW: { type: formulaTypes.DATE, @@ -302,6 +327,7 @@ const formulas: Record = { description: 'Returns the current time and day', syntax: 'NOW()', examples: ['NOW() => 2022-05-19 17:20:43'], + returnType: formulaTypes.DATE, }, REPLACE: { type: formulaTypes.STRING, @@ -313,6 +339,7 @@ const formulas: Record = { description: 'String, after replacing all occurrences of srchStr with rplcStr', syntax: 'REPLACE(str, srchStr, rplcStr)', examples: ['REPLACE("AABBCC", "AA", "BB") => "BBBBCC"', 'REPLACE({column1}, {column2}, {column3})'], + returnType: formulaTypes.STRING, }, SEARCH: { type: formulaTypes.STRING, @@ -324,6 +351,7 @@ const formulas: Record = { description: 'Index of srchStr specified if found, 0 otherwise', syntax: 'SEARCH(str, srchStr)', examples: ['SEARCH("HELLO WORLD", "WORLD") => 7', 'SEARCH({column1}, "abc")'], + returnType: formulaTypes.NUMERIC, }, INT: { type: formulaTypes.NUMERIC, @@ -335,6 +363,7 @@ const formulas: Record = { description: 'Integer value of input parameter', syntax: 'INT(value)', examples: ['INT(3.1415) => 3', 'INT({column1})'], + returnType: formulaTypes.NUMERIC, }, RIGHT: { type: formulaTypes.STRING, @@ -346,6 +375,7 @@ const formulas: Record = { description: 'n characters from the end of input parameter', syntax: 'RIGHT(str, n)', examples: ['RIGHT("HELLO WORLD", 5) => WORLD', 'RIGHT({column1}, 3)'], + returnType: formulaTypes.STRING, }, LEFT: { type: formulaTypes.STRING, @@ -357,6 +387,7 @@ const formulas: Record = { description: 'n characters from the beginning of input parameter', syntax: 'LEFT(str, n)', examples: ['LEFT({column1}, 2)', 'LEFT("ABCD", 2) => "AB"'], + returnType: formulaTypes.STRING, }, SUBSTR: { type: formulaTypes.STRING, @@ -369,6 +400,7 @@ const formulas: Record = { description: 'Substring of length n of input string from the postition specified', syntax: ' SUBTR(str, position, [n])', examples: ['SUBSTR("HELLO WORLD", 7) => WORLD', 'SUBSTR("HELLO WORLD", 7, 3) => WOR', 'SUBSTR({column1}, 7, 5)'], + returnType: formulaTypes.STRING, }, MID: { type: formulaTypes.STRING, @@ -380,6 +412,7 @@ const formulas: Record = { description: 'Alias for SUBSTR', syntax: 'MID(str, position, [count])', examples: ['MID("NocoDB", 3, 2) => "co"', 'MID({column1}, 3, 2)'], + returnType: formulaTypes.STRING, }, IF: { type: formulaTypes.COND_EXP, @@ -392,6 +425,7 @@ const formulas: Record = { description: 'SuccessCase if expr evaluates to TRUE, elseCase otherwise', syntax: 'IF(expr, successCase, elseCase)', examples: ['IF(5 > 1, "YES", "NO") => "YES"', 'IF({column} > 1, "YES", "NO")'], + returnType: formulaTypes.STRING, }, SWITCH: { type: formulaTypes.COND_EXP, @@ -408,6 +442,10 @@ const formulas: Record = { 'SWITCH(3, 1, "One", 2, "Two", "N/A") => "N/A"', 'SWITCH({column1}, 1, "One", 2, "Two", "N/A")', ], + // todo: resolve return type based on the args + returnType: () => { + return formulaTypes.STRING; + }, }, URL: { type: formulaTypes.STRING, @@ -419,6 +457,7 @@ const formulas: Record = { description: 'Convert to a hyperlink if it is a valid URL', syntax: 'URL(str)', examples: ['URL("https://github.com/nocodb/nocodb")', 'URL({column1})'], + returnType: formulaTypes.STRING, }, WEEKDAY: { type: formulaTypes.NUMERIC, @@ -431,6 +470,7 @@ const formulas: Record = { description: 'Returns the day of the week as an integer between 0 and 6 inclusive starting from Monday by default', syntax: 'WEEKDAY(date, [startDayOfWeek])', examples: ['WEEKDAY("2021-06-09")', 'WEEKDAY(NOW(), "sunday")'], + returnType: formulaTypes.NUMERIC, }, TRUE: { @@ -443,6 +483,7 @@ const formulas: Record = { description: 'Returns 1', syntax: 'TRUE()', examples: ['TRUE()'], + returnType: formulaTypes.NUMERIC, }, FALSE: { @@ -455,6 +496,7 @@ const formulas: Record = { description: 'Returns 0', syntax: 'FALSE()', examples: ['FALSE()'], + returnType: formulaTypes.NUMERIC, }, REGEX_MATCH: { @@ -467,6 +509,7 @@ const formulas: Record = { description: 'Returns 1 if the input text matches a regular expression or 0 if it does not.', syntax: 'REGEX_MATCH(string, regex)', examples: ['REGEX_MATCH({title}, "abc.*")'], + returnType: formulaTypes.NUMERIC, }, REGEX_EXTRACT: { @@ -479,6 +522,7 @@ const formulas: Record = { description: 'Returns the first match of a regular expression in a string.', syntax: 'REGEX_EXTRACT(string, regex)', examples: ['REGEX_EXTRACT({title}, "abc.*")'], + returnType: formulaTypes.STRING, }, REGEX_REPLACE: { type: formulaTypes.STRING, @@ -490,6 +534,7 @@ const formulas: Record = { description: 'Replaces all matches of a regular expression in a string with a replacement string', syntax: 'REGEX_MATCH(string, regex, replacement)', examples: ['REGEX_EXTRACT({title}, "abc.*", "abcd")'], + returnType: formulaTypes.STRING, }, BLANK: { type: formulaTypes.STRING, @@ -501,6 +546,7 @@ const formulas: Record = { description: 'Returns a blank value(null)', syntax: 'BLANK()', examples: ['BLANK()'], + returnType: formulaTypes.NULL, }, XOR: { type: formulaTypes.NUMERIC, @@ -512,6 +558,7 @@ const formulas: Record = { description: 'Returns true if an odd number of arguments are true, and false otherwise.', syntax: 'XOR(expression, [exp2, ...])', examples: ['XOR(TRUE(), FALSE(), TRUE())'], + returnType: formulaTypes.BOOLEAN, }, EVEN: { type: formulaTypes.NUMERIC, @@ -523,6 +570,7 @@ const formulas: Record = { description: 'Returns the nearest even integer that is greater than or equal to the specified value', syntax: 'EVEN(value)', examples: ['EVEN({column})'], + returnType: formulaTypes.NUMERIC, }, ODD: { type: formulaTypes.NUMERIC, @@ -534,6 +582,7 @@ const formulas: Record = { description: 'Returns the nearest odd integer that is greater than or equal to the specified value', syntax: 'ODD(value)', examples: ['ODD({column})'], + returnType: formulaTypes.NUMERIC, }, RECORD_ID: { validation: { @@ -544,6 +593,11 @@ const formulas: Record = { description: 'Returns the record id of the current record', syntax: 'RECORD_ID()', examples: ['RECORD_ID()'], + + // todo: resolve return type based on the args + returnType: () => { + return formulaTypes.STRING; + }, }, COUNTA: { validation: { @@ -554,6 +608,7 @@ const formulas: Record = { description: 'Counts the number of non-empty arguments', syntax: 'COUNTA(value1, [value2, ...])', examples: ['COUNTA({field1}, {field2})'], + returnType: formulaTypes.NUMERIC, }, COUNT: { validation: { @@ -564,6 +619,7 @@ const formulas: Record = { description: 'Count the number of arguments that are numbers', syntax: 'COUNT(value1, [value2, ...])', examples: ['COUNT({field1}, {field2})'], + returnType: formulaTypes.NUMERIC, }, COUNTALL: { validation: { @@ -574,6 +630,7 @@ const formulas: Record = { description: 'Counts the number of arguments', syntax: 'COUNTALL(value1, [value2, ...])', examples: ['COUNTALL({field1}, {field2})'], + returnType: formulaTypes.NUMERIC, }, ROUNDDOWN: { type: formulaTypes.NUMERIC, @@ -587,6 +644,7 @@ const formulas: Record = { 'Round down the value after the decimal point to the number of decimal places given by "precision"(default is 0)', syntax: 'ROUNDDOWN(value, [precision])', examples: ['ROUNDDOWN({field1})', 'ROUNDDOWN({field1}, 2)'], + returnType: formulaTypes.NUMERIC, }, ROUNDUP: { type: formulaTypes.NUMERIC, @@ -599,6 +657,7 @@ const formulas: Record = { description: 'Round up the value after the decimal point to the number of decimal places given by "precision"(default is 0)', syntax: 'ROUNDUP(value, [precision])', examples: ['ROUNDUP({field1})', 'ROUNDUP({field1}, 2)'], + returnType: formulaTypes.NUMERIC, }, VALUE: { validation: { @@ -610,6 +669,7 @@ const formulas: Record = { 'Extract the numeric value from a string, if `%` or `-` is present, it will handle it accordingly and return the numeric value', syntax: 'VALUE(value)', examples: ['VALUE({field})', 'VALUE("abc10000%")', 'VALUE("$10000")'], + returnType: formulaTypes.NUMERIC, }, // Disabling these functions for now; these act as alias for CreatedAt & UpdatedAt fields; // Issue: Error noticed if CreatedAt & UpdatedAt fields are removed from the table after creating these formulas From e2e2f74183797acc4b57928314e3c93f8e3d9f96 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:57 +0000 Subject: [PATCH 003/188] feat: define function return types - WIP --- packages/nc-gui/utils/formulaUtils.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/nc-gui/utils/formulaUtils.ts b/packages/nc-gui/utils/formulaUtils.ts index 30a8625795..e0291f80ea 100644 --- a/packages/nc-gui/utils/formulaUtils.ts +++ b/packages/nc-gui/utils/formulaUtils.ts @@ -26,7 +26,6 @@ interface FormulaMeta { } const formulas: Record = { - AVG: { type: formulaTypes.NUMERIC, validation: { @@ -425,7 +424,17 @@ const formulas: Record = { description: 'SuccessCase if expr evaluates to TRUE, elseCase otherwise', syntax: 'IF(expr, successCase, elseCase)', examples: ['IF(5 > 1, "YES", "NO") => "YES"', 'IF({column} > 1, "YES", "NO")'], - returnType: formulaTypes.STRING, + returnType: (argsTypes: formulaTypes[]) => { + if (argsTypes.slice(1).includes(formulaTypes.STRING)) { + return formulaTypes.STRING + } else if (argsTypes.slice(1).includes(formulaTypes.NUMERIC)) { + return formulaTypes.NUMERIC + } else if (argsTypes.slice(1).includes(formulaTypes.BOOLEAN)) { + return formulaTypes.BOOLEAN + } + + return argsTypes[1] + }, }, SWITCH: { type: formulaTypes.COND_EXP, @@ -443,8 +452,8 @@ const formulas: Record = { 'SWITCH({column1}, 1, "One", 2, "Two", "N/A")', ], // todo: resolve return type based on the args - returnType: () => { - return formulaTypes.STRING; + returnType: (argTypes: formulaTypes[]) => { + return formulaTypes.STRING }, }, URL: { @@ -596,7 +605,7 @@ const formulas: Record = { // todo: resolve return type based on the args returnType: () => { - return formulaTypes.STRING; + return formulaTypes.STRING }, }, COUNTA: { From 814b1eed1ecf7bf9b49d2a3dcac15a846efda3cd Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:57 +0000 Subject: [PATCH 004/188] feat: define function return types - WIP --- .../smartsheet/column/FormulaOptions.vue | 4 +++- packages/nc-gui/utils/formulaUtils.ts | 23 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue index f87585f490..c656659de3 100644 --- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue +++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue @@ -190,6 +190,7 @@ function parseAndValidateFormula(formula: string) { } function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = new Set()) { + let type: formulaTypes; if (parsedTree.type === JSEPNode.CALL_EXP) { const calleeName = parsedTree.callee.name.toUpperCase() // validate function name @@ -207,6 +208,7 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n errors.add(t('msg.formula.maxRequiredArgumentsFormula', { maxRequiredArguments: validation.args.max, calleeName })) } } + parsedTree.arguments.map((arg: Record) => validateAgainstMeta(arg, errors)) // validate data type @@ -445,7 +447,7 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n } else { errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')) } - return errors + return {errors, type} } function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set()) { diff --git a/packages/nc-gui/utils/formulaUtils.ts b/packages/nc-gui/utils/formulaUtils.ts index e0291f80ea..c4f4fef23b 100644 --- a/packages/nc-gui/utils/formulaUtils.ts +++ b/packages/nc-gui/utils/formulaUtils.ts @@ -435,6 +435,17 @@ const formulas: Record = { return argsTypes[1] }, + returnType: (argTypes: formulaTypes[]) => { + if (argTypes.slice(1).includes(formulaTypes.STRING)) { + return formulaTypes.STRING + } else if (argTypes.slice(1).includes(formulaTypes.NUMERIC)) { + return formulaTypes.NUMERIC + } else if (argTypes.slice(1).includes(formulaTypes.BOOLEAN)) { + return formulaTypes.BOOLEAN + } + + return argTypes[1] + }, }, SWITCH: { type: formulaTypes.COND_EXP, @@ -453,7 +464,17 @@ const formulas: Record = { ], // todo: resolve return type based on the args returnType: (argTypes: formulaTypes[]) => { - return formulaTypes.STRING + const returnArgTypes = argTypes.slice(2).filter((_, i) => i % 2 === 1) + + if (returnArgTypes.slice(1).includes(formulaTypes.STRING)) { + return formulaTypes.STRING + } else if (returnArgTypes.slice(1).includes(formulaTypes.NUMERIC)) { + return formulaTypes.NUMERIC + } else if (returnArgTypes.slice(1).includes(formulaTypes.BOOLEAN)) { + return formulaTypes.BOOLEAN + } + + return returnArgTypes[1] }, }, URL: { From 8cbe49cd4d3fedd4b18394046afae4adde2d2445 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:57 +0000 Subject: [PATCH 005/188] fix: add formula validation - WIP --- .../smartsheet/column/FormulaOptions.vue | 65 +++++++++++++++++-- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue index c656659de3..6e5039b0f7 100644 --- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue +++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue @@ -190,7 +190,7 @@ function parseAndValidateFormula(formula: string) { } function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = new Set()) { - let type: formulaTypes; + let returnType: formulaTypes if (parsedTree.type === JSEPNode.CALL_EXP) { const calleeName = parsedTree.callee.name.toUpperCase() // validate function name @@ -203,17 +203,41 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n if (validation.args.rqd !== undefined && validation.args.rqd !== parsedTree.arguments.length) { errors.add(t('msg.formula.requiredArgumentsFormula', { requiredArguments: validation.args.rqd, calleeName })) } else if (validation.args.min !== undefined && validation.args.min > parsedTree.arguments.length) { - errors.add(t('msg.formula.minRequiredArgumentsFormula', { minRequiredArguments: validation.args.min, calleeName })) + errors.add( + t('msg.formula.minRequiredArgumentsFormula', { + minRequiredArguments: validation.args.min, + calleeName, + }), + ) } else if (validation.args.max !== undefined && validation.args.max < parsedTree.arguments.length) { - errors.add(t('msg.formula.maxRequiredArgumentsFormula', { maxRequiredArguments: validation.args.max, calleeName })) + errors.add( + t('msg.formula.maxRequiredArgumentsFormula', { + maxRequiredArguments: validation.args.max, + calleeName, + }), + ) } } parsedTree.arguments.map((arg: Record) => validateAgainstMeta(arg, errors)) + // get args type and validate + const validateResult = parsedTree.arguments.map((arg) => { + return validateAgainstMeta(arg, errors, typeErrors) + }) + + const argsTypes = validateResult.map((v: any) => v.returnType); + + if (typeof validateResult[0].returnType === 'function') { + returnType = formulas[calleeName].returnType(argsTypes) + } else if (validateResult[0]) { + returnType = formulas[calleeName].returnType + } + // validate data type if (parsedTree.callee.type === JSEPNode.IDENTIFIER) { const expectedType = formulas[calleeName.toUpperCase()].type + if (expectedType === formulaTypes.NUMERIC) { if (calleeName === 'WEEKDAY') { // parsedTree.arguments[0] = date @@ -242,7 +266,7 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n typeErrors, ) } else { - parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors)) + parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors, argsTypes)) } } else if (expectedType === formulaTypes.DATE) { if (calleeName === 'DATEADD') { @@ -438,7 +462,21 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n } validateAgainstMeta(parsedTree.left, errors) validateAgainstMeta(parsedTree.right, errors) + + // todo: type extraction for binary exps + returnType = formulaTypes.NUMERIC } else if (parsedTree.type === JSEPNode.LITERAL || parsedTree.type === JSEPNode.UNARY_EXP) { + if (parsedTree.type === JSEPNode.LITERAL) { + if (typeof parsedTree.value === 'number') { + returnType = formulaTypes.NUMERIC + } else if (typeof parsedTree.value === 'string') { + returnType = formulaTypes.STRING + } else if (typeof parsedTree.value === 'boolean') { + returnType = formulaTypes.BOOLEAN + } else { + returnType = formulaTypes.STRING + } + } // do nothing } else if (parsedTree.type === JSEPNode.COMPOUND) { if (parsedTree.body.length) { @@ -447,10 +485,11 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n } else { errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')) } - return {errors, type} + return { errors, returnType } } -function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set()) { +function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set(), argTypes: formulaTypes = []) { + let type if (parsedTree === false || typeof parsedTree === 'undefined') { return typeErrors } @@ -460,10 +499,14 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t } else if (expectedType === formulaTypes.NUMERIC) { if (typeof parsedTree.value !== 'number') { typeErrors.add(t('msg.formula.numericTypeIsExpected')) + } else { + type = formulaTypes.NUMERIC } } else if (expectedType === formulaTypes.STRING) { if (typeof parsedTree.value !== 'string') { typeErrors.add(t('msg.formula.stringTypeIsExpected')) + } else { + type = formulaTypes.STRING } } } else if (parsedTree.type === JSEPNode.IDENTIFIER) { @@ -475,6 +518,7 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t if (col.uidt === UITypes.Formula) { const foundType = getRootDataType(jsep(col.colOptions?.formula_raw)) + type = foundType if (foundType === 'N/A') { typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: col.title })) } else if (expectedType !== foundType) { @@ -504,6 +548,7 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t }), ) } + type = formulaTypes.STRING break // numeric @@ -523,6 +568,7 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t }), ) } + type = formulaTypes.NUMERIC break // date @@ -539,6 +585,7 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t }), ) } + type = formulaTypes.DATE break case UITypes.Rollup: { @@ -616,6 +663,8 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t }), ) } + + type = formulaTypes.NUMERIC } else if (parsedTree.type === JSEPNode.CALL_EXP) { const calleeName = parsedTree.callee.name.toUpperCase() if (formulas[calleeName]?.type && expectedType !== formulas[calleeName].type) { @@ -626,8 +675,10 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t }), ) } + // todo: derive type from returnType + type = formulas[calleeName]?.type } - return typeErrors + return { type, typeErrors } } function getRootDataType(parsedTree: any): any { From 3ce53b879167e8fe62c50535d5082697a89de04d Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:58 +0000 Subject: [PATCH 006/188] chore: add jest unit test in nocodb-sdk --- packages/nocodb-sdk/jest.config.js | 5 + .../nocodb-sdk/src/lib/formulaHelpers.spec.ts | 6 + packages/nocodb-sdk/src/lib/formulaHelpers.ts | 528 ++++++++++++++++++ packages/nocodb-sdk/tsconfig.json | 2 +- 4 files changed, 540 insertions(+), 1 deletion(-) create mode 100644 packages/nocodb-sdk/jest.config.js create mode 100644 packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts diff --git a/packages/nocodb-sdk/jest.config.js b/packages/nocodb-sdk/jest.config.js new file mode 100644 index 0000000000..b413e106db --- /dev/null +++ b/packages/nocodb-sdk/jest.config.js @@ -0,0 +1,5 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', +}; \ No newline at end of file diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts new file mode 100644 index 0000000000..6fce60edb8 --- /dev/null +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts @@ -0,0 +1,6 @@ +describe('auth', () => { + it('Formula parsing and type validation', async () => { + const response = {userId: 'fakeUserId'}; + expect(response).toEqual({ userId: 'fakeUserId' }); + }); +}); diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 5a28417d84..8d5cbcbd15 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1,6 +1,7 @@ import jsep from 'jsep'; import { ColumnType } from './Api'; +import {UITypes} from "../../build/main"; export const jsepCurlyHook = { name: 'curly', @@ -189,3 +190,530 @@ function escapeLiteral(v: string) { .replace(/'/g, `\\'`) ); } + + + + +function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = new Set()) { + let returnType: formulaTypes + if (parsedTree.type === JSEPNode.CALL_EXP) { + const calleeName = parsedTree.callee.name.toUpperCase() + // validate function name + if (!availableFunctions.includes(calleeName)) { + errors.add(t('msg.formula.functionNotAvailable', { function: calleeName })) + } + // validate arguments + const validation = formulas[calleeName] && formulas[calleeName].validation + if (validation && validation.args) { + if (validation.args.rqd !== undefined && validation.args.rqd !== parsedTree.arguments.length) { + errors.add(t('msg.formula.requiredArgumentsFormula', { requiredArguments: validation.args.rqd, calleeName })) + } else if (validation.args.min !== undefined && validation.args.min > parsedTree.arguments.length) { + errors.add( + t('msg.formula.minRequiredArgumentsFormula', { + minRequiredArguments: validation.args.min, + calleeName, + }), + ) + } else if (validation.args.max !== undefined && validation.args.max < parsedTree.arguments.length) { + errors.add( + t('msg.formula.maxRequiredArgumentsFormula', { + maxRequiredArguments: validation.args.max, + calleeName, + }), + ) + } + } + + parsedTree.arguments.map((arg: Record) => validateAgainstMeta(arg, errors)) + + // get args type and validate + const validateResult = parsedTree.arguments.map((arg) => { + return validateAgainstMeta(arg, errors, typeErrors) + }) + + const argsTypes = validateResult.map((v: any) => v.returnType); + + if (typeof validateResult[0].returnType === 'function') { + returnType = formulas[calleeName].returnType(argsTypes) + } else if (validateResult[0]) { + returnType = formulas[calleeName].returnType + } + + // validate data type + if (parsedTree.callee.type === JSEPNode.IDENTIFIER) { + const expectedType = formulas[calleeName.toUpperCase()].type + + if (expectedType === formulaTypes.NUMERIC) { + if (calleeName === 'WEEKDAY') { + // parsedTree.arguments[0] = date + validateAgainstType( + parsedTree.arguments[0], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate')) + } + }, + typeErrors, + ) + // parsedTree.arguments[1] = startDayOfWeek (optional) + validateAgainstType( + parsedTree.arguments[1], + formulaTypes.STRING, + (v: any) => { + if ( + typeof v !== 'string' || + !['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'].includes(v.toLowerCase()) + ) { + typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate')) + } + }, + typeErrors, + ) + } else { + parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors, argsTypes)) + } + } else if (expectedType === formulaTypes.DATE) { + if (calleeName === 'DATEADD') { + // parsedTree.arguments[0] = date + validateAgainstType( + parsedTree.arguments[0], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.firstParamDateAddHaveDate')) + } + }, + typeErrors, + ) + // parsedTree.arguments[1] = numeric + validateAgainstType( + parsedTree.arguments[1], + formulaTypes.NUMERIC, + (v: any) => { + if (typeof v !== 'number') { + typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber')) + } + }, + typeErrors, + ) + // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"] + validateAgainstType( + parsedTree.arguments[2], + formulaTypes.STRING, + (v: any) => { + if (!['day', 'week', 'month', 'year'].includes(v)) { + typeErrors.add(typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate'))) + } + }, + typeErrors, + ) + } else if (calleeName === 'DATETIME_DIFF') { + // parsedTree.arguments[0] = date + validateAgainstType( + parsedTree.arguments[0], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate')) + } + }, + typeErrors, + ) + // parsedTree.arguments[1] = date + validateAgainstType( + parsedTree.arguments[1], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate')) + } + }, + typeErrors, + ) + // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"] + validateAgainstType( + parsedTree.arguments[2], + formulaTypes.STRING, + (v: any) => { + if ( + ![ + 'milliseconds', + 'ms', + 'seconds', + 's', + 'minutes', + 'm', + 'hours', + 'h', + 'days', + 'd', + 'weeks', + 'w', + 'months', + 'M', + 'quarters', + 'Q', + 'years', + 'y', + ].includes(v) + ) { + typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate')) + } + }, + typeErrors, + ) + } + } + } + + errors = new Set([...errors, ...typeErrors]) + } else if (parsedTree.type === JSEPNode.IDENTIFIER) { + if (supportedColumns.value.filter((c) => !column || column.value?.id !== c.id).every((c) => c.title !== parsedTree.name)) { + errors.add( + t('msg.formula.columnNotAvailable', { + columnName: parsedTree.name, + }), + ) + } + + // check circular reference + // e.g. formula1 -> formula2 -> formula1 should return circular reference error + + // get all formula columns excluding itself + const formulaPaths = supportedColumns.value + .filter((c) => c.id !== column.value?.id && c.uidt === UITypes.Formula) + .reduce((res: Record[], c: Record) => { + // in `formula`, get all the (unique) target neighbours + // i.e. all column id (e.g. cl_xxxxxxxxxxxxxx) with formula type + const neighbours = [ + ...new Set( + (c.colOptions.formula.match(/cl_\w{14}/g) || []).filter( + (colId: string) => + supportedColumns.value.filter((col: ColumnType) => col.id === colId && col.uidt === UITypes.Formula).length, + ), + ), + ] + if (neighbours.length > 0) { + // e.g. formula column 1 -> [formula column 2, formula column3] + res.push({ [c.id]: neighbours }) + } + return res + }, []) + // include target formula column (i.e. the one to be saved if applicable) + const targetFormulaCol = supportedColumns.value.find( + (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula, + ) + + if (targetFormulaCol && column.value?.id) { + formulaPaths.push({ + [column.value?.id as string]: [targetFormulaCol.id], + }) + } + const vertices = formulaPaths.length + if (vertices > 0) { + // perform kahn's algo for cycle detection + const adj = new Map() + const inDegrees = new Map() + // init adjacency list & indegree + + for (const [_, v] of Object.entries(formulaPaths)) { + const src = Object.keys(v)[0] + const neighbours = v[src] + inDegrees.set(src, inDegrees.get(src) || 0) + for (const neighbour of neighbours) { + adj.set(src, (adj.get(src) || new Set()).add(neighbour)) + inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1) + } + } + const queue: string[] = [] + // put all vertices with in-degree = 0 (i.e. no incoming edges) to queue + inDegrees.forEach((inDegree, col) => { + if (inDegree === 0) { + // in-degree = 0 means we start traversing from this node + queue.push(col) + } + }) + // init count of visited vertices + let visited = 0 + // BFS + while (queue.length !== 0) { + // remove a vertex from the queue + const src = queue.shift() + // if this node has neighbours, increase visited by 1 + const neighbours = adj.get(src) || new Set() + if (neighbours.size > 0) { + visited += 1 + } + // iterate each neighbouring nodes + neighbours.forEach((neighbour: string) => { + // decrease in-degree of its neighbours by 1 + inDegrees.set(neighbour, inDegrees.get(neighbour) - 1) + // if in-degree becomes 0 + if (inDegrees.get(neighbour) === 0) { + // then put the neighboring node to the queue + queue.push(neighbour) + } + }) + } + // vertices not same as visited = cycle found + if (vertices !== visited) { + errors.add(t('msg.formula.cantSaveCircularReference')) + } + } + } else if (parsedTree.type === JSEPNode.BINARY_EXP) { + if (!availableBinOps.includes(parsedTree.operator)) { + errors.add(t('msg.formula.operationNotAvailable', { operation: parsedTree.operator })) + } + validateAgainstMeta(parsedTree.left, errors) + validateAgainstMeta(parsedTree.right, errors) + + // todo: type extraction for binary exps + returnType = formulaTypes.NUMERIC + } else if (parsedTree.type === JSEPNode.LITERAL || parsedTree.type === JSEPNode.UNARY_EXP) { + if (parsedTree.type === JSEPNode.LITERAL) { + if (typeof parsedTree.value === 'number') { + returnType = formulaTypes.NUMERIC + } else if (typeof parsedTree.value === 'string') { + returnType = formulaTypes.STRING + } else if (typeof parsedTree.value === 'boolean') { + returnType = formulaTypes.BOOLEAN + } else { + returnType = formulaTypes.STRING + } + } + // do nothing + } else if (parsedTree.type === JSEPNode.COMPOUND) { + if (parsedTree.body.length) { + errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')) + } + } else { + errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')) + } + return { errors, returnType } +} + +function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set(), argTypes: formulaTypes = []) { + let type + if (parsedTree === false || typeof parsedTree === 'undefined') { + return typeErrors + } + if (parsedTree.type === JSEPNode.LITERAL) { + if (typeof func === 'function') { + func(parsedTree.value) + } else if (expectedType === formulaTypes.NUMERIC) { + if (typeof parsedTree.value !== 'number') { + typeErrors.add(t('msg.formula.numericTypeIsExpected')) + } else { + type = formulaTypes.NUMERIC + } + } else if (expectedType === formulaTypes.STRING) { + if (typeof parsedTree.value !== 'string') { + typeErrors.add(t('msg.formula.stringTypeIsExpected')) + } else { + type = formulaTypes.STRING + } + } + } else if (parsedTree.type === JSEPNode.IDENTIFIER) { + const col = supportedColumns.value.find((c) => c.title === parsedTree.name) + + if (col === undefined) { + return + } + + if (col.uidt === UITypes.Formula) { + const foundType = getRootDataType(jsep(col.colOptions?.formula_raw)) + type = foundType + if (foundType === 'N/A') { + typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: col.title })) + } else if (expectedType !== foundType) { + typeErrors.add( + t('msg.formula.typeIsExpectedButFound', { + type: expectedType, + found: foundType, + }), + ) + } + } else { + switch (col.uidt) { + // string + case UITypes.SingleLineText: + case UITypes.LongText: + case UITypes.MultiSelect: + case UITypes.SingleSelect: + case UITypes.PhoneNumber: + case UITypes.Email: + case UITypes.URL: + if (expectedType !== formulaTypes.STRING) { + typeErrors.add( + t('msg.formula.columnWithTypeFoundButExpected', { + columnName: parsedTree.name, + columnType: formulaTypes.STRING, + expectedType, + }), + ) + } + type = formulaTypes.STRING + break + + // numeric + case UITypes.Year: + case UITypes.Number: + case UITypes.Decimal: + case UITypes.Rating: + case UITypes.Count: + case UITypes.AutoNumber: + case UITypes.Currency: + if (expectedType !== formulaTypes.NUMERIC) { + typeErrors.add( + t('msg.formula.columnWithTypeFoundButExpected', { + columnName: parsedTree.name, + columnType: formulaTypes.NUMERIC, + expectedType, + }), + ) + } + type = formulaTypes.NUMERIC + break + + // date + case UITypes.Date: + case UITypes.DateTime: + case UITypes.CreateTime: + case UITypes.LastModifiedTime: + if (expectedType !== formulaTypes.DATE) { + typeErrors.add( + t('msg.formula.columnWithTypeFoundButExpected', { + columnName: parsedTree.name, + columnType: formulaTypes.DATE, + expectedType, + }), + ) + } + type = formulaTypes.DATE + break + + // not supported + case UITypes.ForeignKey: + case UITypes.Attachment: + case UITypes.ID: + case UITypes.Time: + case UITypes.Percent: + case UITypes.Duration: + case UITypes.Rollup: + case UITypes.Lookup: + case UITypes.Barcode: + case UITypes.Button: + case UITypes.Checkbox: + case UITypes.Collaborator: + case UITypes.QrCode: + default: + typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: parsedTree.name })) + break + } + } + } else if (parsedTree.type === JSEPNode.UNARY_EXP || parsedTree.type === JSEPNode.BINARY_EXP) { + if (expectedType !== formulaTypes.NUMERIC) { + // parsedTree.name won't be available here + typeErrors.add( + t('msg.formula.typeIsExpectedButFound', { + type: formulaTypes.NUMERIC, + found: expectedType, + }), + ) + } + + type = formulaTypes.NUMERIC + } else if (parsedTree.type === JSEPNode.CALL_EXP) { + const calleeName = parsedTree.callee.name.toUpperCase() + if (formulas[calleeName]?.type && expectedType !== formulas[calleeName].type) { + typeErrors.add( + t('msg.formula.typeIsExpectedButFound', { + type: expectedType, + found: formulas[calleeName].type, + }), + ) + } + // todo: derive type from returnType + type = formulas[calleeName]?.type + } + return { type, typeErrors } +} + +function getRootDataType(parsedTree: any): any { + // given a parse tree, return the data type of it + if (parsedTree.type === JSEPNode.CALL_EXP) { + return formulas[parsedTree.callee.name.toUpperCase()].type + } else if (parsedTree.type === JSEPNode.IDENTIFIER) { + const col = supportedColumns.value.find((c) => c.title === parsedTree.name) as Record + if (col?.uidt === UITypes.Formula) { + return getRootDataType(jsep(col?.formula_raw)) + } else { + switch (col?.uidt) { + // string + case UITypes.SingleLineText: + case UITypes.LongText: + case UITypes.MultiSelect: + case UITypes.SingleSelect: + case UITypes.PhoneNumber: + case UITypes.Email: + case UITypes.URL: + return formulaTypes.STRING + + // numeric + case UITypes.Year: + case UITypes.Number: + case UITypes.Decimal: + case UITypes.Rating: + case UITypes.Count: + case UITypes.AutoNumber: + return formulaTypes.NUMERIC + + // date + case UITypes.Date: + case UITypes.DateTime: + case UITypes.CreateTime: + case UITypes.LastModifiedTime: + return formulaTypes.DATE + + // not supported + case UITypes.ForeignKey: + case UITypes.Attachment: + case UITypes.ID: + case UITypes.Time: + case UITypes.Currency: + case UITypes.Percent: + case UITypes.Duration: + case UITypes.Rollup: + case UITypes.Lookup: + case UITypes.Barcode: + case UITypes.Button: + case UITypes.Checkbox: + case UITypes.Collaborator: + case UITypes.QrCode: + default: + return 'N/A' + } + } + } else if (parsedTree.type === JSEPNode.BINARY_EXP || parsedTree.type === JSEPNode.UNARY_EXP) { + return formulaTypes.NUMERIC + } else if (parsedTree.type === JSEPNode.LITERAL) { + return typeof parsedTree.value + } else { + return 'N/A' + } +} + +function isCurlyBracketBalanced() { + // count number of opening curly brackets and closing curly brackets + const cntCurlyBrackets = (formulaRef.value.$el.value.match(/\{|}/g) || []).reduce( + (acc: Record, cur: number) => { + acc[cur] = (acc[cur] || 0) + 1 + return acc + }, + {}, + ) + return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0) +} + + diff --git a/packages/nocodb-sdk/tsconfig.json b/packages/nocodb-sdk/tsconfig.json index 8af501e5e0..0bd395cc39 100644 --- a/packages/nocodb-sdk/tsconfig.json +++ b/packages/nocodb-sdk/tsconfig.json @@ -38,7 +38,7 @@ // "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */, "lib": ["es2017","dom"], - "types": [], + "types": ["jest", "node"], "typeRoots": ["node_modules/@types", "src/types"], "baseUrl": "./src", "paths": { From a6e0bedc6fe1f8b30355f2b6a871a563b0b8337f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:58 +0000 Subject: [PATCH 007/188] feat: add method to validate and extract type --- .../nocodb-sdk/src/lib/formulaHelpers.spec.ts | 67 +- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 1398 +++++++++++++++-- 2 files changed, 1293 insertions(+), 172 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts index 6fce60edb8..071733e6fb 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts @@ -1,6 +1,65 @@ -describe('auth', () => { - it('Formula parsing and type validation', async () => { - const response = {userId: 'fakeUserId'}; - expect(response).toEqual({ userId: 'fakeUserId' }); +import { + FormulaDataTypes, + validateFormulaAndExtractTreeWithType, +} from './formulaHelpers'; +import UITypes from './UITypes'; + +describe('Formula parsing and type validation', () => { + it('Simple formula', async () => { + const result = validateFormulaAndExtractTreeWithType('1 + 2', []); + + expect(result.dataType).toEqual(FormulaDataTypes.NUMERIC); + }); + + it('Formula with IF condition', async () => { + const result = validateFormulaAndExtractTreeWithType( + 'IF({column}, "Found", BLANK())', + [ + { + id: 'cid', + title: 'column', + uidt: UITypes.Number, + }, + ] + ); + + expect(result.dataType).toEqual(FormulaDataTypes.STRING); + }); + it('Complex formula', async () => { + const result = validateFormulaAndExtractTreeWithType( + 'SWITCH({column2},"value1",IF({column1}, "Found", BLANK()),"value2", 2)', + [ + { + id: 'id1', + title: 'column1', + uidt: UITypes.Number, + }, + { + id: 'id2', + title: 'column2', + uidt: UITypes.SingleLineText, + }, + ] + ); + + expect(result.dataType).toEqual(FormulaDataTypes.STRING); + + const result1 = validateFormulaAndExtractTreeWithType( + 'SWITCH({column2},"value1",IF({column1}, 1, 2),"value2", 2)', + [ + { + id: 'id1', + title: 'column1', + uidt: UITypes.Number, + }, + { + id: 'id2', + title: 'column2', + uidt: UITypes.SingleLineText, + }, + ] + ); + + expect(result1.dataType).toEqual(FormulaDataTypes.NUMERIC); }); }); diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 8d5cbcbd15..c1c19b2562 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1,7 +1,7 @@ import jsep from 'jsep'; import { ColumnType } from './Api'; -import {UITypes} from "../../build/main"; +import UITypes from './UITypes'; export const jsepCurlyHook = { name: 'curly', @@ -191,150 +191,955 @@ function escapeLiteral(v: string) { ); } +export enum FormulaDataTypes { + NUMERIC = 'numeric', + STRING = 'string', + DATE = 'date', + LOGICAL = 'logical', + COND_EXP = 'conditional_expression', + NULL = 'null', + BOOLEAN = 'boolean', +} + +export enum JSEPNode { + COMPOUND = 'Compound', + IDENTIFIER = 'Identifier', + MEMBER_EXP = 'MemberExpression', + LITERAL = 'Literal', + THIS_EXP = 'ThisExpression', + CALL_EXP = 'CallExpression', + UNARY_EXP = 'UnaryExpression', + BINARY_EXP = 'BinaryExpression', + ARRAY_EXP = 'ArrayExpression', +} + +interface FormulaMeta { + type?: string; + validation?: { + args?: { + min?: number; + max?: number; + rqd?: number; + }; + }; + description?: string; + syntax?: string; + examples?: string[]; + returnType?: ((args: any[]) => FormulaDataTypes) | FormulaDataTypes; +} + +const formulas: Record = { + AVG: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + min: 1, + }, + }, + description: 'Average of input parameters', + syntax: 'AVG(value1, [value2, ...])', + examples: [ + 'AVG(10, 5) => 7.5', + 'AVG({column1}, {column2})', + 'AVG({column1}, {column2}, {column3})', + ], + returnType: FormulaDataTypes.NUMERIC, + }, + ADD: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + min: 1, + }, + }, + description: 'Sum of input parameters', + syntax: 'ADD(value1, [value2, ...])', + examples: [ + 'ADD(5, 5) => 10', + 'ADD({column1}, {column2})', + 'ADD({column1}, {column2}, {column3})', + ], + returnType: FormulaDataTypes.NUMERIC, + }, + DATEADD: { + type: FormulaDataTypes.DATE, + validation: { + args: { + rqd: 3, + }, + }, + description: 'Adds a "count" units to Datetime.', + syntax: + 'DATEADD(date | datetime, value, ["day" | "week" | "month" | "year"])', + examples: [ + 'DATEADD({column1}, 2, "day")', + 'DATEADD({column1}, -2, "day")', + 'DATEADD({column1}, 2, "week")', + 'DATEADD({column1}, -2, "week")', + 'DATEADD({column1}, 2, "month")', + 'DATEADD({column1}, -2, "month")', + 'DATEADD({column1}, 2, "year")', + 'DATEADD({column1}, -2, "year")', + ], + returnType: FormulaDataTypes.DATE, + }, + DATETIME_DIFF: { + type: FormulaDataTypes.DATE, + validation: { + args: { + min: 2, + max: 3, + }, + }, + description: + 'Calculate the difference of two given date / datetime in specified units.', + syntax: + 'DATETIME_DIFF(date | datetime, date | datetime, ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"])', + examples: [ + 'DATEDIFF({column1}, {column2})', + 'DATEDIFF({column1}, {column2}, "seconds")', + 'DATEDIFF({column1}, {column2}, "s")', + 'DATEDIFF({column1}, {column2}, "years")', + 'DATEDIFF({column1}, {column2}, "y")', + 'DATEDIFF({column1}, {column2}, "minutes")', + 'DATEDIFF({column1}, {column2}, "m")', + 'DATEDIFF({column1}, {column2}, "days")', + 'DATEDIFF({column1}, {column2}, "d")', + ], + returnType: FormulaDataTypes.NUMERIC, + }, + AND: { + type: FormulaDataTypes.COND_EXP, + validation: { + args: { + min: 1, + }, + }, + description: 'TRUE if all expr evaluate to TRUE', + syntax: 'AND(expr1, [expr2, ...])', + examples: ['AND(5 > 2, 5 < 10) => 1', 'AND({column1} > 2, {column2} < 10)'], + returnType: FormulaDataTypes.COND_EXP, + }, + OR: { + type: FormulaDataTypes.COND_EXP, + validation: { + args: { + min: 1, + }, + }, + description: 'TRUE if at least one expr evaluates to TRUE', + syntax: 'OR(expr1, [expr2, ...])', + examples: ['OR(5 > 2, 5 < 10) => 1', 'OR({column1} > 2, {column2} < 10)'], + returnType: FormulaDataTypes.COND_EXP, + }, + CONCAT: { + type: FormulaDataTypes.STRING, + validation: { + args: { + min: 1, + }, + }, + description: 'Concatenated string of input parameters', + syntax: 'CONCAT(str1, [str2, ...])', + examples: [ + 'CONCAT("AA", "BB", "CC") => "AABBCC"', + 'CONCAT({column1}, {column2}, {column3})', + ], + returnType: FormulaDataTypes.STRING, + }, + TRIM: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 1, + }, + }, + description: 'Remove trailing and leading whitespaces from input parameter', + syntax: 'TRIM(str)', + examples: [ + 'TRIM(" HELLO WORLD ") => "HELLO WORLD"', + 'TRIM({column1})', + ], + returnType: FormulaDataTypes.STRING, + }, + UPPER: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 1, + }, + }, + description: 'Upper case converted string of input parameter', + syntax: 'UPPER(str)', + examples: ['UPPER("nocodb") => "NOCODB"', 'UPPER({column1})'], + returnType: FormulaDataTypes.STRING, + }, + LOWER: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 1, + }, + }, + description: 'Lower case converted string of input parameter', + syntax: 'LOWER(str)', + examples: ['LOWER("NOCODB") => "nocodb"', 'LOWER({column1})'], + returnType: FormulaDataTypes.STRING, + }, + LEN: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 1, + }, + }, + description: 'Input parameter character length', + syntax: 'LEN(value)', + examples: ['LEN("NocoDB") => 6', 'LEN({column1})'], + returnType: FormulaDataTypes.NUMERIC, + }, + MIN: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + min: 1, + }, + }, + description: 'Minimum value amongst input parameters', + syntax: 'MIN(value1, [value2, ...])', + examples: ['MIN(1000, 2000) => 1000', 'MIN({column1}, {column2})'], + returnType: FormulaDataTypes.NUMERIC, + }, + MAX: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + min: 1, + }, + }, + description: 'Maximum value amongst input parameters', + syntax: 'MAX(value1, [value2, ...])', + examples: ['MAX(1000, 2000) => 2000', 'MAX({column1}, {column2})'], + returnType: FormulaDataTypes.NUMERIC, + }, + CEILING: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + rqd: 1, + }, + }, + description: 'Rounded next largest integer value of input parameter', + syntax: 'CEILING(value)', + examples: ['CEILING(1.01) => 2', 'CEILING({column1})'], + returnType: FormulaDataTypes.NUMERIC, + }, + FLOOR: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + rqd: 1, + }, + }, + description: + 'Rounded largest integer less than or equal to input parameter', + syntax: 'FLOOR(value)', + examples: ['FLOOR(3.1415) => 3', 'FLOOR({column1})'], + returnType: FormulaDataTypes.NUMERIC, + }, + ROUND: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + min: 1, + max: 2, + }, + }, + description: + 'Rounded number to a specified number of decimal places or the nearest integer if not specified', + syntax: 'ROUND(value, precision), ROUND(value)', + examples: [ + 'ROUND(3.1415) => 3', + 'ROUND(3.1415, 2) => 3.14', + 'ROUND({column1}, 3)', + ], + returnType: FormulaDataTypes.NUMERIC, + }, + MOD: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + rqd: 2, + }, + }, + description: 'Remainder after integer division of input parameters', + syntax: 'MOD(value1, value2)', + examples: ['MOD(1024, 1000) => 24', 'MOD({column}, 2)'], + returnType: FormulaDataTypes.NUMERIC, + }, + REPEAT: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 2, + }, + }, + description: + 'Specified copies of the input parameter string concatenated together', + syntax: 'REPEAT(str, count)', + examples: ['REPEAT("A", 5) => "AAAAA"', 'REPEAT({column}, 5)'], + returnType: FormulaDataTypes.STRING, + }, + LOG: { + type: FormulaDataTypes.NUMERIC, + validation: {}, + description: + 'Logarithm of input parameter to the base (default = e) specified', + syntax: 'LOG([base], value)', + examples: ['LOG(2, 1024) => 10', 'LOG(2, {column1})'], + returnType: FormulaDataTypes.NUMERIC, + }, + EXP: { + type: FormulaDataTypes.NUMERIC, + validation: {}, + description: 'Exponential value of input parameter (e ^ power)', + syntax: 'EXP(power)', + examples: ['EXP(1) => 2.718281828459045', 'EXP({column1})'], + returnType: FormulaDataTypes.NUMERIC, + }, + POWER: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + rqd: 2, + }, + }, + description: 'base to the exponent power, as in base ^ exponent', + syntax: 'POWER(base, exponent)', + examples: ['POWER(2, 10) => 1024', 'POWER({column1}, 10)'], + returnType: FormulaDataTypes.NUMERIC, + }, + SQRT: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + rqd: 1, + }, + }, + description: 'Square root of the input parameter', + syntax: 'SQRT(value)', + examples: ['SQRT(100) => 10', 'SQRT({column1})'], + returnType: FormulaDataTypes.NUMERIC, + }, + ABS: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + rqd: 1, + }, + }, + description: 'Absolute value of the input parameter', + syntax: 'ABS(value)', + examples: ['ABS({column1})'], + returnType: FormulaDataTypes.NUMERIC, + }, + NOW: { + type: FormulaDataTypes.DATE, + validation: { + args: { + rqd: 0, + }, + }, + description: 'Returns the current time and day', + syntax: 'NOW()', + examples: ['NOW() => 2022-05-19 17:20:43'], + returnType: FormulaDataTypes.DATE, + }, + REPLACE: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 3, + }, + }, + description: + 'String, after replacing all occurrences of srchStr with rplcStr', + syntax: 'REPLACE(str, srchStr, rplcStr)', + examples: [ + 'REPLACE("AABBCC", "AA", "BB") => "BBBBCC"', + 'REPLACE({column1}, {column2}, {column3})', + ], + returnType: FormulaDataTypes.STRING, + }, + SEARCH: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 2, + }, + }, + description: 'Index of srchStr specified if found, 0 otherwise', + syntax: 'SEARCH(str, srchStr)', + examples: [ + 'SEARCH("HELLO WORLD", "WORLD") => 7', + 'SEARCH({column1}, "abc")', + ], + returnType: FormulaDataTypes.NUMERIC, + }, + INT: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + rqd: 1, + }, + }, + description: 'Integer value of input parameter', + syntax: 'INT(value)', + examples: ['INT(3.1415) => 3', 'INT({column1})'], + returnType: FormulaDataTypes.NUMERIC, + }, + RIGHT: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 2, + }, + }, + description: 'n characters from the end of input parameter', + syntax: 'RIGHT(str, n)', + examples: ['RIGHT("HELLO WORLD", 5) => WORLD', 'RIGHT({column1}, 3)'], + returnType: FormulaDataTypes.STRING, + }, + LEFT: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 2, + }, + }, + description: 'n characters from the beginning of input parameter', + syntax: 'LEFT(str, n)', + examples: ['LEFT({column1}, 2)', 'LEFT("ABCD", 2) => "AB"'], + returnType: FormulaDataTypes.STRING, + }, + SUBSTR: { + type: FormulaDataTypes.STRING, + validation: { + args: { + min: 2, + max: 3, + }, + }, + description: + 'Substring of length n of input string from the postition specified', + syntax: ' SUBTR(str, position, [n])', + examples: [ + 'SUBSTR("HELLO WORLD", 7) => WORLD', + 'SUBSTR("HELLO WORLD", 7, 3) => WOR', + 'SUBSTR({column1}, 7, 5)', + ], + returnType: FormulaDataTypes.STRING, + }, + MID: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 3, + }, + }, + description: 'Alias for SUBSTR', + syntax: 'MID(str, position, [count])', + examples: ['MID("NocoDB", 3, 2) => "co"', 'MID({column1}, 3, 2)'], + returnType: FormulaDataTypes.STRING, + }, + IF: { + type: FormulaDataTypes.COND_EXP, + validation: { + args: { + min: 2, + max: 3, + }, + }, + description: 'SuccessCase if expr evaluates to TRUE, elseCase otherwise', + syntax: 'IF(expr, successCase, elseCase)', + examples: [ + 'IF(5 > 1, "YES", "NO") => "YES"', + 'IF({column} > 1, "YES", "NO")', + ], + returnType: (argsTypes: FormulaDataTypes[]) => { + if (argsTypes.slice(1).includes(FormulaDataTypes.STRING)) { + return FormulaDataTypes.STRING; + } else if (argsTypes.slice(1).includes(FormulaDataTypes.NUMERIC)) { + return FormulaDataTypes.NUMERIC; + } else if (argsTypes.slice(1).includes(FormulaDataTypes.BOOLEAN)) { + return FormulaDataTypes.BOOLEAN; + } + + return argsTypes[1]; + }, + }, + SWITCH: { + type: FormulaDataTypes.COND_EXP, + validation: { + args: { + min: 3, + }, + }, + description: 'Switch case value based on expr output', + syntax: 'SWITCH(expr, [pattern, value, ..., default])', + examples: [ + 'SWITCH(1, 1, "One", 2, "Two", "N/A") => "One""', + 'SWITCH(2, 1, "One", 2, "Two", "N/A") => "Two"', + 'SWITCH(3, 1, "One", 2, "Two", "N/A") => "N/A"', + 'SWITCH({column1}, 1, "One", 2, "Two", "N/A")', + ], + // todo: resolve return type based on the args + returnType: (argTypes: FormulaDataTypes[]) => { + const returnArgTypes = argTypes.slice(2).filter((_, i) => i % 2 === 0); + + if (returnArgTypes.includes(FormulaDataTypes.STRING)) { + return FormulaDataTypes.STRING; + } else if (returnArgTypes.includes(FormulaDataTypes.NUMERIC)) { + return FormulaDataTypes.NUMERIC; + } else if (returnArgTypes.includes(FormulaDataTypes.BOOLEAN)) { + return FormulaDataTypes.BOOLEAN; + } + + return returnArgTypes[0]; + }, + }, + URL: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 1, + }, + }, + description: 'Convert to a hyperlink if it is a valid URL', + syntax: 'URL(str)', + examples: ['URL("https://github.com/nocodb/nocodb")', 'URL({column1})'], + returnType: FormulaDataTypes.STRING, + }, + WEEKDAY: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + min: 1, + max: 2, + }, + }, + description: + 'Returns the day of the week as an integer between 0 and 6 inclusive starting from Monday by default', + syntax: 'WEEKDAY(date, [startDayOfWeek])', + examples: ['WEEKDAY("2021-06-09")', 'WEEKDAY(NOW(), "sunday")'], + returnType: FormulaDataTypes.NUMERIC, + }, + + TRUE: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + max: 0, + }, + }, + description: 'Returns 1', + syntax: 'TRUE()', + examples: ['TRUE()'], + returnType: FormulaDataTypes.NUMERIC, + }, + + FALSE: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + max: 0, + }, + }, + description: 'Returns 0', + syntax: 'FALSE()', + examples: ['FALSE()'], + returnType: FormulaDataTypes.NUMERIC, + }, + + REGEX_MATCH: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 2, + }, + }, + description: + 'Returns 1 if the input text matches a regular expression or 0 if it does not.', + syntax: 'REGEX_MATCH(string, regex)', + examples: ['REGEX_MATCH({title}, "abc.*")'], + returnType: FormulaDataTypes.NUMERIC, + }, + REGEX_EXTRACT: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 2, + }, + }, + description: 'Returns the first match of a regular expression in a string.', + syntax: 'REGEX_EXTRACT(string, regex)', + examples: ['REGEX_EXTRACT({title}, "abc.*")'], + returnType: FormulaDataTypes.STRING, + }, + REGEX_REPLACE: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 3, + }, + }, + description: + 'Replaces all matches of a regular expression in a string with a replacement string', + syntax: 'REGEX_MATCH(string, regex, replacement)', + examples: ['REGEX_EXTRACT({title}, "abc.*", "abcd")'], + returnType: FormulaDataTypes.STRING, + }, + BLANK: { + type: FormulaDataTypes.STRING, + validation: { + args: { + rqd: 0, + }, + }, + description: 'Returns a blank value(null)', + syntax: 'BLANK()', + examples: ['BLANK()'], + returnType: FormulaDataTypes.NULL, + }, + XOR: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + min: 1, + }, + }, + description: + 'Returns true if an odd number of arguments are true, and false otherwise.', + syntax: 'XOR(expression, [exp2, ...])', + examples: ['XOR(TRUE(), FALSE(), TRUE())'], + returnType: FormulaDataTypes.BOOLEAN, + }, + EVEN: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + rqd: 1, + }, + }, + description: + 'Returns the nearest even integer that is greater than or equal to the specified value', + syntax: 'EVEN(value)', + examples: ['EVEN({column})'], + returnType: FormulaDataTypes.NUMERIC, + }, + ODD: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + rqd: 1, + }, + }, + description: + 'Returns the nearest odd integer that is greater than or equal to the specified value', + syntax: 'ODD(value)', + examples: ['ODD({column})'], + returnType: FormulaDataTypes.NUMERIC, + }, + RECORD_ID: { + validation: { + args: { + rqd: 0, + }, + }, + description: 'Returns the record id of the current record', + syntax: 'RECORD_ID()', + examples: ['RECORD_ID()'], + // todo: resolve return type based on the args + returnType: () => { + return FormulaDataTypes.STRING; + }, + }, + COUNTA: { + validation: { + args: { + min: 1, + }, + }, + description: 'Counts the number of non-empty arguments', + syntax: 'COUNTA(value1, [value2, ...])', + examples: ['COUNTA({field1}, {field2})'], + returnType: FormulaDataTypes.NUMERIC, + }, + COUNT: { + validation: { + args: { + min: 1, + }, + }, + description: 'Count the number of arguments that are numbers', + syntax: 'COUNT(value1, [value2, ...])', + examples: ['COUNT({field1}, {field2})'], + returnType: FormulaDataTypes.NUMERIC, + }, + COUNTALL: { + validation: { + args: { + min: 1, + }, + }, + description: 'Counts the number of arguments', + syntax: 'COUNTALL(value1, [value2, ...])', + examples: ['COUNTALL({field1}, {field2})'], + returnType: FormulaDataTypes.NUMERIC, + }, + ROUNDDOWN: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + min: 1, + max: 2, + }, + }, + description: + 'Round down the value after the decimal point to the number of decimal places given by "precision"(default is 0)', + syntax: 'ROUNDDOWN(value, [precision])', + examples: ['ROUNDDOWN({field1})', 'ROUNDDOWN({field1}, 2)'], + returnType: FormulaDataTypes.NUMERIC, + }, + ROUNDUP: { + type: FormulaDataTypes.NUMERIC, + validation: { + args: { + min: 1, + max: 2, + }, + }, + description: + 'Round up the value after the decimal point to the number of decimal places given by "precision"(default is 0)', + syntax: 'ROUNDUP(value, [precision])', + examples: ['ROUNDUP({field1})', 'ROUNDUP({field1}, 2)'], + returnType: FormulaDataTypes.NUMERIC, + }, + VALUE: { + validation: { + args: { + rqd: 1, + }, + }, + description: + 'Extract the numeric value from a string, if `%` or `-` is present, it will handle it accordingly and return the numeric value', + syntax: 'VALUE(value)', + examples: ['VALUE({field})', 'VALUE("abc10000%")', 'VALUE("$10000")'], + returnType: FormulaDataTypes.NUMERIC, + }, + // Disabling these functions for now; these act as alias for CreatedAt & UpdatedAt fields; + // Issue: Error noticed if CreatedAt & UpdatedAt fields are removed from the table after creating these formulas + // + // CREATED_TIME: { + // validation: { + // args: { + // rqd: 0, + // }, + // }, + // description: 'Returns the created time of the current record if it exists', + // syntax: 'CREATED_TIME()', + // examples: ['CREATED_TIME()'], + // }, + // LAST_MODIFIED_TIME: { + // validation: { + // args: { + // rqd: 0, + // }, + // }, + // description: 'Returns the last modified time of the current record if it exists', + // syntax: ' LAST_MODIFIED_TIME()', + // examples: [' LAST_MODIFIED_TIME()'], + // }, +}; -function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = new Set()) { - let returnType: formulaTypes +/* +function validateAgainstMeta( + parsedTree: any, + errors = new Set(), + typeErrors = new Set() +) { + let returnType: FormulaDataTypes; if (parsedTree.type === JSEPNode.CALL_EXP) { - const calleeName = parsedTree.callee.name.toUpperCase() + const calleeName = parsedTree.callee.name.toUpperCase(); // validate function name if (!availableFunctions.includes(calleeName)) { - errors.add(t('msg.formula.functionNotAvailable', { function: calleeName })) + errors.add( + t('msg.formula.functionNotAvailable', { function: calleeName }) + ); } // validate arguments - const validation = formulas[calleeName] && formulas[calleeName].validation + const validation = formulas[calleeName] && formulas[calleeName].validation; if (validation && validation.args) { - if (validation.args.rqd !== undefined && validation.args.rqd !== parsedTree.arguments.length) { - errors.add(t('msg.formula.requiredArgumentsFormula', { requiredArguments: validation.args.rqd, calleeName })) - } else if (validation.args.min !== undefined && validation.args.min > parsedTree.arguments.length) { + if ( + validation.args.rqd !== undefined && + validation.args.rqd !== parsedTree.arguments.length + ) { + errors.add( + t('msg.formula.requiredArgumentsFormula', { + requiredArguments: validation.args.rqd, + calleeName, + }) + ); + } else if ( + validation.args.min !== undefined && + validation.args.min > parsedTree.arguments.length + ) { errors.add( t('msg.formula.minRequiredArgumentsFormula', { minRequiredArguments: validation.args.min, calleeName, - }), - ) - } else if (validation.args.max !== undefined && validation.args.max < parsedTree.arguments.length) { + }) + ); + } else if ( + validation.args.max !== undefined && + validation.args.max < parsedTree.arguments.length + ) { errors.add( t('msg.formula.maxRequiredArgumentsFormula', { maxRequiredArguments: validation.args.max, calleeName, - }), - ) + }) + ); } } - parsedTree.arguments.map((arg: Record) => validateAgainstMeta(arg, errors)) + parsedTree.arguments.map((arg: Record) => + validateAgainstMeta(arg, errors) + ); // get args type and validate const validateResult = parsedTree.arguments.map((arg) => { - return validateAgainstMeta(arg, errors, typeErrors) - }) + return validateAgainstMeta(arg, errors, typeErrors); + }); const argsTypes = validateResult.map((v: any) => v.returnType); if (typeof validateResult[0].returnType === 'function') { - returnType = formulas[calleeName].returnType(argsTypes) + returnType = formulas[calleeName].returnType(argsTypes); } else if (validateResult[0]) { - returnType = formulas[calleeName].returnType + returnType = formulas[calleeName].returnType; } // validate data type if (parsedTree.callee.type === JSEPNode.IDENTIFIER) { - const expectedType = formulas[calleeName.toUpperCase()].type + const expectedType = formulas[calleeName.toUpperCase()].type; - if (expectedType === formulaTypes.NUMERIC) { + if (expectedType === FormulaDataTypes.NUMERIC) { if (calleeName === 'WEEKDAY') { // parsedTree.arguments[0] = date validateAgainstType( parsedTree.arguments[0], - formulaTypes.DATE, + FormulaDataTypes.DATE, (v: any) => { if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate')) + typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate')); } }, - typeErrors, - ) + typeErrors + ); // parsedTree.arguments[1] = startDayOfWeek (optional) validateAgainstType( parsedTree.arguments[1], - formulaTypes.STRING, + FormulaDataTypes.STRING, (v: any) => { if ( typeof v !== 'string' || - !['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'].includes(v.toLowerCase()) + ![ + 'sunday', + 'monday', + 'tuesday', + 'wednesday', + 'thursday', + 'friday', + 'saturday', + ].includes(v.toLowerCase()) ) { - typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate')) + typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate')); } }, - typeErrors, - ) + typeErrors + ); } else { - parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors, argsTypes)) + parsedTree.arguments.map((arg: Record) => + validateAgainstType(arg, expectedType, null, typeErrors, argsTypes) + ); } - } else if (expectedType === formulaTypes.DATE) { + } else if (expectedType === FormulaDataTypes.DATE) { if (calleeName === 'DATEADD') { // parsedTree.arguments[0] = date validateAgainstType( parsedTree.arguments[0], - formulaTypes.DATE, + FormulaDataTypes.DATE, (v: any) => { if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamDateAddHaveDate')) + typeErrors.add(t('msg.formula.firstParamDateAddHaveDate')); } }, - typeErrors, - ) + typeErrors + ); // parsedTree.arguments[1] = numeric validateAgainstType( parsedTree.arguments[1], - formulaTypes.NUMERIC, + FormulaDataTypes.NUMERIC, (v: any) => { if (typeof v !== 'number') { - typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber')) + typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber')); } }, - typeErrors, - ) + typeErrors + ); // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"] validateAgainstType( parsedTree.arguments[2], - formulaTypes.STRING, + FormulaDataTypes.STRING, (v: any) => { if (!['day', 'week', 'month', 'year'].includes(v)) { - typeErrors.add(typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate'))) + typeErrors.add( + typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate')) + ); } }, - typeErrors, - ) + typeErrors + ); } else if (calleeName === 'DATETIME_DIFF') { // parsedTree.arguments[0] = date validateAgainstType( parsedTree.arguments[0], - formulaTypes.DATE, + FormulaDataTypes.DATE, (v: any) => { if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate')) + typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate')); } }, - typeErrors, - ) + typeErrors + ); // parsedTree.arguments[1] = date validateAgainstType( parsedTree.arguments[1], - formulaTypes.DATE, + FormulaDataTypes.DATE, (v: any) => { if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate')) + typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate')); } }, - typeErrors, - ) + typeErrors + ); // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"] validateAgainstType( parsedTree.arguments[2], - formulaTypes.STRING, + FormulaDataTypes.STRING, (v: any) => { if ( ![ @@ -358,23 +1163,27 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n 'y', ].includes(v) ) { - typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate')) + typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate')); } }, - typeErrors, - ) + typeErrors + ); } } } - errors = new Set([...errors, ...typeErrors]) + errors = new Set([...errors, ...typeErrors]); } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - if (supportedColumns.value.filter((c) => !column || column.value?.id !== c.id).every((c) => c.title !== parsedTree.name)) { + if ( + supportedColumns.value + .filter((c) => !column || column.value?.id !== c.id) + .every((c) => c.title !== parsedTree.name) + ) { errors.add( t('msg.formula.columnNotAvailable', { columnName: parsedTree.name, - }), - ) + }) + ); } // check circular reference @@ -390,149 +1199,170 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n ...new Set( (c.colOptions.formula.match(/cl_\w{14}/g) || []).filter( (colId: string) => - supportedColumns.value.filter((col: ColumnType) => col.id === colId && col.uidt === UITypes.Formula).length, - ), + supportedColumns.value.filter( + (col: ColumnType) => + col.id === colId && col.uidt === UITypes.Formula + ).length + ) ), - ] + ]; if (neighbours.length > 0) { // e.g. formula column 1 -> [formula column 2, formula column3] - res.push({ [c.id]: neighbours }) + res.push({ [c.id]: neighbours }); } - return res - }, []) + return res; + }, []); // include target formula column (i.e. the one to be saved if applicable) const targetFormulaCol = supportedColumns.value.find( - (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula, - ) + (c: ColumnType) => + c.title === parsedTree.name && c.uidt === UITypes.Formula + ); if (targetFormulaCol && column.value?.id) { formulaPaths.push({ [column.value?.id as string]: [targetFormulaCol.id], - }) + }); } - const vertices = formulaPaths.length + const vertices = formulaPaths.length; if (vertices > 0) { // perform kahn's algo for cycle detection - const adj = new Map() - const inDegrees = new Map() + const adj = new Map(); + const inDegrees = new Map(); // init adjacency list & indegree for (const [_, v] of Object.entries(formulaPaths)) { - const src = Object.keys(v)[0] - const neighbours = v[src] - inDegrees.set(src, inDegrees.get(src) || 0) + const src = Object.keys(v)[0]; + const neighbours = v[src]; + inDegrees.set(src, inDegrees.get(src) || 0); for (const neighbour of neighbours) { - adj.set(src, (adj.get(src) || new Set()).add(neighbour)) - inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1) + adj.set(src, (adj.get(src) || new Set()).add(neighbour)); + inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1); } } - const queue: string[] = [] + const queue: string[] = []; // put all vertices with in-degree = 0 (i.e. no incoming edges) to queue inDegrees.forEach((inDegree, col) => { if (inDegree === 0) { // in-degree = 0 means we start traversing from this node - queue.push(col) + queue.push(col); } - }) + }); // init count of visited vertices - let visited = 0 + let visited = 0; // BFS while (queue.length !== 0) { // remove a vertex from the queue - const src = queue.shift() + const src = queue.shift(); // if this node has neighbours, increase visited by 1 - const neighbours = adj.get(src) || new Set() + const neighbours = adj.get(src) || new Set(); if (neighbours.size > 0) { - visited += 1 + visited += 1; } // iterate each neighbouring nodes neighbours.forEach((neighbour: string) => { // decrease in-degree of its neighbours by 1 - inDegrees.set(neighbour, inDegrees.get(neighbour) - 1) + inDegrees.set(neighbour, inDegrees.get(neighbour) - 1); // if in-degree becomes 0 if (inDegrees.get(neighbour) === 0) { // then put the neighboring node to the queue - queue.push(neighbour) + queue.push(neighbour); } - }) + }); } // vertices not same as visited = cycle found if (vertices !== visited) { - errors.add(t('msg.formula.cantSaveCircularReference')) + errors.add(t('msg.formula.cantSaveCircularReference')); } } } else if (parsedTree.type === JSEPNode.BINARY_EXP) { if (!availableBinOps.includes(parsedTree.operator)) { - errors.add(t('msg.formula.operationNotAvailable', { operation: parsedTree.operator })) + errors.add( + t('msg.formula.operationNotAvailable', { + operation: parsedTree.operator, + }) + ); } - validateAgainstMeta(parsedTree.left, errors) - validateAgainstMeta(parsedTree.right, errors) + validateAgainstMeta(parsedTree.left, errors); + validateAgainstMeta(parsedTree.right, errors); // todo: type extraction for binary exps - returnType = formulaTypes.NUMERIC - } else if (parsedTree.type === JSEPNode.LITERAL || parsedTree.type === JSEPNode.UNARY_EXP) { + returnType = FormulaDataTypes.NUMERIC; + } else if ( + parsedTree.type === JSEPNode.LITERAL || + parsedTree.type === JSEPNode.UNARY_EXP + ) { if (parsedTree.type === JSEPNode.LITERAL) { if (typeof parsedTree.value === 'number') { - returnType = formulaTypes.NUMERIC + returnType = FormulaDataTypes.NUMERIC; } else if (typeof parsedTree.value === 'string') { - returnType = formulaTypes.STRING + returnType = FormulaDataTypes.STRING; } else if (typeof parsedTree.value === 'boolean') { - returnType = formulaTypes.BOOLEAN + returnType = FormulaDataTypes.BOOLEAN; } else { - returnType = formulaTypes.STRING + returnType = FormulaDataTypes.STRING; } } // do nothing } else if (parsedTree.type === JSEPNode.COMPOUND) { if (parsedTree.body.length) { - errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')) + errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')); } } else { - errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')) + errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')); } - return { errors, returnType } + return { errors, returnType }; } -function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set(), argTypes: formulaTypes = []) { - let type +function validateAgainstType( + parsedTree: any, + expectedType: string, + func: any, + typeErrors = new Set(), + argTypes: FormulaDataTypes = [] +) { + let type; if (parsedTree === false || typeof parsedTree === 'undefined') { - return typeErrors + return typeErrors; } if (parsedTree.type === JSEPNode.LITERAL) { if (typeof func === 'function') { - func(parsedTree.value) - } else if (expectedType === formulaTypes.NUMERIC) { + func(parsedTree.value); + } else if (expectedType === FormulaDataTypes.NUMERIC) { if (typeof parsedTree.value !== 'number') { - typeErrors.add(t('msg.formula.numericTypeIsExpected')) + typeErrors.add(t('msg.formula.numericTypeIsExpected')); } else { - type = formulaTypes.NUMERIC + type = FormulaDataTypes.NUMERIC; } - } else if (expectedType === formulaTypes.STRING) { + } else if (expectedType === FormulaDataTypes.STRING) { if (typeof parsedTree.value !== 'string') { - typeErrors.add(t('msg.formula.stringTypeIsExpected')) + typeErrors.add(t('msg.formula.stringTypeIsExpected')); } else { - type = formulaTypes.STRING + type = FormulaDataTypes.STRING; } } } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - const col = supportedColumns.value.find((c) => c.title === parsedTree.name) + const col = supportedColumns.value.find((c) => c.title === parsedTree.name); if (col === undefined) { - return + return; } if (col.uidt === UITypes.Formula) { - const foundType = getRootDataType(jsep(col.colOptions?.formula_raw)) - type = foundType + const foundType = getRootDataType(jsep(col.colOptions?.formula_raw)); + type = foundType; if (foundType === 'N/A') { - typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: col.title })) + typeErrors.add( + t('msg.formula.notSupportedToReferenceColumn', { + columnName: col.title, + }) + ); } else if (expectedType !== foundType) { typeErrors.add( t('msg.formula.typeIsExpectedButFound', { type: expectedType, found: foundType, - }), - ) + }) + ); } } else { switch (col.uidt) { @@ -544,17 +1374,17 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t case UITypes.PhoneNumber: case UITypes.Email: case UITypes.URL: - if (expectedType !== formulaTypes.STRING) { + if (expectedType !== FormulaDataTypes.STRING) { typeErrors.add( t('msg.formula.columnWithTypeFoundButExpected', { columnName: parsedTree.name, - columnType: formulaTypes.STRING, + columnType: FormulaDataTypes.STRING, expectedType, - }), - ) + }) + ); } - type = formulaTypes.STRING - break + type = FormulaDataTypes.STRING; + break; // numeric case UITypes.Year: @@ -564,34 +1394,34 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t case UITypes.Count: case UITypes.AutoNumber: case UITypes.Currency: - if (expectedType !== formulaTypes.NUMERIC) { + if (expectedType !== FormulaDataTypes.NUMERIC) { typeErrors.add( t('msg.formula.columnWithTypeFoundButExpected', { columnName: parsedTree.name, - columnType: formulaTypes.NUMERIC, + columnType: FormulaDataTypes.NUMERIC, expectedType, - }), - ) + }) + ); } - type = formulaTypes.NUMERIC - break + type = FormulaDataTypes.NUMERIC; + break; // date case UITypes.Date: case UITypes.DateTime: case UITypes.CreateTime: case UITypes.LastModifiedTime: - if (expectedType !== formulaTypes.DATE) { + if (expectedType !== FormulaDataTypes.DATE) { typeErrors.add( t('msg.formula.columnWithTypeFoundButExpected', { columnName: parsedTree.name, - columnType: formulaTypes.DATE, + columnType: FormulaDataTypes.DATE, expectedType, - }), - ) + }) + ); } - type = formulaTypes.DATE - break + type = FormulaDataTypes.DATE; + break; // not supported case UITypes.ForeignKey: @@ -608,46 +1438,58 @@ function validateAgainstType(parsedTree: any, expectedType: string, func: any, t case UITypes.Collaborator: case UITypes.QrCode: default: - typeErrors.add(t('msg.formula.notSupportedToReferenceColumn', { columnName: parsedTree.name })) - break + typeErrors.add( + t('msg.formula.notSupportedToReferenceColumn', { + columnName: parsedTree.name, + }) + ); + break; } } - } else if (parsedTree.type === JSEPNode.UNARY_EXP || parsedTree.type === JSEPNode.BINARY_EXP) { - if (expectedType !== formulaTypes.NUMERIC) { + } else if ( + parsedTree.type === JSEPNode.UNARY_EXP || + parsedTree.type === JSEPNode.BINARY_EXP + ) { + if (expectedType !== FormulaDataTypes.NUMERIC) { // parsedTree.name won't be available here typeErrors.add( t('msg.formula.typeIsExpectedButFound', { - type: formulaTypes.NUMERIC, + type: FormulaDataTypes.NUMERIC, found: expectedType, - }), - ) + }) + ); } - type = formulaTypes.NUMERIC + type = FormulaDataTypes.NUMERIC; } else if (parsedTree.type === JSEPNode.CALL_EXP) { - const calleeName = parsedTree.callee.name.toUpperCase() - if (formulas[calleeName]?.type && expectedType !== formulas[calleeName].type) { + const calleeName = parsedTree.callee.name.toUpperCase(); + if ( + formulas[calleeName]?.type && + expectedType !== formulas[calleeName].type + ) { typeErrors.add( t('msg.formula.typeIsExpectedButFound', { type: expectedType, found: formulas[calleeName].type, - }), - ) + }) + ); } // todo: derive type from returnType - type = formulas[calleeName]?.type + type = formulas[calleeName]?.type; } - return { type, typeErrors } + return { type, typeErrors }; } function getRootDataType(parsedTree: any): any { // given a parse tree, return the data type of it if (parsedTree.type === JSEPNode.CALL_EXP) { - return formulas[parsedTree.callee.name.toUpperCase()].type + return formulas[parsedTree.callee.name.toUpperCase()].type; } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - const col = supportedColumns.value.find((c) => c.title === parsedTree.name) as Record + const col = supportedColumns.value.find( + (c) => c.title === parsedTree.name + ) as Record; if (col?.uidt === UITypes.Formula) { - return getRootDataType(jsep(col?.formula_raw)) + return getRootDataType(jsep(col?.formula_raw)); } else { switch (col?.uidt) { // string @@ -658,7 +1500,7 @@ function getRootDataType(parsedTree: any): any { case UITypes.PhoneNumber: case UITypes.Email: case UITypes.URL: - return formulaTypes.STRING + return FormulaDataTypes.STRING; // numeric case UITypes.Year: @@ -667,14 +1509,14 @@ function getRootDataType(parsedTree: any): any { case UITypes.Rating: case UITypes.Count: case UITypes.AutoNumber: - return formulaTypes.NUMERIC + return FormulaDataTypes.NUMERIC; // date case UITypes.Date: case UITypes.DateTime: case UITypes.CreateTime: case UITypes.LastModifiedTime: - return formulaTypes.DATE + return FormulaDataTypes.DATE; // not supported case UITypes.ForeignKey: @@ -692,28 +1534,248 @@ function getRootDataType(parsedTree: any): any { case UITypes.Collaborator: case UITypes.QrCode: default: - return 'N/A' + return 'N/A'; } } - } else if (parsedTree.type === JSEPNode.BINARY_EXP || parsedTree.type === JSEPNode.UNARY_EXP) { - return formulaTypes.NUMERIC + } else if ( + parsedTree.type === JSEPNode.BINARY_EXP || + parsedTree.type === JSEPNode.UNARY_EXP + ) { + return FormulaDataTypes.NUMERIC; } else if (parsedTree.type === JSEPNode.LITERAL) { - return typeof parsedTree.value + return typeof parsedTree.value; } else { - return 'N/A' + return 'N/A'; } } function isCurlyBracketBalanced() { // count number of opening curly brackets and closing curly brackets - const cntCurlyBrackets = (formulaRef.value.$el.value.match(/\{|}/g) || []).reduce( - (acc: Record, cur: number) => { - acc[cur] = (acc[cur] || 0) + 1 - return acc - }, - {}, - ) - return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0) + const cntCurlyBrackets = ( + formulaRef.value.$el.value.match(/\{|}/g) || [] + ).reduce((acc: Record, cur: number) => { + acc[cur] = (acc[cur] || 0) + 1; + return acc; + }, {}); + return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0); +} +*/ + +enum FormulaErrorType { + NOT_AVAILABLE = 'NOT_AVAILABLE', + NOT_SUPPORTED = 'NOT_SUPPORTED', + 'MIN_ARG' = 'MIN_ARG', + 'MAX_ARG' = 'MAX_ARG', + 'TYPE_MISMATCH' = 'TYPE_MISMATCH', + 'INVALID_ARG' = 'INVALID_ARG', + 'INVALID_ARG_TYPE' = 'INVALID_ARG_TYPE', + 'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE', + 'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT', } +class FormulaError extends Error { + public name: string; + public type: FormulaErrorType; + constructor( + type: FormulaErrorType, + name: string, + message: string = 'Formula Error' + ) { + super(message); + this.name = name; + this.type = type; + } +} + +export function validateFormulaAndExtractTreeWithType( + formula, + columns: ColumnType[] +) { + const colAliasToColMap = {}; + const colIdToColMap = {}; + + for (const col of columns) { + colAliasToColMap[col.title] = col; + colIdToColMap[col.id] = col; + } + + const validateAndExtract = (parsedTree: any) => { + const res: { + dataType?: FormulaDataTypes; + errors?: Set; + [key: string]: any; + } = { ...parsedTree }; + + if (parsedTree.type === JSEPNode.CALL_EXP) { + const calleeName = parsedTree.callee.name.toUpperCase(); + // validate function name + if (!formulas[calleeName]) { + throw new FormulaError( + FormulaErrorType.INVALID_ARG_TYPE, + 'Function not available' + ); + //t('msg.formula.functionNotAvailable', { function: calleeName }) + } + // validate arguments + const validation = + formulas[calleeName] && formulas[calleeName].validation; + if (validation && validation.args) { + if ( + validation.args.rqd !== undefined && + validation.args.rqd !== parsedTree.arguments.length + ) { + throw new FormulaError( + FormulaErrorType.INVALID_ARG, + calleeName, + 'Required arguments missing' + ); + + // errors.add( + // t('msg.formula.requiredArgumentsFormula', { + // requiredArguments: validation.args.rqd, + // calleeName, + // }) + // ); + } else if ( + validation.args.min !== undefined && + validation.args.min > parsedTree.arguments.length + ) { + throw new FormulaError( + FormulaErrorType.MIN_ARG, + calleeName, + 'Minimum arguments required' + ); + + // errors.add( + // t('msg.formula.minRequiredArgumentsFormula', { + // minRequiredArguments: validation.args.min, + // calleeName, + // }) + // ); + } else if ( + validation.args.max !== undefined && + validation.args.max < parsedTree.arguments.length + ) { + throw new FormulaError( + FormulaErrorType.MAX_ARG, + calleeName, + 'Maximum arguments required' + ); + + // errors.add( + // t('msg.formula.maxRequiredArgumentsFormula', { + // maxRequiredArguments: validation.args.max, + // calleeName, + // }) + // ); + } + } + // get args type and validate + const validateResult = res.arguments = parsedTree.arguments.map((arg) => { + return validateAndExtract(arg); + }); + + const argsTypes = validateResult.map((v: any) => v.dataType); + + if (typeof formulas[calleeName].returnType === 'function') { + res.dataType = (formulas[calleeName].returnType as any)?.( + argsTypes + ) as FormulaDataTypes; + } else if (formulas[calleeName].returnType) { + res.dataType = formulas[calleeName].returnType as FormulaDataTypes; + } + } else if (parsedTree.type === JSEPNode.IDENTIFIER) { + const col = columns.find[parsedTree.name] as Record< + string, + any + >; + res.name = col.id; + + if (col?.uidt === UITypes.Formula) { + // todo: check for circular reference + + // todo: extract the type and return + const formulaRes = validateFormulaAndExtractTreeWithType( + col.colOptions.formula, + columns + ); + + res.dataType = formulaRes as any; + } else { + + switch (col?.uidt) { + // string + case UITypes.SingleLineText: + case UITypes.LongText: + case UITypes.MultiSelect: + case UITypes.SingleSelect: + case UITypes.PhoneNumber: + case UITypes.Email: + case UITypes.URL: + res.dataType = FormulaDataTypes.STRING; + break; + // numeric + case UITypes.Year: + case UITypes.Number: + case UITypes.Decimal: + case UITypes.Rating: + case UITypes.Count: + case UITypes.AutoNumber: + res.dataType = FormulaDataTypes.NUMERIC; + break; + // date + case UITypes.Date: + case UITypes.DateTime: + case UITypes.CreateTime: + case UITypes.LastModifiedTime: + res.dataType = FormulaDataTypes.DATE; + break; + // not supported + case UITypes.ForeignKey: + case UITypes.Attachment: + case UITypes.ID: + case UITypes.Time: + case UITypes.Currency: + case UITypes.Percent: + case UITypes.Duration: + case UITypes.Rollup: + case UITypes.Lookup: + case UITypes.Barcode: + case UITypes.Button: + case UITypes.Checkbox: + case UITypes.Collaborator: + case UITypes.QrCode: + default: + throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, ''); + } + } + } else if (parsedTree.type === JSEPNode.LITERAL) { + if (typeof parsedTree.value === 'number') { + res.dataType = FormulaDataTypes.NUMERIC; + } else if (typeof parsedTree.value === 'string') { + res.dataType = FormulaDataTypes.STRING; + } else if (typeof parsedTree.value === 'boolean') { + res.dataType = FormulaDataTypes.BOOLEAN; + } else { + res.dataType = FormulaDataTypes.STRING; + } + } else if ( + parsedTree.type === JSEPNode.BINARY_EXP || + parsedTree.type === JSEPNode.UNARY_EXP + ) { + res.left = validateAndExtract(parsedTree.left); + res.right = validateAndExtract(parsedTree.right); + res.dataType = FormulaDataTypes.NUMERIC; + } else { + // res.type= 'N/A'; + } + + return res; + }; + // register jsep curly hook + jsep.plugins.register(jsepCurlyHook); + const parsedFormula = jsep(formula); + const result = validateAndExtract(parsedFormula); + return result; +} From f4944a2dd72d93df4f9df7b21c43ea2494f7ae20 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:58 +0000 Subject: [PATCH 008/188] feat: support non conditional_expression in if formula --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 2 +- packages/nocodb-sdk/tsconfig.json | 2 +- .../src/db/formulav2/formulaQueryBuilderv2.ts | 15 ++++++++++--- .../src/db/functionMappings/commonFns.ts | 21 ++++++++++++++++--- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index c1c19b2562..5d0dba7305 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1685,7 +1685,7 @@ export function validateFormulaAndExtractTreeWithType( res.dataType = formulas[calleeName].returnType as FormulaDataTypes; } } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - const col = columns.find[parsedTree.name] as Record< + const col = (colIdToColMap[parsedTree.name] || colAliasToColMap[parsedTree.name]) as Record< string, any >; diff --git a/packages/nocodb-sdk/tsconfig.json b/packages/nocodb-sdk/tsconfig.json index 0bd395cc39..86fdaa2404 100644 --- a/packages/nocodb-sdk/tsconfig.json +++ b/packages/nocodb-sdk/tsconfig.json @@ -38,7 +38,7 @@ // "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */, "lib": ["es2017","dom"], - "types": ["jest", "node"], + "types": ["jest"], "typeRoots": ["node_modules/@types", "src/types"], "baseUrl": "./src", "paths": { diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts index 390c733025..e8bd658f97 100644 --- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts @@ -2,6 +2,7 @@ import jsep from 'jsep'; import { jsepCurlyHook, UITypes, + validateFormulaAndExtractTreeWithType, validateDateWithUnknownFormat, } from 'nocodb-sdk'; import mapFunctionName from '../mapFunctionName'; @@ -14,7 +15,10 @@ import type LookupColumn from '~/models/LookupColumn'; import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; import NocoCache from '~/cache/NocoCache'; import { CacheGetType, CacheScope } from '~/utils/globals'; -import { convertDateFormatForConcat } from '~/helpers/formulaFnHelper'; +import { + convertDateFormatForConcat, + validateDateWithUnknownFormat, +} from '~/helpers/formulaFnHelper'; import FormulaColumn from '~/models/FormulaColumn'; // todo: switch function based on database @@ -62,14 +66,19 @@ async function _formulaQueryBuilder( ) { const knex = baseModelSqlv2.dbDriver; + const columns = await model.getColumns(); // formula may include double curly brackets in previous version // convert to single curly bracket here for compatibility - const tree = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}')); + // const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}')); + const tree = validateFormulaAndExtractTreeWithType( + _tree.replaceAll('{{', '{').replaceAll('}}', '}'), + columns, + ); const columnIdToUidt = {}; // todo: improve - implement a common solution for filter, sort, formula, etc - for (const col of await model.getColumns()) { + for (const col of columns) { columnIdToUidt[col.id] = col.uidt; if (col.id in aliasToColumn) continue; switch (col.uidt) { diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts index 547f65796e..6f87f5f84d 100644 --- a/packages/nocodb/src/db/functionMappings/commonFns.ts +++ b/packages/nocodb/src/db/functionMappings/commonFns.ts @@ -1,3 +1,4 @@ +import { FormulaDataTypes } from 'nocodb-sdk'; import type { MapFnArgs } from '../mapFunctionName'; import { NcError } from '~/helpers/catchError'; @@ -36,11 +37,25 @@ export default { }; }, IF: async (args: MapFnArgs) => { + const condArg = (await args.fn(args.pt.arguments[0])).builder.toQuery(); + + let cond = condArg; + + switch (args.pt.arguments[0].dataType as FormulaDataTypes) { + case FormulaDataTypes.NUMERIC: + cond = `(${condArg}) IS NOT NULL OR (${condArg}) != 0`; + break; + case FormulaDataTypes.STRING: + cond = `(${condArg}) IS NOT NULL OR (${condArg}) != ''`; + break; + case FormulaDataTypes.BOOLEAN: + cond = `(${condArg}) IS NOT NULL OR (${condArg}) != false`; + break; + } + let query = args.knex .raw( - `\n\tWHEN ${( - await args.fn(args.pt.arguments[0]) - ).builder.toQuery()} THEN ${( + `\n\tWHEN ${cond} THEN ${( await args.fn(args.pt.arguments[1]) ).builder.toQuery()}`, ) From 019e5b0919536b18157fa42385f4f964f8a98251 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:58 +0000 Subject: [PATCH 009/188] fix: correction in IF formula --- packages/nocodb/src/db/functionMappings/commonFns.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts index 6f87f5f84d..8e39d1f4de 100644 --- a/packages/nocodb/src/db/functionMappings/commonFns.ts +++ b/packages/nocodb/src/db/functionMappings/commonFns.ts @@ -43,13 +43,13 @@ export default { switch (args.pt.arguments[0].dataType as FormulaDataTypes) { case FormulaDataTypes.NUMERIC: - cond = `(${condArg}) IS NOT NULL OR (${condArg}) != 0`; + cond = `(${condArg}) IS NOT NULL AND (${condArg}) != 0`; break; case FormulaDataTypes.STRING: - cond = `(${condArg}) IS NOT NULL OR (${condArg}) != ''`; + cond = `(${condArg}) IS NOT NULL AND (${condArg}) != ''`; break; case FormulaDataTypes.BOOLEAN: - cond = `(${condArg}) IS NOT NULL OR (${condArg}) != false`; + cond = `(${condArg}) IS NOT NULL AND (${condArg}) != false`; break; } From 82e8bcd52368ccfc845dce4f73902259878fc702 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:58 +0000 Subject: [PATCH 010/188] fix: DATE value with if --- packages/nocodb/src/db/functionMappings/commonFns.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts index 8e39d1f4de..d5d987c866 100644 --- a/packages/nocodb/src/db/functionMappings/commonFns.ts +++ b/packages/nocodb/src/db/functionMappings/commonFns.ts @@ -51,6 +51,9 @@ export default { case FormulaDataTypes.BOOLEAN: cond = `(${condArg}) IS NOT NULL AND (${condArg}) != false`; break; + case FormulaDataTypes.DATE: + cond = `(${condArg}) IS NOT NULL`; + break; } let query = args.knex From 36277d55093890b318eca2796b8c416f8baa676f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:58 +0000 Subject: [PATCH 011/188] feat: add migration and hide parsed tree data from api response --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 713 +++--------------- .../src/db/formulav2/formulaQueryBuilderv2.ts | 24 +- .../v2/nc_038_formula_parsed_tree_column.ts | 16 + packages/nocodb/src/models/FormulaColumn.ts | 24 +- .../nocodb/src/services/columns.service.ts | 6 +- 5 files changed, 162 insertions(+), 621 deletions(-) create mode 100644 packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 5d0dba7305..94b3dcc62d 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -2,6 +2,7 @@ import jsep from 'jsep'; import { ColumnType } from './Api'; import UITypes from './UITypes'; +import { formulaTypes } from '../../../nc-gui/utils'; export const jsepCurlyHook = { name: 'curly', @@ -220,6 +221,7 @@ interface FormulaMeta { min?: number; max?: number; rqd?: number; + validator?: (args: formulaTypes[]) => boolean; }; }; description?: string; @@ -230,7 +232,6 @@ interface FormulaMeta { const formulas: Record = { AVG: { - type: FormulaDataTypes.NUMERIC, validation: { args: { min: 1, @@ -309,7 +310,6 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, AND: { - type: FormulaDataTypes.COND_EXP, validation: { args: { min: 1, @@ -321,7 +321,6 @@ const formulas: Record = { returnType: FormulaDataTypes.COND_EXP, }, OR: { - type: FormulaDataTypes.COND_EXP, validation: { args: { min: 1, @@ -964,603 +963,6 @@ const formulas: Record = { // }, }; -/* -function validateAgainstMeta( - parsedTree: any, - errors = new Set(), - typeErrors = new Set() -) { - let returnType: FormulaDataTypes; - if (parsedTree.type === JSEPNode.CALL_EXP) { - const calleeName = parsedTree.callee.name.toUpperCase(); - // validate function name - if (!availableFunctions.includes(calleeName)) { - errors.add( - t('msg.formula.functionNotAvailable', { function: calleeName }) - ); - } - // validate arguments - const validation = formulas[calleeName] && formulas[calleeName].validation; - if (validation && validation.args) { - if ( - validation.args.rqd !== undefined && - validation.args.rqd !== parsedTree.arguments.length - ) { - errors.add( - t('msg.formula.requiredArgumentsFormula', { - requiredArguments: validation.args.rqd, - calleeName, - }) - ); - } else if ( - validation.args.min !== undefined && - validation.args.min > parsedTree.arguments.length - ) { - errors.add( - t('msg.formula.minRequiredArgumentsFormula', { - minRequiredArguments: validation.args.min, - calleeName, - }) - ); - } else if ( - validation.args.max !== undefined && - validation.args.max < parsedTree.arguments.length - ) { - errors.add( - t('msg.formula.maxRequiredArgumentsFormula', { - maxRequiredArguments: validation.args.max, - calleeName, - }) - ); - } - } - - parsedTree.arguments.map((arg: Record) => - validateAgainstMeta(arg, errors) - ); - - // get args type and validate - const validateResult = parsedTree.arguments.map((arg) => { - return validateAgainstMeta(arg, errors, typeErrors); - }); - - const argsTypes = validateResult.map((v: any) => v.returnType); - - if (typeof validateResult[0].returnType === 'function') { - returnType = formulas[calleeName].returnType(argsTypes); - } else if (validateResult[0]) { - returnType = formulas[calleeName].returnType; - } - - // validate data type - if (parsedTree.callee.type === JSEPNode.IDENTIFIER) { - const expectedType = formulas[calleeName.toUpperCase()].type; - - if (expectedType === FormulaDataTypes.NUMERIC) { - if (calleeName === 'WEEKDAY') { - // parsedTree.arguments[0] = date - validateAgainstType( - parsedTree.arguments[0], - FormulaDataTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[1] = startDayOfWeek (optional) - validateAgainstType( - parsedTree.arguments[1], - FormulaDataTypes.STRING, - (v: any) => { - if ( - typeof v !== 'string' || - ![ - 'sunday', - 'monday', - 'tuesday', - 'wednesday', - 'thursday', - 'friday', - 'saturday', - ].includes(v.toLowerCase()) - ) { - typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate')); - } - }, - typeErrors - ); - } else { - parsedTree.arguments.map((arg: Record) => - validateAgainstType(arg, expectedType, null, typeErrors, argsTypes) - ); - } - } else if (expectedType === FormulaDataTypes.DATE) { - if (calleeName === 'DATEADD') { - // parsedTree.arguments[0] = date - validateAgainstType( - parsedTree.arguments[0], - FormulaDataTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamDateAddHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[1] = numeric - validateAgainstType( - parsedTree.arguments[1], - FormulaDataTypes.NUMERIC, - (v: any) => { - if (typeof v !== 'number') { - typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber')); - } - }, - typeErrors - ); - // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"] - validateAgainstType( - parsedTree.arguments[2], - FormulaDataTypes.STRING, - (v: any) => { - if (!['day', 'week', 'month', 'year'].includes(v)) { - typeErrors.add( - typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate')) - ); - } - }, - typeErrors - ); - } else if (calleeName === 'DATETIME_DIFF') { - // parsedTree.arguments[0] = date - validateAgainstType( - parsedTree.arguments[0], - FormulaDataTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[1] = date - validateAgainstType( - parsedTree.arguments[1], - FormulaDataTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"] - validateAgainstType( - parsedTree.arguments[2], - FormulaDataTypes.STRING, - (v: any) => { - if ( - ![ - 'milliseconds', - 'ms', - 'seconds', - 's', - 'minutes', - 'm', - 'hours', - 'h', - 'days', - 'd', - 'weeks', - 'w', - 'months', - 'M', - 'quarters', - 'Q', - 'years', - 'y', - ].includes(v) - ) { - typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate')); - } - }, - typeErrors - ); - } - } - } - - errors = new Set([...errors, ...typeErrors]); - } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - if ( - supportedColumns.value - .filter((c) => !column || column.value?.id !== c.id) - .every((c) => c.title !== parsedTree.name) - ) { - errors.add( - t('msg.formula.columnNotAvailable', { - columnName: parsedTree.name, - }) - ); - } - - // check circular reference - // e.g. formula1 -> formula2 -> formula1 should return circular reference error - - // get all formula columns excluding itself - const formulaPaths = supportedColumns.value - .filter((c) => c.id !== column.value?.id && c.uidt === UITypes.Formula) - .reduce((res: Record[], c: Record) => { - // in `formula`, get all the (unique) target neighbours - // i.e. all column id (e.g. cl_xxxxxxxxxxxxxx) with formula type - const neighbours = [ - ...new Set( - (c.colOptions.formula.match(/cl_\w{14}/g) || []).filter( - (colId: string) => - supportedColumns.value.filter( - (col: ColumnType) => - col.id === colId && col.uidt === UITypes.Formula - ).length - ) - ), - ]; - if (neighbours.length > 0) { - // e.g. formula column 1 -> [formula column 2, formula column3] - res.push({ [c.id]: neighbours }); - } - return res; - }, []); - // include target formula column (i.e. the one to be saved if applicable) - const targetFormulaCol = supportedColumns.value.find( - (c: ColumnType) => - c.title === parsedTree.name && c.uidt === UITypes.Formula - ); - - if (targetFormulaCol && column.value?.id) { - formulaPaths.push({ - [column.value?.id as string]: [targetFormulaCol.id], - }); - } - const vertices = formulaPaths.length; - if (vertices > 0) { - // perform kahn's algo for cycle detection - const adj = new Map(); - const inDegrees = new Map(); - // init adjacency list & indegree - - for (const [_, v] of Object.entries(formulaPaths)) { - const src = Object.keys(v)[0]; - const neighbours = v[src]; - inDegrees.set(src, inDegrees.get(src) || 0); - for (const neighbour of neighbours) { - adj.set(src, (adj.get(src) || new Set()).add(neighbour)); - inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1); - } - } - const queue: string[] = []; - // put all vertices with in-degree = 0 (i.e. no incoming edges) to queue - inDegrees.forEach((inDegree, col) => { - if (inDegree === 0) { - // in-degree = 0 means we start traversing from this node - queue.push(col); - } - }); - // init count of visited vertices - let visited = 0; - // BFS - while (queue.length !== 0) { - // remove a vertex from the queue - const src = queue.shift(); - // if this node has neighbours, increase visited by 1 - const neighbours = adj.get(src) || new Set(); - if (neighbours.size > 0) { - visited += 1; - } - // iterate each neighbouring nodes - neighbours.forEach((neighbour: string) => { - // decrease in-degree of its neighbours by 1 - inDegrees.set(neighbour, inDegrees.get(neighbour) - 1); - // if in-degree becomes 0 - if (inDegrees.get(neighbour) === 0) { - // then put the neighboring node to the queue - queue.push(neighbour); - } - }); - } - // vertices not same as visited = cycle found - if (vertices !== visited) { - errors.add(t('msg.formula.cantSaveCircularReference')); - } - } - } else if (parsedTree.type === JSEPNode.BINARY_EXP) { - if (!availableBinOps.includes(parsedTree.operator)) { - errors.add( - t('msg.formula.operationNotAvailable', { - operation: parsedTree.operator, - }) - ); - } - validateAgainstMeta(parsedTree.left, errors); - validateAgainstMeta(parsedTree.right, errors); - - // todo: type extraction for binary exps - returnType = FormulaDataTypes.NUMERIC; - } else if ( - parsedTree.type === JSEPNode.LITERAL || - parsedTree.type === JSEPNode.UNARY_EXP - ) { - if (parsedTree.type === JSEPNode.LITERAL) { - if (typeof parsedTree.value === 'number') { - returnType = FormulaDataTypes.NUMERIC; - } else if (typeof parsedTree.value === 'string') { - returnType = FormulaDataTypes.STRING; - } else if (typeof parsedTree.value === 'boolean') { - returnType = FormulaDataTypes.BOOLEAN; - } else { - returnType = FormulaDataTypes.STRING; - } - } - // do nothing - } else if (parsedTree.type === JSEPNode.COMPOUND) { - if (parsedTree.body.length) { - errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')); - } - } else { - errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')); - } - return { errors, returnType }; -} - -function validateAgainstType( - parsedTree: any, - expectedType: string, - func: any, - typeErrors = new Set(), - argTypes: FormulaDataTypes = [] -) { - let type; - if (parsedTree === false || typeof parsedTree === 'undefined') { - return typeErrors; - } - if (parsedTree.type === JSEPNode.LITERAL) { - if (typeof func === 'function') { - func(parsedTree.value); - } else if (expectedType === FormulaDataTypes.NUMERIC) { - if (typeof parsedTree.value !== 'number') { - typeErrors.add(t('msg.formula.numericTypeIsExpected')); - } else { - type = FormulaDataTypes.NUMERIC; - } - } else if (expectedType === FormulaDataTypes.STRING) { - if (typeof parsedTree.value !== 'string') { - typeErrors.add(t('msg.formula.stringTypeIsExpected')); - } else { - type = FormulaDataTypes.STRING; - } - } - } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - const col = supportedColumns.value.find((c) => c.title === parsedTree.name); - - if (col === undefined) { - return; - } - - if (col.uidt === UITypes.Formula) { - const foundType = getRootDataType(jsep(col.colOptions?.formula_raw)); - type = foundType; - if (foundType === 'N/A') { - typeErrors.add( - t('msg.formula.notSupportedToReferenceColumn', { - columnName: col.title, - }) - ); - } else if (expectedType !== foundType) { - typeErrors.add( - t('msg.formula.typeIsExpectedButFound', { - type: expectedType, - found: foundType, - }) - ); - } - } else { - switch (col.uidt) { - // string - case UITypes.SingleLineText: - case UITypes.LongText: - case UITypes.MultiSelect: - case UITypes.SingleSelect: - case UITypes.PhoneNumber: - case UITypes.Email: - case UITypes.URL: - if (expectedType !== FormulaDataTypes.STRING) { - typeErrors.add( - t('msg.formula.columnWithTypeFoundButExpected', { - columnName: parsedTree.name, - columnType: FormulaDataTypes.STRING, - expectedType, - }) - ); - } - type = FormulaDataTypes.STRING; - break; - - // numeric - case UITypes.Year: - case UITypes.Number: - case UITypes.Decimal: - case UITypes.Rating: - case UITypes.Count: - case UITypes.AutoNumber: - case UITypes.Currency: - if (expectedType !== FormulaDataTypes.NUMERIC) { - typeErrors.add( - t('msg.formula.columnWithTypeFoundButExpected', { - columnName: parsedTree.name, - columnType: FormulaDataTypes.NUMERIC, - expectedType, - }) - ); - } - type = FormulaDataTypes.NUMERIC; - break; - - // date - case UITypes.Date: - case UITypes.DateTime: - case UITypes.CreateTime: - case UITypes.LastModifiedTime: - if (expectedType !== FormulaDataTypes.DATE) { - typeErrors.add( - t('msg.formula.columnWithTypeFoundButExpected', { - columnName: parsedTree.name, - columnType: FormulaDataTypes.DATE, - expectedType, - }) - ); - } - type = FormulaDataTypes.DATE; - break; - - // not supported - case UITypes.ForeignKey: - case UITypes.Attachment: - case UITypes.ID: - case UITypes.Time: - case UITypes.Percent: - case UITypes.Duration: - case UITypes.Rollup: - case UITypes.Lookup: - case UITypes.Barcode: - case UITypes.Button: - case UITypes.Checkbox: - case UITypes.Collaborator: - case UITypes.QrCode: - default: - typeErrors.add( - t('msg.formula.notSupportedToReferenceColumn', { - columnName: parsedTree.name, - }) - ); - break; - } - } - } else if ( - parsedTree.type === JSEPNode.UNARY_EXP || - parsedTree.type === JSEPNode.BINARY_EXP - ) { - if (expectedType !== FormulaDataTypes.NUMERIC) { - // parsedTree.name won't be available here - typeErrors.add( - t('msg.formula.typeIsExpectedButFound', { - type: FormulaDataTypes.NUMERIC, - found: expectedType, - }) - ); - } - - type = FormulaDataTypes.NUMERIC; - } else if (parsedTree.type === JSEPNode.CALL_EXP) { - const calleeName = parsedTree.callee.name.toUpperCase(); - if ( - formulas[calleeName]?.type && - expectedType !== formulas[calleeName].type - ) { - typeErrors.add( - t('msg.formula.typeIsExpectedButFound', { - type: expectedType, - found: formulas[calleeName].type, - }) - ); - } - // todo: derive type from returnType - type = formulas[calleeName]?.type; - } - return { type, typeErrors }; -} - -function getRootDataType(parsedTree: any): any { - // given a parse tree, return the data type of it - if (parsedTree.type === JSEPNode.CALL_EXP) { - return formulas[parsedTree.callee.name.toUpperCase()].type; - } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - const col = supportedColumns.value.find( - (c) => c.title === parsedTree.name - ) as Record; - if (col?.uidt === UITypes.Formula) { - return getRootDataType(jsep(col?.formula_raw)); - } else { - switch (col?.uidt) { - // string - case UITypes.SingleLineText: - case UITypes.LongText: - case UITypes.MultiSelect: - case UITypes.SingleSelect: - case UITypes.PhoneNumber: - case UITypes.Email: - case UITypes.URL: - return FormulaDataTypes.STRING; - - // numeric - case UITypes.Year: - case UITypes.Number: - case UITypes.Decimal: - case UITypes.Rating: - case UITypes.Count: - case UITypes.AutoNumber: - return FormulaDataTypes.NUMERIC; - - // date - case UITypes.Date: - case UITypes.DateTime: - case UITypes.CreateTime: - case UITypes.LastModifiedTime: - return FormulaDataTypes.DATE; - - // not supported - case UITypes.ForeignKey: - case UITypes.Attachment: - case UITypes.ID: - case UITypes.Time: - case UITypes.Currency: - case UITypes.Percent: - case UITypes.Duration: - case UITypes.Rollup: - case UITypes.Lookup: - case UITypes.Barcode: - case UITypes.Button: - case UITypes.Checkbox: - case UITypes.Collaborator: - case UITypes.QrCode: - default: - return 'N/A'; - } - } - } else if ( - parsedTree.type === JSEPNode.BINARY_EXP || - parsedTree.type === JSEPNode.UNARY_EXP - ) { - return FormulaDataTypes.NUMERIC; - } else if (parsedTree.type === JSEPNode.LITERAL) { - return typeof parsedTree.value; - } else { - return 'N/A'; - } -} - -function isCurlyBracketBalanced() { - // count number of opening curly brackets and closing curly brackets - const cntCurlyBrackets = ( - formulaRef.value.$el.value.match(/\{|}/g) || [] - ).reduce((acc: Record, cur: number) => { - acc[cur] = (acc[cur] || 0) + 1; - return acc; - }, {}); - return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0); -} -*/ - enum FormulaErrorType { NOT_AVAILABLE = 'NOT_AVAILABLE', NOT_SUPPORTED = 'NOT_SUPPORTED', @@ -1571,11 +973,13 @@ enum FormulaErrorType { 'INVALID_ARG_TYPE' = 'INVALID_ARG_TYPE', 'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE', 'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT', + CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE', } class FormulaError extends Error { public name: string; public type: FormulaErrorType; + constructor( type: FormulaErrorType, name: string, @@ -1671,9 +1075,11 @@ export function validateFormulaAndExtractTreeWithType( } } // get args type and validate - const validateResult = res.arguments = parsedTree.arguments.map((arg) => { - return validateAndExtract(arg); - }); + const validateResult = (res.arguments = parsedTree.arguments.map( + (arg) => { + return validateAndExtract(arg); + } + )); const argsTypes = validateResult.map((v: any) => v.dataType); @@ -1685,10 +1091,8 @@ export function validateFormulaAndExtractTreeWithType( res.dataType = formulas[calleeName].returnType as FormulaDataTypes; } } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - const col = (colIdToColMap[parsedTree.name] || colAliasToColMap[parsedTree.name]) as Record< - string, - any - >; + const col = (colIdToColMap[parsedTree.name] || + colAliasToColMap[parsedTree.name]) as Record; res.name = col.id; if (col?.uidt === UITypes.Formula) { @@ -1702,7 +1106,6 @@ export function validateFormulaAndExtractTreeWithType( res.dataType = formulaRes as any; } else { - switch (col?.uidt) { // string case UITypes.SingleLineText: @@ -1766,8 +1169,6 @@ export function validateFormulaAndExtractTreeWithType( res.left = validateAndExtract(parsedTree.left); res.right = validateAndExtract(parsedTree.right); res.dataType = FormulaDataTypes.NUMERIC; - } else { - // res.type= 'N/A'; } return res; @@ -1779,3 +1180,95 @@ export function validateFormulaAndExtractTreeWithType( const result = validateAndExtract(parsedFormula); return result; } + +function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { + // check circular reference + // e.g. formula1 -> formula2 -> formula1 should return circular reference error + + // get all formula columns excluding itself + const formulaPaths = columns.value + .filter((c) => c.id !== formulaCol.value?.id && c.uidt === UITypes.Formula) + .reduce((res: Record[], c: Record) => { + // in `formula`, get all the (unique) target neighbours + // i.e. all column id (e.g. cxxxxxxxxxxxxxx) with formula type + const neighbours = [ + ...new Set( + (c.colOptions.formula.match(/c\w{15}/g) || []).filter( + (colId: string) => + columns.value.filter( + (col: ColumnType) => + col.id === colId && col.uidt === UITypes.Formula + ).length + ) + ), + ]; + if (neighbours.length > 0) { + // e.g. formula column 1 -> [formula column 2, formula column3] + res.push({ [c.id]: neighbours }); + } + return res; + }, []); + + // include target formula column (i.e. the one to be saved if applicable) + const targetFormulaCol = columns.value.find( + (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula + ); + + if (targetFormulaCol && formulaCol.value?.id) { + formulaPaths.push({ + [formulaCol.value?.id as string]: [targetFormulaCol.id], + }); + } + const vertices = formulaPaths.length; + if (vertices > 0) { + // perform kahn's algo for cycle detection + const adj = new Map(); + const inDegrees = new Map(); + // init adjacency list & indegree + + for (const [_, v] of Object.entries(formulaPaths)) { + const src = Object.keys(v)[0]; + const neighbours = v[src]; + inDegrees.set(src, inDegrees.get(src) || 0); + for (const neighbour of neighbours) { + adj.set(src, (adj.get(src) || new Set()).add(neighbour)); + inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1); + } + } + const queue: string[] = []; + // put all vertices with in-degree = 0 (i.e. no incoming edges) to queue + inDegrees.forEach((inDegree, col) => { + if (inDegree === 0) { + // in-degree = 0 means we start traversing from this node + queue.push(col); + } + }); + // init count of visited vertices + let visited = 0; + // BFS + while (queue.length !== 0) { + // remove a vertex from the queue + const src = queue.shift(); + // if this node has neighbours, increase visited by 1 + const neighbours = adj.get(src) || new Set(); + if (neighbours.size > 0) { + visited += 1; + } + // iterate each neighbouring nodes + neighbours.forEach((neighbour: string) => { + // decrease in-degree of its neighbours by 1 + inDegrees.set(neighbour, inDegrees.get(neighbour) - 1); + // if in-degree becomes 0 + if (inDegrees.get(neighbour) === 0) { + // then put the neighboring node to the queue + queue.push(neighbour); + } + }); + } + // vertices not same as visited = cycle found + if (vertices !== visited) { + // errors.add(t('msg.formula.cantSaveCircularReference')) + throw new FormulaError(FormulaErrorType.CIRCULAR_REFERENCE, ''); + } + } +} diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts index e8bd658f97..f1ac991183 100644 --- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts @@ -63,17 +63,22 @@ async function _formulaQueryBuilder( model: Model, aliasToColumn: Record Promise<{ builder: any }>> = {}, tableAlias?: string, + parsedTree?: any, ) { const knex = baseModelSqlv2.dbDriver; const columns = await model.getColumns(); - // formula may include double curly brackets in previous version - // convert to single curly bracket here for compatibility - // const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}')); - const tree = validateFormulaAndExtractTreeWithType( - _tree.replaceAll('{{', '{').replaceAll('}}', '}'), - columns, - ); + + let tree = parsedTree; + if (!tree) { + // formula may include double curly brackets in previous version + // convert to single curly bracket here for compatibility + // const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}')); + tree = validateFormulaAndExtractTreeWithType( + _tree.replaceAll('{{', '{').replaceAll('}}', '}'), + columns, + ); + } const columnIdToUidt = {}; @@ -93,6 +98,7 @@ async function _formulaQueryBuilder( model, { ...aliasToColumn, [col.id]: null }, tableAlias, + formulOption.getParsedTree(), ); builder.sql = '(' + builder.sql + ')'; return { @@ -413,6 +419,7 @@ async function _formulaQueryBuilder( '', lookupModel, aliasToColumn, + formulaOption.getParsedTree() ); if (isMany) { const qb = selectQb; @@ -968,6 +975,9 @@ export default async function formulaQueryBuilderv2( model, aliasToColumn, tableAlias, + await column + ?.getColOptions() + .then((formula) => formula?.getParsedTree()), ); if (!validateFormula) return qb; diff --git a/packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts b/packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts new file mode 100644 index 0000000000..5e50af058c --- /dev/null +++ b/packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts @@ -0,0 +1,16 @@ +import type { Knex } from 'knex'; +import { MetaTable } from '~/utils/globals'; + +const up = async (knex: Knex) => { + await knex.schema.alterTable(MetaTable.COL_FORMULA, (table) => { + table.text('parsed_tree'); + }); +}; + +const down = async (knex: Knex) => { + await knex.schema.alterTable(MetaTable.COL_FORMULA, (table) => { + table.dropColumn('parsed_tree'); + }); +}; + +export { up, down }; diff --git a/packages/nocodb/src/models/FormulaColumn.ts b/packages/nocodb/src/models/FormulaColumn.ts index 955d480b11..d3f1234a44 100644 --- a/packages/nocodb/src/models/FormulaColumn.ts +++ b/packages/nocodb/src/models/FormulaColumn.ts @@ -2,19 +2,21 @@ import Noco from '~/Noco'; import NocoCache from '~/cache/NocoCache'; import { extractProps } from '~/helpers/extractProps'; import { CacheGetType, CacheScope, MetaTable } from '~/utils/globals'; +import { parseMetaProp, stringifyMetaProp } from '~/utils/modelUtils'; export default class FormulaColumn { formula: string; formula_raw: string; fk_column_id: string; error: string; + private parsed_tree?: any; constructor(data: Partial) { Object.assign(this, data); } public static async insert( - formulaColumn: Partial, + formulaColumn: Partial, ncMeta = Noco.ncMeta, ) { const insertObj = extractProps(formulaColumn, [ @@ -22,11 +24,16 @@ export default class FormulaColumn { 'formula_raw', 'formula', 'error', + 'parsed_tree', ]); + + insertObj.parsed_tree = stringifyMetaProp(insertObj, 'parsed_tree'); + await ncMeta.metaInsert2(null, null, MetaTable.COL_FORMULA, insertObj); return this.read(formulaColumn.fk_column_id, ncMeta); } + public static async read(columnId: string, ncMeta = Noco.ncMeta) { let column = columnId && @@ -41,7 +48,10 @@ export default class FormulaColumn { MetaTable.COL_FORMULA, { fk_column_id: columnId }, ); - await NocoCache.set(`${CacheScope.COL_FORMULA}:${columnId}`, column); + if (column) { + column.parsed_tree = parseMetaProp(column, 'parsed_tree'); + await NocoCache.set(`${CacheScope.COL_FORMULA}:${columnId}`, column); + } } return column ? new FormulaColumn(column) : null; @@ -51,7 +61,7 @@ export default class FormulaColumn { static async update( id: string, - formula: Partial, + formula: Partial & { parsed_tree?: any }, ncMeta = Noco.ncMeta, ) { const updateObj = extractProps(formula, [ @@ -59,7 +69,11 @@ export default class FormulaColumn { 'formula_raw', 'fk_column_id', 'error', + 'parsed_tree', ]); + + updateObj.parsed_tree = stringifyMetaProp(insertObj, 'parsed_tree'); + // get existing cache const key = `${CacheScope.COL_FORMULA}:${id}`; let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); @@ -71,4 +85,8 @@ export default class FormulaColumn { // set meta await ncMeta.metaUpdate(null, null, MetaTable.COL_FORMULA, updateObj, id); } + + public getParsedTree() { + return this.parsed_tree; + } } diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index 09cecbd6e6..48c2f30464 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -5,7 +5,7 @@ import { isVirtualCol, substituteColumnAliasWithIdInFormula, substituteColumnIdWithAliasInFormula, - UITypes, + UITypes, validateFormulaAndExtractTreeWithType, } from 'nocodb-sdk'; import { pluralize, singularize } from 'inflection'; import hash from 'object-hash'; @@ -1197,6 +1197,10 @@ export class ColumnsService { colBody.formula_raw || colBody.formula, table.columns, ); + colBody.parsed_tree = validateFormulaAndExtractTreeWithType( + colBody.formula_raw || colBody.formula, + table.columns, + ); try { const baseModel = await reuseOrSave('baseModel', reuse, async () => From 106858fc61da4f62644a147ca4dce41e8366507f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:58 +0000 Subject: [PATCH 012/188] fix: corrections --- packages/nocodb/src/models/Column.ts | 6 +++++- packages/nocodb/src/models/FormulaColumn.ts | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts index 484ae6579b..f9996aea0a 100644 --- a/packages/nocodb/src/models/Column.ts +++ b/packages/nocodb/src/models/Column.ts @@ -713,7 +713,11 @@ export default class Column implements ColumnType { title: col?.title, }) ) - await FormulaColumn.update(formulaCol.id, formula, ncMeta); + await FormulaColumn.update( + formulaCol.id, + formula as FormulaColumn & { parsed_tree?: any }, + ncMeta, + ); } } diff --git a/packages/nocodb/src/models/FormulaColumn.ts b/packages/nocodb/src/models/FormulaColumn.ts index d3f1234a44..4c786125e9 100644 --- a/packages/nocodb/src/models/FormulaColumn.ts +++ b/packages/nocodb/src/models/FormulaColumn.ts @@ -16,7 +16,7 @@ export default class FormulaColumn { } public static async insert( - formulaColumn: Partial, + formulaColumn: Partial & { parsed_tree?: any }, ncMeta = Noco.ncMeta, ) { const insertObj = extractProps(formulaColumn, [ @@ -72,7 +72,7 @@ export default class FormulaColumn { 'parsed_tree', ]); - updateObj.parsed_tree = stringifyMetaProp(insertObj, 'parsed_tree'); + updateObj.parsed_tree = stringifyMetaProp(updateObj, 'parsed_tree'); // get existing cache const key = `${CacheScope.COL_FORMULA}:${id}`; From 60983af77cacab8aece2ff0873fcc2367665a82b Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:58 +0000 Subject: [PATCH 013/188] refactor: exclude parsed tree from response --- .../src/db/formulav2/formulaQueryBuilderv2.ts | 10 ++++++---- .../src/meta/migrations/XcMigrationSourcev2.ts | 4 ++++ packages/nocodb/src/models/Column.ts | 1 + packages/nocodb/src/models/FormulaColumn.ts | 6 ++++-- packages/nocodb/src/services/columns.service.ts | 14 +++++++++++++- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts index f1ac991183..3a6dcb8775 100644 --- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts @@ -419,7 +419,7 @@ async function _formulaQueryBuilder( '', lookupModel, aliasToColumn, - formulaOption.getParsedTree() + formulaOption.getParsedTree(), ); if (isMany) { const qb = selectQb; @@ -963,6 +963,7 @@ export default async function formulaQueryBuilderv2( aliasToColumn = {}, tableAlias?: string, validateFormula = false, + parsedTree?: any, ) { const knex = baseModelSqlv2.dbDriver; // register jsep curly hook once only @@ -975,9 +976,10 @@ export default async function formulaQueryBuilderv2( model, aliasToColumn, tableAlias, - await column - ?.getColOptions() - .then((formula) => formula?.getParsedTree()), + parsedTree ?? + (await column + ?.getColOptions() + .then((formula) => formula?.getParsedTree())), ); if (!validateFormula) return qb; diff --git a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts index 6dedcf2843..2df5d90b5a 100644 --- a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts +++ b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts @@ -24,6 +24,7 @@ import * as nc_034_erd_filter_and_notification from '~/meta/migrations/v2/nc_034 import * as nc_035_add_username_to_users from '~/meta/migrations/v2/nc_035_add_username_to_users'; import * as nc_036_base_deleted from '~/meta/migrations/v2/nc_036_base_deleted'; import * as nc_037_rename_project_and_base from '~/meta/migrations/v2/nc_037_rename_project_and_base'; +import * as nc_038_formula_parsed_tree_column from '~/meta/migrations/v2/nc_038_formula_parsed_tree_column'; // Create a custom migration source class export default class XcMigrationSourcev2 { @@ -59,6 +60,7 @@ export default class XcMigrationSourcev2 { 'nc_035_add_username_to_users', 'nc_036_base_deleted', 'nc_037_rename_project_and_base', + 'nc_038_formula_parsed_tree_column' ]); } @@ -120,6 +122,8 @@ export default class XcMigrationSourcev2 { return nc_036_base_deleted; case 'nc_037_rename_project_and_base': return nc_037_rename_project_and_base; + case 'nc_038_formula_parsed_tree_column': + return nc_038_formula_parsed_tree_column; } } } diff --git a/packages/nocodb/src/models/Column.ts b/packages/nocodb/src/models/Column.ts index f9996aea0a..2d76ed8860 100644 --- a/packages/nocodb/src/models/Column.ts +++ b/packages/nocodb/src/models/Column.ts @@ -308,6 +308,7 @@ export default class Column implements ColumnType { fk_column_id: colId, formula: column.formula, formula_raw: column.formula_raw, + parsed_tree: column.parsed_tree, }, ncMeta, ); diff --git a/packages/nocodb/src/models/FormulaColumn.ts b/packages/nocodb/src/models/FormulaColumn.ts index 4c786125e9..f807737286 100644 --- a/packages/nocodb/src/models/FormulaColumn.ts +++ b/packages/nocodb/src/models/FormulaColumn.ts @@ -11,8 +11,10 @@ export default class FormulaColumn { error: string; private parsed_tree?: any; - constructor(data: Partial) { - Object.assign(this, data); + constructor(data: Partial & { parsed_tree?: any }) { + const { parsed_tree, ...rest } = data; + this.parsed_tree = parsed_tree; + Object.assign(this, rest); } public static async insert( diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index 48c2f30464..edace7a76d 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -5,7 +5,8 @@ import { isVirtualCol, substituteColumnAliasWithIdInFormula, substituteColumnIdWithAliasInFormula, - UITypes, validateFormulaAndExtractTreeWithType, + UITypes, + validateFormulaAndExtractTreeWithType, } from 'nocodb-sdk'; import { pluralize, singularize } from 'inflection'; import hash from 'object-hash'; @@ -184,6 +185,7 @@ export class ColumnsService { let colBody = { ...param.column } as Column & { formula?: string; formula_raw?: string; + parsed_tree?: any; }; if ( [ @@ -208,6 +210,10 @@ export class ColumnsService { colBody.formula_raw || colBody.formula, table.columns, ); + colBody.parsed_tree = validateFormulaAndExtractTreeWithType( + colBody.formula_raw || colBody.formula, + table.columns, + ); try { const baseModel = await reuseOrSave('baseModel', reuse, async () => @@ -227,6 +233,7 @@ export class ColumnsService { {}, null, true, + colBody.parsed_tree ); } catch (e) { console.error(e); @@ -931,6 +938,7 @@ export class ColumnsService { ]); await FormulaColumn.update(c.id, { formula_raw: new_formula_raw, + parsed_tree: validateFormulaAndExtractTreeWithType(new_formula_raw, table.columns) }); } } @@ -992,6 +1000,10 @@ export class ColumnsService { ]); await FormulaColumn.update(c.id, { formula_raw: new_formula_raw, + parsed_tree: validateFormulaAndExtractTreeWithType( + new_formula_raw, + table.columns, + ), }); } } From 24ee8745e158f0339ff344ac9abd85dfb246fef7 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:59 +0000 Subject: [PATCH 014/188] feat:populate and save parsed tree if missing --- packages/nc-gui/utils/formulaUtils.ts | 11 ----------- .../src/db/formulav2/formulaQueryBuilderv2.ts | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/nc-gui/utils/formulaUtils.ts b/packages/nc-gui/utils/formulaUtils.ts index c4f4fef23b..13788c1a07 100644 --- a/packages/nc-gui/utils/formulaUtils.ts +++ b/packages/nc-gui/utils/formulaUtils.ts @@ -435,17 +435,6 @@ const formulas: Record = { return argsTypes[1] }, - returnType: (argTypes: formulaTypes[]) => { - if (argTypes.slice(1).includes(formulaTypes.STRING)) { - return formulaTypes.STRING - } else if (argTypes.slice(1).includes(formulaTypes.NUMERIC)) { - return formulaTypes.NUMERIC - } else if (argTypes.slice(1).includes(formulaTypes.BOOLEAN)) { - return formulaTypes.BOOLEAN - } - - return argTypes[1] - }, }, SWITCH: { type: formulaTypes.COND_EXP, diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts index 3a6dcb8775..f6826bb8fb 100644 --- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts @@ -7,12 +7,12 @@ import { } from 'nocodb-sdk'; import mapFunctionName from '../mapFunctionName'; import genRollupSelectv2 from '../genRollupSelectv2'; -import type Column from '~/models/Column'; import type Model from '~/models/Model'; import type RollupColumn from '~/models/RollupColumn'; import type LinkToAnotherRecordColumn from '~/models/LinkToAnotherRecordColumn'; import type LookupColumn from '~/models/LookupColumn'; import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2'; +import type Column from '~/models/Column'; import NocoCache from '~/cache/NocoCache'; import { CacheGetType, CacheScope } from '~/utils/globals'; import { @@ -21,7 +21,7 @@ import { } from '~/helpers/formulaFnHelper'; import FormulaColumn from '~/models/FormulaColumn'; -// todo: switch function based on database +const logger = new Logger('FormulaQueryBuilderv2'); // @ts-ignore const getAggregateFn: (fnName: string) => (args: { qb; knex?; cn }) => any = ( @@ -64,6 +64,7 @@ async function _formulaQueryBuilder( aliasToColumn: Record Promise<{ builder: any }>> = {}, tableAlias?: string, parsedTree?: any, + column: Column = null, ) { const knex = baseModelSqlv2.dbDriver; @@ -78,6 +79,18 @@ async function _formulaQueryBuilder( _tree.replaceAll('{{', '{').replaceAll('}}', '}'), columns, ); + + // populate and save parsedTree to column if not exist + if (column) { + FormulaColumn.update(column.id, { parsed_tree: tree }).then( + () => { + // ignore + }, + (err) => { + logger.error(err); + }, + ); + } } const columnIdToUidt = {}; From c64cf83bba57733dafd559dbf00789407b859c78 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:59 +0000 Subject: [PATCH 015/188] fix: corrections --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 94b3dcc62d..d14bc44c18 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -2,7 +2,6 @@ import jsep from 'jsep'; import { ColumnType } from './Api'; import UITypes from './UITypes'; -import { formulaTypes } from '../../../nc-gui/utils'; export const jsepCurlyHook = { name: 'curly', @@ -221,7 +220,7 @@ interface FormulaMeta { min?: number; max?: number; rqd?: number; - validator?: (args: formulaTypes[]) => boolean; + validator?: (args: FormulaDataTypes[]) => boolean; }; }; description?: string; @@ -1180,6 +1179,7 @@ export function validateFormulaAndExtractTreeWithType( const result = validateAndExtract(parsedFormula); return result; } +/* function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { // check circular reference @@ -1272,3 +1272,4 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { } } } +*/ From 3412a0e54af3ce272191272a80e4f2dc846aa8bd Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:59 +0000 Subject: [PATCH 016/188] fix: missing import statement --- packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts index f6826bb8fb..f6688082a3 100644 --- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts @@ -5,6 +5,7 @@ import { validateFormulaAndExtractTreeWithType, validateDateWithUnknownFormat, } from 'nocodb-sdk'; +import { Logger } from '@nestjs/common'; import mapFunctionName from '../mapFunctionName'; import genRollupSelectv2 from '../genRollupSelectv2'; import type Model from '~/models/Model'; From 0457968fc2e549cf9e237eef2147167efaa92d74 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:59 +0000 Subject: [PATCH 017/188] fix: replace double curly brace --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 7 ++++--- packages/nocodb/src/services/columns.service.ts | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index d14bc44c18..0085b617d2 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1097,9 +1097,10 @@ export function validateFormulaAndExtractTreeWithType( if (col?.uidt === UITypes.Formula) { // todo: check for circular reference - // todo: extract the type and return - const formulaRes = validateFormulaAndExtractTreeWithType( - col.colOptions.formula, + const formulaRes = col.colOptions?.parsed_tree || validateFormulaAndExtractTreeWithType( + // formula may include double curly brackets in previous version + // convert to single curly bracket here for compatibility + col.colOptions.formula.replaceAll('{{', '{').replaceAll('}}', '}'), columns ); diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index edace7a76d..f9731fc115 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -1210,7 +1210,9 @@ export class ColumnsService { table.columns, ); colBody.parsed_tree = validateFormulaAndExtractTreeWithType( - colBody.formula_raw || colBody.formula, + // formula may include double curly brackets in previous version + // convert to single curly bracket here for compatibility + colBody.formula_raw || colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'), table.columns, ); From 7b80cabe4b806615b415a6ce5c53ad6bd0451cd2 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:59 +0000 Subject: [PATCH 018/188] fix: circular reference validation --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 0085b617d2..5285d8eebb 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1095,7 +1095,8 @@ export function validateFormulaAndExtractTreeWithType( res.name = col.id; if (col?.uidt === UITypes.Formula) { - // todo: check for circular reference + // check for circular reference + checkForCircularFormulaRef(col, parsedTree, columns); const formulaRes = col.colOptions?.parsed_tree || validateFormulaAndExtractTreeWithType( // formula may include double curly brackets in previous version @@ -1180,7 +1181,6 @@ export function validateFormulaAndExtractTreeWithType( const result = validateAndExtract(parsedFormula); return result; } -/* function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { // check circular reference @@ -1188,15 +1188,15 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { // get all formula columns excluding itself const formulaPaths = columns.value - .filter((c) => c.id !== formulaCol.value?.id && c.uidt === UITypes.Formula) + .filter((c) => c.id !== formulaCol?.id && c.uidt === UITypes.Formula) .reduce((res: Record[], c: Record) => { // in `formula`, get all the (unique) target neighbours // i.e. all column id (e.g. cxxxxxxxxxxxxxx) with formula type const neighbours = [ ...new Set( - (c.colOptions.formula.match(/c\w{15}/g) || []).filter( + (c.colOptions.formula.match(/c_?\w{14,15}/g) || []).filter( (colId: string) => - columns.value.filter( + columns.filter( (col: ColumnType) => col.id === colId && col.uidt === UITypes.Formula ).length @@ -1211,13 +1211,13 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { }, []); // include target formula column (i.e. the one to be saved if applicable) - const targetFormulaCol = columns.value.find( + const targetFormulaCol = columns.find( (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula ); - if (targetFormulaCol && formulaCol.value?.id) { + if (targetFormulaCol && formulaCol?.id) { formulaPaths.push({ - [formulaCol.value?.id as string]: [targetFormulaCol.id], + [formulaCol?.id as string]: [targetFormulaCol.id], }); } const vertices = formulaPaths.length; @@ -1273,4 +1273,4 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { } } } -*/ + From c7b265e429a15af9f3a6fd64bf4f475400a967f9 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:59 +0000 Subject: [PATCH 019/188] fix: formula errors --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 75 +++++++++---------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 5285d8eebb..95b98d8b0d 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -976,17 +976,19 @@ enum FormulaErrorType { } class FormulaError extends Error { - public name: string; public type: FormulaErrorType; + public extra: Record; constructor( type: FormulaErrorType, - name: string, + extra: { + [key: string]: any; + }, message: string = 'Formula Error' ) { super(message); - this.name = name; this.type = type; + this.extra = extra; } } @@ -1029,48 +1031,39 @@ export function validateFormulaAndExtractTreeWithType( ) { throw new FormulaError( FormulaErrorType.INVALID_ARG, - calleeName, + { + key: 'msg.formula.requiredArgumentsFormula', + requiredArguments: validation.args.rqd, + calleeName, + }, 'Required arguments missing' ); - - // errors.add( - // t('msg.formula.requiredArgumentsFormula', { - // requiredArguments: validation.args.rqd, - // calleeName, - // }) - // ); } else if ( validation.args.min !== undefined && validation.args.min > parsedTree.arguments.length ) { throw new FormulaError( FormulaErrorType.MIN_ARG, - calleeName, + { + key: 'msg.formula.minRequiredArgumentsFormula', + minRequiredArguments: validation.args.min, + calleeName, + }, 'Minimum arguments required' ); - - // errors.add( - // t('msg.formula.minRequiredArgumentsFormula', { - // minRequiredArguments: validation.args.min, - // calleeName, - // }) - // ); } else if ( validation.args.max !== undefined && validation.args.max < parsedTree.arguments.length ) { throw new FormulaError( - FormulaErrorType.MAX_ARG, - calleeName, - 'Maximum arguments required' + FormulaErrorType.INVALID_ARG, + { + key: 'msg.formula.maxRequiredArgumentsFormula', + maxRequiredArguments: validation.args.max, + calleeName, + }, + 'Maximum arguments missing' ); - - // errors.add( - // t('msg.formula.maxRequiredArgumentsFormula', { - // maxRequiredArguments: validation.args.max, - // calleeName, - // }) - // ); } } // get args type and validate @@ -1098,12 +1091,14 @@ export function validateFormulaAndExtractTreeWithType( // check for circular reference checkForCircularFormulaRef(col, parsedTree, columns); - const formulaRes = col.colOptions?.parsed_tree || validateFormulaAndExtractTreeWithType( - // formula may include double curly brackets in previous version - // convert to single curly bracket here for compatibility - col.colOptions.formula.replaceAll('{{', '{').replaceAll('}}', '}'), - columns - ); + const formulaRes = + col.colOptions?.parsed_tree || + validateFormulaAndExtractTreeWithType( + // formula may include double curly brackets in previous version + // convert to single curly bracket here for compatibility + col.colOptions.formula.replaceAll('{{', '{').replaceAll('}}', '}'), + columns + ); res.dataType = formulaRes as any; } else { @@ -1268,9 +1263,13 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { } // vertices not same as visited = cycle found if (vertices !== visited) { - // errors.add(t('msg.formula.cantSaveCircularReference')) - throw new FormulaError(FormulaErrorType.CIRCULAR_REFERENCE, ''); + throw new FormulaError( + FormulaErrorType.CIRCULAR_REFERENCE, + { + key: 'msg.formula.cantSaveCircularReference', + }, + 'Circular reference detected' + ); } } } - From 1a632051bd401a0f855427aae0d2604cf405fc66 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:59 +0000 Subject: [PATCH 020/188] fix: typo --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 95b98d8b0d..f2281c89f1 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -973,6 +973,7 @@ enum FormulaErrorType { 'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE', 'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT', CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE', + INVALID_FUNCTION_NAME = 'INVALID_FUNCTION_NAME', } class FormulaError extends Error { @@ -1016,7 +1017,8 @@ export function validateFormulaAndExtractTreeWithType( // validate function name if (!formulas[calleeName]) { throw new FormulaError( - FormulaErrorType.INVALID_ARG_TYPE, + FormulaErrorType.INVALID_FUNCTION_NAME, + {}, 'Function not available' ); //t('msg.formula.functionNotAvailable', { function: calleeName }) From 2f26553ccd324bb780593a4003062d37837cefec Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:59 +0000 Subject: [PATCH 021/188] refactor: SWITCH and IF type extraction correction --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 64 +++++++++++++------ 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index f2281c89f1..6bdabf0dcc 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -664,16 +664,28 @@ const formulas: Record = { 'IF(5 > 1, "YES", "NO") => "YES"', 'IF({column} > 1, "YES", "NO")', ], - returnType: (argsTypes: FormulaDataTypes[]) => { - if (argsTypes.slice(1).includes(FormulaDataTypes.STRING)) { + returnType: (argTypes: FormulaDataTypes[]) => { + // extract all return types except NULL, since null can be returned by any type + const returnValueTypes = new Set( + argTypes.slice(1).filter((type) => type !== FormulaDataTypes.NULL) + ); + // if there are more than one return types or if there is a string return type + // return type as string else return the type + if ( + returnValueTypes.size > 1 || + returnValueTypes.has(FormulaDataTypes.STRING) + ) { return FormulaDataTypes.STRING; - } else if (argsTypes.slice(1).includes(FormulaDataTypes.NUMERIC)) { + } else if (returnValueTypes.has(FormulaDataTypes.NUMERIC)) { return FormulaDataTypes.NUMERIC; - } else if (argsTypes.slice(1).includes(FormulaDataTypes.BOOLEAN)) { + } else if (returnValueTypes.has(FormulaDataTypes.BOOLEAN)) { return FormulaDataTypes.BOOLEAN; + } else if (returnValueTypes.has(FormulaDataTypes.DATE)) { + return FormulaDataTypes.DATE; } - return argsTypes[1]; + // if none of the above conditions are met, return the first return argument type + return argTypes[1]; }, }, SWITCH: { @@ -681,7 +693,7 @@ const formulas: Record = { validation: { args: { min: 3, - }, + } }, description: 'Switch case value based on expr output', syntax: 'SWITCH(expr, [pattern, value, ..., default])', @@ -691,19 +703,29 @@ const formulas: Record = { 'SWITCH(3, 1, "One", 2, "Two", "N/A") => "N/A"', 'SWITCH({column1}, 1, "One", 2, "Two", "N/A")', ], - // todo: resolve return type based on the args returnType: (argTypes: FormulaDataTypes[]) => { - const returnArgTypes = argTypes.slice(2).filter((_, i) => i % 2 === 0); + // extract all return types except NULL, since null can be returned by any type + const returnValueTypes = new Set( + argTypes.slice(2).filter((_, i) => i % 2 === 0) + ); - if (returnArgTypes.includes(FormulaDataTypes.STRING)) { + // if there are more than one return types or if there is a string return type + // return type as string else return the type + if ( + returnValueTypes.size > 1 || + returnValueTypes.has(FormulaDataTypes.STRING) + ) { return FormulaDataTypes.STRING; - } else if (returnArgTypes.includes(FormulaDataTypes.NUMERIC)) { + } else if (returnValueTypes.has(FormulaDataTypes.NUMERIC)) { return FormulaDataTypes.NUMERIC; - } else if (returnArgTypes.includes(FormulaDataTypes.BOOLEAN)) { + } else if (returnValueTypes.has(FormulaDataTypes.BOOLEAN)) { return FormulaDataTypes.BOOLEAN; + } else if (returnValueTypes.has(FormulaDataTypes.DATE)) { + return FormulaDataTypes.DATE; } - return returnArgTypes[0]; + // if none of the above conditions are met, return the first return argument type + return argTypes[1]; }, }, URL: { @@ -965,13 +987,13 @@ const formulas: Record = { enum FormulaErrorType { NOT_AVAILABLE = 'NOT_AVAILABLE', NOT_SUPPORTED = 'NOT_SUPPORTED', - 'MIN_ARG' = 'MIN_ARG', - 'MAX_ARG' = 'MAX_ARG', - 'TYPE_MISMATCH' = 'TYPE_MISMATCH', - 'INVALID_ARG' = 'INVALID_ARG', - 'INVALID_ARG_TYPE' = 'INVALID_ARG_TYPE', - 'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE', - 'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT', + MIN_ARG = 'MIN_ARG', + MAX_ARG = 'MAX_ARG', + TYPE_MISMATCH = 'TYPE_MISMATCH', + INVALID_ARG = 'INVALID_ARG', + INVALID_ARG_TYPE = 'INVALID_ARG_TYPE', + INVALID_ARG_VALUE = 'INVALID_ARG_VALUE', + INVALID_ARG_COUNT = 'INVALID_ARG_COUNT', CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE', INVALID_FUNCTION_NAME = 'INVALID_FUNCTION_NAME', } @@ -1075,11 +1097,11 @@ export function validateFormulaAndExtractTreeWithType( } )); - const argsTypes = validateResult.map((v: any) => v.dataType); + const argTypes = validateResult.map((v: any) => v.dataType); if (typeof formulas[calleeName].returnType === 'function') { res.dataType = (formulas[calleeName].returnType as any)?.( - argsTypes + argTypes ) as FormulaDataTypes; } else if (formulas[calleeName].returnType) { res.dataType = formulas[calleeName].returnType as FormulaDataTypes; From 9e8871cf406abd1e2b5ec5c11d2b304c479c9ad1 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:00 +0000 Subject: [PATCH 022/188] fix: treat values in AND/OR based on type --- .../src/db/functionMappings/commonFns.ts | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts index d5d987c866..d8f1929ac5 100644 --- a/packages/nocodb/src/db/functionMappings/commonFns.ts +++ b/packages/nocodb/src/db/functionMappings/commonFns.ts @@ -2,6 +2,31 @@ import { FormulaDataTypes } from 'nocodb-sdk'; import type { MapFnArgs } from '../mapFunctionName'; import { NcError } from '~/helpers/catchError'; +async function treatArgAsConditionalExp( + args: MapFnArgs, + argument = args.pt?.arguments?.[0], +) { + const condArg = (await args.fn(argument)).builder.toQuery(); + + let cond = condArg; + + switch (argument.dataType as FormulaDataTypes) { + case FormulaDataTypes.NUMERIC: + cond = `(${condArg}) IS NOT NULL AND (${condArg}) != 0`; + break; + case FormulaDataTypes.STRING: + cond = `(${condArg}) IS NOT NULL AND (${condArg}) != ''`; + break; + case FormulaDataTypes.BOOLEAN: + cond = `(${condArg}) IS NOT NULL AND (${condArg}) != false`; + break; + case FormulaDataTypes.DATE: + cond = `(${condArg}) IS NOT NULL`; + break; + } + return { builder: args.knex.raw(cond) }; +} + export default { // todo: handle default case SWITCH: async (args: MapFnArgs) => { @@ -37,24 +62,7 @@ export default { }; }, IF: async (args: MapFnArgs) => { - const condArg = (await args.fn(args.pt.arguments[0])).builder.toQuery(); - - let cond = condArg; - - switch (args.pt.arguments[0].dataType as FormulaDataTypes) { - case FormulaDataTypes.NUMERIC: - cond = `(${condArg}) IS NOT NULL AND (${condArg}) != 0`; - break; - case FormulaDataTypes.STRING: - cond = `(${condArg}) IS NOT NULL AND (${condArg}) != ''`; - break; - case FormulaDataTypes.BOOLEAN: - cond = `(${condArg}) IS NOT NULL AND (${condArg}) != false`; - break; - case FormulaDataTypes.DATE: - cond = `(${condArg}) IS NOT NULL`; - break; - } + const cond = await treatArgAsConditionalExp(args); let query = args.knex .raw( @@ -80,7 +88,7 @@ export default { `${( await Promise.all( args.pt.arguments.map(async (ar) => - (await args.fn(ar)).builder.toQuery(), + (await treatArgAsConditionalExp(args, ar)).builder.toQuery(), ), ) ).join(' AND ')}`, @@ -98,7 +106,7 @@ export default { `${( await Promise.all( args.pt.arguments.map(async (ar) => - (await args.fn(ar)).builder.toQuery(), + (await treatArgAsConditionalExp(args, ar)).builder.toQuery(), ), ) ).join(' OR ')}`, From ef36baa15e26bf6a0d6890a6b1ec325e88eb5057 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:00 +0000 Subject: [PATCH 023/188] fix: treat values in AND/OR based on type --- packages/nocodb/src/db/functionMappings/commonFns.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts index d8f1929ac5..205bb2f339 100644 --- a/packages/nocodb/src/db/functionMappings/commonFns.ts +++ b/packages/nocodb/src/db/functionMappings/commonFns.ts @@ -10,6 +10,11 @@ async function treatArgAsConditionalExp( let cond = condArg; + // based on the data type of the argument, we need to handle the condition + // if string - value is not null and not empty then true + // if number - value is not null and not 0 then true + // if boolean - value is not null and not false then true + // if date - value is not null then true switch (argument.dataType as FormulaDataTypes) { case FormulaDataTypes.NUMERIC: cond = `(${condArg}) IS NOT NULL AND (${condArg}) != 0`; From 19be58a420570762b19137eebddd4e925c6f0ee2 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:00 +0000 Subject: [PATCH 024/188] feat: cast values to string if there are different types values in IF --- .../src/db/functionMappings/commonFns.ts | 54 ++++++++++++++----- .../nocodb/src/db/functionMappings/mysql.ts | 9 ++++ packages/nocodb/src/db/functionMappings/pg.ts | 9 ++++ 3 files changed, 59 insertions(+), 13 deletions(-) diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts index 205bb2f339..e6382a3a07 100644 --- a/packages/nocodb/src/db/functionMappings/commonFns.ts +++ b/packages/nocodb/src/db/functionMappings/commonFns.ts @@ -67,24 +67,52 @@ export default { }; }, IF: async (args: MapFnArgs) => { - const cond = await treatArgAsConditionalExp(args); + const cond = (await treatArgAsConditionalExp(args)).builder; + let thenArg; + let elseArg; + const returnArgsType = new Set( + [args.pt.arguments[1].dataType, args.pt.arguments[2].dataType].filter( + (type) => type !== FormulaDataTypes.NULL, + ), + ); + // cast to string if the return value types are different + if (returnArgsType.size > 1) { + thenArg = ( + await args.fn({ + type: 'CallExpression', + arguments: [args.pt.arguments[1]], + callee: { + type: 'Identifier', + name: 'STRING', + }, + } as any) + ).builder; + elseArg = ( + await args.fn({ + type: 'CallExpression', + arguments: [args.pt.arguments[2]], + callee: { + type: 'Identifier', + name: 'STRING', + }, + } as any) + ).builder; + } else { + thenArg = (await args.fn(args.pt.arguments[1])).builder.toQuery(); + elseArg = (await args.fn(args.pt.arguments[1])).builder.toQuery(); + } - let query = args.knex - .raw( - `\n\tWHEN ${cond} THEN ${( - await args.fn(args.pt.arguments[1]) - ).builder.toQuery()}`, - ) - .toQuery(); + let query = args.knex.raw(`\n\tWHEN ${cond} THEN ${thenArg}`).toQuery(); if (args.pt.arguments[2]) { - query += args.knex - .raw( - `\n\tELSE ${(await args.fn(args.pt.arguments[2])).builder.toQuery()}`, - ) - .toQuery(); + query += args.knex.raw(`\n\tELSE ${elseArg}`).toQuery(); } return { builder: args.knex.raw(`CASE ${query}\n END${args.colAlias}`) }; }, + // used only for casting to string internally, this one is dummy function + // and will work as fallback for dbs which don't support/implemented CAST + STRING(args: MapFnArgs) { + return args.fn(args.pt?.arguments?.[0]); + }, AND: async (args: MapFnArgs) => { return { builder: args.knex.raw( diff --git a/packages/nocodb/src/db/functionMappings/mysql.ts b/packages/nocodb/src/db/functionMappings/mysql.ts index 8026806876..588f51aa3a 100644 --- a/packages/nocodb/src/db/functionMappings/mysql.ts +++ b/packages/nocodb/src/db/functionMappings/mysql.ts @@ -160,6 +160,15 @@ END) ${colAlias}`, ), }; }, + STRING: async (args: MapFnArgs) => { + return { + builder: args.knex.raw( + `CAST(${(await args.fn(args.pt.arguments[0])).builder} AS CHAR) ${ + args.colAlias + }`, + ), + }; + }, }; export default mysql2; diff --git a/packages/nocodb/src/db/functionMappings/pg.ts b/packages/nocodb/src/db/functionMappings/pg.ts index a26180ab34..9ef7dff338 100644 --- a/packages/nocodb/src/db/functionMappings/pg.ts +++ b/packages/nocodb/src/db/functionMappings/pg.ts @@ -299,6 +299,15 @@ END) ${colAlias}`, ), }; }, + STRING: async (args: MapFnArgs) => { + return { + builder: args.knex.raw( + `(${(await args.fn(args.pt.arguments[0])).builder})::text ${ + args.colAlias + }`, + ), + }; + }, }; export default pg; From 23270c9078095021c0f8168509128f21020a78cb Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:00 +0000 Subject: [PATCH 025/188] feat: cast values to string if there are different types values in SWITCH --- .../src/db/functionMappings/commonFns.ts | 63 ++++++++++++++++--- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts index e6382a3a07..d8648b954e 100644 --- a/packages/nocodb/src/db/functionMappings/commonFns.ts +++ b/packages/nocodb/src/db/functionMappings/commonFns.ts @@ -38,27 +38,70 @@ export default { const count = Math.floor((args.pt.arguments.length - 1) / 2); let query = ''; + const returnArgsType = new Set( + args.pt.arguments + .filter( + (type, i) => i > 1 && i % 2 === 0 && type !== FormulaDataTypes.NULL, + ) + .map((type) => type.dataType), + ); + + // if else case present then push that to types + if (args.pt.arguments.length % 2 === 0) { + returnArgsType.add( + args.pt.arguments[args.pt.arguments.length - 1].dataType, + ); + } + const switchVal = (await args.fn(args.pt.arguments[0])).builder.toQuery(); for (let i = 0; i < count; i++) { + let val; + // cast to string if the return value types are different + if (returnArgsType.size > 1) { + val = ( + await args.fn({ + type: 'CallExpression', + arguments: [args.pt.arguments[i * 2 + 2]], + callee: { + type: 'Identifier', + name: 'STRING', + }, + } as any) + ).builder.toQuery(); + } else { + val = (await args.fn(args.pt.arguments[i * 2 + 2])).builder.toQuery(); + } + query += args.knex .raw( `\n\tWHEN ${( await args.fn(args.pt.arguments[i * 2 + 1]) - ).builder.toQuery()} THEN ${( - await args.fn(args.pt.arguments[i * 2 + 2]) - ).builder.toQuery()}`, + ).builder.toQuery()} THEN ${val}`, ) .toQuery(); } if (args.pt.arguments.length % 2 === 0) { - query += args.knex - .raw( - `\n\tELSE ${( - await args.fn(args.pt.arguments[args.pt.arguments.length - 1]) - ).builder.toQuery()}`, - ) - .toQuery(); + let val; + // cast to string if the return value types are different + if (returnArgsType.size > 1) { + val = ( + await args.fn({ + type: 'CallExpression', + arguments: [args.pt.arguments[args.pt.arguments.length - 1]], + callee: { + type: 'Identifier', + name: 'STRING', + }, + } as any) + ).builder.toQuery(); + } else { + val = ( + await args.fn(args.pt.arguments[args.pt.arguments.length - 1]) + ).builder.toQuery(); + } + + query += `\n\tELSE ${val}`; } return { builder: args.knex.raw( From a4df91819f7bb67e71595199f4fe63da9a7ed569 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:00 +0000 Subject: [PATCH 026/188] fix: formula if condition validation correction --- .../smartsheet/column/FormulaOptions.vue | 27 ++++++++++++++----- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 9 ++++--- .../src/db/functionMappings/commonFns.ts | 2 +- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue index 6e5039b0f7..45624cf039 100644 --- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue +++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue @@ -10,7 +10,7 @@ import { isSystemColumn, jsepCurlyHook, substituteColumnIdWithAliasInFormula, - validateDateWithUnknownFormat, + validateFormulaAndExtractTreeWithType } from 'nocodb-sdk' import { MetaInj, @@ -99,10 +99,15 @@ const validators = { validator: (_: any, formula: any) => { return new Promise((resolve, reject) => { if (!formula?.trim()) return reject(new Error('Required')) - const res = parseAndValidateFormula(formula) - if (res !== true) { - return reject(new Error(res)) + + try { + validateFormulaAndExtractTreeWithType(formula, supportedColumns.value) + } catch (e: any) { + return reject(new Error(e.message)) } + // if (res !== true) { + // return reject(new Error(res)) + // } resolve() }) }, @@ -226,7 +231,7 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n return validateAgainstMeta(arg, errors, typeErrors) }) - const argsTypes = validateResult.map((v: any) => v.returnType); + const argsTypes = validateResult.map((v: any) => v.returnType) if (typeof validateResult[0].returnType === 'function') { returnType = formulas[calleeName].returnType(argsTypes) @@ -266,7 +271,9 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n typeErrors, ) } else { - parsedTree.arguments.map((arg: Record) => validateAgainstType(arg, expectedType, null, typeErrors, argsTypes)) + parsedTree.arguments.map((arg: Record) => + validateAgainstType(arg, expectedType, null, typeErrors, argsTypes), + ) } } else if (expectedType === formulaTypes.DATE) { if (calleeName === 'DATEADD') { @@ -488,7 +495,13 @@ function validateAgainstMeta(parsedTree: any, errors = new Set(), typeErrors = n return { errors, returnType } } -function validateAgainstType(parsedTree: any, expectedType: string, func: any, typeErrors = new Set(), argTypes: formulaTypes = []) { +function validateAgainstType( + parsedTree: any, + expectedType: string, + func: any, + typeErrors = new Set(), + argTypes: formulaTypes = [], +) { let type if (parsedTree === false || typeof parsedTree === 'undefined') { return typeErrors diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 6bdabf0dcc..c93a564bb5 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -693,7 +693,7 @@ const formulas: Record = { validation: { args: { min: 3, - } + }, }, description: 'Switch case value based on expr output', syntax: 'SWITCH(expr, [pattern, value, ..., default])', @@ -1169,7 +1169,7 @@ export function validateFormulaAndExtractTreeWithType( case UITypes.Collaborator: case UITypes.QrCode: default: - throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, ''); + throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, {}); } } } else if (parsedTree.type === JSEPNode.LITERAL) { @@ -1188,7 +1188,10 @@ export function validateFormulaAndExtractTreeWithType( ) { res.left = validateAndExtract(parsedTree.left); res.right = validateAndExtract(parsedTree.right); - res.dataType = FormulaDataTypes.NUMERIC; + + if (['==', '<', '>', '<=', '>=', '!='].includes(parsedTree.operator)) { + res.dataType = FormulaDataTypes.COND_EXP; + } else res.dataType = FormulaDataTypes.NUMERIC; } return res; diff --git a/packages/nocodb/src/db/functionMappings/commonFns.ts b/packages/nocodb/src/db/functionMappings/commonFns.ts index d8648b954e..ef3f8b5db3 100644 --- a/packages/nocodb/src/db/functionMappings/commonFns.ts +++ b/packages/nocodb/src/db/functionMappings/commonFns.ts @@ -142,7 +142,7 @@ export default { ).builder; } else { thenArg = (await args.fn(args.pt.arguments[1])).builder.toQuery(); - elseArg = (await args.fn(args.pt.arguments[1])).builder.toQuery(); + elseArg = (await args.fn(args.pt.arguments[2])).builder.toQuery(); } let query = args.knex.raw(`\n\tWHEN ${cond} THEN ${thenArg}`).toQuery(); From aa7bc17c1c8c195c6fa0b6a258b7aaf1daa0df4f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:00 +0000 Subject: [PATCH 027/188] refactor: WEEKDAY validation --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 207 ++++++++++++++++-- 1 file changed, 192 insertions(+), 15 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index c93a564bb5..63b970b327 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -2,6 +2,7 @@ import jsep from 'jsep'; import { ColumnType } from './Api'; import UITypes from './UITypes'; +import { validateDateWithUnknownFormat } from '../../../nc-gui/utils'; export const jsepCurlyHook = { name: 'curly', @@ -73,6 +74,20 @@ export async function substituteColumnAliasWithIdInFormula( return jsepTreeToFormula(parsedFormula); } +enum FormulaErrorType { + NOT_AVAILABLE = 'NOT_AVAILABLE', + NOT_SUPPORTED = 'NOT_SUPPORTED', + MIN_ARG = 'MIN_ARG', + MAX_ARG = 'MAX_ARG', + TYPE_MISMATCH = 'TYPE_MISMATCH', + INVALID_ARG = 'INVALID_ARG', + INVALID_ARG_TYPE = 'INVALID_ARG_TYPE', + INVALID_ARG_VALUE = 'INVALID_ARG_VALUE', + INVALID_ARG_COUNT = 'INVALID_ARG_COUNT', + CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE', + INVALID_FUNCTION_NAME = 'INVALID_FUNCTION_NAME', +} + export function substituteColumnIdWithAliasInFormula( formula, columns: ColumnType[], @@ -747,6 +762,40 @@ const formulas: Record = { min: 1, max: 2, }, + validator(argTypes: FormulaDataTypes[], parsedTree: any) { + if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { + if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { + throw new FormulaError( + FormulaErrorType.TYPE_MISMATCH, + { key: 'msg.formula.firstParamWeekDayHaveDate' }, + 'First parameter of WEEKDY should be a date' + ); + } + } + + if (parsedTree.arguments[1].type === JSEPNode.LITERAL) { + const value = parsedTree.arguments[0].value; + if ( + typeof value !== 'string' || + ![ + 'sunday', + 'monday', + 'tuesday', + 'wednesday', + 'thursday', + 'friday', + 'saturday', + ].includes(value.toLowerCase()) + ) { + typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate')); + throw new FormulaError( + FormulaErrorType.TYPE_MISMATCH, + { key: 'msg.formula.secondParamWeekDayHaveDate' }, + 'Second parameter of WEEKDY should be day of week string' + ); + } + } + }, }, description: 'Returns the day of the week as an integer between 0 and 6 inclusive starting from Monday by default', @@ -984,20 +1033,6 @@ const formulas: Record = { // }, }; -enum FormulaErrorType { - NOT_AVAILABLE = 'NOT_AVAILABLE', - NOT_SUPPORTED = 'NOT_SUPPORTED', - MIN_ARG = 'MIN_ARG', - MAX_ARG = 'MAX_ARG', - TYPE_MISMATCH = 'TYPE_MISMATCH', - INVALID_ARG = 'INVALID_ARG', - INVALID_ARG_TYPE = 'INVALID_ARG_TYPE', - INVALID_ARG_VALUE = 'INVALID_ARG_VALUE', - INVALID_ARG_COUNT = 'INVALID_ARG_COUNT', - CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE', - INVALID_FUNCTION_NAME = 'INVALID_FUNCTION_NAME', -} - class FormulaError extends Error { public type: FormulaErrorType; public extra: Record; @@ -1043,7 +1078,149 @@ export function validateFormulaAndExtractTreeWithType( {}, 'Function not available' ); - //t('msg.formula.functionNotAvailable', { function: calleeName }) + + // validate data type + if (parsedTree.callee.type === JSEPNode.IDENTIFIER) { + const expectedType = formulas[calleeName.toUpperCase()].type; + if (expectedType === formulaTypes.NUMERIC) { + if (calleeName === 'WEEKDAY') { + // parsedTree.arguments[0] = date + validateAgainstType( + parsedTree.arguments[0], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate')); + } + }, + typeErrors + ); + // parsedTree.arguments[1] = startDayOfWeek (optional) + validateAgainstType( + parsedTree.arguments[1], + formulaTypes.STRING, + (v: any) => { + if ( + typeof v !== 'string' || + ![ + 'sunday', + 'monday', + 'tuesday', + 'wednesday', + 'thursday', + 'friday', + 'saturday', + ].includes(v.toLowerCase()) + ) { + typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate')); + } + }, + typeErrors + ); + } else { + parsedTree.arguments.map((arg: Record) => + validateAgainstType(arg, expectedType, null, typeErrors) + ); + } + } else if (expectedType === formulaTypes.DATE) { + if (calleeName === 'DATEADD') { + // parsedTree.arguments[0] = date + validateAgainstType( + parsedTree.arguments[0], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.firstParamDateAddHaveDate')); + } + }, + typeErrors + ); + // parsedTree.arguments[1] = numeric + validateAgainstType( + parsedTree.arguments[1], + formulaTypes.NUMERIC, + (v: any) => { + if (typeof v !== 'number') { + typeErrors.add( + t('msg.formula.secondParamDateAddHaveNumber') + ); + } + }, + typeErrors + ); + // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"] + validateAgainstType( + parsedTree.arguments[2], + formulaTypes.STRING, + (v: any) => { + if (!['day', 'week', 'month', 'year'].includes(v)) { + typeErrors.add( + typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate')) + ); + } + }, + typeErrors + ); + } else if (calleeName === 'DATETIME_DIFF') { + // parsedTree.arguments[0] = date + validateAgainstType( + parsedTree.arguments[0], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate')); + } + }, + typeErrors + ); + // parsedTree.arguments[1] = date + validateAgainstType( + parsedTree.arguments[1], + formulaTypes.DATE, + (v: any) => { + if (!validateDateWithUnknownFormat(v)) { + typeErrors.add( + t('msg.formula.secondParamDateDiffHaveDate') + ); + } + }, + typeErrors + ); + // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"] + validateAgainstType( + parsedTree.arguments[2], + formulaTypes.STRING, + (v: any) => { + if ( + ![ + 'milliseconds', + 'ms', + 'seconds', + 's', + 'minutes', + 'm', + 'hours', + 'h', + 'days', + 'd', + 'weeks', + 'w', + 'months', + 'M', + 'quarters', + 'Q', + 'years', + 'y', + ].includes(v) + ) { + typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate')); + } + }, + typeErrors + ); + } + } + } } // validate arguments const validation = From c3142e00a9329b9c92b7ffcd0b1f2cf13ca7b3e3 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:00 +0000 Subject: [PATCH 028/188] refactor: DATEADD validation --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 63b970b327..01c7619831 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -282,6 +282,40 @@ const formulas: Record = { args: { rqd: 3, }, + validator: (args: FormulaDataTypes[], parsedTree: any) => { + if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { + if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { + throw new FormulaError( + FormulaErrorType.TYPE_MISMATCH, + { key: 'msg.formula.firstParamDateAddHaveDate' }, + 'First parameter of DATEADD should be a date' + ); + } + } + + if (parsedTree.arguments[1].type === JSEPNode.LITERAL) { + if (typeof parsedTree.arguments[1].value !== 'number') { + throw new FormulaError( + FormulaErrorType.TYPE_MISMATCH, + { key: 'msg.formula.secondParamDateAddHaveNumber' }, + 'Second parameter of DATEADD should be a number' + ); + } + } + if (parsedTree.arguments[2].type === JSEPNode.LITERAL) { + if ( + !['day', 'week', 'month', 'year'].includes( + parsedTree.arguments[2].value + ) + ) { + throw new FormulaError( + FormulaErrorType.TYPE_MISMATCH, + { key: 'msg.formula.thirdParamDateAddHaveDate' }, + "Third parameter of DATEADD should be one of 'day', 'week', 'month', 'year'" + ); + } + } + }, }, description: 'Adds a "count" units to Datetime.', syntax: @@ -768,7 +802,7 @@ const formulas: Record = { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.firstParamWeekDayHaveDate' }, - 'First parameter of WEEKDY should be a date' + 'First parameter of WEEKDAY should be a date' ); } } @@ -787,11 +821,10 @@ const formulas: Record = { 'saturday', ].includes(value.toLowerCase()) ) { - typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate')); throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, { key: 'msg.formula.secondParamWeekDayHaveDate' }, - 'Second parameter of WEEKDY should be day of week string' + 'Second parameter of WEEKDAY should be day of week string' ); } } From 7f02ac7c2dc606bca2793976fefe189101b37122 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:00 +0000 Subject: [PATCH 029/188] refactor: DATETIME_DIFF validation --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 01c7619831..2180052c72 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -339,6 +339,56 @@ const formulas: Record = { min: 2, max: 3, }, + validator: (args: FormulaDataTypes[], parsedTree: any) => { + + if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { + if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { + throw new FormulaError( + FormulaErrorType.TYPE_MISMATCH, + { key: 'msg.formula.firstParamDateDiffHaveDate' }, + 'First parameter of DATETIME_DIFF should be a date' + ); + } + } + + if (parsedTree.arguments[1].type === JSEPNode.LITERAL) { + if (!validateDateWithUnknownFormat(parsedTree.arguments[1].value)) { + throw new FormulaError( + FormulaErrorType.TYPE_MISMATCH, + { key: 'msg.formula.secondParamDateDiffHaveDate' }, + 'Second parameter of DATETIME_DIFF should be a date' + ); + } + } + if (parsedTree.arguments[2].type === JSEPNode.LITERAL) { + if (![ + 'milliseconds', + 'ms', + 'seconds', + 's', + 'minutes', + 'm', + 'hours', + 'h', + 'days', + 'd', + 'weeks', + 'w', + 'months', + 'M', + 'quarters', + 'Q', + 'years', + 'y', + ].includes(parsedTree.arguments[0].value)) { + throw new FormulaError( + FormulaErrorType.TYPE_MISMATCH, + { key: 'msg.formula.thirdParamDateDiffHaveDate' }, + 'Third parameter of DATETIME_DIFF should be one of \'milliseconds\', \'ms\', \'seconds\', \'s\', \'minutes\', \'m\', \'hours\', \'h\', \'days\', \'d\', \'weeks\', \'w\', \'months\', \'M\', \'quarters\', \'Q\', \'years\', \'y\''' + ); + } + } + } }, description: 'Calculate the difference of two given date / datetime in specified units.', From 248155ec0d8205e8ea3eebf221fe3f5200d7efb8 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:00 +0000 Subject: [PATCH 030/188] refactor: move custom validation to formula meta object --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 189 +++--------------- 1 file changed, 26 insertions(+), 163 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 2180052c72..1fd1d78e62 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1,8 +1,8 @@ import jsep from 'jsep'; -import { ColumnType } from './Api'; +import {ColumnType} from './Api'; import UITypes from './UITypes'; -import { validateDateWithUnknownFormat } from '../../../nc-gui/utils'; +import {validateDateWithUnknownFormat} from '../../../nc-gui/utils'; export const jsepCurlyHook = { name: 'curly', @@ -11,7 +11,7 @@ export const jsepCurlyHook = { const OCURLY_CODE = 123; // { const CCURLY_CODE = 125; // } let start = -1; - const { context } = env; + const {context} = env; if ( !jsep.isIdentifierStart(context.code) && context.code === OCURLY_CODE @@ -28,10 +28,10 @@ export const jsepCurlyHook = { name: /{{(.*?)}}/.test(context.expr) ? // start would be the position of the first curly bracket // add 2 to point to the first character for expressions like {{col1}} - context.expr.slice(start + 2, context.index - 1) + context.expr.slice(start + 2, context.index - 1) : // start would be the position of the first curly bracket // add 1 to point to the first character for expressions like {col1} - context.expr.slice(start + 1, context.index - 1), + context.expr.slice(start + 1, context.index - 1), }; return env.node; } else { @@ -235,8 +235,8 @@ interface FormulaMeta { min?: number; max?: number; rqd?: number; - validator?: (args: FormulaDataTypes[]) => boolean; }; + custom?: (args: FormulaDataTypes[], parseTree: any) => void; }; description?: string; syntax?: string; @@ -282,12 +282,12 @@ const formulas: Record = { args: { rqd: 3, }, - validator: (args: FormulaDataTypes[], parsedTree: any) => { + custom: (args: FormulaDataTypes[], parsedTree: any) => { if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - { key: 'msg.formula.firstParamDateAddHaveDate' }, + {key: 'msg.formula.firstParamDateAddHaveDate'}, 'First parameter of DATEADD should be a date' ); } @@ -297,7 +297,7 @@ const formulas: Record = { if (typeof parsedTree.arguments[1].value !== 'number') { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - { key: 'msg.formula.secondParamDateAddHaveNumber' }, + {key: 'msg.formula.secondParamDateAddHaveNumber'}, 'Second parameter of DATEADD should be a number' ); } @@ -310,7 +310,7 @@ const formulas: Record = { ) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - { key: 'msg.formula.thirdParamDateAddHaveDate' }, + {key: 'msg.formula.thirdParamDateAddHaveDate'}, "Third parameter of DATEADD should be one of 'day', 'week', 'month', 'year'" ); } @@ -339,13 +339,13 @@ const formulas: Record = { min: 2, max: 3, }, - validator: (args: FormulaDataTypes[], parsedTree: any) => { + custom: (args: FormulaDataTypes[], parsedTree: any) => { if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - { key: 'msg.formula.firstParamDateDiffHaveDate' }, + {key: 'msg.formula.firstParamDateDiffHaveDate'}, 'First parameter of DATETIME_DIFF should be a date' ); } @@ -355,7 +355,7 @@ const formulas: Record = { if (!validateDateWithUnknownFormat(parsedTree.arguments[1].value)) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - { key: 'msg.formula.secondParamDateDiffHaveDate' }, + {key: 'msg.formula.secondParamDateDiffHaveDate'}, 'Second parameter of DATETIME_DIFF should be a date' ); } @@ -383,8 +383,8 @@ const formulas: Record = { ].includes(parsedTree.arguments[0].value)) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - { key: 'msg.formula.thirdParamDateDiffHaveDate' }, - 'Third parameter of DATETIME_DIFF should be one of \'milliseconds\', \'ms\', \'seconds\', \'s\', \'minutes\', \'m\', \'hours\', \'h\', \'days\', \'d\', \'weeks\', \'w\', \'months\', \'M\', \'quarters\', \'Q\', \'years\', \'y\''' + {key: 'msg.formula.thirdParamDateDiffHaveDate'}, + 'Third parameter of DATETIME_DIFF should be one of \'milliseconds\', \'ms\', \'seconds\', \'s\', \'minutes\', \'m\', \'hours\', \'h\', \'days\', \'d\', \'weeks\', \'w\', \'months\', \'M\', \'quarters\', \'Q\', \'years\', \'y\'' ); } } @@ -846,12 +846,12 @@ const formulas: Record = { min: 1, max: 2, }, - validator(argTypes: FormulaDataTypes[], parsedTree: any) { + custom(argTypes: FormulaDataTypes[], parsedTree: any) { if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - { key: 'msg.formula.firstParamWeekDayHaveDate' }, + {key: 'msg.formula.firstParamWeekDayHaveDate'}, 'First parameter of WEEKDAY should be a date' ); } @@ -873,7 +873,7 @@ const formulas: Record = { ) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - { key: 'msg.formula.secondParamWeekDayHaveDate' }, + {key: 'msg.formula.secondParamWeekDayHaveDate'}, 'Second parameter of WEEKDAY should be day of week string' ); } @@ -1150,7 +1150,7 @@ export function validateFormulaAndExtractTreeWithType( dataType?: FormulaDataTypes; errors?: Set; [key: string]: any; - } = { ...parsedTree }; + } = {...parsedTree}; if (parsedTree.type === JSEPNode.CALL_EXP) { const calleeName = parsedTree.callee.name.toUpperCase(); @@ -1161,150 +1161,8 @@ export function validateFormulaAndExtractTreeWithType( {}, 'Function not available' ); - - // validate data type - if (parsedTree.callee.type === JSEPNode.IDENTIFIER) { - const expectedType = formulas[calleeName.toUpperCase()].type; - if (expectedType === formulaTypes.NUMERIC) { - if (calleeName === 'WEEKDAY') { - // parsedTree.arguments[0] = date - validateAgainstType( - parsedTree.arguments[0], - formulaTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[1] = startDayOfWeek (optional) - validateAgainstType( - parsedTree.arguments[1], - formulaTypes.STRING, - (v: any) => { - if ( - typeof v !== 'string' || - ![ - 'sunday', - 'monday', - 'tuesday', - 'wednesday', - 'thursday', - 'friday', - 'saturday', - ].includes(v.toLowerCase()) - ) { - typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate')); - } - }, - typeErrors - ); - } else { - parsedTree.arguments.map((arg: Record) => - validateAgainstType(arg, expectedType, null, typeErrors) - ); - } - } else if (expectedType === formulaTypes.DATE) { - if (calleeName === 'DATEADD') { - // parsedTree.arguments[0] = date - validateAgainstType( - parsedTree.arguments[0], - formulaTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamDateAddHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[1] = numeric - validateAgainstType( - parsedTree.arguments[1], - formulaTypes.NUMERIC, - (v: any) => { - if (typeof v !== 'number') { - typeErrors.add( - t('msg.formula.secondParamDateAddHaveNumber') - ); - } - }, - typeErrors - ); - // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"] - validateAgainstType( - parsedTree.arguments[2], - formulaTypes.STRING, - (v: any) => { - if (!['day', 'week', 'month', 'year'].includes(v)) { - typeErrors.add( - typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate')) - ); - } - }, - typeErrors - ); - } else if (calleeName === 'DATETIME_DIFF') { - // parsedTree.arguments[0] = date - validateAgainstType( - parsedTree.arguments[0], - formulaTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[1] = date - validateAgainstType( - parsedTree.arguments[1], - formulaTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add( - t('msg.formula.secondParamDateDiffHaveDate') - ); - } - }, - typeErrors - ); - // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"] - validateAgainstType( - parsedTree.arguments[2], - formulaTypes.STRING, - (v: any) => { - if ( - ![ - 'milliseconds', - 'ms', - 'seconds', - 's', - 'minutes', - 'm', - 'hours', - 'h', - 'days', - 'd', - 'weeks', - 'w', - 'months', - 'M', - 'quarters', - 'Q', - 'years', - 'y', - ].includes(v) - ) { - typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate')); - } - }, - typeErrors - ); - } - } - } } + // validate arguments const validation = formulas[calleeName] && formulas[calleeName].validation; @@ -1359,6 +1217,11 @@ export function validateFormulaAndExtractTreeWithType( const argTypes = validateResult.map((v: any) => v.dataType); + // if validation function is present, call it + if (formulas[calleeName].validation?.custom) { + formulas[calleeName].validation?.custom(argTypes, parsedTree); + } + if (typeof formulas[calleeName].returnType === 'function') { res.dataType = (formulas[calleeName].returnType as any)?.( argTypes @@ -1487,7 +1350,7 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { ]; if (neighbours.length > 0) { // e.g. formula column 1 -> [formula column 2, formula column3] - res.push({ [c.id]: neighbours }); + res.push({[c.id]: neighbours}); } return res; }, []); From f7b53e0d45fdcf1d4c3731c86dd54c045de467c1 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:00 +0000 Subject: [PATCH 031/188] fix: import statement corrections --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 104 +++++++++++------- 1 file changed, 66 insertions(+), 38 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 1fd1d78e62..efb3031713 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1,8 +1,35 @@ import jsep from 'jsep'; -import {ColumnType} from './Api'; +import { ColumnType } from './Api'; import UITypes from './UITypes'; -import {validateDateWithUnknownFormat} from '../../../nc-gui/utils'; +import dayjs from 'dayjs'; + +// todo: move to date utils and export, remove duplicate from gui + +export const dateFormats = [ + 'YYYY-MM-DD', + 'YYYY/MM/DD', + 'DD-MM-YYYY', + 'MM-DD-YYYY', + 'DD/MM/YYYY', + 'MM/DD/YYYY', + 'DD MM YYYY', + 'MM DD YYYY', + 'YYYY MM DD', +]; +function validateDateWithUnknownFormat(v: string) { + for (const format of dateFormats) { + if (dayjs(v, format, true).isValid() as any) { + return true; + } + for (const timeFormat of ['HH:mm', 'HH:mm:ss', 'HH:mm:ss.SSS']) { + if (dayjs(v, `${format} ${timeFormat}`, true).isValid() as any) { + return true; + } + } + } + return false; +} export const jsepCurlyHook = { name: 'curly', @@ -11,7 +38,7 @@ export const jsepCurlyHook = { const OCURLY_CODE = 123; // { const CCURLY_CODE = 125; // } let start = -1; - const {context} = env; + const { context } = env; if ( !jsep.isIdentifierStart(context.code) && context.code === OCURLY_CODE @@ -28,10 +55,10 @@ export const jsepCurlyHook = { name: /{{(.*?)}}/.test(context.expr) ? // start would be the position of the first curly bracket // add 2 to point to the first character for expressions like {{col1}} - context.expr.slice(start + 2, context.index - 1) + context.expr.slice(start + 2, context.index - 1) : // start would be the position of the first curly bracket // add 1 to point to the first character for expressions like {col1} - context.expr.slice(start + 1, context.index - 1), + context.expr.slice(start + 1, context.index - 1), }; return env.node; } else { @@ -287,7 +314,7 @@ const formulas: Record = { if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - {key: 'msg.formula.firstParamDateAddHaveDate'}, + { key: 'msg.formula.firstParamDateAddHaveDate' }, 'First parameter of DATEADD should be a date' ); } @@ -297,7 +324,7 @@ const formulas: Record = { if (typeof parsedTree.arguments[1].value !== 'number') { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - {key: 'msg.formula.secondParamDateAddHaveNumber'}, + { key: 'msg.formula.secondParamDateAddHaveNumber' }, 'Second parameter of DATEADD should be a number' ); } @@ -310,7 +337,7 @@ const formulas: Record = { ) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - {key: 'msg.formula.thirdParamDateAddHaveDate'}, + { key: 'msg.formula.thirdParamDateAddHaveDate' }, "Third parameter of DATEADD should be one of 'day', 'week', 'month', 'year'" ); } @@ -340,12 +367,11 @@ const formulas: Record = { max: 3, }, custom: (args: FormulaDataTypes[], parsedTree: any) => { - if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - {key: 'msg.formula.firstParamDateDiffHaveDate'}, + { key: 'msg.formula.firstParamDateDiffHaveDate' }, 'First parameter of DATETIME_DIFF should be a date' ); } @@ -355,40 +381,42 @@ const formulas: Record = { if (!validateDateWithUnknownFormat(parsedTree.arguments[1].value)) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - {key: 'msg.formula.secondParamDateDiffHaveDate'}, + { key: 'msg.formula.secondParamDateDiffHaveDate' }, 'Second parameter of DATETIME_DIFF should be a date' ); } } if (parsedTree.arguments[2].type === JSEPNode.LITERAL) { - if (![ - 'milliseconds', - 'ms', - 'seconds', - 's', - 'minutes', - 'm', - 'hours', - 'h', - 'days', - 'd', - 'weeks', - 'w', - 'months', - 'M', - 'quarters', - 'Q', - 'years', - 'y', - ].includes(parsedTree.arguments[0].value)) { + if ( + ![ + 'milliseconds', + 'ms', + 'seconds', + 's', + 'minutes', + 'm', + 'hours', + 'h', + 'days', + 'd', + 'weeks', + 'w', + 'months', + 'M', + 'quarters', + 'Q', + 'years', + 'y', + ].includes(parsedTree.arguments[0].value) + ) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - {key: 'msg.formula.thirdParamDateDiffHaveDate'}, - 'Third parameter of DATETIME_DIFF should be one of \'milliseconds\', \'ms\', \'seconds\', \'s\', \'minutes\', \'m\', \'hours\', \'h\', \'days\', \'d\', \'weeks\', \'w\', \'months\', \'M\', \'quarters\', \'Q\', \'years\', \'y\'' + { key: 'msg.formula.thirdParamDateDiffHaveDate' }, + "Third parameter of DATETIME_DIFF should be one of 'milliseconds', 'ms', 'seconds', 's', 'minutes', 'm', 'hours', 'h', 'days', 'd', 'weeks', 'w', 'months', 'M', 'quarters', 'Q', 'years', 'y'" ); } } - } + }, }, description: 'Calculate the difference of two given date / datetime in specified units.', @@ -851,7 +879,7 @@ const formulas: Record = { if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - {key: 'msg.formula.firstParamWeekDayHaveDate'}, + { key: 'msg.formula.firstParamWeekDayHaveDate' }, 'First parameter of WEEKDAY should be a date' ); } @@ -873,7 +901,7 @@ const formulas: Record = { ) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, - {key: 'msg.formula.secondParamWeekDayHaveDate'}, + { key: 'msg.formula.secondParamWeekDayHaveDate' }, 'Second parameter of WEEKDAY should be day of week string' ); } @@ -1150,7 +1178,7 @@ export function validateFormulaAndExtractTreeWithType( dataType?: FormulaDataTypes; errors?: Set; [key: string]: any; - } = {...parsedTree}; + } = { ...parsedTree }; if (parsedTree.type === JSEPNode.CALL_EXP) { const calleeName = parsedTree.callee.name.toUpperCase(); @@ -1350,7 +1378,7 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { ]; if (neighbours.length > 0) { // e.g. formula column 1 -> [formula column 2, formula column3] - res.push({[c.id]: neighbours}); + res.push({ [c.id]: neighbours }); } return res; }, []); From 27529b39d82e01317988b2b2b712c57f574431c0 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:01 +0000 Subject: [PATCH 032/188] fix: prefix unused vars with _ --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index efb3031713..1a95a477d9 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -309,7 +309,7 @@ const formulas: Record = { args: { rqd: 3, }, - custom: (args: FormulaDataTypes[], parsedTree: any) => { + custom: (_argTypes: FormulaDataTypes[], parsedTree: any) => { if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { throw new FormulaError( @@ -366,7 +366,7 @@ const formulas: Record = { min: 2, max: 3, }, - custom: (args: FormulaDataTypes[], parsedTree: any) => { + custom: (_argTypes: FormulaDataTypes[], parsedTree: any) => { if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { throw new FormulaError( @@ -874,7 +874,7 @@ const formulas: Record = { min: 1, max: 2, }, - custom(argTypes: FormulaDataTypes[], parsedTree: any) { + custom(_argTypes: FormulaDataTypes[], parsedTree: any) { if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { if (!validateDateWithUnknownFormat(parsedTree.arguments[0].value)) { throw new FormulaError( From ad4ac116a0c5f2bf52033ed7091b7daab1775fca Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:01 +0000 Subject: [PATCH 033/188] refactor: move arg type --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 99 ++++++++++--------- 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 1a95a477d9..64628670fb 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -256,12 +256,13 @@ export enum JSEPNode { } interface FormulaMeta { - type?: string; validation?: { args?: { min?: number; max?: number; rqd?: number; + + type?: FormulaDataTypes; }; custom?: (args: FormulaDataTypes[], parseTree: any) => void; }; @@ -276,6 +277,7 @@ const formulas: Record = { validation: { args: { min: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: 'Average of input parameters', @@ -288,10 +290,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, ADD: { - type: FormulaDataTypes.NUMERIC, validation: { args: { min: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: 'Sum of input parameters', @@ -304,10 +306,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, DATEADD: { - type: FormulaDataTypes.DATE, validation: { args: { rqd: 3, + type: FormulaDataTypes.DATE, }, custom: (_argTypes: FormulaDataTypes[], parsedTree: any) => { if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { @@ -360,11 +362,11 @@ const formulas: Record = { returnType: FormulaDataTypes.DATE, }, DATETIME_DIFF: { - type: FormulaDataTypes.DATE, validation: { args: { min: 2, max: 3, + type: FormulaDataTypes.DATE, }, custom: (_argTypes: FormulaDataTypes[], parsedTree: any) => { if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { @@ -458,10 +460,10 @@ const formulas: Record = { returnType: FormulaDataTypes.COND_EXP, }, CONCAT: { - type: FormulaDataTypes.STRING, validation: { args: { min: 1, + type: FormulaDataTypes.STRING, }, }, description: 'Concatenated string of input parameters', @@ -473,10 +475,10 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, TRIM: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 1, + type: FormulaDataTypes.STRING, }, }, description: 'Remove trailing and leading whitespaces from input parameter', @@ -488,10 +490,10 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, UPPER: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 1, + type: FormulaDataTypes.STRING, }, }, description: 'Upper case converted string of input parameter', @@ -500,10 +502,10 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, LOWER: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 1, + type: FormulaDataTypes.STRING, }, }, description: 'Lower case converted string of input parameter', @@ -512,10 +514,10 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, LEN: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 1, + type: FormulaDataTypes.STRING, }, }, description: 'Input parameter character length', @@ -524,10 +526,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, MIN: { - type: FormulaDataTypes.NUMERIC, validation: { args: { min: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: 'Minimum value amongst input parameters', @@ -536,10 +538,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, MAX: { - type: FormulaDataTypes.NUMERIC, validation: { args: { min: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: 'Maximum value amongst input parameters', @@ -548,10 +550,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, CEILING: { - type: FormulaDataTypes.NUMERIC, validation: { args: { rqd: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: 'Rounded next largest integer value of input parameter', @@ -560,10 +562,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, FLOOR: { - type: FormulaDataTypes.NUMERIC, validation: { args: { rqd: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: @@ -573,11 +575,11 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, ROUND: { - type: FormulaDataTypes.NUMERIC, validation: { args: { min: 1, max: 2, + type: FormulaDataTypes.NUMERIC, }, }, description: @@ -591,10 +593,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, MOD: { - type: FormulaDataTypes.NUMERIC, validation: { args: { rqd: 2, + type: FormulaDataTypes.NUMERIC, }, }, description: 'Remainder after integer division of input parameters', @@ -603,10 +605,11 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, REPEAT: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 2, + + type: FormulaDataTypes.STRING, }, }, description: @@ -616,8 +619,11 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, LOG: { - type: FormulaDataTypes.NUMERIC, - validation: {}, + validation: { + args: { + type: FormulaDataTypes.NUMERIC, + }, + }, description: 'Logarithm of input parameter to the base (default = e) specified', syntax: 'LOG([base], value)', @@ -625,18 +631,21 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, EXP: { - type: FormulaDataTypes.NUMERIC, - validation: {}, + validation: { + args: { + type: FormulaDataTypes.NUMERIC, + }, + }, description: 'Exponential value of input parameter (e ^ power)', syntax: 'EXP(power)', examples: ['EXP(1) => 2.718281828459045', 'EXP({column1})'], returnType: FormulaDataTypes.NUMERIC, }, POWER: { - type: FormulaDataTypes.NUMERIC, validation: { args: { rqd: 2, + type: FormulaDataTypes.NUMERIC, }, }, description: 'base to the exponent power, as in base ^ exponent', @@ -645,10 +654,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, SQRT: { - type: FormulaDataTypes.NUMERIC, validation: { args: { rqd: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: 'Square root of the input parameter', @@ -657,10 +666,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, ABS: { - type: FormulaDataTypes.NUMERIC, validation: { args: { rqd: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: 'Absolute value of the input parameter', @@ -669,10 +678,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, NOW: { - type: FormulaDataTypes.DATE, validation: { args: { rqd: 0, + type: FormulaDataTypes.DATE, }, }, description: 'Returns the current time and day', @@ -681,10 +690,10 @@ const formulas: Record = { returnType: FormulaDataTypes.DATE, }, REPLACE: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 3, + type: FormulaDataTypes.STRING, }, }, description: @@ -697,10 +706,10 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, SEARCH: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 2, + type: FormulaDataTypes.STRING, }, }, description: 'Index of srchStr specified if found, 0 otherwise', @@ -712,10 +721,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, INT: { - type: FormulaDataTypes.NUMERIC, validation: { args: { rqd: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: 'Integer value of input parameter', @@ -724,10 +733,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, RIGHT: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 2, + type: FormulaDataTypes.STRING, }, }, description: 'n characters from the end of input parameter', @@ -736,10 +745,10 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, LEFT: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 2, + type: FormulaDataTypes.STRING, }, }, description: 'n characters from the beginning of input parameter', @@ -748,11 +757,11 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, SUBSTR: { - type: FormulaDataTypes.STRING, validation: { args: { min: 2, max: 3, + type: FormulaDataTypes.STRING, }, }, description: @@ -766,10 +775,10 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, MID: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 3, + type: FormulaDataTypes.STRING, }, }, description: 'Alias for SUBSTR', @@ -778,7 +787,6 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, IF: { - type: FormulaDataTypes.COND_EXP, validation: { args: { min: 2, @@ -816,11 +824,13 @@ const formulas: Record = { }, }, SWITCH: { - type: FormulaDataTypes.COND_EXP, validation: { args: { min: 3, }, + custom: (_argTypes: any[], _parseTree) => { + // Todo: Add validation for switch + }, }, description: 'Switch case value based on expr output', syntax: 'SWITCH(expr, [pattern, value, ..., default])', @@ -856,10 +866,10 @@ const formulas: Record = { }, }, URL: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 1, + type: FormulaDataTypes.STRING, }, }, description: 'Convert to a hyperlink if it is a valid URL', @@ -868,11 +878,11 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, WEEKDAY: { - type: FormulaDataTypes.NUMERIC, validation: { args: { min: 1, max: 2, + type: FormulaDataTypes.NUMERIC, }, custom(_argTypes: FormulaDataTypes[], parsedTree: any) { if (parsedTree.arguments[0].type === JSEPNode.LITERAL) { @@ -916,7 +926,6 @@ const formulas: Record = { }, TRUE: { - type: FormulaDataTypes.NUMERIC, validation: { args: { max: 0, @@ -929,7 +938,6 @@ const formulas: Record = { }, FALSE: { - type: FormulaDataTypes.NUMERIC, validation: { args: { max: 0, @@ -942,10 +950,10 @@ const formulas: Record = { }, REGEX_MATCH: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 2, + type: FormulaDataTypes.STRING, }, }, description: @@ -956,10 +964,10 @@ const formulas: Record = { }, REGEX_EXTRACT: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 2, + type: FormulaDataTypes.STRING, }, }, description: 'Returns the first match of a regular expression in a string.', @@ -968,10 +976,10 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, REGEX_REPLACE: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 3, + type: FormulaDataTypes.STRING, }, }, description: @@ -981,7 +989,6 @@ const formulas: Record = { returnType: FormulaDataTypes.STRING, }, BLANK: { - type: FormulaDataTypes.STRING, validation: { args: { rqd: 0, @@ -993,11 +1000,11 @@ const formulas: Record = { returnType: FormulaDataTypes.NULL, }, XOR: { - type: FormulaDataTypes.NUMERIC, validation: { args: { min: 1, }, + // todo: validation for boolean }, description: 'Returns true if an odd number of arguments are true, and false otherwise.', @@ -1006,10 +1013,10 @@ const formulas: Record = { returnType: FormulaDataTypes.BOOLEAN, }, EVEN: { - type: FormulaDataTypes.NUMERIC, validation: { args: { rqd: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: @@ -1019,10 +1026,10 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, ODD: { - type: FormulaDataTypes.NUMERIC, validation: { args: { rqd: 1, + type: FormulaDataTypes.NUMERIC, }, }, description: @@ -1080,11 +1087,11 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, ROUNDDOWN: { - type: FormulaDataTypes.NUMERIC, validation: { args: { min: 1, max: 2, + type: FormulaDataTypes.NUMERIC, }, }, description: @@ -1094,11 +1101,11 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, ROUNDUP: { - type: FormulaDataTypes.NUMERIC, validation: { args: { min: 1, max: 2, + type: FormulaDataTypes.NUMERIC, }, }, description: From c3b5db43df382236fe8066c5233309b762721274 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:01 +0000 Subject: [PATCH 034/188] refactor: arg validation logic --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 64628670fb..44fb1e2a09 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -17,6 +17,7 @@ export const dateFormats = [ 'MM DD YYYY', 'YYYY MM DD', ]; + function validateDateWithUnknownFormat(v: string) { for (const format of dateFormats) { if (dayjs(v, format, true).isValid() as any) { @@ -1256,6 +1257,24 @@ export function validateFormulaAndExtractTreeWithType( if (formulas[calleeName].validation?.custom) { formulas[calleeName].validation?.custom(argTypes, parsedTree); } + // validate against expected arg types if present + else if (formulas[calleeName].validation?.args?.type) { + const expectedArgType = formulas[calleeName].validation.args.type; + if ( + argTypes.some( + (argType) => + argType !== expectedArgType && argType !== FormulaDataTypes.NULL + ) + ) + throw new FormulaError( + FormulaErrorType.INVALID_ARG, + { + key: 'msg.formula.invalidArgumentType', + calleeName, + }, + 'Invalid argument type' + ); + } if (typeof formulas[calleeName].returnType === 'function') { res.dataType = (formulas[calleeName].returnType as any)?.( From 83933e920c9b1b6adbccacb20378cc6608ed599f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:01 +0000 Subject: [PATCH 035/188] fix: typo correction - remove unnecessary `.value` --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 44fb1e2a09..2299641d74 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1386,7 +1386,7 @@ function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { // e.g. formula1 -> formula2 -> formula1 should return circular reference error // get all formula columns excluding itself - const formulaPaths = columns.value + const formulaPaths = columns .filter((c) => c.id !== formulaCol?.id && c.uidt === UITypes.Formula) .reduce((res: Record[], c: Record) => { // in `formula`, get all the (unique) target neighbours From d3a84fd5d04f887b65d7f592da5066d808a4ab3a Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:01 +0000 Subject: [PATCH 036/188] fix: extract datatype from nested formula properly --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 2299641d74..5f365fbab2 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1301,7 +1301,7 @@ export function validateFormulaAndExtractTreeWithType( columns ); - res.dataType = formulaRes as any; + res.dataType = (formulaRes as any)?.dataType; } else { switch (col?.uidt) { // string From cb3c3d7b006c315804a7b53bfed117cb5b0ccbca Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:01 +0000 Subject: [PATCH 037/188] fix: error message correction --- .../smartsheet/column/FormulaOptions.vue | 16 ++++++++++---- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 21 +++++++++++++++---- .../nocodb/src/services/columns.service.ts | 14 ++++++++++--- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue index 45624cf039..1cd4dfcb3e 100644 --- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue +++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue @@ -2,6 +2,14 @@ import type { Ref } from 'vue' import type { ListItem as AntListItem } from 'ant-design-vue' import jsep from 'jsep' +import type { ColumnType, FormulaType } from 'nocodb-sdk' +import { + FormulaError, + UITypes, + jsepCurlyHook, + substituteColumnIdWithAliasInFormula, + validateFormulaAndExtractTreeWithType, +} from 'nocodb-sdk' import type { ColumnType, FormulaType, LinkToAnotherRecordType, TableType } from 'nocodb-sdk' import { UITypes, @@ -52,10 +60,6 @@ const { setAdditionalValidations, validateInfos, sqlUi, column } = useColumnCrea const { t } = useI18n() -const baseStore = useBase() - -const { tables } = storeToRefs(baseStore) - const { predictFunction: _predictFunction } = useNocoEe() enum JSEPNode { @@ -103,6 +107,10 @@ const validators = { try { validateFormulaAndExtractTreeWithType(formula, supportedColumns.value) } catch (e: any) { + if (e instanceof FormulaError && e.extra?.key) { + return reject(new Error(t(e.extra.key, e.extra))) + } + return reject(new Error(e.message)) } // if (res !== true) { diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 5f365fbab2..7198412eec 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -102,7 +102,7 @@ export async function substituteColumnAliasWithIdInFormula( return jsepTreeToFormula(parsedFormula); } -enum FormulaErrorType { +export enum FormulaErrorType { NOT_AVAILABLE = 'NOT_AVAILABLE', NOT_SUPPORTED = 'NOT_SUPPORTED', MIN_ARG = 'MIN_ARG', @@ -1152,7 +1152,7 @@ const formulas: Record = { // }, }; -class FormulaError extends Error { +export class FormulaError extends Error { public type: FormulaErrorType; public extra: Record; @@ -1265,15 +1265,28 @@ export function validateFormulaAndExtractTreeWithType( (argType) => argType !== expectedArgType && argType !== FormulaDataTypes.NULL ) - ) + ) { + let key = ''; + + if (expectedArgType === FormulaDataTypes.NUMERIC) { + key = 'msg.formula.numericTypeIsExpected'; + } else if (expectedArgType === FormulaDataTypes.STRING) { + key = 'msg.formula.stringTypeIsExpected'; + } else if (expectedArgType === FormulaDataTypes.BOOLEAN) { + key = 'msg.formula.booleanTypeIsExpected'; + } else if (expectedArgType === FormulaDataTypes.DATE) { + key = 'msg.formula.dateTypeIsExpected'; + } + throw new FormulaError( FormulaErrorType.INVALID_ARG, { - key: 'msg.formula.invalidArgumentType', + key, calleeName, }, 'Invalid argument type' ); + } } if (typeof formulas[calleeName].returnType === 'function') { diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index f9731fc115..81c261f421 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -233,7 +233,7 @@ export class ColumnsService { {}, null, true, - colBody.parsed_tree + colBody.parsed_tree, ); } catch (e) { console.error(e); @@ -938,7 +938,10 @@ export class ColumnsService { ]); await FormulaColumn.update(c.id, { formula_raw: new_formula_raw, - parsed_tree: validateFormulaAndExtractTreeWithType(new_formula_raw, table.columns) + parsed_tree: validateFormulaAndExtractTreeWithType( + new_formula_raw, + table.columns, + ), }); } } @@ -1209,10 +1212,15 @@ export class ColumnsService { colBody.formula_raw || colBody.formula, table.columns, ); + console.log( + colBody.formula_raw || + colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'), + ); colBody.parsed_tree = validateFormulaAndExtractTreeWithType( // formula may include double curly brackets in previous version // convert to single curly bracket here for compatibility - colBody.formula_raw || colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'), + colBody.formula_raw || + colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'), table.columns, ); From bc9f106b44c7a56c727ba72c7b4561642e8cd522 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:01 +0000 Subject: [PATCH 038/188] fix: validation corrections --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 43 ++++++++++++++----- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 7198412eec..8ed974081c 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -389,7 +389,10 @@ const formulas: Record = { ); } } - if (parsedTree.arguments[2].type === JSEPNode.LITERAL) { + if ( + parsedTree.arguments[2] && + parsedTree.arguments[2].type === JSEPNode.LITERAL + ) { if ( ![ 'milliseconds', @@ -410,7 +413,7 @@ const formulas: Record = { 'Q', 'years', 'y', - ].includes(parsedTree.arguments[0].value) + ].includes(parsedTree.arguments[2].value) ) { throw new FormulaError( FormulaErrorType.TYPE_MISMATCH, @@ -896,8 +899,12 @@ const formulas: Record = { } } - if (parsedTree.arguments[1].type === JSEPNode.LITERAL) { - const value = parsedTree.arguments[0].value; + // if second argument is present and literal then validate it + if ( + parsedTree.arguments[1] && + parsedTree.arguments[1].type === JSEPNode.LITERAL + ) { + const value = parsedTree.arguments[1].value; if ( typeof value !== 'string' || ![ @@ -1343,23 +1350,37 @@ export function validateFormulaAndExtractTreeWithType( case UITypes.LastModifiedTime: res.dataType = FormulaDataTypes.DATE; break; - // not supported - case UITypes.ForeignKey: - case UITypes.Attachment: - case UITypes.ID: - case UITypes.Time: + case UITypes.Currency: case UITypes.Percent: case UITypes.Duration: + case UITypes.Links: case UITypes.Rollup: + res.dataType = FormulaDataTypes.NUMERIC; + break; + + case UITypes.Attachment: + res.dataType = FormulaDataTypes.STRING; + break; + case UITypes.Checkbox: + res.dataType = FormulaDataTypes.NUMERIC; + break; + case UITypes.ID: + case UITypes.ForeignKey: + { + res.dataType = FormulaDataTypes.NUMERIC; + } + break; + // not supported + case UITypes.Time: case UITypes.Lookup: case UITypes.Barcode: case UITypes.Button: - case UITypes.Checkbox: case UITypes.Collaborator: case UITypes.QrCode: default: - throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, {}); + break; + // throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, {}); } } } else if (parsedTree.type === JSEPNode.LITERAL) { From 0ad6b17298e8941ac7da2e506ffc7eece868a34c Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:01 +0000 Subject: [PATCH 039/188] fix: extract type of id/foreign key using sqlui --- .../smartsheet/column/FormulaOptions.vue | 3 +- .../nocodb-sdk/src/lib/formulaHelpers.spec.ts | 40 ++++++----- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 72 ++++++++++++++++--- .../nocodb/src/services/columns.service.ts | 37 +++++----- 4 files changed, 111 insertions(+), 41 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue index 1cd4dfcb3e..049d5dab30 100644 --- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue +++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue @@ -105,8 +105,9 @@ const validators = { if (!formula?.trim()) return reject(new Error('Required')) try { - validateFormulaAndExtractTreeWithType(formula, supportedColumns.value) + validateFormulaAndExtractTreeWithType({ formula, columns: supportedColumns.value, clientOrSqlUi: sqlUi.value }) } catch (e: any) { + console.log(e) if (e instanceof FormulaError && e.extra?.key) { return reject(new Error(t(e.extra.key, e.extra))) } diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts index 071733e6fb..f0fcb1f697 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.spec.ts @@ -6,29 +6,35 @@ import UITypes from './UITypes'; describe('Formula parsing and type validation', () => { it('Simple formula', async () => { - const result = validateFormulaAndExtractTreeWithType('1 + 2', []); + const result = validateFormulaAndExtractTreeWithType({ + formula: '1 + 2', + columns: [], + clientOrSqlUi: 'mysql2', + }); expect(result.dataType).toEqual(FormulaDataTypes.NUMERIC); }); it('Formula with IF condition', async () => { - const result = validateFormulaAndExtractTreeWithType( - 'IF({column}, "Found", BLANK())', - [ + const result = validateFormulaAndExtractTreeWithType({ + formula: 'IF({column}, "Found", BLANK())', + columns: [ { id: 'cid', title: 'column', uidt: UITypes.Number, }, - ] - ); + ], + clientOrSqlUi: 'mysql2', + }); expect(result.dataType).toEqual(FormulaDataTypes.STRING); }); it('Complex formula', async () => { - const result = validateFormulaAndExtractTreeWithType( - 'SWITCH({column2},"value1",IF({column1}, "Found", BLANK()),"value2", 2)', - [ + const result = validateFormulaAndExtractTreeWithType({ + formula: + 'SWITCH({column2},"value1",IF({column1}, "Found", BLANK()),"value2", 2)', + columns: [ { id: 'id1', title: 'column1', @@ -39,14 +45,15 @@ describe('Formula parsing and type validation', () => { title: 'column2', uidt: UITypes.SingleLineText, }, - ] - ); + ], + clientOrSqlUi: 'mysql2', + }); expect(result.dataType).toEqual(FormulaDataTypes.STRING); - const result1 = validateFormulaAndExtractTreeWithType( - 'SWITCH({column2},"value1",IF({column1}, 1, 2),"value2", 2)', - [ + const result1 = validateFormulaAndExtractTreeWithType({ + formula: 'SWITCH({column2},"value1",IF({column1}, 1, 2),"value2", 2)', + columns: [ { id: 'id1', title: 'column1', @@ -57,8 +64,9 @@ describe('Formula parsing and type validation', () => { title: 'column2', uidt: UITypes.SingleLineText, }, - ] - ); + ], + clientOrSqlUi: 'mysql2', + }); expect(result1.dataType).toEqual(FormulaDataTypes.NUMERIC); }); diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 8ed974081c..fe8db215bf 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -3,6 +3,14 @@ import jsep from 'jsep'; import { ColumnType } from './Api'; import UITypes from './UITypes'; import dayjs from 'dayjs'; +import { + MssqlUi, + MysqlUi, + OracleUi, + PgUi, + SnowflakeUi, + SqlUiFactory, +} from './sqlUi'; // todo: move to date utils and export, remove duplicate from gui @@ -242,6 +250,7 @@ export enum FormulaDataTypes { COND_EXP = 'conditional_expression', NULL = 'null', BOOLEAN = 'boolean', + UNKNOWN = 'unknown', } export enum JSEPNode { @@ -1176,10 +1185,30 @@ export class FormulaError extends Error { } } -export function validateFormulaAndExtractTreeWithType( +export function validateFormulaAndExtractTreeWithType({ formula, - columns: ColumnType[] -) { + columns, + clientOrSqlUi, + getMeta, +}: { + formula: string; + columns: ColumnType[]; + clientOrSqlUi: + | 'mysql' + | 'pg' + | 'sqlite3' + | 'mssql' + | 'mysql2' + | 'oracledb' + | 'mariadb' + | 'sqlite' + | MysqlUi + | MssqlUi + | SnowflakeUi + | PgUi + | OracleUi; + getMeta?: (tableId: string) => Promise; +}) { const colAliasToColMap = {}; const colIdToColMap = {}; @@ -1270,7 +1299,9 @@ export function validateFormulaAndExtractTreeWithType( if ( argTypes.some( (argType) => - argType !== expectedArgType && argType !== FormulaDataTypes.NULL + argType !== expectedArgType && + argType !== FormulaDataTypes.NULL && + argType !== FormulaDataTypes.UNKNOWN ) ) { let key = ''; @@ -1317,8 +1348,14 @@ export function validateFormulaAndExtractTreeWithType( validateFormulaAndExtractTreeWithType( // formula may include double curly brackets in previous version // convert to single curly bracket here for compatibility - col.colOptions.formula.replaceAll('{{', '{').replaceAll('}}', '}'), - columns + { + formula: col.colOptions.formula + .replaceAll('{{', '{') + .replaceAll('}}', '}'), + columns, + clientOrSqlUi, + getMeta, + } ); res.dataType = (formulaRes as any)?.dataType; @@ -1368,7 +1405,26 @@ export function validateFormulaAndExtractTreeWithType( case UITypes.ID: case UITypes.ForeignKey: { - res.dataType = FormulaDataTypes.NUMERIC; + const sqlUI = + typeof clientOrSqlUi === 'string' + ? SqlUiFactory.create(clientOrSqlUi) + : clientOrSqlUi; + if (sqlUI) { + const abstractType = sqlUI.getAbstractType(col); + if (['integer', 'float', 'decimal'].includes(abstractType)) { + res.dataType = FormulaDataTypes.NUMERIC; + } else if (['boolean'].includes(abstractType)) { + res.dataType = FormulaDataTypes.BOOLEAN; + } else if ( + ['date', 'datetime', 'time', 'year'].includes(abstractType) + ) { + res.dataType = FormulaDataTypes.DATE; + } else { + res.dataType = FormulaDataTypes.STRING; + } + } else { + res.dataType = FormulaDataTypes.UNKNOWN; + } } break; // not supported @@ -1379,8 +1435,8 @@ export function validateFormulaAndExtractTreeWithType( case UITypes.Collaborator: case UITypes.QrCode: default: + res.dataType = FormulaDataTypes.UNKNOWN; break; - // throw new FormulaError(FormulaErrorType.NOT_SUPPORTED, {}); } } } else if (parsedTree.type === JSEPNode.LITERAL) { diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index 81c261f421..c39c312fe7 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -210,10 +210,11 @@ export class ColumnsService { colBody.formula_raw || colBody.formula, table.columns, ); - colBody.parsed_tree = validateFormulaAndExtractTreeWithType( - colBody.formula_raw || colBody.formula, - table.columns, - ); + colBody.parsed_tree = validateFormulaAndExtractTreeWithType({ + formula: colBody.formula_raw || colBody.formula, + columns: table.columns, + clientOrSqlUi: source.type, + }); try { const baseModel = await reuseOrSave('baseModel', reuse, async () => @@ -938,10 +939,11 @@ export class ColumnsService { ]); await FormulaColumn.update(c.id, { formula_raw: new_formula_raw, - parsed_tree: validateFormulaAndExtractTreeWithType( - new_formula_raw, - table.columns, - ), + parsed_tree: validateFormulaAndExtractTreeWithType({ + formula: new_formula_raw, + columns: table.columns, + clientOrSqlUi: source.type, + }), }); } } @@ -1003,10 +1005,11 @@ export class ColumnsService { ]); await FormulaColumn.update(c.id, { formula_raw: new_formula_raw, - parsed_tree: validateFormulaAndExtractTreeWithType( - new_formula_raw, - table.columns, - ), + parsed_tree: validateFormulaAndExtractTreeWithType({ + formula: new_formula_raw, + columns: table.columns, + clientOrSqlUi: source.type, + }), }); } } @@ -1216,13 +1219,15 @@ export class ColumnsService { colBody.formula_raw || colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'), ); - colBody.parsed_tree = validateFormulaAndExtractTreeWithType( + colBody.parsed_tree = validateFormulaAndExtractTreeWithType({ // formula may include double curly brackets in previous version // convert to single curly bracket here for compatibility - colBody.formula_raw || + formula: + colBody.formula_raw || colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'), - table.columns, - ); + columns: table.columns, + clientOrSqlUi: source.type, + }); try { const baseModel = await reuseOrSave('baseModel', reuse, async () => From 59e5ba83bda63224e90c29700bce6b0d0922050a Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:01 +0000 Subject: [PATCH 040/188] fix: type definition correction --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index fe8db215bf..b7f96413f6 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1202,11 +1202,11 @@ export function validateFormulaAndExtractTreeWithType({ | 'oracledb' | 'mariadb' | 'sqlite' - | MysqlUi - | MssqlUi - | SnowflakeUi - | PgUi - | OracleUi; + | typeof MysqlUi + | typeof MssqlUi + | typeof SnowflakeUi + | typeof PgUi + | typeof OracleUi; getMeta?: (tableId: string) => Promise; }) { const colAliasToColMap = {}; From 20f1c04e7e65ec507c8801ac4ac3177969262a3a Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:02 +0000 Subject: [PATCH 041/188] fix: method usage correction --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 1 + .../nocodb/src/db/formulav2/formulaQueryBuilderv2.ts | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index b7f96413f6..1775c57a8a 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1202,6 +1202,7 @@ export function validateFormulaAndExtractTreeWithType({ | 'oracledb' | 'mariadb' | 'sqlite' + | 'snowflake' | typeof MysqlUi | typeof MssqlUi | typeof SnowflakeUi diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts index f6688082a3..b9294e6591 100644 --- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts @@ -21,6 +21,7 @@ import { validateDateWithUnknownFormat, } from '~/helpers/formulaFnHelper'; import FormulaColumn from '~/models/FormulaColumn'; +import { Source } from '~/models'; const logger = new Logger('FormulaQueryBuilderv2'); @@ -76,10 +77,13 @@ async function _formulaQueryBuilder( // formula may include double curly brackets in previous version // convert to single curly bracket here for compatibility // const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}')); - tree = validateFormulaAndExtractTreeWithType( - _tree.replaceAll('{{', '{').replaceAll('}}', '}'), + tree = validateFormulaAndExtractTreeWithType({ + formula: _tree.replaceAll('{{', '{').replaceAll('}}', '}'), columns, - ); + clientOrSqlUi: await Source.get(column.source_id).then( + (source) => source.type, + ), + }); // populate and save parsedTree to column if not exist if (column) { From 748322ab669c50934fbbff990c828c6b550fbeae Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:02 +0000 Subject: [PATCH 042/188] fix: extract source id from model/column --- packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts index b9294e6591..993dedf412 100644 --- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts @@ -80,9 +80,9 @@ async function _formulaQueryBuilder( tree = validateFormulaAndExtractTreeWithType({ formula: _tree.replaceAll('{{', '{').replaceAll('}}', '}'), columns, - clientOrSqlUi: await Source.get(column.source_id).then( - (source) => source.type, - ), + clientOrSqlUi: await Source.get( + model?.source_id ?? column?.source_id, + ).then((source) => source.type), }); // populate and save parsedTree to column if not exist From e0435b5804dc62a39f236f86d1a7e6f775d98740 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:02 +0000 Subject: [PATCH 043/188] fix: pass argument as object and extract client from basemodelSql --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 5 ++++- .../src/db/formulav2/formulaQueryBuilderv2.ts | 13 ++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 1775c57a8a..fc3858dc47 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -46,6 +46,9 @@ export const jsepCurlyHook = { jsep.hooks.add('gobble-token', function gobbleCurlyLiteral(env) { const OCURLY_CODE = 123; // { const CCURLY_CODE = 125; // } + // jsep.addIdentifierChar('.'); + // jsep.addIdentifierChar('*'); + // jsep.addIdentifierChar('?'); let start = -1; const { context } = env; if ( @@ -1408,7 +1411,7 @@ export function validateFormulaAndExtractTreeWithType({ { const sqlUI = typeof clientOrSqlUi === 'string' - ? SqlUiFactory.create(clientOrSqlUi) + ? SqlUiFactory.create({ client: clientOrSqlUi }) : clientOrSqlUi; if (sqlUI) { const abstractType = sqlUI.getAbstractType(col); diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts index 993dedf412..e7b056c5bf 100644 --- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts @@ -80,9 +80,16 @@ async function _formulaQueryBuilder( tree = validateFormulaAndExtractTreeWithType({ formula: _tree.replaceAll('{{', '{').replaceAll('}}', '}'), columns, - clientOrSqlUi: await Source.get( - model?.source_id ?? column?.source_id, - ).then((source) => source.type), + clientOrSqlUi: baseModelSqlv2.clientType as + | 'mysql' + | 'pg' + | 'sqlite3' + | 'mssql' + | 'mysql2' + | 'oracledb' + | 'mariadb' + | 'sqlite' + | 'snowflake', }); // populate and save parsedTree to column if not exist From 8f3550a8eed25b4935cf3c9224acd7e048705b78 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:02 +0000 Subject: [PATCH 044/188] fix: extract type of specificDBtype column using sqlui --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index fc3858dc47..1e6ff74aac 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -1408,6 +1408,7 @@ export function validateFormulaAndExtractTreeWithType({ break; case UITypes.ID: case UITypes.ForeignKey: + case UITypes.SpecificDBType: { const sqlUI = typeof clientOrSqlUi === 'string' From 568eb90b1a6e403e745f1a56cb5373fc75cb05b4 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:02 +0000 Subject: [PATCH 045/188] fix: add support for special chars in identifier if wrapped within curly bracket --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 66 ++++++++++++++++++- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 1e6ff74aac..3d59f668a8 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -26,6 +26,54 @@ export const dateFormats = [ 'YYYY MM DD', ]; +// opening and closing string code +const OCURLY_CODE = 123; // '{' +const CCURLY_CODE = 125; // '}' + +export const jsepCurlyHook = { + name: 'curly', + init(jsep) { + // Match identifier in following pattern: {abc-cde} + jsep.hooks.add('gobble-token', function escapedIdentifier(env) { + // check if the current token is an opening curly bracket + if (this.code === OCURLY_CODE) { + const patternIndex = this.index; + // move to the next character until we find a closing curly bracket + while (this.index < this.expr.length) { + ++this.index; + if (this.code === CCURLY_CODE) { + let identifier = this.expr.slice(patternIndex, ++this.index); + + // if starting with double curley brace then check for ending double curley brace + // if found include with the identifier + if ( + identifier.startsWith('{{') && + this.expr.slice(patternIndex, this.index + 1).endsWith('}') + ) { + identifier = this.expr.slice(patternIndex, ++this.index); + } + env.node = { + type: jsep.IDENTIFIER, + name: /^{{.*}}$/.test(identifier) + ? // start would be the position of the first curly bracket + // add 2 to point to the first character for expressions like {{col1}} + identifier.slice(2, -2) + : // start would be the position of the first curly bracket + // add 1 to point to the first character for expressions like {col1} + identifier.slice(1, -1), + raw: identifier, + }; + + // env.node = this.gobbleTokenProperty(env.node); + return env.node; + } + } + this.throwError('Unclosed }'); + } + }); + }, +} as jsep.IPlugin; + function validateDateWithUnknownFormat(v: string) { for (const format of dateFormats) { if (dayjs(v, format, true).isValid() as any) { @@ -40,15 +88,13 @@ function validateDateWithUnknownFormat(v: string) { return false; } +/* export const jsepCurlyHook = { name: 'curly', init(jsep) { jsep.hooks.add('gobble-token', function gobbleCurlyLiteral(env) { const OCURLY_CODE = 123; // { const CCURLY_CODE = 125; // } - // jsep.addIdentifierChar('.'); - // jsep.addIdentifierChar('*'); - // jsep.addIdentifierChar('?'); let start = -1; const { context } = env; if ( @@ -80,6 +126,7 @@ export const jsepCurlyHook = { }); }, } as jsep.IPlugin; +*/ export async function substituteColumnAliasWithIdInFormula( formula, @@ -125,6 +172,7 @@ export enum FormulaErrorType { INVALID_ARG_COUNT = 'INVALID_ARG_COUNT', CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE', INVALID_FUNCTION_NAME = 'INVALID_FUNCTION_NAME', + INVALID_COLUMN = 'INVALID_COLUMN', } export function substituteColumnIdWithAliasInFormula( @@ -1341,6 +1389,18 @@ export function validateFormulaAndExtractTreeWithType({ } else if (parsedTree.type === JSEPNode.IDENTIFIER) { const col = (colIdToColMap[parsedTree.name] || colAliasToColMap[parsedTree.name]) as Record; + + if (!col) { + throw new FormulaError( + FormulaErrorType.INVALID_COLUMN, + { + key: 'msg.formula.invalidColumn', + column: parsedTree.name, + }, + `Invalid column name/id ${JSON.stringify(parsedTree.name)} in formula` + ); + } + res.name = col.id; if (col?.uidt === UITypes.Formula) { From ff172ae5e3f0a236d483651abb1df195f391040f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:02 +0000 Subject: [PATCH 046/188] fix: avoid passing index as arg --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 8 ++++++-- packages/nocodb/src/services/columns.service.ts | 4 ---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 3d59f668a8..956ac11fe0 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -255,13 +255,17 @@ export function jsepTreeToFormula(node, isCallExpId = false) { return ( jsepTreeToFormula(node.callee, true) + '(' + - node.arguments.map(jsepTreeToFormula).join(', ') + + node.arguments.map((argPt) => jsepTreeToFormula(argPt)).join(', ') + ')' ); } if (node.type === 'ArrayExpression') { - return '[' + node.elements.map(jsepTreeToFormula).join(', ') + ']'; + return ( + '[' + + node.elements.map((elePt) => jsepTreeToFormula(elePt)).join(', ') + + ']' + ); } if (node.type === 'Compound') { diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index c39c312fe7..3e3f0c4073 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -1215,10 +1215,6 @@ export class ColumnsService { colBody.formula_raw || colBody.formula, table.columns, ); - console.log( - colBody.formula_raw || - colBody.formula?.replaceAll('{{', '{').replaceAll('}}', '}'), - ); colBody.parsed_tree = validateFormulaAndExtractTreeWithType({ // formula may include double curly brackets in previous version // convert to single curly bracket here for compatibility From ece0749f549ec233daf3488841779c772211f53f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:17:02 +0000 Subject: [PATCH 047/188] chore: cleanup --- .../smartsheet/column/FormulaOptions.vue | 69 +------------------ packages/nocodb-sdk/src/lib/formulaHelpers.ts | 40 ----------- 2 files changed, 1 insertion(+), 108 deletions(-) diff --git a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue index 049d5dab30..319a5319cc 100644 --- a/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue +++ b/packages/nc-gui/components/smartsheet/column/FormulaOptions.vue @@ -25,7 +25,6 @@ import { NcAutocompleteTree, computed, formulaList, - formulaTypes, formulas, getUIDTIcon, getWordUntilCaret, @@ -62,18 +61,6 @@ const { t } = useI18n() const { predictFunction: _predictFunction } = useNocoEe() -enum JSEPNode { - COMPOUND = 'Compound', - IDENTIFIER = 'Identifier', - MEMBER_EXP = 'MemberExpression', - LITERAL = 'Literal', - THIS_EXP = 'ThisExpression', - CALL_EXP = 'CallExpression', - UNARY_EXP = 'UnaryExpression', - BINARY_EXP = 'BinaryExpression', - ARRAY_EXP = 'ArrayExpression', -} - const meta = inject(MetaInj, ref()) const supportedColumns = computed( @@ -107,16 +94,12 @@ const validators = { try { validateFormulaAndExtractTreeWithType({ formula, columns: supportedColumns.value, clientOrSqlUi: sqlUi.value }) } catch (e: any) { - console.log(e) if (e instanceof FormulaError && e.extra?.key) { return reject(new Error(t(e.extra.key, e.extra))) } return reject(new Error(e.message)) } - // if (res !== true) { - // return reject(new Error(res)) - // } resolve() }) }, @@ -610,53 +593,6 @@ function validateAgainstType( type = formulaTypes.DATE break - case UITypes.Rollup: { - const rollupFunction = col.colOptions.rollup_function - if (['count', 'avg', 'sum', 'countDistinct', 'sumDistinct', 'avgDistinct'].includes(rollupFunction)) { - // these functions produce a numeric value, which can be used in numeric functions - if (expectedType !== formulaTypes.NUMERIC) { - typeErrors.add( - t('msg.formula.columnWithTypeFoundButExpected', { - columnName: parsedTree.name, - columnType: formulaTypes.NUMERIC, - expectedType, - }), - ) - } - } else { - // the value is based on the foreign rollup column type - const selectedTable = refTables.value.find((t) => t.column.id === col.colOptions.fk_relation_column_id) - const refTableColumns = metas.value[selectedTable.id].columns.filter( - (c: ColumnType) => - vModel.value.fk_lookup_column_id === c.id || - (!isSystemColumn(c) && c.id !== vModel.value.id && c.uidt !== UITypes.Links), - ) - const childFieldColumn = refTableColumns.find( - (column: ColumnType) => column.id === col.colOptions.fk_rollup_column_id, - ) - const abstractType = sqlUi.value.getAbstractType(childFieldColumn) - - if (expectedType === formulaTypes.DATE && !isDate(childFieldColumn, sqlUi.value.getAbstractType(childFieldColumn))) { - typeErrors.add( - t('msg.formula.columnWithTypeFoundButExpected', { - columnName: parsedTree.name, - columnType: abstractType, - expectedType, - }), - ) - } else if (expectedType === formulaTypes.NUMERIC && !isNumericCol(childFieldColumn)) { - typeErrors.add( - t('msg.formula.columnWithTypeFoundButExpected', { - columnName: parsedTree.name, - columnType: abstractType, - expectedType, - }), - ) - } - } - break - } - // not supported case UITypes.ForeignKey: case UITypes.Attachment: @@ -664,6 +600,7 @@ function validateAgainstType( case UITypes.Time: case UITypes.Percent: case UITypes.Duration: + case UITypes.Rollup: case UITypes.Lookup: case UITypes.Barcode: case UITypes.Button: @@ -871,10 +808,6 @@ setAdditionalValidations({ onMounted(() => { jsep.plugins.register(jsepCurlyHook) }) - -// const predictFunction = async () => { -// await _predictFunction(formState, meta, supportedColumns, suggestionsList, vModel) -// } diff --git a/packages/nc-gui/components/project/View.vue b/packages/nc-gui/components/project/View.vue index 0571b7e67c..15aa847b0a 100644 --- a/packages/nc-gui/components/project/View.vue +++ b/packages/nc-gui/components/project/View.vue @@ -5,7 +5,7 @@ import { isEeUI } from '#imports' const basesStore = useBases() -const { getProjectUsers } = basesStore +const { getBaseUsers } = basesStore const { openedProject, activeProjectId, baseUserCount } = storeToRefs(basesStore) const { activeTables } = storeToRefs(useTablesStore()) @@ -38,7 +38,7 @@ const updateBaseUserCount = async () => { if (!baseUserCount || !isUIAllowed('newUser')) return try { - const { totalRows } = await getProjectUsers({ + const { totalRows } = await getBaseUsers({ baseId: activeProjectId.value!, page: 1, searchText: undefined, diff --git a/packages/nc-gui/store/base.ts b/packages/nc-gui/store/base.ts index 93ab4d3092..1e6913dad0 100644 --- a/packages/nc-gui/store/base.ts +++ b/packages/nc-gui/store/base.ts @@ -172,6 +172,10 @@ export const useBase = defineStore('baseStore', () => { await loadTables() + await basesStore.getBaseUsers({ + baseId: base.value.id || baseId.value, + }) + // if (withTheme) setTheme(baseMeta.value?.theme) return baseLoadedHook.trigger(base.value) diff --git a/packages/nc-gui/store/bases.ts b/packages/nc-gui/store/bases.ts index b1a74e6857..aae911ce5f 100644 --- a/packages/nc-gui/store/bases.ts +++ b/packages/nc-gui/store/bases.ts @@ -12,6 +12,8 @@ export const useBases = defineStore('basesStore', () => { const bases = ref>(new Map()) const basesList = computed(() => Array.from(bases.value.values()).sort((a, b) => a.updated_at - b.updated_at)) + const basesUser = ref>(new Map()) + const baseUserCount = ref(undefined) const router = useRouter() @@ -48,33 +50,35 @@ export const useBases = defineStore('basesStore', () => { const isProjectsLoading = ref(false) - async function getProjectUsers({ - baseId, - limit, - page, - searchText, - }: { - baseId: string - limit: number - page: number - searchText: string | undefined - }) { + async function getBaseUsers({ baseId, searchText, force = false }: { baseId: string; searchText?: string; force?: boolean }) { + if (!force && !limit && !page && basesUser.value.has(baseId)) { + const users = basesUser.value.get(baseId) + return { + users, + totalRows: users?.length ?? 0, + } + } + const response: any = await api.auth.baseUserList(baseId, { query: { - limit, - offset: (page - 1) * limit, query: searchText, }, } as RequestParams) const totalRows = response.users.pageInfo.totalRows ?? 0 + basesUser.value.set(baseId, response.users.list) + return { users: response.users.list, totalRows, } } + const clearBasesUser = () => { + basesUser.value.clear() + } + const createProjectUser = async (baseId: string, user: User) => { await api.auth.baseUserAdd(baseId, user as ProjectUserReqType) } @@ -312,13 +316,15 @@ export const useBases = defineStore('basesStore', () => { activeProjectId, openedProject, openedProjectBasesMap, - getProjectUsers, + getBaseUsers, createProjectUser, updateProjectUser, navigateToProject, removeProjectUser, navigateToFirstProjectOrHome, toggleStarred, + basesUser, + clearBasesUser, } }) diff --git a/packages/nc-gui/store/users.ts b/packages/nc-gui/store/users.ts index d209429a41..4a7cad25b9 100644 --- a/packages/nc-gui/store/users.ts +++ b/packages/nc-gui/store/users.ts @@ -4,6 +4,7 @@ export const useUsers = defineStore('userStore', () => { const { api } = useApi() const { user } = useGlobal() const { loadRoles } = useRoles() + const basesStore = useBases() const updateUserProfile = async ({ attrs, @@ -20,6 +21,8 @@ export const useUsers = defineStore('userStore', () => { ...user.value, ...attrs, } + + basesStore.clearBasesUser() } const loadCurrentUser = loadRoles diff --git a/packages/nocodb/src/models/Base.ts b/packages/nocodb/src/models/Base.ts index 294eadb562..61fc986e84 100644 --- a/packages/nocodb/src/models/Base.ts +++ b/packages/nocodb/src/models/Base.ts @@ -295,8 +295,6 @@ export default class Base implements BaseType { let base = await this.get(baseId); const users = await BaseUser.getUsersList({ base_id: baseId, - offset: 0, - limit: 1000, }); for (const user of users) { diff --git a/packages/nocodb/src/models/BaseUser.ts b/packages/nocodb/src/models/BaseUser.ts index 3792557dde..c58b01e5ee 100644 --- a/packages/nocodb/src/models/BaseUser.ts +++ b/packages/nocodb/src/models/BaseUser.ts @@ -3,7 +3,6 @@ import type { BaseType } from 'nocodb-sdk'; import User from '~/models/User'; import Base from '~/models/Base'; import { - CacheDelDirection, // CacheDelDirection, CacheGetType, CacheScope, @@ -67,27 +66,37 @@ export default class BaseUser { `${CacheScope.BASE_USER}:${baseId}:${userId}`, CacheGetType.TYPE_OBJECT, )); - if (!baseUser) { - baseUser = await ncMeta.metaGet2(null, null, MetaTable.PROJECT_USERS, { - fk_user_id: userId, - base_id: baseId, + if (!baseUser || !baseUser.roles) { + const queryBuilder = ncMeta + .knex(MetaTable.USERS) + .select( + `${MetaTable.USERS}.id`, + `${MetaTable.USERS}.email`, + `${MetaTable.USERS}.display_name`, + `${MetaTable.USERS}.invite_token`, + `${MetaTable.USERS}.roles as main_roles`, + `${MetaTable.USERS}.created_at as created_at`, + `${MetaTable.PROJECT_USERS}.base_id`, + `${MetaTable.PROJECT_USERS}.roles as roles`, + ); + + queryBuilder.leftJoin(MetaTable.PROJECT_USERS, function () { + this.on( + `${MetaTable.PROJECT_USERS}.fk_user_id`, + '=', + `${MetaTable.USERS}.id`, + ).andOn( + `${MetaTable.PROJECT_USERS}.base_id`, + '=', + ncMeta.knex.raw('?', [baseId]), + ); }); - if (baseUser) { - const { - id, - email, - invite_token, - roles: main_roles, - } = await User.get(userId, ncMeta); - - baseUser = { - ...baseUser, - id, - email, - invite_token, - main_roles, - }; + queryBuilder.where(`${MetaTable.USERS}.id`, userId); + + baseUser = await queryBuilder.first(); + + if (baseUser) { await NocoCache.set( `${CacheScope.BASE_USER}:${baseId}:${userId}`, baseUser, @@ -100,13 +109,9 @@ export default class BaseUser { public static async getUsersList( { base_id, - limit = 25, - offset = 0, query, }: { base_id: string; - limit?: number; - offset?: number; query?: string; }, ncMeta = Noco.ncMeta, @@ -120,6 +125,7 @@ export default class BaseUser { .select( `${MetaTable.USERS}.id`, `${MetaTable.USERS}.email`, + `${MetaTable.USERS}.display_name`, `${MetaTable.USERS}.invite_token`, `${MetaTable.USERS}.roles as main_roles`, `${MetaTable.USERS}.created_at as created_at`, @@ -127,10 +133,6 @@ export default class BaseUser { `${MetaTable.PROJECT_USERS}.roles as roles`, ); - if (limit) { - queryBuilder.offset(offset).limit(limit); - } - if (query) { queryBuilder.where('email', 'like', `%${query.toLowerCase?.()}%`); } @@ -259,12 +261,9 @@ export default class BaseUser { }, ); - // delete cache - await NocoCache.deepDel( - CacheScope.BASE_USER, - `${CacheScope.BASE_USER}:${baseId}:${userId}`, - CacheDelDirection.CHILD_TO_PARENT, - ); + // delete list cache to refresh list + await NocoCache.del(`${CacheScope.BASE_USER}:${baseId}:${userId}`); + await NocoCache.del(`${CacheScope.BASE_USER}:${baseId}:list`); return response; } diff --git a/packages/nocodb/src/services/base-users/base-users.service.ts b/packages/nocodb/src/services/base-users/base-users.service.ts index 16dcf07ebc..7d95258273 100644 --- a/packages/nocodb/src/services/base-users/base-users.service.ts +++ b/packages/nocodb/src/services/base-users/base-users.service.ts @@ -28,19 +28,15 @@ export class BaseUsersService { constructor(protected appHooksService: AppHooksService) {} async userList(param: { baseId: string; query: any }) { - return new PagedResponseImpl( - await BaseUser.getUsersList({ - ...param.query, - base_id: param.baseId, - }), - { - ...param.query, - count: await BaseUser.getUsersCount({ - base_id: param.baseId, - ...param.query, - }), - }, - ); + const baseUsers = await BaseUser.getUsersList({ + ...param.query, + base_id: param.baseId, + }); + + return new PagedResponseImpl(baseUsers, { + ...param.query, + count: baseUsers.length, + }); } async userInvite(param: { @@ -110,7 +106,7 @@ export class BaseUsersService { return NcError.badRequest('Invalid base id'); } - if (baseUser) { + if (baseUser && baseUser.roles) { NcError.badRequest( `${user.email} with role ${baseUser.roles} already exists in this base`, ); From b6c398f3f2f1948e2012773c3a52bdb4d20088f4 Mon Sep 17 00:00:00 2001 From: mertmit Date: Mon, 11 Dec 2023 15:58:34 +0000 Subject: [PATCH 105/188] feat: group by with user field --- .../nc-gui/components/smartsheet/grid/GroupBy.vue | 12 +++++++++++- packages/nc-gui/composables/useViewGroupBy.ts | 11 +++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/nc-gui/components/smartsheet/grid/GroupBy.vue b/packages/nc-gui/components/smartsheet/grid/GroupBy.vue index ddbf216c5b..510c902cb2 100644 --- a/packages/nc-gui/components/smartsheet/grid/GroupBy.vue +++ b/packages/nc-gui/components/smartsheet/grid/GroupBy.vue @@ -167,11 +167,21 @@ const parseKey = (group: Group) => { if (key && group.column?.uidt === UITypes.Time && dayjs(key).isValid()) { return [dayjs(key).format(timeFormats[0])] } + + if (key && group.column?.uidt === UITypes.User) { + try { + const parsedKey = JSON.parse(key) + return [parsedKey] + } catch { + return null + } + } + return [key] } const shouldRenderCell = (column) => - [UITypes.Lookup, UITypes.Attachment, UITypes.Barcode, UITypes.QrCode, UITypes.Links].includes(column?.uidt) + [UITypes.Lookup, UITypes.Attachment, UITypes.Barcode, UITypes.QrCode, UITypes.Links, UITypes.User].includes(column?.uidt)