From 36277d55093890b318eca2796b8c416f8baa676f Mon Sep 17 00:00:00 2001 From: Pranav C Date: Thu, 21 Dec 2023 09:16:58 +0000 Subject: [PATCH] feat: add migration and hide parsed tree data from api response --- packages/nocodb-sdk/src/lib/formulaHelpers.ts | 713 +++--------------- .../src/db/formulav2/formulaQueryBuilderv2.ts | 24 +- .../v2/nc_038_formula_parsed_tree_column.ts | 16 + packages/nocodb/src/models/FormulaColumn.ts | 24 +- .../nocodb/src/services/columns.service.ts | 6 +- 5 files changed, 162 insertions(+), 621 deletions(-) create mode 100644 packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts diff --git a/packages/nocodb-sdk/src/lib/formulaHelpers.ts b/packages/nocodb-sdk/src/lib/formulaHelpers.ts index 5d0dba7305..94b3dcc62d 100644 --- a/packages/nocodb-sdk/src/lib/formulaHelpers.ts +++ b/packages/nocodb-sdk/src/lib/formulaHelpers.ts @@ -2,6 +2,7 @@ import jsep from 'jsep'; import { ColumnType } from './Api'; import UITypes from './UITypes'; +import { formulaTypes } from '../../../nc-gui/utils'; export const jsepCurlyHook = { name: 'curly', @@ -220,6 +221,7 @@ interface FormulaMeta { min?: number; max?: number; rqd?: number; + validator?: (args: formulaTypes[]) => boolean; }; }; description?: string; @@ -230,7 +232,6 @@ interface FormulaMeta { const formulas: Record = { AVG: { - type: FormulaDataTypes.NUMERIC, validation: { args: { min: 1, @@ -309,7 +310,6 @@ const formulas: Record = { returnType: FormulaDataTypes.NUMERIC, }, AND: { - type: FormulaDataTypes.COND_EXP, validation: { args: { min: 1, @@ -321,7 +321,6 @@ const formulas: Record = { returnType: FormulaDataTypes.COND_EXP, }, OR: { - type: FormulaDataTypes.COND_EXP, validation: { args: { min: 1, @@ -964,603 +963,6 @@ const formulas: Record = { // }, }; -/* -function validateAgainstMeta( - parsedTree: any, - errors = new Set(), - typeErrors = new Set() -) { - let returnType: FormulaDataTypes; - if (parsedTree.type === JSEPNode.CALL_EXP) { - const calleeName = parsedTree.callee.name.toUpperCase(); - // validate function name - if (!availableFunctions.includes(calleeName)) { - errors.add( - t('msg.formula.functionNotAvailable', { function: calleeName }) - ); - } - // validate arguments - const validation = formulas[calleeName] && formulas[calleeName].validation; - if (validation && validation.args) { - if ( - validation.args.rqd !== undefined && - validation.args.rqd !== parsedTree.arguments.length - ) { - errors.add( - t('msg.formula.requiredArgumentsFormula', { - requiredArguments: validation.args.rqd, - calleeName, - }) - ); - } else if ( - validation.args.min !== undefined && - validation.args.min > parsedTree.arguments.length - ) { - errors.add( - t('msg.formula.minRequiredArgumentsFormula', { - minRequiredArguments: validation.args.min, - calleeName, - }) - ); - } else if ( - validation.args.max !== undefined && - validation.args.max < parsedTree.arguments.length - ) { - errors.add( - t('msg.formula.maxRequiredArgumentsFormula', { - maxRequiredArguments: validation.args.max, - calleeName, - }) - ); - } - } - - parsedTree.arguments.map((arg: Record) => - validateAgainstMeta(arg, errors) - ); - - // get args type and validate - const validateResult = parsedTree.arguments.map((arg) => { - return validateAgainstMeta(arg, errors, typeErrors); - }); - - const argsTypes = validateResult.map((v: any) => v.returnType); - - if (typeof validateResult[0].returnType === 'function') { - returnType = formulas[calleeName].returnType(argsTypes); - } else if (validateResult[0]) { - returnType = formulas[calleeName].returnType; - } - - // validate data type - if (parsedTree.callee.type === JSEPNode.IDENTIFIER) { - const expectedType = formulas[calleeName.toUpperCase()].type; - - if (expectedType === FormulaDataTypes.NUMERIC) { - if (calleeName === 'WEEKDAY') { - // parsedTree.arguments[0] = date - validateAgainstType( - parsedTree.arguments[0], - FormulaDataTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[1] = startDayOfWeek (optional) - validateAgainstType( - parsedTree.arguments[1], - FormulaDataTypes.STRING, - (v: any) => { - if ( - typeof v !== 'string' || - ![ - 'sunday', - 'monday', - 'tuesday', - 'wednesday', - 'thursday', - 'friday', - 'saturday', - ].includes(v.toLowerCase()) - ) { - typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate')); - } - }, - typeErrors - ); - } else { - parsedTree.arguments.map((arg: Record) => - validateAgainstType(arg, expectedType, null, typeErrors, argsTypes) - ); - } - } else if (expectedType === FormulaDataTypes.DATE) { - if (calleeName === 'DATEADD') { - // parsedTree.arguments[0] = date - validateAgainstType( - parsedTree.arguments[0], - FormulaDataTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamDateAddHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[1] = numeric - validateAgainstType( - parsedTree.arguments[1], - FormulaDataTypes.NUMERIC, - (v: any) => { - if (typeof v !== 'number') { - typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber')); - } - }, - typeErrors - ); - // parsedTree.arguments[2] = ["day" | "week" | "month" | "year"] - validateAgainstType( - parsedTree.arguments[2], - FormulaDataTypes.STRING, - (v: any) => { - if (!['day', 'week', 'month', 'year'].includes(v)) { - typeErrors.add( - typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate')) - ); - } - }, - typeErrors - ); - } else if (calleeName === 'DATETIME_DIFF') { - // parsedTree.arguments[0] = date - validateAgainstType( - parsedTree.arguments[0], - FormulaDataTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[1] = date - validateAgainstType( - parsedTree.arguments[1], - FormulaDataTypes.DATE, - (v: any) => { - if (!validateDateWithUnknownFormat(v)) { - typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate')); - } - }, - typeErrors - ); - // parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"] - validateAgainstType( - parsedTree.arguments[2], - FormulaDataTypes.STRING, - (v: any) => { - if ( - ![ - 'milliseconds', - 'ms', - 'seconds', - 's', - 'minutes', - 'm', - 'hours', - 'h', - 'days', - 'd', - 'weeks', - 'w', - 'months', - 'M', - 'quarters', - 'Q', - 'years', - 'y', - ].includes(v) - ) { - typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate')); - } - }, - typeErrors - ); - } - } - } - - errors = new Set([...errors, ...typeErrors]); - } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - if ( - supportedColumns.value - .filter((c) => !column || column.value?.id !== c.id) - .every((c) => c.title !== parsedTree.name) - ) { - errors.add( - t('msg.formula.columnNotAvailable', { - columnName: parsedTree.name, - }) - ); - } - - // check circular reference - // e.g. formula1 -> formula2 -> formula1 should return circular reference error - - // get all formula columns excluding itself - const formulaPaths = supportedColumns.value - .filter((c) => c.id !== column.value?.id && c.uidt === UITypes.Formula) - .reduce((res: Record[], c: Record) => { - // in `formula`, get all the (unique) target neighbours - // i.e. all column id (e.g. cl_xxxxxxxxxxxxxx) with formula type - const neighbours = [ - ...new Set( - (c.colOptions.formula.match(/cl_\w{14}/g) || []).filter( - (colId: string) => - supportedColumns.value.filter( - (col: ColumnType) => - col.id === colId && col.uidt === UITypes.Formula - ).length - ) - ), - ]; - if (neighbours.length > 0) { - // e.g. formula column 1 -> [formula column 2, formula column3] - res.push({ [c.id]: neighbours }); - } - return res; - }, []); - // include target formula column (i.e. the one to be saved if applicable) - const targetFormulaCol = supportedColumns.value.find( - (c: ColumnType) => - c.title === parsedTree.name && c.uidt === UITypes.Formula - ); - - if (targetFormulaCol && column.value?.id) { - formulaPaths.push({ - [column.value?.id as string]: [targetFormulaCol.id], - }); - } - const vertices = formulaPaths.length; - if (vertices > 0) { - // perform kahn's algo for cycle detection - const adj = new Map(); - const inDegrees = new Map(); - // init adjacency list & indegree - - for (const [_, v] of Object.entries(formulaPaths)) { - const src = Object.keys(v)[0]; - const neighbours = v[src]; - inDegrees.set(src, inDegrees.get(src) || 0); - for (const neighbour of neighbours) { - adj.set(src, (adj.get(src) || new Set()).add(neighbour)); - inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1); - } - } - const queue: string[] = []; - // put all vertices with in-degree = 0 (i.e. no incoming edges) to queue - inDegrees.forEach((inDegree, col) => { - if (inDegree === 0) { - // in-degree = 0 means we start traversing from this node - queue.push(col); - } - }); - // init count of visited vertices - let visited = 0; - // BFS - while (queue.length !== 0) { - // remove a vertex from the queue - const src = queue.shift(); - // if this node has neighbours, increase visited by 1 - const neighbours = adj.get(src) || new Set(); - if (neighbours.size > 0) { - visited += 1; - } - // iterate each neighbouring nodes - neighbours.forEach((neighbour: string) => { - // decrease in-degree of its neighbours by 1 - inDegrees.set(neighbour, inDegrees.get(neighbour) - 1); - // if in-degree becomes 0 - if (inDegrees.get(neighbour) === 0) { - // then put the neighboring node to the queue - queue.push(neighbour); - } - }); - } - // vertices not same as visited = cycle found - if (vertices !== visited) { - errors.add(t('msg.formula.cantSaveCircularReference')); - } - } - } else if (parsedTree.type === JSEPNode.BINARY_EXP) { - if (!availableBinOps.includes(parsedTree.operator)) { - errors.add( - t('msg.formula.operationNotAvailable', { - operation: parsedTree.operator, - }) - ); - } - validateAgainstMeta(parsedTree.left, errors); - validateAgainstMeta(parsedTree.right, errors); - - // todo: type extraction for binary exps - returnType = FormulaDataTypes.NUMERIC; - } else if ( - parsedTree.type === JSEPNode.LITERAL || - parsedTree.type === JSEPNode.UNARY_EXP - ) { - if (parsedTree.type === JSEPNode.LITERAL) { - if (typeof parsedTree.value === 'number') { - returnType = FormulaDataTypes.NUMERIC; - } else if (typeof parsedTree.value === 'string') { - returnType = FormulaDataTypes.STRING; - } else if (typeof parsedTree.value === 'boolean') { - returnType = FormulaDataTypes.BOOLEAN; - } else { - returnType = FormulaDataTypes.STRING; - } - } - // do nothing - } else if (parsedTree.type === JSEPNode.COMPOUND) { - if (parsedTree.body.length) { - errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')); - } - } else { - errors.add(t('msg.formula.cantSaveFieldFormulaInvalid')); - } - return { errors, returnType }; -} - -function validateAgainstType( - parsedTree: any, - expectedType: string, - func: any, - typeErrors = new Set(), - argTypes: FormulaDataTypes = [] -) { - let type; - if (parsedTree === false || typeof parsedTree === 'undefined') { - return typeErrors; - } - if (parsedTree.type === JSEPNode.LITERAL) { - if (typeof func === 'function') { - func(parsedTree.value); - } else if (expectedType === FormulaDataTypes.NUMERIC) { - if (typeof parsedTree.value !== 'number') { - typeErrors.add(t('msg.formula.numericTypeIsExpected')); - } else { - type = FormulaDataTypes.NUMERIC; - } - } else if (expectedType === FormulaDataTypes.STRING) { - if (typeof parsedTree.value !== 'string') { - typeErrors.add(t('msg.formula.stringTypeIsExpected')); - } else { - type = FormulaDataTypes.STRING; - } - } - } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - const col = supportedColumns.value.find((c) => c.title === parsedTree.name); - - if (col === undefined) { - return; - } - - if (col.uidt === UITypes.Formula) { - const foundType = getRootDataType(jsep(col.colOptions?.formula_raw)); - type = foundType; - if (foundType === 'N/A') { - typeErrors.add( - t('msg.formula.notSupportedToReferenceColumn', { - columnName: col.title, - }) - ); - } else if (expectedType !== foundType) { - typeErrors.add( - t('msg.formula.typeIsExpectedButFound', { - type: expectedType, - found: foundType, - }) - ); - } - } else { - switch (col.uidt) { - // string - case UITypes.SingleLineText: - case UITypes.LongText: - case UITypes.MultiSelect: - case UITypes.SingleSelect: - case UITypes.PhoneNumber: - case UITypes.Email: - case UITypes.URL: - if (expectedType !== FormulaDataTypes.STRING) { - typeErrors.add( - t('msg.formula.columnWithTypeFoundButExpected', { - columnName: parsedTree.name, - columnType: FormulaDataTypes.STRING, - expectedType, - }) - ); - } - type = FormulaDataTypes.STRING; - break; - - // numeric - case UITypes.Year: - case UITypes.Number: - case UITypes.Decimal: - case UITypes.Rating: - case UITypes.Count: - case UITypes.AutoNumber: - case UITypes.Currency: - if (expectedType !== FormulaDataTypes.NUMERIC) { - typeErrors.add( - t('msg.formula.columnWithTypeFoundButExpected', { - columnName: parsedTree.name, - columnType: FormulaDataTypes.NUMERIC, - expectedType, - }) - ); - } - type = FormulaDataTypes.NUMERIC; - break; - - // date - case UITypes.Date: - case UITypes.DateTime: - case UITypes.CreateTime: - case UITypes.LastModifiedTime: - if (expectedType !== FormulaDataTypes.DATE) { - typeErrors.add( - t('msg.formula.columnWithTypeFoundButExpected', { - columnName: parsedTree.name, - columnType: FormulaDataTypes.DATE, - expectedType, - }) - ); - } - type = FormulaDataTypes.DATE; - break; - - // not supported - case UITypes.ForeignKey: - case UITypes.Attachment: - case UITypes.ID: - case UITypes.Time: - case UITypes.Percent: - case UITypes.Duration: - case UITypes.Rollup: - case UITypes.Lookup: - case UITypes.Barcode: - case UITypes.Button: - case UITypes.Checkbox: - case UITypes.Collaborator: - case UITypes.QrCode: - default: - typeErrors.add( - t('msg.formula.notSupportedToReferenceColumn', { - columnName: parsedTree.name, - }) - ); - break; - } - } - } else if ( - parsedTree.type === JSEPNode.UNARY_EXP || - parsedTree.type === JSEPNode.BINARY_EXP - ) { - if (expectedType !== FormulaDataTypes.NUMERIC) { - // parsedTree.name won't be available here - typeErrors.add( - t('msg.formula.typeIsExpectedButFound', { - type: FormulaDataTypes.NUMERIC, - found: expectedType, - }) - ); - } - - type = FormulaDataTypes.NUMERIC; - } else if (parsedTree.type === JSEPNode.CALL_EXP) { - const calleeName = parsedTree.callee.name.toUpperCase(); - if ( - formulas[calleeName]?.type && - expectedType !== formulas[calleeName].type - ) { - typeErrors.add( - t('msg.formula.typeIsExpectedButFound', { - type: expectedType, - found: formulas[calleeName].type, - }) - ); - } - // todo: derive type from returnType - type = formulas[calleeName]?.type; - } - return { type, typeErrors }; -} - -function getRootDataType(parsedTree: any): any { - // given a parse tree, return the data type of it - if (parsedTree.type === JSEPNode.CALL_EXP) { - return formulas[parsedTree.callee.name.toUpperCase()].type; - } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - const col = supportedColumns.value.find( - (c) => c.title === parsedTree.name - ) as Record; - if (col?.uidt === UITypes.Formula) { - return getRootDataType(jsep(col?.formula_raw)); - } else { - switch (col?.uidt) { - // string - case UITypes.SingleLineText: - case UITypes.LongText: - case UITypes.MultiSelect: - case UITypes.SingleSelect: - case UITypes.PhoneNumber: - case UITypes.Email: - case UITypes.URL: - return FormulaDataTypes.STRING; - - // numeric - case UITypes.Year: - case UITypes.Number: - case UITypes.Decimal: - case UITypes.Rating: - case UITypes.Count: - case UITypes.AutoNumber: - return FormulaDataTypes.NUMERIC; - - // date - case UITypes.Date: - case UITypes.DateTime: - case UITypes.CreateTime: - case UITypes.LastModifiedTime: - return FormulaDataTypes.DATE; - - // not supported - case UITypes.ForeignKey: - case UITypes.Attachment: - case UITypes.ID: - case UITypes.Time: - case UITypes.Currency: - case UITypes.Percent: - case UITypes.Duration: - case UITypes.Rollup: - case UITypes.Lookup: - case UITypes.Barcode: - case UITypes.Button: - case UITypes.Checkbox: - case UITypes.Collaborator: - case UITypes.QrCode: - default: - return 'N/A'; - } - } - } else if ( - parsedTree.type === JSEPNode.BINARY_EXP || - parsedTree.type === JSEPNode.UNARY_EXP - ) { - return FormulaDataTypes.NUMERIC; - } else if (parsedTree.type === JSEPNode.LITERAL) { - return typeof parsedTree.value; - } else { - return 'N/A'; - } -} - -function isCurlyBracketBalanced() { - // count number of opening curly brackets and closing curly brackets - const cntCurlyBrackets = ( - formulaRef.value.$el.value.match(/\{|}/g) || [] - ).reduce((acc: Record, cur: number) => { - acc[cur] = (acc[cur] || 0) + 1; - return acc; - }, {}); - return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0); -} -*/ - enum FormulaErrorType { NOT_AVAILABLE = 'NOT_AVAILABLE', NOT_SUPPORTED = 'NOT_SUPPORTED', @@ -1571,11 +973,13 @@ enum FormulaErrorType { 'INVALID_ARG_TYPE' = 'INVALID_ARG_TYPE', 'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE', 'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT', + CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE', } class FormulaError extends Error { public name: string; public type: FormulaErrorType; + constructor( type: FormulaErrorType, name: string, @@ -1671,9 +1075,11 @@ export function validateFormulaAndExtractTreeWithType( } } // get args type and validate - const validateResult = res.arguments = parsedTree.arguments.map((arg) => { - return validateAndExtract(arg); - }); + const validateResult = (res.arguments = parsedTree.arguments.map( + (arg) => { + return validateAndExtract(arg); + } + )); const argsTypes = validateResult.map((v: any) => v.dataType); @@ -1685,10 +1091,8 @@ export function validateFormulaAndExtractTreeWithType( res.dataType = formulas[calleeName].returnType as FormulaDataTypes; } } else if (parsedTree.type === JSEPNode.IDENTIFIER) { - const col = (colIdToColMap[parsedTree.name] || colAliasToColMap[parsedTree.name]) as Record< - string, - any - >; + const col = (colIdToColMap[parsedTree.name] || + colAliasToColMap[parsedTree.name]) as Record; res.name = col.id; if (col?.uidt === UITypes.Formula) { @@ -1702,7 +1106,6 @@ export function validateFormulaAndExtractTreeWithType( res.dataType = formulaRes as any; } else { - switch (col?.uidt) { // string case UITypes.SingleLineText: @@ -1766,8 +1169,6 @@ export function validateFormulaAndExtractTreeWithType( res.left = validateAndExtract(parsedTree.left); res.right = validateAndExtract(parsedTree.right); res.dataType = FormulaDataTypes.NUMERIC; - } else { - // res.type= 'N/A'; } return res; @@ -1779,3 +1180,95 @@ export function validateFormulaAndExtractTreeWithType( const result = validateAndExtract(parsedFormula); return result; } + +function checkForCircularFormulaRef(formulaCol, parsedTree, columns) { + // check circular reference + // e.g. formula1 -> formula2 -> formula1 should return circular reference error + + // get all formula columns excluding itself + const formulaPaths = columns.value + .filter((c) => c.id !== formulaCol.value?.id && c.uidt === UITypes.Formula) + .reduce((res: Record[], c: Record) => { + // in `formula`, get all the (unique) target neighbours + // i.e. all column id (e.g. cxxxxxxxxxxxxxx) with formula type + const neighbours = [ + ...new Set( + (c.colOptions.formula.match(/c\w{15}/g) || []).filter( + (colId: string) => + columns.value.filter( + (col: ColumnType) => + col.id === colId && col.uidt === UITypes.Formula + ).length + ) + ), + ]; + if (neighbours.length > 0) { + // e.g. formula column 1 -> [formula column 2, formula column3] + res.push({ [c.id]: neighbours }); + } + return res; + }, []); + + // include target formula column (i.e. the one to be saved if applicable) + const targetFormulaCol = columns.value.find( + (c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula + ); + + if (targetFormulaCol && formulaCol.value?.id) { + formulaPaths.push({ + [formulaCol.value?.id as string]: [targetFormulaCol.id], + }); + } + const vertices = formulaPaths.length; + if (vertices > 0) { + // perform kahn's algo for cycle detection + const adj = new Map(); + const inDegrees = new Map(); + // init adjacency list & indegree + + for (const [_, v] of Object.entries(formulaPaths)) { + const src = Object.keys(v)[0]; + const neighbours = v[src]; + inDegrees.set(src, inDegrees.get(src) || 0); + for (const neighbour of neighbours) { + adj.set(src, (adj.get(src) || new Set()).add(neighbour)); + inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1); + } + } + const queue: string[] = []; + // put all vertices with in-degree = 0 (i.e. no incoming edges) to queue + inDegrees.forEach((inDegree, col) => { + if (inDegree === 0) { + // in-degree = 0 means we start traversing from this node + queue.push(col); + } + }); + // init count of visited vertices + let visited = 0; + // BFS + while (queue.length !== 0) { + // remove a vertex from the queue + const src = queue.shift(); + // if this node has neighbours, increase visited by 1 + const neighbours = adj.get(src) || new Set(); + if (neighbours.size > 0) { + visited += 1; + } + // iterate each neighbouring nodes + neighbours.forEach((neighbour: string) => { + // decrease in-degree of its neighbours by 1 + inDegrees.set(neighbour, inDegrees.get(neighbour) - 1); + // if in-degree becomes 0 + if (inDegrees.get(neighbour) === 0) { + // then put the neighboring node to the queue + queue.push(neighbour); + } + }); + } + // vertices not same as visited = cycle found + if (vertices !== visited) { + // errors.add(t('msg.formula.cantSaveCircularReference')) + throw new FormulaError(FormulaErrorType.CIRCULAR_REFERENCE, ''); + } + } +} diff --git a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts index e8bd658f97..f1ac991183 100644 --- a/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts +++ b/packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts @@ -63,17 +63,22 @@ async function _formulaQueryBuilder( model: Model, aliasToColumn: Record Promise<{ builder: any }>> = {}, tableAlias?: string, + parsedTree?: any, ) { const knex = baseModelSqlv2.dbDriver; const columns = await model.getColumns(); - // formula may include double curly brackets in previous version - // convert to single curly bracket here for compatibility - // const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}')); - const tree = validateFormulaAndExtractTreeWithType( - _tree.replaceAll('{{', '{').replaceAll('}}', '}'), - columns, - ); + + let tree = parsedTree; + if (!tree) { + // formula may include double curly brackets in previous version + // convert to single curly bracket here for compatibility + // const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}')); + tree = validateFormulaAndExtractTreeWithType( + _tree.replaceAll('{{', '{').replaceAll('}}', '}'), + columns, + ); + } const columnIdToUidt = {}; @@ -93,6 +98,7 @@ async function _formulaQueryBuilder( model, { ...aliasToColumn, [col.id]: null }, tableAlias, + formulOption.getParsedTree(), ); builder.sql = '(' + builder.sql + ')'; return { @@ -413,6 +419,7 @@ async function _formulaQueryBuilder( '', lookupModel, aliasToColumn, + formulaOption.getParsedTree() ); if (isMany) { const qb = selectQb; @@ -968,6 +975,9 @@ export default async function formulaQueryBuilderv2( model, aliasToColumn, tableAlias, + await column + ?.getColOptions() + .then((formula) => formula?.getParsedTree()), ); if (!validateFormula) return qb; diff --git a/packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts b/packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts new file mode 100644 index 0000000000..5e50af058c --- /dev/null +++ b/packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts @@ -0,0 +1,16 @@ +import type { Knex } from 'knex'; +import { MetaTable } from '~/utils/globals'; + +const up = async (knex: Knex) => { + await knex.schema.alterTable(MetaTable.COL_FORMULA, (table) => { + table.text('parsed_tree'); + }); +}; + +const down = async (knex: Knex) => { + await knex.schema.alterTable(MetaTable.COL_FORMULA, (table) => { + table.dropColumn('parsed_tree'); + }); +}; + +export { up, down }; diff --git a/packages/nocodb/src/models/FormulaColumn.ts b/packages/nocodb/src/models/FormulaColumn.ts index 955d480b11..d3f1234a44 100644 --- a/packages/nocodb/src/models/FormulaColumn.ts +++ b/packages/nocodb/src/models/FormulaColumn.ts @@ -2,19 +2,21 @@ import Noco from '~/Noco'; import NocoCache from '~/cache/NocoCache'; import { extractProps } from '~/helpers/extractProps'; import { CacheGetType, CacheScope, MetaTable } from '~/utils/globals'; +import { parseMetaProp, stringifyMetaProp } from '~/utils/modelUtils'; export default class FormulaColumn { formula: string; formula_raw: string; fk_column_id: string; error: string; + private parsed_tree?: any; constructor(data: Partial) { Object.assign(this, data); } public static async insert( - formulaColumn: Partial, + formulaColumn: Partial, ncMeta = Noco.ncMeta, ) { const insertObj = extractProps(formulaColumn, [ @@ -22,11 +24,16 @@ export default class FormulaColumn { 'formula_raw', 'formula', 'error', + 'parsed_tree', ]); + + insertObj.parsed_tree = stringifyMetaProp(insertObj, 'parsed_tree'); + await ncMeta.metaInsert2(null, null, MetaTable.COL_FORMULA, insertObj); return this.read(formulaColumn.fk_column_id, ncMeta); } + public static async read(columnId: string, ncMeta = Noco.ncMeta) { let column = columnId && @@ -41,7 +48,10 @@ export default class FormulaColumn { MetaTable.COL_FORMULA, { fk_column_id: columnId }, ); - await NocoCache.set(`${CacheScope.COL_FORMULA}:${columnId}`, column); + if (column) { + column.parsed_tree = parseMetaProp(column, 'parsed_tree'); + await NocoCache.set(`${CacheScope.COL_FORMULA}:${columnId}`, column); + } } return column ? new FormulaColumn(column) : null; @@ -51,7 +61,7 @@ export default class FormulaColumn { static async update( id: string, - formula: Partial, + formula: Partial & { parsed_tree?: any }, ncMeta = Noco.ncMeta, ) { const updateObj = extractProps(formula, [ @@ -59,7 +69,11 @@ export default class FormulaColumn { 'formula_raw', 'fk_column_id', 'error', + 'parsed_tree', ]); + + updateObj.parsed_tree = stringifyMetaProp(insertObj, 'parsed_tree'); + // get existing cache const key = `${CacheScope.COL_FORMULA}:${id}`; let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); @@ -71,4 +85,8 @@ export default class FormulaColumn { // set meta await ncMeta.metaUpdate(null, null, MetaTable.COL_FORMULA, updateObj, id); } + + public getParsedTree() { + return this.parsed_tree; + } } diff --git a/packages/nocodb/src/services/columns.service.ts b/packages/nocodb/src/services/columns.service.ts index 09cecbd6e6..48c2f30464 100644 --- a/packages/nocodb/src/services/columns.service.ts +++ b/packages/nocodb/src/services/columns.service.ts @@ -5,7 +5,7 @@ import { isVirtualCol, substituteColumnAliasWithIdInFormula, substituteColumnIdWithAliasInFormula, - UITypes, + UITypes, validateFormulaAndExtractTreeWithType, } from 'nocodb-sdk'; import { pluralize, singularize } from 'inflection'; import hash from 'object-hash'; @@ -1197,6 +1197,10 @@ export class ColumnsService { colBody.formula_raw || colBody.formula, table.columns, ); + colBody.parsed_tree = validateFormulaAndExtractTreeWithType( + colBody.formula_raw || colBody.formula, + table.columns, + ); try { const baseModel = await reuseOrSave('baseModel', reuse, async () =>