github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: add migration and hide parsed tree data from api response

pull/7268/head
Pranav C 11 months ago
parent
82e8bcd523
commit
36277d5509
5 changed files with 162 additions and 621 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 711
      packages/nocodb-sdk/src/lib/formulaHelpers.ts
  2. 12
      packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
  3. 16
      packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts
  4. 22
      packages/nocodb/src/models/FormulaColumn.ts
  5. 6
      packages/nocodb/src/services/columns.service.ts

711
packages/nocodb-sdk/src/lib/formulaHelpers.ts
Unescape View File

@ -2,6 +2,7 @@ import jsep from 'jsep';
import { ColumnType } from './Api'; import { ColumnType } from './Api';
import UITypes from './UITypes'; import UITypes from './UITypes';
import { formulaTypes } from '../../../nc-gui/utils';
export const jsepCurlyHook = { export const jsepCurlyHook = {
name: 'curly', name: 'curly',
@ -220,6 +221,7 @@ interface FormulaMeta {
min?: number; min?: number;
max?: number; max?: number;
rqd?: number; rqd?: number;
validator?: (args: formulaTypes[]) => boolean;
}; };
}; };
description?: string; description?: string;
@ -230,7 +232,6 @@ interface FormulaMeta {
const formulas: Record<string, FormulaMeta> = { const formulas: Record<string, FormulaMeta> = {
AVG: { AVG: {
type: FormulaDataTypes.NUMERIC,
validation: { validation: {
args: { args: {
min: 1, min: 1,
@ -309,7 +310,6 @@ const formulas: Record<string, FormulaMeta> = {
returnType: FormulaDataTypes.NUMERIC, returnType: FormulaDataTypes.NUMERIC,
}, },
AND: { AND: {
type: FormulaDataTypes.COND_EXP,
validation: { validation: {
args: { args: {
min: 1, min: 1,
@ -321,7 +321,6 @@ const formulas: Record<string, FormulaMeta> = {
returnType: FormulaDataTypes.COND_EXP, returnType: FormulaDataTypes.COND_EXP,
}, },
OR: { OR: {
type: FormulaDataTypes.COND_EXP,
validation: { validation: {
args: { args: {
min: 1, min: 1,
@ -964,603 +963,6 @@ const formulas: Record<string, FormulaMeta> = {
// }, // },
}; };
/*
function validateAgainstMeta(
parsedTree: any,
errors = new Set(),
typeErrors = new Set()
) {
let returnType: FormulaDataTypes;
if (parsedTree.type === JSEPNode.CALL_EXP) {
const calleeName = parsedTree.callee.name.toUpperCase();
// validate function name
if (!availableFunctions.includes(calleeName)) {
errors.add(
t('msg.formula.functionNotAvailable', { function: calleeName })
);
}
// validate arguments
const validation = formulas[calleeName] && formulas[calleeName].validation;
if (validation && validation.args) {
if (
validation.args.rqd !== undefined &&
validation.args.rqd !== parsedTree.arguments.length
) {
errors.add(
t('msg.formula.requiredArgumentsFormula', {
requiredArguments: validation.args.rqd,
calleeName,
})
);
} else if (
validation.args.min !== undefined &&
validation.args.min > parsedTree.arguments.length
) {
errors.add(
t('msg.formula.minRequiredArgumentsFormula', {
minRequiredArguments: validation.args.min,
calleeName,
})
);
} else if (
validation.args.max !== undefined &&
validation.args.max < parsedTree.arguments.length
) {
errors.add(
t('msg.formula.maxRequiredArgumentsFormula', {
maxRequiredArguments: validation.args.max,
calleeName,
})
);
}
}
parsedTree.arguments.map((arg: Record<string, any>) =>
validateAgainstMeta(arg, errors)
);
// get args type and validate
const validateResult = parsedTree.arguments.map((arg) => {
return validateAgainstMeta(arg, errors, typeErrors);
});
const argsTypes = validateResult.map((v: any) => v.returnType);
if (typeof validateResult[0].returnType === 'function') {
returnType = formulas[calleeName].returnType(argsTypes);
} else if (validateResult[0]) {
returnType = formulas[calleeName].returnType;
}
// validate data type
if (parsedTree.callee.type === JSEPNode.IDENTIFIER) {
const expectedType = formulas[calleeName.toUpperCase()].type;
if (expectedType === FormulaDataTypes.NUMERIC) {
if (calleeName === 'WEEKDAY') {
// parsedTree.arguments[0] = date
validateAgainstType(
parsedTree.arguments[0],
FormulaDataTypes.DATE,
(v: any) => {
if (!validateDateWithUnknownFormat(v)) {
typeErrors.add(t('msg.formula.firstParamWeekDayHaveDate'));
}
},
typeErrors
);
// parsedTree.arguments[1] = startDayOfWeek (optional)
validateAgainstType(
parsedTree.arguments[1],
FormulaDataTypes.STRING,
(v: any) => {
if (
typeof v !== 'string' ||
![
'sunday',
'monday',
'tuesday',
'wednesday',
'thursday',
'friday',
'saturday',
].includes(v.toLowerCase())
) {
typeErrors.add(t('msg.formula.secondParamWeekDayHaveDate'));
}
},
typeErrors
);
} else {
parsedTree.arguments.map((arg: Record<string, any>) =>
validateAgainstType(arg, expectedType, null, typeErrors, argsTypes)
);
}
} else if (expectedType === FormulaDataTypes.DATE) {
if (calleeName === 'DATEADD') {
// parsedTree.arguments[0] = date
validateAgainstType(
parsedTree.arguments[0],
FormulaDataTypes.DATE,
(v: any) => {
if (!validateDateWithUnknownFormat(v)) {
typeErrors.add(t('msg.formula.firstParamDateAddHaveDate'));
}
},
typeErrors
);
// parsedTree.arguments[1] = numeric
validateAgainstType(
parsedTree.arguments[1],
FormulaDataTypes.NUMERIC,
(v: any) => {
if (typeof v !== 'number') {
typeErrors.add(t('msg.formula.secondParamDateAddHaveNumber'));
}
},
typeErrors
);
// parsedTree.arguments[2] = ["day" | "week" | "month" | "year"]
validateAgainstType(
parsedTree.arguments[2],
FormulaDataTypes.STRING,
(v: any) => {
if (!['day', 'week', 'month', 'year'].includes(v)) {
typeErrors.add(
typeErrors.add(t('msg.formula.thirdParamDateAddHaveDate'))
);
}
},
typeErrors
);
} else if (calleeName === 'DATETIME_DIFF') {
// parsedTree.arguments[0] = date
validateAgainstType(
parsedTree.arguments[0],
FormulaDataTypes.DATE,
(v: any) => {
if (!validateDateWithUnknownFormat(v)) {
typeErrors.add(t('msg.formula.firstParamDateDiffHaveDate'));
}
},
typeErrors
);
// parsedTree.arguments[1] = date
validateAgainstType(
parsedTree.arguments[1],
FormulaDataTypes.DATE,
(v: any) => {
if (!validateDateWithUnknownFormat(v)) {
typeErrors.add(t('msg.formula.secondParamDateDiffHaveDate'));
}
},
typeErrors
);
// parsedTree.arguments[2] = ["milliseconds" | "ms" | "seconds" | "s" | "minutes" | "m" | "hours" | "h" | "days" | "d" | "weeks" | "w" | "months" | "M" | "quarters" | "Q" | "years" | "y"]
validateAgainstType(
parsedTree.arguments[2],
FormulaDataTypes.STRING,
(v: any) => {
if (
![
'milliseconds',
'ms',
'seconds',
's',
'minutes',
'm',
'hours',
'h',
'days',
'd',
'weeks',
'w',
'months',
'M',
'quarters',
'Q',
'years',
'y',
].includes(v)
) {
typeErrors.add(t('msg.formula.thirdParamDateDiffHaveDate'));
}
},
typeErrors
);
}
}
}
errors = new Set([...errors, ...typeErrors]);
} else if (parsedTree.type === JSEPNode.IDENTIFIER) {
if (
supportedColumns.value
.filter((c) => !column || column.value?.id !== c.id)
.every((c) => c.title !== parsedTree.name)
) {
errors.add(
t('msg.formula.columnNotAvailable', {
columnName: parsedTree.name,
})
);
}
// check circular reference
// e.g. formula1 -> formula2 -> formula1 should return circular reference error
// get all formula columns excluding itself
const formulaPaths = supportedColumns.value
.filter((c) => c.id !== column.value?.id && c.uidt === UITypes.Formula)
.reduce((res: Record<string, any>[], c: Record<string, any>) => {
// in `formula`, get all the (unique) target neighbours
// i.e. all column id (e.g. cl_xxxxxxxxxxxxxx) with formula type
const neighbours = [
...new Set(
(c.colOptions.formula.match(/cl_\w{14}/g) || []).filter(
(colId: string) =>
supportedColumns.value.filter(
(col: ColumnType) =>
col.id === colId && col.uidt === UITypes.Formula
).length
)
),
];
if (neighbours.length > 0) {
// e.g. formula column 1 -> [formula column 2, formula column3]
res.push({ [c.id]: neighbours });
}
return res;
}, []);
// include target formula column (i.e. the one to be saved if applicable)
const targetFormulaCol = supportedColumns.value.find(
(c: ColumnType) =>
c.title === parsedTree.name && c.uidt === UITypes.Formula
);
if (targetFormulaCol && column.value?.id) {
formulaPaths.push({
[column.value?.id as string]: [targetFormulaCol.id],
});
}
const vertices = formulaPaths.length;
if (vertices > 0) {
// perform kahn's algo for cycle detection
const adj = new Map();
const inDegrees = new Map();
// init adjacency list & indegree
for (const [_, v] of Object.entries(formulaPaths)) {
const src = Object.keys(v)[0];
const neighbours = v[src];
inDegrees.set(src, inDegrees.get(src) || 0);
for (const neighbour of neighbours) {
adj.set(src, (adj.get(src) || new Set()).add(neighbour));
inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1);
}
}
const queue: string[] = [];
// put all vertices with in-degree = 0 (i.e. no incoming edges) to queue
inDegrees.forEach((inDegree, col) => {
if (inDegree === 0) {
// in-degree = 0 means we start traversing from this node
queue.push(col);
}
});
// init count of visited vertices
let visited = 0;
// BFS
while (queue.length !== 0) {
// remove a vertex from the queue
const src = queue.shift();
// if this node has neighbours, increase visited by 1
const neighbours = adj.get(src) || new Set();
if (neighbours.size > 0) {
visited += 1;
}
// iterate each neighbouring nodes
neighbours.forEach((neighbour: string) => {
// decrease in-degree of its neighbours by 1
inDegrees.set(neighbour, inDegrees.get(neighbour) - 1);
// if in-degree becomes 0
if (inDegrees.get(neighbour) === 0) {
// then put the neighboring node to the queue
queue.push(neighbour);
}
});
}
// vertices not same as visited = cycle found
if (vertices !== visited) {
errors.add(t('msg.formula.cantSaveCircularReference'));
}
}
} else if (parsedTree.type === JSEPNode.BINARY_EXP) {
if (!availableBinOps.includes(parsedTree.operator)) {
errors.add(
t('msg.formula.operationNotAvailable', {
operation: parsedTree.operator,
})
);
}
validateAgainstMeta(parsedTree.left, errors);
validateAgainstMeta(parsedTree.right, errors);
// todo: type extraction for binary exps
returnType = FormulaDataTypes.NUMERIC;
} else if (
parsedTree.type === JSEPNode.LITERAL ||
parsedTree.type === JSEPNode.UNARY_EXP
) {
if (parsedTree.type === JSEPNode.LITERAL) {
if (typeof parsedTree.value === 'number') {
returnType = FormulaDataTypes.NUMERIC;
} else if (typeof parsedTree.value === 'string') {
returnType = FormulaDataTypes.STRING;
} else if (typeof parsedTree.value === 'boolean') {
returnType = FormulaDataTypes.BOOLEAN;
} else {
returnType = FormulaDataTypes.STRING;
}
}
// do nothing
} else if (parsedTree.type === JSEPNode.COMPOUND) {
if (parsedTree.body.length) {
errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'));
}
} else {
errors.add(t('msg.formula.cantSaveFieldFormulaInvalid'));
}
return { errors, returnType };
}
function validateAgainstType(
parsedTree: any,
expectedType: string,
func: any,
typeErrors = new Set(),
argTypes: FormulaDataTypes = []
) {
let type;
if (parsedTree === false || typeof parsedTree === 'undefined') {
return typeErrors;
}
if (parsedTree.type === JSEPNode.LITERAL) {
if (typeof func === 'function') {
func(parsedTree.value);
} else if (expectedType === FormulaDataTypes.NUMERIC) {
if (typeof parsedTree.value !== 'number') {
typeErrors.add(t('msg.formula.numericTypeIsExpected'));
} else {
type = FormulaDataTypes.NUMERIC;
}
} else if (expectedType === FormulaDataTypes.STRING) {
if (typeof parsedTree.value !== 'string') {
typeErrors.add(t('msg.formula.stringTypeIsExpected'));
} else {
type = FormulaDataTypes.STRING;
}
}
} else if (parsedTree.type === JSEPNode.IDENTIFIER) {
const col = supportedColumns.value.find((c) => c.title === parsedTree.name);
if (col === undefined) {
return;
}
if (col.uidt === UITypes.Formula) {
const foundType = getRootDataType(jsep(col.colOptions?.formula_raw));
type = foundType;
if (foundType === 'N/A') {
typeErrors.add(
t('msg.formula.notSupportedToReferenceColumn', {
columnName: col.title,
})
);
} else if (expectedType !== foundType) {
typeErrors.add(
t('msg.formula.typeIsExpectedButFound', {
type: expectedType,
found: foundType,
})
);
}
} else {
switch (col.uidt) {
// string
case UITypes.SingleLineText:
case UITypes.LongText:
case UITypes.MultiSelect:
case UITypes.SingleSelect:
case UITypes.PhoneNumber:
case UITypes.Email:
case UITypes.URL:
if (expectedType !== FormulaDataTypes.STRING) {
typeErrors.add(
t('msg.formula.columnWithTypeFoundButExpected', {
columnName: parsedTree.name,
columnType: FormulaDataTypes.STRING,
expectedType,
})
);
}
type = FormulaDataTypes.STRING;
break;
// numeric
case UITypes.Year:
case UITypes.Number:
case UITypes.Decimal:
case UITypes.Rating:
case UITypes.Count:
case UITypes.AutoNumber:
case UITypes.Currency:
if (expectedType !== FormulaDataTypes.NUMERIC) {
typeErrors.add(
t('msg.formula.columnWithTypeFoundButExpected', {
columnName: parsedTree.name,
columnType: FormulaDataTypes.NUMERIC,
expectedType,
})
);
}
type = FormulaDataTypes.NUMERIC;
break;
// date
case UITypes.Date:
case UITypes.DateTime:
case UITypes.CreateTime:
case UITypes.LastModifiedTime:
if (expectedType !== FormulaDataTypes.DATE) {
typeErrors.add(
t('msg.formula.columnWithTypeFoundButExpected', {
columnName: parsedTree.name,
columnType: FormulaDataTypes.DATE,
expectedType,
})
);
}
type = FormulaDataTypes.DATE;
break;
// not supported
case UITypes.ForeignKey:
case UITypes.Attachment:
case UITypes.ID:
case UITypes.Time:
case UITypes.Percent:
case UITypes.Duration:
case UITypes.Rollup:
case UITypes.Lookup:
case UITypes.Barcode:
case UITypes.Button:
case UITypes.Checkbox:
case UITypes.Collaborator:
case UITypes.QrCode:
default:
typeErrors.add(
t('msg.formula.notSupportedToReferenceColumn', {
columnName: parsedTree.name,
})
);
break;
}
}
} else if (
parsedTree.type === JSEPNode.UNARY_EXP ||
parsedTree.type === JSEPNode.BINARY_EXP
) {
if (expectedType !== FormulaDataTypes.NUMERIC) {
// parsedTree.name won't be available here
typeErrors.add(
t('msg.formula.typeIsExpectedButFound', {
type: FormulaDataTypes.NUMERIC,
found: expectedType,
})
);
}
type = FormulaDataTypes.NUMERIC;
} else if (parsedTree.type === JSEPNode.CALL_EXP) {
const calleeName = parsedTree.callee.name.toUpperCase();
if (
formulas[calleeName]?.type &&
expectedType !== formulas[calleeName].type
) {
typeErrors.add(
t('msg.formula.typeIsExpectedButFound', {
type: expectedType,
found: formulas[calleeName].type,
})
);
}
// todo: derive type from returnType
type = formulas[calleeName]?.type;
}
return { type, typeErrors };
}
function getRootDataType(parsedTree: any): any {
// given a parse tree, return the data type of it
if (parsedTree.type === JSEPNode.CALL_EXP) {
return formulas[parsedTree.callee.name.toUpperCase()].type;
} else if (parsedTree.type === JSEPNode.IDENTIFIER) {
const col = supportedColumns.value.find(
(c) => c.title === parsedTree.name
) as Record<string, any>;
if (col?.uidt === UITypes.Formula) {
return getRootDataType(jsep(col?.formula_raw));
} else {
switch (col?.uidt) {
// string
case UITypes.SingleLineText:
case UITypes.LongText:
case UITypes.MultiSelect:
case UITypes.SingleSelect:
case UITypes.PhoneNumber:
case UITypes.Email:
case UITypes.URL:
return FormulaDataTypes.STRING;
// numeric
case UITypes.Year:
case UITypes.Number:
case UITypes.Decimal:
case UITypes.Rating:
case UITypes.Count:
case UITypes.AutoNumber:
return FormulaDataTypes.NUMERIC;
// date
case UITypes.Date:
case UITypes.DateTime:
case UITypes.CreateTime:
case UITypes.LastModifiedTime:
return FormulaDataTypes.DATE;
// not supported
case UITypes.ForeignKey:
case UITypes.Attachment:
case UITypes.ID:
case UITypes.Time:
case UITypes.Currency:
case UITypes.Percent:
case UITypes.Duration:
case UITypes.Rollup:
case UITypes.Lookup:
case UITypes.Barcode:
case UITypes.Button:
case UITypes.Checkbox:
case UITypes.Collaborator:
case UITypes.QrCode:
default:
return 'N/A';
}
}
} else if (
parsedTree.type === JSEPNode.BINARY_EXP ||
parsedTree.type === JSEPNode.UNARY_EXP
) {
return FormulaDataTypes.NUMERIC;
} else if (parsedTree.type === JSEPNode.LITERAL) {
return typeof parsedTree.value;
} else {
return 'N/A';
}
}
function isCurlyBracketBalanced() {
// count number of opening curly brackets and closing curly brackets
const cntCurlyBrackets = (
formulaRef.value.$el.value.match(/\{|}/g) || []
).reduce((acc: Record<number, number>, cur: number) => {
acc[cur] = (acc[cur] || 0) + 1;
return acc;
}, {});
return (cntCurlyBrackets['{'] || 0) === (cntCurlyBrackets['}'] || 0);
}
*/
enum FormulaErrorType { enum FormulaErrorType {
NOT_AVAILABLE = 'NOT_AVAILABLE', NOT_AVAILABLE = 'NOT_AVAILABLE',
NOT_SUPPORTED = 'NOT_SUPPORTED', NOT_SUPPORTED = 'NOT_SUPPORTED',
@ -1571,11 +973,13 @@ enum FormulaErrorType {
'INVALID_ARG_TYPE' = 'INVALID_ARG_TYPE', 'INVALID_ARG_TYPE' = 'INVALID_ARG_TYPE',
'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE', 'INVALID_ARG_VALUE' = 'INVALID_ARG_VALUE',
'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT', 'INVALID_ARG_COUNT' = 'INVALID_ARG_COUNT',
CIRCULAR_REFERENCE = 'CIRCULAR_REFERENCE',
} }
class FormulaError extends Error { class FormulaError extends Error {
public name: string; public name: string;
public type: FormulaErrorType; public type: FormulaErrorType;
constructor( constructor(
type: FormulaErrorType, type: FormulaErrorType,
name: string, name: string,
@ -1671,9 +1075,11 @@ export function validateFormulaAndExtractTreeWithType(
} }
} }
// get args type and validate // get args type and validate
const validateResult = res.arguments = parsedTree.arguments.map((arg) => { const validateResult = (res.arguments = parsedTree.arguments.map(
(arg) => {
return validateAndExtract(arg); return validateAndExtract(arg);
}); }
));
const argsTypes = validateResult.map((v: any) => v.dataType); const argsTypes = validateResult.map((v: any) => v.dataType);
@ -1685,10 +1091,8 @@ export function validateFormulaAndExtractTreeWithType(
res.dataType = formulas[calleeName].returnType as FormulaDataTypes; res.dataType = formulas[calleeName].returnType as FormulaDataTypes;
} }
} else if (parsedTree.type === JSEPNode.IDENTIFIER) { } else if (parsedTree.type === JSEPNode.IDENTIFIER) {
const col = (colIdToColMap[parsedTree.name] || colAliasToColMap[parsedTree.name]) as Record< const col = (colIdToColMap[parsedTree.name] ||
string, colAliasToColMap[parsedTree.name]) as Record<string, any>;
any
>;
res.name = col.id; res.name = col.id;
if (col?.uidt === UITypes.Formula) { if (col?.uidt === UITypes.Formula) {
@ -1702,7 +1106,6 @@ export function validateFormulaAndExtractTreeWithType(
res.dataType = formulaRes as any; res.dataType = formulaRes as any;
} else { } else {
switch (col?.uidt) { switch (col?.uidt) {
// string // string
case UITypes.SingleLineText: case UITypes.SingleLineText:
@ -1766,8 +1169,6 @@ export function validateFormulaAndExtractTreeWithType(
res.left = validateAndExtract(parsedTree.left); res.left = validateAndExtract(parsedTree.left);
res.right = validateAndExtract(parsedTree.right); res.right = validateAndExtract(parsedTree.right);
res.dataType = FormulaDataTypes.NUMERIC; res.dataType = FormulaDataTypes.NUMERIC;
} else {
// res.type= 'N/A';
} }
return res; return res;
@ -1779,3 +1180,95 @@ export function validateFormulaAndExtractTreeWithType(
const result = validateAndExtract(parsedFormula); const result = validateAndExtract(parsedFormula);
return result; return result;
} }
function checkForCircularFormulaRef(formulaCol, parsedTree, columns) {
// check circular reference
// e.g. formula1 -> formula2 -> formula1 should return circular reference error
// get all formula columns excluding itself
const formulaPaths = columns.value
.filter((c) => c.id !== formulaCol.value?.id && c.uidt === UITypes.Formula)
.reduce((res: Record<string, any>[], c: Record<string, any>) => {
// in `formula`, get all the (unique) target neighbours
// i.e. all column id (e.g. cxxxxxxxxxxxxxx) with formula type
const neighbours = [
...new Set(
(c.colOptions.formula.match(/c\w{15}/g) || []).filter(
(colId: string) =>
columns.value.filter(
(col: ColumnType) =>
col.id === colId && col.uidt === UITypes.Formula
).length
)
),
];
if (neighbours.length > 0) {
// e.g. formula column 1 -> [formula column 2, formula column3]
res.push({ [c.id]: neighbours });
}
return res;
}, []);
// include target formula column (i.e. the one to be saved if applicable)
const targetFormulaCol = columns.value.find(
(c: ColumnType) => c.title === parsedTree.name && c.uidt === UITypes.Formula
);
if (targetFormulaCol && formulaCol.value?.id) {
formulaPaths.push({
[formulaCol.value?.id as string]: [targetFormulaCol.id],
});
}
const vertices = formulaPaths.length;
if (vertices > 0) {
// perform kahn's algo for cycle detection
const adj = new Map();
const inDegrees = new Map();
// init adjacency list & indegree
for (const [_, v] of Object.entries(formulaPaths)) {
const src = Object.keys(v)[0];
const neighbours = v[src];
inDegrees.set(src, inDegrees.get(src) || 0);
for (const neighbour of neighbours) {
adj.set(src, (adj.get(src) || new Set()).add(neighbour));
inDegrees.set(neighbour, (inDegrees.get(neighbour) || 0) + 1);
}
}
const queue: string[] = [];
// put all vertices with in-degree = 0 (i.e. no incoming edges) to queue
inDegrees.forEach((inDegree, col) => {
if (inDegree === 0) {
// in-degree = 0 means we start traversing from this node
queue.push(col);
}
});
// init count of visited vertices
let visited = 0;
// BFS
while (queue.length !== 0) {
// remove a vertex from the queue
const src = queue.shift();
// if this node has neighbours, increase visited by 1
const neighbours = adj.get(src) || new Set();
if (neighbours.size > 0) {
visited += 1;
}
// iterate each neighbouring nodes
neighbours.forEach((neighbour: string) => {
// decrease in-degree of its neighbours by 1
inDegrees.set(neighbour, inDegrees.get(neighbour) - 1);
// if in-degree becomes 0
if (inDegrees.get(neighbour) === 0) {
// then put the neighboring node to the queue
queue.push(neighbour);
}
});
}
// vertices not same as visited = cycle found
if (vertices !== visited) {
// errors.add(t('msg.formula.cantSaveCircularReference'))
throw new FormulaError(FormulaErrorType.CIRCULAR_REFERENCE, '');
}
}
}

12
packages/nocodb/src/db/formulav2/formulaQueryBuilderv2.ts
Unescape View File

@ -63,17 +63,22 @@ async function _formulaQueryBuilder(
model: Model, model: Model,
aliasToColumn: Record<string, () => Promise<{ builder: any }>> = {}, aliasToColumn: Record<string, () => Promise<{ builder: any }>> = {},
tableAlias?: string, tableAlias?: string,
parsedTree?: any,
) { ) {
const knex = baseModelSqlv2.dbDriver; const knex = baseModelSqlv2.dbDriver;
const columns = await model.getColumns(); const columns = await model.getColumns();
let tree = parsedTree;
if (!tree) {
// formula may include double curly brackets in previous version // formula may include double curly brackets in previous version
// convert to single curly bracket here for compatibility // convert to single curly bracket here for compatibility
// const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}')); // const _tree1 = jsep(_tree.replaceAll('{{', '{').replaceAll('}}', '}'));
const tree = validateFormulaAndExtractTreeWithType( tree = validateFormulaAndExtractTreeWithType(
_tree.replaceAll('{{', '{').replaceAll('}}', '}'), _tree.replaceAll('{{', '{').replaceAll('}}', '}'),
columns, columns,
); );
}
const columnIdToUidt = {}; const columnIdToUidt = {};
@ -93,6 +98,7 @@ async function _formulaQueryBuilder(
model, model,
{ ...aliasToColumn, [col.id]: null }, { ...aliasToColumn, [col.id]: null },
tableAlias, tableAlias,
formulOption.getParsedTree(),
); );
builder.sql = '(' + builder.sql + ')'; builder.sql = '(' + builder.sql + ')';
return { return {
@ -413,6 +419,7 @@ async function _formulaQueryBuilder(
'', '',
lookupModel, lookupModel,
aliasToColumn, aliasToColumn,
formulaOption.getParsedTree()
); );
if (isMany) { if (isMany) {
const qb = selectQb; const qb = selectQb;
@ -968,6 +975,9 @@ export default async function formulaQueryBuilderv2(
model, model,
aliasToColumn, aliasToColumn,
tableAlias, tableAlias,
await column
?.getColOptions<FormulaColumn>()
.then((formula) => formula?.getParsedTree()),
); );
if (!validateFormula) return qb; if (!validateFormula) return qb;

16
packages/nocodb/src/meta/migrations/v2/nc_038_formula_parsed_tree_column.ts
Unescape View File

@ -0,0 +1,16 @@
import type { Knex } from 'knex';
import { MetaTable } from '~/utils/globals';
const up = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.COL_FORMULA, (table) => {
table.text('parsed_tree');
});
};
const down = async (knex: Knex) => {
await knex.schema.alterTable(MetaTable.COL_FORMULA, (table) => {
table.dropColumn('parsed_tree');
});
};
export { up, down };

22
packages/nocodb/src/models/FormulaColumn.ts
Unescape View File

@ -2,19 +2,21 @@ import Noco from '~/Noco';
import NocoCache from '~/cache/NocoCache'; import NocoCache from '~/cache/NocoCache';
import { extractProps } from '~/helpers/extractProps'; import { extractProps } from '~/helpers/extractProps';
import { CacheGetType, CacheScope, MetaTable } from '~/utils/globals'; import { CacheGetType, CacheScope, MetaTable } from '~/utils/globals';
import { parseMetaProp, stringifyMetaProp } from '~/utils/modelUtils';
export default class FormulaColumn { export default class FormulaColumn {
formula: string; formula: string;
formula_raw: string; formula_raw: string;
fk_column_id: string; fk_column_id: string;
error: string; error: string;
private parsed_tree?: any;
constructor(data: Partial<FormulaColumn>) { constructor(data: Partial<FormulaColumn>) {
Object.assign(this, data); Object.assign(this, data);
} }
public static async insert( public static async insert(
formulaColumn: Partial<FormulaColumn>, formulaColumn: Partial<FormulaColumn & { parsed_tree?: any }>,
ncMeta = Noco.ncMeta, ncMeta = Noco.ncMeta,
) { ) {
const insertObj = extractProps(formulaColumn, [ const insertObj = extractProps(formulaColumn, [
@ -22,11 +24,16 @@ export default class FormulaColumn {
'formula_raw', 'formula_raw',
'formula', 'formula',
'error', 'error',
'parsed_tree',
]); ]);
insertObj.parsed_tree = stringifyMetaProp(insertObj, 'parsed_tree');
await ncMeta.metaInsert2(null, null, MetaTable.COL_FORMULA, insertObj); await ncMeta.metaInsert2(null, null, MetaTable.COL_FORMULA, insertObj);
return this.read(formulaColumn.fk_column_id, ncMeta); return this.read(formulaColumn.fk_column_id, ncMeta);
} }
public static async read(columnId: string, ncMeta = Noco.ncMeta) { public static async read(columnId: string, ncMeta = Noco.ncMeta) {
let column = let column =
columnId && columnId &&
@ -41,8 +48,11 @@ export default class FormulaColumn {
MetaTable.COL_FORMULA, MetaTable.COL_FORMULA,
{ fk_column_id: columnId }, { fk_column_id: columnId },
); );
if (column) {
column.parsed_tree = parseMetaProp(column, 'parsed_tree');
await NocoCache.set(`${CacheScope.COL_FORMULA}:${columnId}`, column); await NocoCache.set(`${CacheScope.COL_FORMULA}:${columnId}`, column);
} }
}
return column ? new FormulaColumn(column) : null; return column ? new FormulaColumn(column) : null;
} }
@ -51,7 +61,7 @@ export default class FormulaColumn {
static async update( static async update(
id: string, id: string,
formula: Partial<FormulaColumn>, formula: Partial<FormulaColumn> & { parsed_tree?: any },
ncMeta = Noco.ncMeta, ncMeta = Noco.ncMeta,
) { ) {
const updateObj = extractProps(formula, [ const updateObj = extractProps(formula, [
@ -59,7 +69,11 @@ export default class FormulaColumn {
'formula_raw', 'formula_raw',
'fk_column_id', 'fk_column_id',
'error', 'error',
'parsed_tree',
]); ]);
updateObj.parsed_tree = stringifyMetaProp(insertObj, 'parsed_tree');
// get existing cache // get existing cache
const key = `${CacheScope.COL_FORMULA}:${id}`; const key = `${CacheScope.COL_FORMULA}:${id}`;
let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT); let o = await NocoCache.get(key, CacheGetType.TYPE_OBJECT);
@ -71,4 +85,8 @@ export default class FormulaColumn {
// set meta // set meta
await ncMeta.metaUpdate(null, null, MetaTable.COL_FORMULA, updateObj, id); await ncMeta.metaUpdate(null, null, MetaTable.COL_FORMULA, updateObj, id);
} }
public getParsedTree() {
return this.parsed_tree;
}
} }

6
packages/nocodb/src/services/columns.service.ts
Unescape View File

@ -5,7 +5,7 @@ import {
isVirtualCol, isVirtualCol,
substituteColumnAliasWithIdInFormula, substituteColumnAliasWithIdInFormula,
substituteColumnIdWithAliasInFormula, substituteColumnIdWithAliasInFormula,
UITypes, UITypes, validateFormulaAndExtractTreeWithType,
} from 'nocodb-sdk'; } from 'nocodb-sdk';
import { pluralize, singularize } from 'inflection'; import { pluralize, singularize } from 'inflection';
import hash from 'object-hash'; import hash from 'object-hash';
@ -1197,6 +1197,10 @@ export class ColumnsService {
colBody.formula_raw || colBody.formula, colBody.formula_raw || colBody.formula,
table.columns, table.columns,
); );
colBody.parsed_tree = validateFormulaAndExtractTreeWithType(
colBody.formula_raw || colBody.formula,
table.columns,
);
try { try {
const baseModel = await reuseOrSave('baseModel', reuse, async () => const baseModel = await reuseOrSave('baseModel', reuse, async () =>

Write Preview
Loading…
Cancel
Save