From 1a0ea4b5593cc48fdbb454be7769a93d358e1a60 Mon Sep 17 00:00:00 2001 From: Matt Greenwood Date: Sat, 27 Jun 2015 20:00:34 -0400 Subject: [PATCH] initial commit of function support providing math / JSON helper routines in path execution --- .../jayway/jsonpath/AggregationMapReduce.java | 23 ++++++ .../java/com/jayway/jsonpath/Function.java | 33 +++++++++ .../internal/function/FunctionFactory.java | 74 +++++++++++++++++++ .../jsonpath/internal/function/Length.java | 23 ++++++ .../internal/function/PassthruFunction.java | 18 +++++ .../function/numeric/AbstractAggregation.java | 52 +++++++++++++ .../internal/function/numeric/Average.java | 26 +++++++ .../internal/function/numeric/Max.java | 22 ++++++ .../internal/function/numeric/Min.java | 22 ++++++ .../function/numeric/StandardDeviation.java | 24 ++++++ .../internal/function/numeric/Sum.java | 20 +++++ .../internal/token/FunctionPathToken.java | 50 +++++++++++++ .../jsonpath/functions/BaseFunctionTest.java | 36 +++++++++ .../functions/JSONEntityFunctionTest.java | 23 ++++++ .../functions/NumericFunctionTest.java | 62 ++++++++++++++++ 15 files changed, 508 insertions(+) create mode 100644 json-path/src/main/java/com/jayway/jsonpath/AggregationMapReduce.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/Function.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/internal/function/FunctionFactory.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/internal/function/Length.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/internal/function/PassthruFunction.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/AbstractAggregation.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Average.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Max.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Min.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/StandardDeviation.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Sum.java create mode 100644 json-path/src/main/java/com/jayway/jsonpath/internal/token/FunctionPathToken.java create mode 100644 json-path/src/test/java/com/jayway/jsonpath/functions/BaseFunctionTest.java create mode 100644 json-path/src/test/java/com/jayway/jsonpath/functions/JSONEntityFunctionTest.java create mode 100644 json-path/src/test/java/com/jayway/jsonpath/functions/NumericFunctionTest.java diff --git a/json-path/src/main/java/com/jayway/jsonpath/AggregationMapReduce.java b/json-path/src/main/java/com/jayway/jsonpath/AggregationMapReduce.java new file mode 100644 index 00000000..f147eb57 --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/AggregationMapReduce.java @@ -0,0 +1,23 @@ +package com.jayway.jsonpath; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * Defines a pattern for taking a collection of input streams and executing the same path operation across all files + * map / reducing the results as they come in to provide an aggregation function on top of the + * + * Created by matt@mjgreenwood.net on 6/26/15. + */ +public class AggregationMapReduce { + + public static void main(String args[]) throws IOException { + ReadContext ctx = JsonPath.parse(new File("/home/mattg/dev/JsonPath/json-path-assert/src/test/resources/lotto.json")); + List numbers = ctx.read("$.lotto.winners..numbers.%sum()"); + + Object value = ctx.read("$.lotto.winners.[?(@.winnerId > $.lotto.winners.%length())].numbers.%avg()"); + System.out.println(numbers); + System.out.println(value); + } +} diff --git a/json-path/src/main/java/com/jayway/jsonpath/Function.java b/json-path/src/main/java/com/jayway/jsonpath/Function.java new file mode 100644 index 00000000..4256f8cf --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/Function.java @@ -0,0 +1,33 @@ +package com.jayway.jsonpath; + +import com.jayway.jsonpath.internal.EvaluationContext; +import com.jayway.jsonpath.internal.PathRef; + +/** + * Defines the pattern by which a function can be executed over the result set in the particular path + * being grabbed. The Function's input is the content of the data from the json path selector and its output + * is defined via the functions behavior. Thus transformations in types can take place. Additionally, functions + * can accept multiple selectors in order to produce their output. + * + * Created by matt@mjgreenwood.net on 6/26/15. + */ +public interface Function { + + /** + * Invoke the function and output a JSON object (or scalar) value which will be the result of executing the path + * + * @param currentPath + * The current path location inclusive of the function name + * @param parent + * The path location above the current function + * + * @param model + * The JSON model as input to this particular function + * + * @param ctx + * Eval context, state bag used as the path is traversed, maintains the result of executing + * + * @return + */ + Object invoke(String currentPath, PathRef parent, Object model, EvaluationContext ctx); +} diff --git a/json-path/src/main/java/com/jayway/jsonpath/internal/function/FunctionFactory.java b/json-path/src/main/java/com/jayway/jsonpath/internal/function/FunctionFactory.java new file mode 100644 index 00000000..f224d679 --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/internal/function/FunctionFactory.java @@ -0,0 +1,74 @@ +package com.jayway.jsonpath.internal.function; + +import com.jayway.jsonpath.Function; +import com.jayway.jsonpath.InvalidPathException; +import com.jayway.jsonpath.internal.function.numeric.*; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Implements a factory that given a name of the function will return the Function implementation, or null + * if the value is not obtained. + * + * Leverages the function's name in order to determine which function to execute which is maintained internally + * here via a static map + * + * Created by mattg on 6/27/15. + */ +public class FunctionFactory { + + public static final Map FUNCTIONS; + + static { + // New functions should be added here and ensure the name is not overridden + Map map = new HashMap(); + + // Math Functions + map.put("avg", Average.class); + map.put("stddev", StandardDeviation.class); + map.put("sum", Sum.class); + map.put("min", Min.class); + map.put("max", Max.class); + + // JSON Entity Functions + map.put("length", Length.class); + + FUNCTIONS = Collections.unmodifiableMap(map); + } + + /** + * Either provides a pass thru function when the function cannot be properly mapped or otherwise returns the function + * implementation based on the name using the internal FUNCTION map + * + * @see #FUNCTIONS + * @see Function + * + * @param pathFragment + * The path fragment that is currently being processed which is believed to be the name of a function + * + * @return + * The implementation of a function + * + * @throws InvalidPathException + */ + public static Function newFunction(String pathFragment) throws InvalidPathException { + Function result = new PassthruFunction(); + if (null != pathFragment) { + String name = pathFragment.replaceAll("['%\\]\\[\\(\\)]", "").trim().toLowerCase(); + + if (null != name && FUNCTIONS.containsKey(name) && Function.class.isAssignableFrom(FUNCTIONS.get(name))) { + try { + result = (Function)FUNCTIONS.get(name).newInstance(); + } catch (InstantiationException e) { + throw new InvalidPathException("Function of name: " + name + " cannot be created", e); + } catch (IllegalAccessException e) { + throw new InvalidPathException("Function of name: " + name + " cannot be created", e); + } + } + } + return result; + + } +} diff --git a/json-path/src/main/java/com/jayway/jsonpath/internal/function/Length.java b/json-path/src/main/java/com/jayway/jsonpath/internal/function/Length.java new file mode 100644 index 00000000..747b93b4 --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/internal/function/Length.java @@ -0,0 +1,23 @@ +package com.jayway.jsonpath.internal.function; + +import com.jayway.jsonpath.Function; +import com.jayway.jsonpath.internal.EvaluationContext; +import com.jayway.jsonpath.internal.PathRef; +import net.minidev.json.JSONArray; + +/** + * Provides the length of a JSONArray Object + * + * Created by mattg on 6/26/15. + */ +public class Length implements Function { + + @Override + public Object invoke(String currentPath, PathRef parent, Object model, EvaluationContext ctx) { + if (model instanceof JSONArray) { + JSONArray array = (JSONArray)model; + return Integer.valueOf(array.size()); + } + return null; + } +} \ No newline at end of file diff --git a/json-path/src/main/java/com/jayway/jsonpath/internal/function/PassthruFunction.java b/json-path/src/main/java/com/jayway/jsonpath/internal/function/PassthruFunction.java new file mode 100644 index 00000000..fadda31d --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/internal/function/PassthruFunction.java @@ -0,0 +1,18 @@ +package com.jayway.jsonpath.internal.function; + +import com.jayway.jsonpath.Function; +import com.jayway.jsonpath.internal.EvaluationContext; +import com.jayway.jsonpath.internal.PathRef; + +/** + * Defines the default behavior which is to return the model that is provided as input as output + * + * Created by mattg on 6/26/15. + */ +public class PassthruFunction implements Function { + + @Override + public Object invoke(String currentPath, PathRef parent, Object model, EvaluationContext ctx) { + return model; + } +} diff --git a/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/AbstractAggregation.java b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/AbstractAggregation.java new file mode 100644 index 00000000..8322194b --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/AbstractAggregation.java @@ -0,0 +1,52 @@ +package com.jayway.jsonpath.internal.function.numeric; + +import com.jayway.jsonpath.Function; +import com.jayway.jsonpath.internal.EvaluationContext; +import com.jayway.jsonpath.internal.PathRef; +import net.minidev.json.JSONArray; + +import java.util.Iterator; + +/** + * Defines the pattern for processing numerical values via an abstract implementation that iterates over the collection + * of JSONArray entities and verifies that each is a numerical value and then passes that along the abstract methods + * + * + * Created by mattg on 6/26/15. + */ +public abstract class AbstractAggregation implements Function { + + /** + * Defines the next value in the array to the mathmatical function + * + * @param value + * The numerical value to process next + */ + protected abstract void next(Number value); + + /** + * Obtains the value generated via the series of next value calls + * + * @return + * A numerical answer based on the input value provided + */ + protected abstract Number getValue(); + + @Override + public Object invoke(String currentPath, PathRef parent, Object model, EvaluationContext ctx) { + if (model instanceof JSONArray) { + JSONArray array = (JSONArray)model; + Double num = 0d; + Iterator it = array.iterator(); + while (it.hasNext()) { + Object next = it.next(); + if (next instanceof Number) { + Number value = (Number) next; + next(value); + } + } + return getValue(); + } + return null; + } +} diff --git a/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Average.java b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Average.java new file mode 100644 index 00000000..f4c6788e --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Average.java @@ -0,0 +1,26 @@ +package com.jayway.jsonpath.internal.function.numeric; + +/** + * Provides the average of a series of numbers in a JSONArray + * + * Created by mattg on 6/26/15. + */ +public class Average extends AbstractAggregation { + + private Double summation = 0d; + private Double count = 0d; + + @Override + protected void next(Number value) { + count++; + summation += value.doubleValue(); + } + + @Override + protected Number getValue() { + if (count != 0d) { + return summation / count; + } + return 0d; + } +} diff --git a/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Max.java b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Max.java new file mode 100644 index 00000000..27570bf6 --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Max.java @@ -0,0 +1,22 @@ +package com.jayway.jsonpath.internal.function.numeric; + +/** + * Defines the summation of a series of JSONArray numerical values + * + * Created by mattg on 6/26/15. + */ +public class Max extends AbstractAggregation { + private Double max = Double.MIN_VALUE; + + @Override + protected void next(Number value) { + if (max < value.doubleValue()) { + max = value.doubleValue(); + } + } + + @Override + protected Number getValue() { + return max; + } +} diff --git a/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Min.java b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Min.java new file mode 100644 index 00000000..d626d48e --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Min.java @@ -0,0 +1,22 @@ +package com.jayway.jsonpath.internal.function.numeric; + +/** + * Defines the summation of a series of JSONArray numerical values + * + * Created by mattg on 6/26/15. + */ +public class Min extends AbstractAggregation { + private Double min = Double.MAX_VALUE; + + @Override + protected void next(Number value) { + if (min < value.doubleValue()) { + min = value.doubleValue(); + } + } + + @Override + protected Number getValue() { + return min; + } +} diff --git a/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/StandardDeviation.java b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/StandardDeviation.java new file mode 100644 index 00000000..d659f2d6 --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/StandardDeviation.java @@ -0,0 +1,24 @@ +package com.jayway.jsonpath.internal.function.numeric; + +/** + * Provides the standard deviation of a series of numbers + * + * Created by mattg on 6/27/15. + */ +public class StandardDeviation extends AbstractAggregation { + private Double sumSq = 0d; + private Double sum = 0d; + private Double count = 0d; + + @Override + protected void next(Number value) { + sum += value.doubleValue(); + sumSq += value.doubleValue()*value.doubleValue(); + count++; + } + + @Override + protected Number getValue() { + return Math.sqrt(sumSq/count - sum*sum/count/count); + } +} diff --git a/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Sum.java b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Sum.java new file mode 100644 index 00000000..3996bb43 --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/internal/function/numeric/Sum.java @@ -0,0 +1,20 @@ +package com.jayway.jsonpath.internal.function.numeric; + +/** + * Defines the summation of a series of JSONArray numerical values + * + * Created by mattg on 6/26/15. + */ +public class Sum extends AbstractAggregation { + private Double summation = 0d; + + @Override + protected void next(Number value) { + summation += value.doubleValue(); + } + + @Override + protected Number getValue() { + return summation; + } +} diff --git a/json-path/src/main/java/com/jayway/jsonpath/internal/token/FunctionPathToken.java b/json-path/src/main/java/com/jayway/jsonpath/internal/token/FunctionPathToken.java new file mode 100644 index 00000000..7765e292 --- /dev/null +++ b/json-path/src/main/java/com/jayway/jsonpath/internal/token/FunctionPathToken.java @@ -0,0 +1,50 @@ +package com.jayway.jsonpath.internal.token; + +import com.jayway.jsonpath.Function; +import com.jayway.jsonpath.internal.PathRef; +import com.jayway.jsonpath.internal.function.FunctionFactory; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Token representing a Function call to one of the functions produced via the FunctionFactory + * + * @see FunctionFactory + * + * Created by mattg on 6/27/15. + */ +public class FunctionPathToken extends PathToken { + + private final String functionName; + private final String pathFragment; + + public FunctionPathToken(String pathFragment) { + this.pathFragment = pathFragment; + Matcher matcher = Pattern.compile(".*?\\%(\\w+)\\(.*?").matcher(pathFragment); + if (matcher.matches()) { + functionName = matcher.group(1); + } + else { + // We'll end up throwing an error from the factory when we get that far + functionName = null; + } + } + + @Override + public void evaluate(String currentPath, PathRef parent, Object model, EvaluationContextImpl ctx) { + Function function = FunctionFactory.newFunction(functionName); + ctx.addResult(currentPath, parent, function.invoke(currentPath, parent, model, ctx)); + } + + @Override + boolean isTokenDefinite() { + return false; + } + + @Override + String getPathFragment() { + return pathFragment; + } + +} diff --git a/json-path/src/test/java/com/jayway/jsonpath/functions/BaseFunctionTest.java b/json-path/src/test/java/com/jayway/jsonpath/functions/BaseFunctionTest.java new file mode 100644 index 00000000..100be413 --- /dev/null +++ b/json-path/src/test/java/com/jayway/jsonpath/functions/BaseFunctionTest.java @@ -0,0 +1,36 @@ +package com.jayway.jsonpath.functions; + +import com.jayway.jsonpath.Configuration; + +import static com.jayway.jsonpath.JsonPath.using; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Created by mattg on 6/27/15. + */ +public class BaseFunctionTest { + protected static final String NUMBER_SERIES = "{\"numbers\" : [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}"; + protected static final String TEXT_SERIES = "{\"text\" : [ \"a\", \"b\", \"c\", \"d\", \"e\", \"f\" ]}"; + + + /** + * Verify the function returns the correct result based on the input expectedValue + * + * @param pathExpr + * The path expression to execute + * + * @param json + * The json document (actual content) to parse + * + * @param expectedValue + * The expected value to be returned from the test + */ + protected void verifyFunction(String pathExpr, String json, Object expectedValue) { + Configuration conf = Configuration.defaultConfiguration(); + assertThat(using(conf).parse(json).read(pathExpr)).isEqualTo(expectedValue); + } + + protected void verifyMathFunction(String pathExpr, Object expectedValue) { + verifyFunction(pathExpr, NUMBER_SERIES, expectedValue); + } +} diff --git a/json-path/src/test/java/com/jayway/jsonpath/functions/JSONEntityFunctionTest.java b/json-path/src/test/java/com/jayway/jsonpath/functions/JSONEntityFunctionTest.java new file mode 100644 index 00000000..254d2e9f --- /dev/null +++ b/json-path/src/test/java/com/jayway/jsonpath/functions/JSONEntityFunctionTest.java @@ -0,0 +1,23 @@ +package com.jayway.jsonpath.functions; + +import org.junit.Test; + +/** + * Verifies methods that are helper implementations of functions for manipulating JSON entities, i.e. + * length, etc. + * + * Created by mattg on 6/27/15. + */ +public class JSONEntityFunctionTest extends BaseFunctionTest { + @Test + public void testLengthOfTextArray() { + // The length of JSONArray is an integer + System.out.println(TEXT_SERIES); + verifyFunction("$['text'].%length()", TEXT_SERIES, 6); + } + @Test + public void testLengthOfNumberArray() { + // The length of JSONArray is an integer + verifyFunction("$.numbers.%length()", NUMBER_SERIES, 10); + } +} diff --git a/json-path/src/test/java/com/jayway/jsonpath/functions/NumericFunctionTest.java b/json-path/src/test/java/com/jayway/jsonpath/functions/NumericFunctionTest.java new file mode 100644 index 00000000..fcfa394b --- /dev/null +++ b/json-path/src/test/java/com/jayway/jsonpath/functions/NumericFunctionTest.java @@ -0,0 +1,62 @@ +package com.jayway.jsonpath.functions; + +import com.jayway.jsonpath.Configuration; +import net.minidev.json.JSONArray; +import org.junit.Test; + +import java.util.Arrays; + +import static com.jayway.jsonpath.JsonPath.using; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Defines functional tests around executing: + * + * - sum + * - avg + * - stddev + * + * for each of the above, executes the test and verifies that the results are as expected based on a static input + * and static output. + * + * Created by mattg on 6/26/15. + */ +public class NumericFunctionTest extends BaseFunctionTest { + + @Test + public void testAverageOfDoubles() { + verifyMathFunction("$.numbers.%average()", (10d * (10d + 1d)) / 2d); + } + + @Test + public void testSumOfDouble() { + verifyMathFunction("$.numbers.%sum()", (10d * (10d + 1d)) / 2d); + } + + @Test + public void testMaxOfDouble() { + verifyMathFunction("$.numbers.%max()", 10d); + } + + @Test + public void testMinOfDouble() { + verifyMathFunction("$.numbers.%min()", 1d); + } + + @Test + public void testStdDevOfDouble() { + verifyMathFunction("$.numbers.%stddev()", 1d); + } + + /** + * Expect that for an invalid function name we'll get back the original input to the function + */ + @Test + public void testInvalidFunctionNameNegative() { + Configuration conf = Configuration.defaultConfiguration(); + JSONArray numberSeries = new JSONArray(); + numberSeries.addAll(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); + assertThat(using(conf).parse(NUMBER_SERIES).read("$.numbers.%foo()")).isEqualTo(numberSeries); + } + +}