Merge pull request #1 from NetNow/master

Split a large JSON payload using streaming API, return results.

json-path/src/main/java/com/jayway/jsonpath/EvaluationCallback.java

@@ -26,12 +26,12 @@ public interface EvaluationCallback {
      * Callback invoked when result is found
      * @param path -- the specific path that was triggered
      */
-    void resultFound(Path path);
+    public void resultFound(Object source, Object obj, Path path) throws Exception;
 
     /**
      * Callback invoked when the parser leaves the region in which the match
      * was found
      * @param path -- the specific path that was untriggered
      */
-    void resultFoundExit(Path path);
+    public void resultFoundExit(Object source, Object obj, Path path) throws Exception;
 }

json-path/src/main/java/com/jayway/jsonpath/internal/token/ArrayToken.java

@@ -5,7 +5,7 @@ package com.jayway.jsonpath.internal.token;
  *
  * @author Hunter Payne
  **/
-public class ArrayToken implements TokenStackElement
+public class ArrayToken extends TokenStackElement
 {
     int currentIndex;
     TokenStackElement value; // can be an object, array, or property

json-path/src/main/java/com/jayway/jsonpath/internal/token/FloatToken.java

@@ -5,7 +5,7 @@ package com.jayway.jsonpath.internal.token;
  *
  * @author Hunter Payne
  **/
-public class FloatToken implements TokenStackElement
+public class FloatToken extends TokenStackElement
 {
     public float value;
 

json-path/src/main/java/com/jayway/jsonpath/internal/token/IntToken.java

@@ -5,7 +5,7 @@ package com.jayway.jsonpath.internal.token;
  *
  * @author Hunter Payne
  **/
-public class IntToken implements TokenStackElement
+public class IntToken extends TokenStackElement
 {
     public int value;
 

json-path/src/main/java/com/jayway/jsonpath/internal/token/ObjectToken.java

@@ -5,7 +5,7 @@ package com.jayway.jsonpath.internal.token;
  *
  * @author Hunter Payne
  **/
-public class ObjectToken implements TokenStackElement
+public class ObjectToken extends TokenStackElement
 {
     String key;
     TokenStackElement value; // can be an array, object, or property

json-path/src/main/java/com/jayway/jsonpath/internal/token/StringToken.java

@@ -5,7 +5,7 @@ package com.jayway.jsonpath.internal.token;
  *
  * @author Hunter Payne
  **/
-public class StringToken implements TokenStackElement
+public class StringToken extends TokenStackElement
 {
     public String value;
 

json-path/src/main/java/com/jayway/jsonpath/internal/token/TokenStack.java

@@ -2,14 +2,13 @@
 package com.jayway.jsonpath.internal.token;
 
 import java.util.*;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 import com.jayway.jsonpath.Configuration;
 import com.jayway.jsonpath.EvaluationCallback;
 import com.jayway.jsonpath.internal.Path;
 import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  *
@@ -17,15 +16,18 @@ import com.fasterxml.jackson.core.JsonToken;
  **/
 public class TokenStack
 {
-    protected static Logger log = Logger.getLogger("com.jayway.jsonpath");
+    private static final Logger log = LoggerFactory.getLogger(TokenStack.class);
 
     protected Configuration conf;
     protected Stack<TokenStackElement> elements;
+    protected Stack<Object> objStack;
     protected List<Path> paths;
     protected Map<TokenStackElement, Path> matchedPaths;
 
     private TokenStackElement curr;
     private Path rootMatch;
+    private Class jsonArrayType;
+    private Class jsonObjectType;
 
     public TokenStack(Configuration conf)
     {
@@ -33,7 +35,11 @@ public class TokenStack
         paths = new ArrayList<Path>();
         matchedPaths = new HashMap<TokenStackElement, Path>();
         elements = new Stack<TokenStackElement>();
+        objStack = new Stack<Object>();
         rootMatch = null;
+        
+        this.jsonArrayType = this.getNewJsonArray().getClass();
+        this.jsonObjectType = this.getNewJsonObject().getClass(); 
     }
 
     public Stack<TokenStackElement> getStack()
@@ -49,28 +55,26 @@ public class TokenStack
         paths.add(path);
     }
 
+    public void read(JsonParser parser, EvaluationCallback callback)
+        throws Exception
+    {
+        read(parser,callback,true);
+    }
+    
     /**
-     * reads from stream and notifies the callback of matched registered paths
+     * reads from stream and notifies the callback of matched registered paths, with results.
+     * Note: GSON not supported
      */
-    public void read(JsonParser parser, EvaluationCallback callback)
+    public void read(JsonParser parser, EvaluationCallback callback, boolean getParents)
         throws Exception
     {
         assert(callback != null);
 
+        Object obj = null;
         boolean needsPathCheck = false;
-        /*
-        if (null == curr && elements.empty()) {
-            // check for $ patterns
-            for (Path path : paths) {
-                if (path.checkForMatch(this)) {
-                    matchedPaths.put(curr, path);
-                    callback.resultFound(path);
-                    rootMatch = path;
-                }
-            }
-        }
-        */
+
         while (parser.nextToken() != null) {
+            //log.debug("type/name/val: " + parser.getCurrentToken() + " " + parser.getCurrentName() + " " + parser.getText());
             boolean saveMatch = false;
             switch (parser.getCurrentToken()) {
             case START_ARRAY:
@@ -85,33 +89,42 @@ public class TokenStack
                 saveMatch = true;
                 needsPathCheck = true;
                 elements.push(curr);
+               
+                obj = stackPush(parser.getCurrentName(),getNewJsonArray());
                 break;
             }
             case END_ARRAY:
             {
                 Path match = matchedPaths.remove(curr);
                 if (match != null) {
-                    callback.resultFoundExit(match);
+                    callback.resultFoundExit(parser.getCurrentToken(), obj, match);
                 }
                 elements.pop();
                 if (elements.empty()) curr = null;
                 else curr = elements.peek();
+                
+                obj = stackPop(callback, curr, getJsonArrayType(), match);
                 break;
             }
             case VALUE_EMBEDDED_OBJECT:
             case START_OBJECT:
             {
-                if (curr != null && curr.getType() == TokenType.ARRAY_TOKEN) {
+                obj = stackPush(parser.getCurrentName(), getNewJsonObject());
+                
+                if (isArray(curr)) {
                     if (((ArrayToken)curr).getValue() != null &&
                         matchedPaths.containsKey(curr))
                     {
                         Path match = matchedPaths.remove(curr);
-                        callback.resultFoundExit(match);
+                        if (getParents) {
+                           callback.resultFoundExit(parser.getCurrentToken(), obj, match);
+                        }
 
                         if (match.checkForMatch(this)) {
 
                             matchedPaths.put(curr, match);
-                            callback.resultFound(match);
+                            //callback.resultFound(match);
+                            curr.setMatched();
                         }
                     }
                 } else if (null == curr && elements.empty()) {
@@ -119,7 +132,7 @@ public class TokenStack
                     for (Path path : paths) {
                         if (path.checkForMatch(this)) {
                             matchedPaths.put(curr, path);
-                            callback.resultFound(path);
+                            //callback.resultFound(path);
                             rootMatch = path;
                         }
                     }
@@ -139,20 +152,24 @@ public class TokenStack
             }
             case END_OBJECT:
             {
+                if (getParents) {
                 if (!"$".equals(curr)) {
                     Path match = matchedPaths.remove(curr);
                     if (match != null) {
-                        callback.resultFoundExit(match);
+                        callback.resultFoundExit(parser.getCurrentToken(), obj, match);
                     }
                 } else {
                     Path match = matchedPaths.get("$");
                     if (match != null) {
-                        callback.resultFoundExit(match);
+                        callback.resultFoundExit(parser.getCurrentToken(), obj, match);
                     }
                 }
+                }
                 elements.pop();
                 if (elements.empty()) curr = null;
                 else curr = elements.peek();
+                
+                obj = stackPop(callback, curr, this.getJsonObjectType(), null);
                 break;
             }
             case FIELD_NAME:
@@ -166,6 +183,7 @@ public class TokenStack
                 StringToken newToken = new StringToken("FALSE");
                 curr.setValue(newToken);
                 needsPathCheck = true;
+                objPutVal(obj, curr, newToken.value);
                 break;
             }
             case VALUE_TRUE:
@@ -173,6 +191,7 @@ public class TokenStack
                 StringToken newToken = new StringToken("TRUE");
                 curr.setValue(newToken);
                 needsPathCheck = true;
+                objPutVal(obj, curr, newToken.value);
                 break;
             }
             case VALUE_NUMBER_FLOAT:
@@ -181,6 +200,7 @@ public class TokenStack
                     new FloatToken((float)parser.getValueAsDouble());
                 curr.setValue(newToken);
                 needsPathCheck = true;
+                objPutVal(obj, curr, newToken.value);
                 break;
             }
             case VALUE_NUMBER_INT:
@@ -188,6 +208,7 @@ public class TokenStack
                 IntToken newToken = new IntToken(parser.getValueAsInt());
                 curr.setValue(newToken);
                 needsPathCheck = true;
+                objPutVal(obj, curr, newToken.value);
                 break;
             }
             case VALUE_STRING:
@@ -195,12 +216,14 @@ public class TokenStack
                 StringToken newToken = new StringToken(parser.getText());
                 curr.setValue(newToken);
                 needsPathCheck = true;
+                objPutVal(obj, curr, parser.getText());
                 break;
             }
             case VALUE_NULL:
             {
                 curr.setValue(null);
                 needsPathCheck = true;
+                objPutVal(obj, curr, null);
                 break;
             }
             default:
@@ -211,18 +234,116 @@ public class TokenStack
                 for (Path path : paths) {
                     if (path.checkForMatch(this)) {
                         if (saveMatch) matchedPaths.put(curr, path);
-                        callback.resultFound(path);
+                        curr.setMatched();
+                        if (getParents) {
+                           callback.resultFound(parser.getCurrentToken(), obj, path);
+                        }
                     }
                 }
                 needsPathCheck = false;
             }
 
-            if (rootMatch != null && elements.empty()) {
-                callback.resultFoundExit(rootMatch);
+            if (rootMatch != null && elements.empty() && getParents) {
+               if (isArray(curr)) {
+                    obj = this.getNewJsonArray();
+                }
+                callback.resultFoundExit(parser.getCurrentToken(), obj, rootMatch);
                 rootMatch = null;
             }
         }
     }
+    
+    private Object getNewJsonObject() {
+        return conf.jsonProvider().createMap();
+    }
+
+    private Object getJsonArrayType() {
+        return this.jsonArrayType;
+    }
+
+    private Object getNewJsonArray() {
+        return conf.jsonProvider().createArray();
+    }
+   
+    private Class<? extends Object> getJsonObjectType() {
+        return this.jsonObjectType;
+    }
+
+    private Object stackPush(String key, Object jsObj) throws Exception {
+        if (jsObj == null) {
+            return null;
+        }
+        
+        if (this.objStack.size() > 0) {
+            Object obj = this.objStack.peek();
+            //log.trace("PP : Push[" + this.objStack.size() + "] " + jsObj.getClass().getSimpleName() + " -> " + obj.getClass());
+            if (obj.getClass() == getJsonArrayType()) {
+                ((List)obj).add(jsObj);
+            } else if (obj.getClass() == getJsonObjectType()) {
+                ((HashMap<String,Object>)obj).put(key, jsObj);
+            } else {
+                throw new Exception("Unhandled type: " + obj.getClass());
+            }
+
+        } else {
+            //log.trace("PP : Push " + jsObj.getClass().getSimpleName() + " -> ROOT");
+        }
+        this.objStack.add(jsObj);
+        
+        return jsObj;
+    }
+
+    private <T> Object stackPop(EvaluationCallback callback, TokenStackElement tse, T jsObj, Path path) throws Exception {
+        if (jsObj == null) {
+            return null;
+        }
+        
+        Object obj = null;
+        Object popObj = null;
+        if (this.objStack.size() > 0) {
+            popObj = this.objStack.peek();
+            
+            //log.trace("PP : Pop " +( popObj != null ? popObj.getClass() : " null"));
+            if (popObj.getClass() != jsObj) {
+                throw new Exception("Unexpected type : " + popObj.getClass());
+            } else {
+                popObj = this.objStack.pop();
+                if (this.objStack.size() > 0) {
+                    obj = objStack.peek();
+                } else {
+                    //log.debug("PP : Now ROOT");
+                    obj = null;
+                }
+            }
+        }
+
+        if (tse != null && tse.getMatched()) {
+            callback.resultFound("Stack", popObj, path);
+        } 
+        
+        return obj;
+    }
+    
+    private boolean isArray(TokenStackElement current) { 
+        return current != null && current.getType() == TokenType.ARRAY_TOKEN;
+    }
+        
+    private void objPutVal(Object objIn, TokenStackElement el, Object value) throws Exception {
+        if (objIn == null) {
+            return;
+        }
+        
+        Class objInCls = objIn.getClass();
+        if (objInCls == this.getJsonObjectType()) {
+            HashMap obj = (HashMap)objIn;
+            obj.put(((ObjectToken)el).key, value);
+        } else if (objInCls == this.getJsonArrayType()) {
+            ArrayList obj = (ArrayList)objIn;
+            obj.add(value);
+        } else {
+            throw new Exception("Unhandled type: " + objInCls);
+        }
+    }
 
     public String toString()
     {

json-path/src/main/java/com/jayway/jsonpath/internal/token/TokenStackElement.java

@@ -1,17 +1,45 @@
 
 package com.jayway.jsonpath.internal.token;
 
+import java.util.logging.Logger;
+
 /**
  *
  * @author Hunter Payne
  **/
-public interface TokenStackElement
+public abstract class TokenStackElement
 {
-    public TokenType getType(); // otherwise its an object
+    private static Logger log = Logger.getLogger(TokenStackElement.class.getName());
+    
+    private boolean matched = false;
+    
+    private TokenStackElement parent;
+    
+    public abstract TokenType getType(); // otherwise its an object
+
+    public abstract TokenStackElement getValue();
+
+    public abstract void setValue(TokenStackElement elem);
+    
+    public TokenStackElement getParent() {
+        if (parent == null) {
+            return this;
+        }
+        //log.trace("parent: " + parent);
+        return parent;
+    }
 
-    public TokenStackElement getValue();
+    public void setParent(TokenStackElement parent) {
+        this.parent = parent;
+    }
 
-    public void setValue(TokenStackElement elem);
+    public void setMatched() {
+        this.matched = true;
+    }
+    
+    public boolean getMatched() {
+        return matched;
+    } 
 }
 
 // End TokenStackElement.java

json-path/src/test/java/com/jayway/jsonpath/CallbackRecorder.java

@@ -44,12 +44,12 @@ public class CallbackRecorder implements EvaluationCallback {
         results = new ArrayList<CallbackEvent>();
     }
 
-    public void resultFound(Path path) {
+    public void resultFound(Object src, Object val, Path path) {
         System.err.println("found result " + path);
         results.add(new CallbackEvent(path, false));
     }
 
-    public void resultFoundExit(Path path) {
+    public void resultFoundExit(Object src, Object val, Path path) {
         System.err.println("exiting result " + path);
         results.add(new CallbackEvent(path, true));
     }

json-path/src/test/java/com/jayway/jsonpath/JacksonTest.java

@@ -9,12 +9,10 @@ import java.nio.charset.Charset;
 
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonFactory;
-
 import com.jayway.jsonpath.EvaluationCallback;
 import com.jayway.jsonpath.internal.Path;
 import com.jayway.jsonpath.internal.PathCompiler;
 import com.jayway.jsonpath.internal.token.*;
-
 import static java.util.Collections.singletonMap;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -607,7 +605,7 @@ public class JacksonTest extends BaseTest implements EvaluationCallback {
                equals(new CallbackRecorder.CallbackEvent(rootPath, true)));
     }
 
-    public void resultFound(Path path) {
+    public void resultFound(Object src, Object val, Path path) {
         if (path == idPath) {
             switch (match++) {
             case 0:
@@ -1040,14 +1038,14 @@ public class JacksonTest extends BaseTest implements EvaluationCallback {
                 break;
             }
         }
-        recorder.resultFound(path);
+        recorder.resultFound(src, val, path);
     }
 
-    public void resultFoundExit(Path path) {
+    public void resultFoundExit(Object src, Object val, Path path) {
         assert(path != idPath);
         assert(path != floatPath);
         assert(path != intPath);
-        recorder.resultFoundExit(path);
+        recorder.resultFoundExit(src, val, path);
     }
 
     protected void checkResult(TokenStack stack, Object expected) {

json-path/src/test/java/com/jayway/jsonpath/JacksonTest_Split.java

@@ -0,0 +1,58 @@
+package com.jayway.jsonpath;
+
+import static org.junit.Assert.*;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.jayway.jsonpath.internal.Path;
+import com.jayway.jsonpath.internal.PathCompiler;
+import com.jayway.jsonpath.internal.token.TokenStack;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.junit.Test;
+
+public class JacksonTest_Split extends BaseTest implements EvaluationCallback {
+    
+    private static final Logger log = LoggerFactory.getLogger(JacksonTest_Split.class);
+    private List<Object> results = new ArrayList<Object>();
+    
+    @Test
+    public void json_Test() throws Exception {
+        jsonSplit_Test(JACKSON_CONFIGURATION);
+        results.clear();
+        jsonSplit_Test(JSON_SMART_CONFIGURATION);
+        results.clear();
+    }
+
+    private void jsonSplit_Test(Configuration jsonProviderCfg) throws JsonParseException, IOException, Exception {
+
+        String res = "json_opsview1.json";
+        try (InputStream stream = getClass().getClassLoader().getResourceAsStream(res)) {
+             Path path = PathCompiler.compile("$.list[*]");
+             
+             TokenStack stack = new TokenStack(jsonProviderCfg);
+           
+             JsonFactory factory = new JsonFactory();
+             stack.registerPath(path);
+             stack.read(factory.createParser(stream), this, false);
+        }
+        log.debug("results: " + results.size());
+        assertTrue(results.size() == 96);
+    }
+
+    @Override
+    public void resultFound(Object source, Object obj, Path path) throws Exception {
+        //log.debug(source + ":" + String.valueOf(obj));
+        results.add(obj);
+    }
+
+    @Override
+    public void resultFoundExit(Object source, Object obj, Path path) throws Exception {
+        //log.debug(source + ":" + String.valueOf(obj));
+    }
+}