Merge pull request #268 in CORE/base-third from final/10.0 to bugfix/10.0

* commit '49e3d226f913d90efe9ea3cdd417da2f40dfb6df':
  REPORT-20372 CVE-2017-5644 fix
  DEC-8889 jackson高危漏洞
  DEC-8889 jackson高危漏洞
  RoaringBitmap单独抽到third
  KERNER-724 j2v8升级
  DEC-8173 脏数据导致quartz启动失败
  DEC-7569 定时调度模块启动异常
  无JIRA任务 打包失败，job未初始化
  DEC-7376 定时调度任务表中存在脏数据的时候，不能影响整个scheduler
  DEC-7376 定时调度任务表中存在脏数据的时候，不能影响整个scheduler
  无 jira 任务， final --> persist 回复一下
  无 jira 任务， final --> persist 少了一个空行
  无 jira 任务， 解决打包冲突 还是有问题， 删掉 gradle
  无 jira 任务， 解决打包冲突
  升级apache工具类
  build.third_step0.gradle edited online with Bitbucket

build.third_step0.gradle

@@ -41,7 +41,7 @@ def buildDir=files[0].path.substring(0,files[0].path.lastIndexOf (java.io.File.s
 def branchName=buildDir.substring(buildDir.lastIndexOf (java.io.File.separator)+1)
 
 def srcDir="."
-def maven_version="${version}-RELEASE-SNAPSHOT"
+def maven_version="${version}-FINAL-SNAPSHOT"
 def jar_version = version
 
 configurations {

build.third_step6.gradle

@@ -46,7 +46,8 @@ sourceSets{
 				"${srcDir}/fine-kryo/src",
 				"${srcDir}/fine-lz4/src",
 				"${srcDir}/fine-log4j/src",
-				"${srcDir}/fine-jgit/src"
+				"${srcDir}/fine-jgit/src",
+				"${srcDir}/fine-roaringbitmap/src"
 				]
 		}
 	}
@@ -121,6 +122,7 @@ task copyFiles(type:Copy,dependsOn:'compileJava'){
 		with dataContent.call("${srcDir}/fine-log4j/resources")
 		with dataContent.call("${srcDir}/fine-jgit/src")
 		with dataContent.call("${srcDir}/fine-jgit/resources")
+		with dataContent.call("${srcDir}/fine-roaringbitmap/src")
 		into "${classesDir}"
 	}
 }

fine-jackson/src/com/fr/third/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java

@@ -78,6 +78,16 @@ public class SubTypeValidator
         s.add("org.apache.openjpa.ee.RegistryManagedRuntime");
         s.add("org.apache.openjpa.ee.JNDIManagedRuntime");
         s.add("org.apache.axis2.transport.jms.JMSOutTransportInfo");
+
+        // [databind#2326] (2.9.9)
+        s.add("com.mysql.cj.jdbc.admin.MiniAdmin");
+
+        // [databind#2334]: logback-core (2.9.9.1)
+        s.add("ch.qos.logback.core.db.DriverManagerConnectionSource");
+
+        // [databind#2341]: jdom/jdom2 (2.9.9.1)
+        s.add("org.jdom.transform.XSLTransformer");
+        s.add("org.jdom2.transform.XSLTransformer");
         DEFAULT_NO_DESER_CLASS_NAMES = Collections.unmodifiableSet(s);
     }
 

fine-poi/src/com/fr/third/v2/org/apache/poi/POIXMLTypeLoader.java

@@ -18,9 +18,11 @@
 package com.fr.third.v2.org.apache.poi;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Reader;
+import java.io.StringReader;
 import java.net.URL;
 import java.util.Collections;
 import java.util.HashMap;
@@ -28,6 +30,7 @@ import java.util.Map;
 
 import javax.xml.stream.XMLStreamReader;
 
+import com.fr.third.v2.org.apache.poi.util.DocumentHelper;
 import com.fr.third.v2.org.apache.xmlbeans.SchemaType;
 import com.fr.third.v2.org.apache.xmlbeans.XmlBeans;
 import com.fr.third.v2.org.apache.xmlbeans.XmlException;
@@ -35,19 +38,26 @@ import com.fr.third.v2.org.apache.xmlbeans.XmlObject;
 import com.fr.third.v2.org.apache.xmlbeans.XmlOptions;
 import com.fr.third.v2.org.apache.xmlbeans.xml.stream.XMLInputStream;
 import com.fr.third.v2.org.apache.xmlbeans.xml.stream.XMLStreamException;
+import org.w3c.dom.Document;
 import org.w3c.dom.Node;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
 
 @SuppressWarnings("deprecation")
 public class POIXMLTypeLoader {
 
     public static final XmlOptions DEFAULT_XML_OPTIONS;
+
     static {
         DEFAULT_XML_OPTIONS = new XmlOptions();
         DEFAULT_XML_OPTIONS.setSaveOuter();
         DEFAULT_XML_OPTIONS.setUseDefaultNamespace();
         DEFAULT_XML_OPTIONS.setSaveAggressiveNamespaces();
         DEFAULT_XML_OPTIONS.setCharacterEncoding("UTF-8");
-        DEFAULT_XML_OPTIONS.setLoadEntityBytesLimit(4096);
+        // Piccolo is disabled for POI builts, i.e. JAXP is used for parsing
+        // so only user code using XmlObject/XmlToken.Factory.parse
+        // directly can bypass the entity check, which is probably unlikely (... and not within our responsibility :))
+        // DEFAULT_XML_OPTIONS.setLoadEntityBytesLimit(4096);
 
         Map<String, String> map = new HashMap<String, String>();
         map.put("http://schemas.openxmlformats.org/drawingml/2006/main", "a");
@@ -68,8 +78,7 @@ public class POIXMLTypeLoader {
     }
 
     private static XmlOptions getXmlOptions(XmlOptions options) {
-        XmlOptions opt = (options == null) ? DEFAULT_XML_OPTIONS : options;
-        return opt;
+        return options == null ? DEFAULT_XML_OPTIONS : options;
     }
 
     public static XmlObject newInstance(SchemaType type, XmlOptions options) {
@@ -77,19 +86,38 @@ public class POIXMLTypeLoader {
     }
 
     public static XmlObject parse(String xmlText, SchemaType type, XmlOptions options) throws XmlException {
-        return XmlBeans.getContextTypeLoader().parse(xmlText, type, getXmlOptions(options));
+        try {
+            return parse(new StringReader(xmlText), type, options);
+        } catch (IOException e) {
+            throw new XmlException("Unable to parse xml bean", e);
+        }
     }
 
     public static XmlObject parse(File file, SchemaType type, XmlOptions options) throws XmlException, IOException {
-        return XmlBeans.getContextTypeLoader().parse(file, type, getXmlOptions(options));
+        InputStream is = new FileInputStream(file);
+        try {
+            return parse(is, type, options);
+        } finally {
+            is.close();
+        }
     }
 
     public static XmlObject parse(URL file, SchemaType type, XmlOptions options) throws XmlException, IOException {
-        return XmlBeans.getContextTypeLoader().parse(file, type, getXmlOptions(options));
+        InputStream is = file.openStream();
+        try {
+            return parse(is, type, options);
+        } finally {
+            is.close();
+        }
     }
 
     public static XmlObject parse(InputStream jiois, SchemaType type, XmlOptions options) throws XmlException, IOException {
-        return XmlBeans.getContextTypeLoader().parse(jiois, type, getXmlOptions(options));
+        try {
+            Document doc = DocumentHelper.readDocument(jiois);
+            return XmlBeans.getContextTypeLoader().parse(doc.getDocumentElement(), type, getXmlOptions(options));
+        } catch (SAXException e) {
+            throw new IOException("Unable to parse xml bean", e);
+        }
     }
 
     public static XmlObject parse(XMLStreamReader xsr, SchemaType type, XmlOptions options) throws XmlException {
@@ -97,7 +125,12 @@ public class POIXMLTypeLoader {
     }
 
     public static XmlObject parse(Reader jior, SchemaType type, XmlOptions options) throws XmlException, IOException {
-        return XmlBeans.getContextTypeLoader().parse(jior, type, getXmlOptions(options));
+        try {
+            Document doc = DocumentHelper.readDocument(new InputSource(jior));
+            return XmlBeans.getContextTypeLoader().parse(doc.getDocumentElement(), type, getXmlOptions(options));
+        } catch (SAXException e) {
+            throw new XmlException("Unable to parse xml bean", e);
+        }
     }
 
     public static XmlObject parse(Node node, SchemaType type, XmlOptions options) throws XmlException {
@@ -108,7 +141,7 @@ public class POIXMLTypeLoader {
         return XmlBeans.getContextTypeLoader().parse(xis, type, getXmlOptions(options));
     }
 
-    public static XMLInputStream newValidatingXMLInputStream ( XMLInputStream xis, SchemaType type, XmlOptions options ) throws XmlException, XMLStreamException {
+    public static XMLInputStream newValidatingXMLInputStream(XMLInputStream xis, SchemaType type, XmlOptions options) throws XmlException, XMLStreamException {
         return XmlBeans.getContextTypeLoader().newValidatingXMLInputStream(xis, type, getXmlOptions(options));
     }
 }

fine-poi/src/com/fr/third/v2/org/apache/poi/util/DocumentHelper.java

@@ -29,12 +29,55 @@ import javax.xml.stream.events.Namespace;
 
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
 
 public final class DocumentHelper {
     private static POILogger logger = POILogFactory.getLogger(DocumentHelper.class);
 
-    private DocumentHelper() {}
+    private DocumentHelper() {
+    }
+
+    private static class DocHelperErrorHandler implements ErrorHandler {
+
+        public void warning(SAXParseException exception) throws SAXException {
+            printError(POILogger.WARN, exception);
+        }
+
+        public void error(SAXParseException exception) throws SAXException {
+            printError(POILogger.ERROR, exception);
+        }
+
+        public void fatalError(SAXParseException exception) throws SAXException {
+            printError(POILogger.FATAL, exception);
+            throw exception;
+        }
+
+        /**
+         * Prints the error message.
+         */
+        private void printError(int type, SAXParseException ex) {
+            StringBuilder sb = new StringBuilder();
+
+            String systemId = ex.getSystemId();
+            if (systemId != null) {
+                int index = systemId.lastIndexOf('/');
+                if (index != -1)
+                    systemId = systemId.substring(index + 1);
+                sb.append(systemId);
+            }
+            sb.append(':');
+            sb.append(ex.getLineNumber());
+            sb.append(':');
+            sb.append(ex.getColumnNumber());
+            sb.append(": ");
+            sb.append(ex.getMessage());
+
+            logger.log(type, sb.toString(), ex);
+        }
+    }
 
     /**
      * Creates a new document builder, with sensible defaults
@@ -43,6 +86,7 @@ public final class DocumentHelper {
         try {
             DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
             documentBuilder.setEntityResolver(SAXHelper.IGNORING_ENTITY_RESOLVER);
+            documentBuilder.setErrorHandler(new DocHelperErrorHandler());
             return documentBuilder;
         } catch (ParserConfigurationException e) {
             throw new IllegalStateException("cannot create a DocumentBuilder", e);
@@ -50,6 +94,7 @@ public final class DocumentHelper {
     }
 
     private static final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
+
     static {
         documentBuilderFactory.setNamespaceAware(true);
         documentBuilderFactory.setValidating(false);
@@ -69,7 +114,7 @@ public final class DocumentHelper {
 
     private static void trySetXercesSecurityManager(DocumentBuilderFactory documentBuilderFactory) {
         // Try built-in JVM one first, standalone if not
-        for (String securityManagerClassName : new String[] {
+        for (String securityManagerClassName : new String[]{
                 "com.sun.org.apache.xerces.internal.util.SecurityManager",
                 "org.apache.xerces.util.SecurityManager"
         }) {
@@ -89,6 +134,7 @@ public final class DocumentHelper {
     /**
      * Parses the given stream via the default (sensible)
      * DocumentBuilder
+     *
      * @param inp Stream to read the XML data from
      * @return the parsed Document
      */
@@ -96,6 +142,17 @@ public final class DocumentHelper {
         return newDocumentBuilder().parse(inp);
     }
 
+    /**
+     * Parses the given stream via the default (sensible)
+     * DocumentBuilder
+     *
+     * @param inp sax source to read the XML data from
+     * @return the parsed Document
+     */
+    public static Document readDocument(InputSource inp) throws IOException, SAXException {
+        return newDocumentBuilder().parse(inp);
+    }
+
     // must only be used to create empty documents, do not use it for parsing!
     private static final DocumentBuilder documentBuilderSingleton = newDocumentBuilder();
 

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/BitSetUtil.java

@@ -0,0 +1,108 @@
+package com.fr.third.bitmap.roaringbitmap;
+
+
+import java.util.Arrays;
+import java.util.BitSet;
+
+
+/***
+ *
+ * This class provides convenience functions to manipulate BitSet and RoaringBitmap objects.
+ *
+ */
+public class BitSetUtil {
+    // todo: add a method to convert a RoaringBitmap to a BitSet using BitSet.valueOf
+
+    // a block consists has a maximum of 1024 words, each representing 64 bits,
+    // thus representing at maximum 65536 bits
+    static final private int BLOCK_LENGTH = BitmapContainer.MAX_CAPACITY / Long.SIZE; //
+    // 64-bit
+    // word
+
+    private static ArrayContainer arrayContainerOf(final int from, final int to,
+                                                   final int cardinality, final long[] words) {
+        // precondition: cardinality is max 4096
+        final short[] content = new short[cardinality];
+        int index = 0;
+
+        for (int i = from, socket = 0; i < to; ++i, socket += Long.SIZE) {
+            long word = words[i];
+            while (word != 0) {
+                long t = word & -word;
+                content[index++] = (short) (socket + Long.bitCount(t - 1));
+                word ^= t;
+            }
+        }
+        return new ArrayContainer(content);
+    }
+
+    /**
+     * Generate a RoaringBitmap out of a long[], each long using little-endian representation of its
+     * bits
+     *
+     * @param words array of longs (will not be modified)
+     * @return roaring bitmap
+     * @see BitSet#toLongArray() for an equivalent
+     */
+    public static RoaringBitmap bitmapOf(final long[] words) {
+        // split long[] into blocks.
+        // each block becomes a single container, if any bit is set
+        final RoaringBitmap ans = new RoaringBitmap();
+        int containerIndex = 0;
+        for (int from = 0; from < words.length; from += BLOCK_LENGTH) {
+            final int to = Math.min(from + BLOCK_LENGTH, words.length);
+            final int blockCardinality = cardinality(from, to, words);
+            if (blockCardinality > 0) {
+                ans.highLowContainer.insertNewKeyValueAt(containerIndex++, Util.highbits(from * Long.SIZE),
+                        BitSetUtil.containerOf(from, to, blockCardinality, words));
+            }
+        }
+        return ans;
+    }
+
+    private static int cardinality(final int from, final int to, final long[] words) {
+        int sum = 0;
+        for (int i = from; i < to; i++) {
+            sum += Long.bitCount(words[i]);
+        }
+        return sum;
+    }
+
+
+    private static Container containerOf(final int from, final int to, final int blockCardinality,
+                                         final long[] words) {
+        // find the best container available
+        if (blockCardinality <= ArrayContainer.DEFAULT_MAX_SIZE) {
+            // containers with DEFAULT_MAX_SIZE or less integers should be
+            // ArrayContainers
+            return arrayContainerOf(from, to, blockCardinality, words);
+        } else {
+            // otherwise use bitmap container
+            return new BitmapContainer(Arrays.copyOfRange(words, from, from + BLOCK_LENGTH),
+                    blockCardinality);
+        }
+    }
+
+
+    /**
+     * Compares a RoaringBitmap and a BitSet. They are equal if and only if they contain the same set
+     * of integers.
+     *
+     * @param bitset first object to be compared
+     * @param bitmap second object to be compared
+     * @return whether they are equals
+     */
+    public static boolean equals(final BitSet bitset, final RoaringBitmap bitmap) {
+        if (bitset.cardinality() != bitmap.getCardinality()) {
+            return false;
+        }
+        final IntIterator it = bitmap.getIntIterator();
+        while (it.hasNext()) {
+            int val = it.next();
+            if (!bitset.get(val)) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/BitmapDataProvider.java

@@ -0,0 +1,38 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+/**
+ * Representing a general bitmap interface.
+ */
+public interface BitmapDataProvider extends ImmutableBitmapDataProvider {
+    /**
+     * set the value to "true", whether it already appears or not.
+     *
+     * @param x integer value
+     */
+    public void add(int x);
+
+    /**
+     * If present remove the specified integers (effectively, sets its bit value to false)
+     *
+     * @param x integer value representing the index in a bitmap
+     */
+    public void remove(int x);
+
+    /**
+     * Return the jth value stored in this bitmap.
+     *
+     * @param j index of the value
+     * @return the value
+     */
+    @Override
+    public int select(int j);
+
+    /**
+     * Recover allocated but unused memory.
+     */
+    public void trim();
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/Container.java

@@ -0,0 +1,808 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+import com.fr.third.bitmap.roaringbitmap.buffer.MappeableContainer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.Externalizable;
+import java.io.IOException;
+
+/**
+ * Base container class.
+ */
+public abstract class Container implements Iterable<Short>, Cloneable, Externalizable {
+
+    /**
+     * Name of the various possible containers
+     */
+    public static String ContainerNames[] = {"bitmap", "array", "run"};
+
+    /**
+     * Create a container initialized with a range of consecutive values
+     *
+     * @param start first index
+     * @param last  last index (range is exclusive)
+     * @return a new container initialized with the specified values
+     */
+    public static Container rangeOfOnes(final int start, final int last) {
+        final int sizeAsArrayContainer = ArrayContainer.serializedSizeInBytes(last - start);
+        final int sizeAsRunContainer = RunContainer.serializedSizeInBytes(1);
+        Container answer =
+                sizeAsRunContainer < sizeAsArrayContainer ? new RunContainer() : new ArrayContainer();
+        answer = answer.iadd(start, last);
+        return answer;
+    }
+
+    /**
+     * Return a new container with all shorts in [begin,end) added using an unsigned interpretation.
+     *
+     * @param begin start of range (inclusive)
+     * @param end   end of range (exclusive)
+     * @return the new container
+     */
+    public abstract Container add(int begin, int end);
+
+    /**
+     * Add a short to the container. May generate a new container.
+     *
+     * @param x short to be added
+     * @return the new container
+     */
+    public abstract Container add(short x);
+
+    /**
+     * Computes the bitwise AND of this container with another (intersection). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container and(ArrayContainer x);
+
+    /**
+     * Computes the bitwise AND of this container with another (intersection). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container and(BitmapContainer x);
+
+    /**
+     * Computes the bitwise AND of this container with another (intersection). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public Container and(Container x) {
+        if (x instanceof ArrayContainer) {
+            return and((ArrayContainer) x);
+        } else if (x instanceof BitmapContainer) {
+            return and((BitmapContainer) x);
+        }
+        return and((RunContainer) x);
+    }
+
+    /**
+     * Computes the bitwise AND of this container with another (intersection). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container and(RunContainer x);
+
+    protected abstract int andCardinality(ArrayContainer x);
+
+    protected abstract int andCardinality(BitmapContainer x);
+
+    protected abstract int andCardinality(RunContainer x);
+
+    /**
+     * Computes the bitwise AND of this container with another (intersection). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public int andCardinality(Container x) {
+        if (this.getCardinality() == 0) {
+            return 0;
+        } else if (x.getCardinality() == 0) {
+            return 0;
+        } else {
+            if (x instanceof ArrayContainer) {
+                return andCardinality((ArrayContainer) x);
+            } else if (x instanceof BitmapContainer) {
+                return andCardinality((BitmapContainer) x);
+            }
+            return andCardinality((RunContainer) x);
+        }
+    }
+
+    /**
+     * Computes the bitwise ANDNOT of this container with another (difference). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container andNot(ArrayContainer x);
+
+    /**
+     * Computes the bitwise ANDNOT of this container with another (difference). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container andNot(BitmapContainer x);
+
+    /**
+     * Computes the bitwise ANDNOT of this container with another (difference). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public Container andNot(Container x) {
+        if (x instanceof ArrayContainer) {
+            return andNot((ArrayContainer) x);
+        } else if (x instanceof BitmapContainer) {
+            return andNot((BitmapContainer) x);
+        }
+        return andNot((RunContainer) x);
+    }
+
+    /**
+     * Computes the bitwise ANDNOT of this container with another (difference). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container andNot(RunContainer x);
+
+    /**
+     * Empties the container
+     */
+    public abstract void clear();
+
+    @Override
+    public abstract Container clone();
+
+    /**
+     * Checks whether the contain contains the provided value
+     *
+     * @param x value to check
+     * @return whether the value is in the container
+     */
+    public abstract boolean contains(short x);
+
+    /**
+     * Deserialize (recover) the container.
+     *
+     * @param in the DataInput stream
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    public abstract void deserialize(DataInput in) throws IOException;
+
+    /**
+     * Fill the least significant 16 bits of the integer array, starting at index i, with the short
+     * values from this container. The caller is responsible to allocate enough room. The most
+     * significant 16 bits of each integer are given by the most significant bits of the provided
+     * mask.
+     *
+     * @param x    provided array
+     * @param i    starting index
+     * @param mask indicates most significant bits
+     */
+    public abstract void fillLeastSignificant16bits(int[] x, int i, int mask);
+
+    /**
+     * Add a short to the container if it is not present, otherwise remove it. May generate a new
+     * container.
+     *
+     * @param x short to be added
+     * @return the new container
+     */
+    public abstract Container flip(short x);
+
+    /**
+     * Size of the underlying array
+     *
+     * @return size in bytes
+     */
+    protected abstract int getArraySizeInBytes();
+
+    /**
+     * Computes the distinct number of short values in the container. Can be expected to run in
+     * constant time.
+     *
+     * @return the cardinality
+     */
+    public abstract int getCardinality();
+
+    /**
+     * Get the name of this container.
+     *
+     * @return name of the container
+     */
+    public String getContainerName() {
+        if (this instanceof BitmapContainer) {
+            return ContainerNames[0];
+        } else if (this instanceof ArrayContainer) {
+            return ContainerNames[1];
+        } else {
+            return ContainerNames[2];
+        }
+    }
+
+    /**
+     * Iterate through the values of this container and pass them
+     * along to the IntConsumer, using msb as the 16 most significant bits.
+     *
+     * @param msb 16 most significant bits
+     * @param ic  consumer
+     */
+    public abstract void forEach(short msb, IntConsumer ic);
+
+    /**
+     * Iterator to visit the short values in the container in descending order.
+     *
+     * @return iterator
+     */
+    public abstract ShortIterator getReverseShortIterator();
+
+    /**
+     * Iterator to visit the short values in the container in ascending order.
+     *
+     * @return iterator
+     */
+    public abstract PeekableShortIterator getShortIterator();
+
+    /**
+     * Computes an estimate of the memory usage of this container. The estimate is not meant to be
+     * exact.
+     *
+     * @return estimated memory usage in bytes
+     */
+    public abstract int getSizeInBytes();
+
+    /**
+     * Add all shorts in [begin,end) using an unsigned interpretation. May generate a new container.
+     *
+     * @param begin start of range (inclusive)
+     * @param end   end of range (exclusive)
+     * @return the new container
+     */
+    public abstract Container iadd(int begin, int end);
+
+    /**
+     * Computes the in-place bitwise AND of this container with another (intersection). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container iand(ArrayContainer x);
+
+
+    /**
+     * Computes the in-place bitwise AND of this container with another (intersection). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container iand(BitmapContainer x);
+
+    /**
+     * Computes the in-place bitwise AND of this container with another (intersection). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public Container iand(Container x) {
+        if (x instanceof ArrayContainer) {
+            return iand((ArrayContainer) x);
+        } else if (x instanceof BitmapContainer) {
+            return iand((BitmapContainer) x);
+        }
+        return iand((RunContainer) x);
+    }
+
+    /**
+     * Computes the in-place bitwise AND of this container with another (intersection). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container iand(RunContainer x);
+
+    /**
+     * Computes the in-place bitwise ANDNOT of this container with another (difference). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container iandNot(ArrayContainer x);
+
+
+    /**
+     * Computes the in-place bitwise ANDNOT of this container with another (difference). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container iandNot(BitmapContainer x);
+
+    /**
+     * Computes the in-place bitwise ANDNOT of this container with another (difference). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public Container iandNot(Container x) {
+        if (x instanceof ArrayContainer) {
+            return iandNot((ArrayContainer) x);
+        } else if (x instanceof BitmapContainer) {
+            return iandNot((BitmapContainer) x);
+        }
+        return iandNot((RunContainer) x);
+    }
+
+    /**
+     * Computes the in-place bitwise ANDNOT of this container with another (difference). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container iandNot(RunContainer x);
+
+
+    /**
+     * Computes the in-place bitwise NOT of this container (complement). Only those bits within the
+     * range are affected. The current container is generally modified. May generate a new container.
+     *
+     * @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
+     * @param rangeEnd   ending of range (exclusive)
+     * @return (partially) complemented container
+     */
+    public abstract Container inot(int rangeStart, int rangeEnd);
+
+    /**
+     * Returns true if the current container intersects the other container.
+     *
+     * @param x other container
+     * @return whether they intersect
+     */
+    public abstract boolean intersects(ArrayContainer x);
+
+    /**
+     * Returns true if the current container intersects the other container.
+     *
+     * @param x other container
+     * @return whether they intersect
+     */
+    public abstract boolean intersects(BitmapContainer x);
+
+    /**
+     * Returns true if the current container intersects the other container.
+     *
+     * @param x other container
+     * @return whether they intersect
+     */
+    public boolean intersects(Container x) {
+        if (x instanceof ArrayContainer) {
+            return intersects((ArrayContainer) x);
+        } else if (x instanceof BitmapContainer) {
+            return intersects((BitmapContainer) x);
+        }
+        return intersects((RunContainer) x);
+    }
+
+    /**
+     * Returns true if the current container intersects the other container.
+     *
+     * @param x other container
+     * @return whether they intersect
+     */
+    public abstract boolean intersects(RunContainer x);
+
+    /**
+     * Computes the in-place bitwise OR of this container with another (union). The current container
+     * is generally modified, whereas the provided container (x) is unaffected. May generate a new
+     * container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container ior(ArrayContainer x);
+
+    /**
+     * Computes the in-place bitwise OR of this container with another (union). The current container
+     * is generally modified, whereas the provided container (x) is unaffected. May generate a new
+     * container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container ior(BitmapContainer x);
+
+    /**
+     * Computes the in-place bitwise OR of this container with another (union). The current container
+     * is generally modified, whereas the provided container (x) is unaffected. May generate a new
+     * container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public Container ior(Container x) {
+        if (x instanceof ArrayContainer) {
+            return ior((ArrayContainer) x);
+        } else if (x instanceof BitmapContainer) {
+            return ior((BitmapContainer) x);
+        }
+        return ior((RunContainer) x);
+    }
+
+    /**
+     * Computes the in-place bitwise OR of this container with another (union). The current container
+     * is generally modified, whereas the provided container (x) is unaffected. May generate a new
+     * container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container ior(RunContainer x);
+
+    /**
+     * Remove shorts in [begin,end) using an unsigned interpretation. May generate a new container.
+     *
+     * @param begin start of range (inclusive)
+     * @param end   end of range (exclusive)
+     * @return the new container
+     */
+    public abstract Container iremove(int begin, int end);
+
+    /**
+     * Computes the in-place bitwise XOR of this container with another (symmetric difference). The
+     * current container is generally modified, whereas the provided container (x) is unaffected. May
+     * generate a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container ixor(ArrayContainer x);
+
+    /**
+     * Computes the in-place bitwise XOR of this container with another (symmetric difference). The
+     * current container is generally modified, whereas the provided container (x) is unaffected. May
+     * generate a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container ixor(BitmapContainer x);
+
+
+    /**
+     * Computes the in-place bitwise OR of this container with another (union). The current container
+     * is generally modified, whereas the provided container (x) is unaffected. May generate a new
+     * container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public Container ixor(Container x) {
+        if (x instanceof ArrayContainer) {
+            return ixor((ArrayContainer) x);
+        } else if (x instanceof BitmapContainer) {
+            return ixor((BitmapContainer) x);
+        }
+        return ixor((RunContainer) x);
+    }
+
+    /**
+     * Computes the in-place bitwise XOR of this container with another (symmetric difference). The
+     * current container is generally modified, whereas the provided container (x) is unaffected. May
+     * generate a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container ixor(RunContainer x);
+
+    /**
+     * Computes the in-place bitwise OR of this container with another (union). The current container
+     * is generally modified, whereas the provided container (x) is unaffected. May generate a new
+     * container. The resulting container may not track its cardinality correctly. The resulting
+     * container may not track its cardinality correctly. This can be fixed as follows:
+     * if(c.getCardinality()&lt;0) ((BitmapContainer)c).computeCardinality();
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public Container lazyIOR(Container x) {
+        if (this instanceof ArrayContainer) {
+            if (x instanceof ArrayContainer) {
+                return ((ArrayContainer) this).lazyor((ArrayContainer) x);
+            } else if (x instanceof BitmapContainer) {
+                return ior((BitmapContainer) x);
+            }
+            return ((RunContainer) x).lazyor((ArrayContainer) this);
+        } else if (this instanceof RunContainer) {
+            if (x instanceof ArrayContainer) {
+                return ((RunContainer) this).ilazyor((ArrayContainer) x);
+            } else if (x instanceof BitmapContainer) {
+                return ior((BitmapContainer) x);
+            }
+            return ior((RunContainer) x);
+        } else {
+            if (x instanceof ArrayContainer) {
+                return ((BitmapContainer) this).ilazyor((ArrayContainer) x);
+            } else if (x instanceof BitmapContainer) {
+                return ((BitmapContainer) this).ilazyor((BitmapContainer) x);
+            }
+            return ((BitmapContainer) this).ilazyor((RunContainer) x);
+        }
+    }
+
+    /**
+     * Computes the bitwise OR of this container with another (union). This container as well as the
+     * provided container are left unaffected. The resulting container may not track its cardinality
+     * correctly. This can be fixed as follows: if(c.getCardinality()&lt;0)
+     * ((BitmapContainer)c).computeCardinality();
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public Container lazyOR(Container x) {
+        if (this instanceof ArrayContainer) {
+            if (x instanceof ArrayContainer) {
+                return ((ArrayContainer) this).lazyor((ArrayContainer) x);
+            } else if (x instanceof BitmapContainer) {
+                return ((BitmapContainer) x).lazyor((ArrayContainer) this);
+            }
+            return ((RunContainer) x).lazyor((ArrayContainer) this);
+        } else if (this instanceof RunContainer) {
+            if (x instanceof ArrayContainer) {
+                return ((RunContainer) this).lazyor((ArrayContainer) x);
+            } else if (x instanceof BitmapContainer) {
+                return ((BitmapContainer) x).lazyor((RunContainer) this);
+            }
+            return or((RunContainer) x);
+        } else {
+            if (x instanceof ArrayContainer) {
+                return ((BitmapContainer) this).lazyor((ArrayContainer) x);
+            } else if (x instanceof BitmapContainer) {
+                return ((BitmapContainer) this).lazyor((BitmapContainer) x);
+            }
+            return ((BitmapContainer) this).lazyor((RunContainer) x);
+        }
+    }
+
+    /**
+     * Create a new Container containing at most maxcardinality integers.
+     *
+     * @param maxcardinality maximal cardinality
+     * @return a new bitmap with cardinality no more than maxcardinality
+     */
+    public abstract Container limit(int maxcardinality);
+
+    /**
+     * Computes the bitwise NOT of this container (complement). Only those bits within the range are
+     * affected. The current container is left unaffected.
+     *
+     * @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
+     * @param rangeEnd   ending of range (exclusive)
+     * @return (partially) complemented container
+     */
+    public abstract Container not(int rangeStart, int rangeEnd);
+
+    abstract int numberOfRuns(); // exact
+
+
+    /**
+     * Computes the bitwise OR of this container with another (union). This container as well as the
+     * provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container or(ArrayContainer x);
+
+    /**
+     * Computes the bitwise OR of this container with another (union). This container as well as the
+     * provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container or(BitmapContainer x);
+
+    /**
+     * Computes the bitwise OR of this container with another (union). This container as well as the
+     * provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public Container or(Container x) {
+        if (x instanceof ArrayContainer) {
+            return or((ArrayContainer) x);
+        } else if (x instanceof BitmapContainer) {
+            return or((BitmapContainer) x);
+        }
+        return or((RunContainer) x);
+    }
+
+    /**
+     * Computes the bitwise OR of this container with another (union). This container as well as the
+     * provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container or(RunContainer x);
+
+
+    /**
+     * Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be
+     * GetCardinality()).
+     *
+     * @param lowbits upper limit
+     * @return the rank
+     */
+    public abstract int rank(short lowbits);
+
+    /**
+     * Return a new container with all shorts in [begin,end) remove using an unsigned interpretation.
+     *
+     * @param begin start of range (inclusive)
+     * @param end   end of range (exclusive)
+     * @return the new container
+     */
+    public abstract Container remove(int begin, int end);
+
+    /**
+     * Remove the short from this container. May create a new container.
+     *
+     * @param x to be removed
+     * @return New container
+     */
+    public abstract Container remove(short x);
+
+    /**
+     * The output of a lazyOR or lazyIOR might be an invalid container, this should be called on it.
+     *
+     * @return a new valid container
+     */
+    public abstract Container repairAfterLazy();
+
+    /**
+     * Convert to RunContainers, when the result is smaller. Overridden by RunContainer to possibility
+     * switch from RunContainer to a smaller alternative. Overridden by BitmapContainer with a more
+     * efficient approach.
+     *
+     * @return the new container
+     */
+    public abstract Container runOptimize();
+
+    /**
+     * Return the jth value
+     *
+     * @param j index of the value
+     * @return the value
+     */
+    public abstract short select(int j);
+
+    /**
+     * Serialize the container.
+     *
+     * @param out the DataOutput stream
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    public abstract void serialize(DataOutput out) throws IOException;
+
+    /**
+     * Report the number of bytes required to serialize this container.
+     *
+     * @return the size in bytes
+     */
+    public abstract int serializedSizeInBytes();
+
+    /**
+     * Convert to a mappeable container.
+     *
+     * @return the mappeable container
+     */
+    public abstract MappeableContainer toMappeableContainer();
+
+
+    /**
+     * If possible, recover wasted memory.
+     */
+    public abstract void trim();
+
+    /**
+     * Write just the underlying array.
+     *
+     * @param out output stream
+     * @throws IOException in case of failure
+     */
+    protected abstract void writeArray(DataOutput out) throws IOException;
+
+
+    /**
+     * Computes the bitwise XOR of this container with another (symmetric difference). This container
+     * as well as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container xor(ArrayContainer x);
+
+    /**
+     * Computes the bitwise XOR of this container with another (symmetric difference). This container
+     * as well as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container xor(BitmapContainer x);
+
+
+    /**
+     * Computes the bitwise OR of this container with another (symmetric difference). This container
+     * as well as the provided container are left unaffected.
+     *
+     * @param x other parameter
+     * @return aggregated container
+     */
+    public Container xor(Container x) {
+        if (x instanceof ArrayContainer) {
+            return xor((ArrayContainer) x);
+        } else if (x instanceof BitmapContainer) {
+            return xor((BitmapContainer) x);
+        }
+        return xor((RunContainer) x);
+    }
+
+    /**
+     * Computes the bitwise XOR of this container with another (symmetric difference). This container
+     * as well as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract Container xor(RunContainer x);
+
+    /**
+     * Convert the current container to a BitmapContainer, if a conversion is needed.
+     * If the container is already a bitmap, the container is returned unchanged.
+     *
+     * @return a bitmap container
+     */
+    public abstract BitmapContainer toBitmapContainer();
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ContainerPointer.java

@@ -0,0 +1,60 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+/**
+ * This interface allows you to iterate over the containers in a roaring bitmap.
+ */
+public interface ContainerPointer extends Comparable<ContainerPointer>, Cloneable {
+    /**
+     * Move to the next container
+     */
+    void advance();
+
+
+    /**
+     * Create a copy
+     *
+     * @return return a clone of this pointer
+     */
+    ContainerPointer clone();
+
+    /**
+     * Return the cardinality of the current container
+     *
+     * @return the cardinality
+     */
+    int getCardinality();
+
+    /**
+     * This method can be used to check whether there is current a valid container as it returns null
+     * when there is not.
+     *
+     * @return null or the current container
+     */
+    Container getContainer();
+
+    /**
+     * Check whether the current container is a bitmap container.
+     *
+     * @return whether it is a bitmap container
+     */
+    boolean isBitmapContainer();
+
+    /**
+     * Check whether the current container is a run container.
+     *
+     * @return whether it is a run container
+     */
+    boolean isRunContainer();
+
+    /**
+     * The key is a 16-bit integer that indicates the position of the container in the roaring bitmap.
+     * To be interpreted as an unsigned integer.
+     *
+     * @return the key
+     */
+    short key();
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/FastAggregation.java

@@ -0,0 +1,532 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+
+/**
+ * Fast algorithms to aggregate many bitmaps.
+ *
+ * @author Daniel Lemire
+ */
+public final class FastAggregation {
+
+
+    /**
+     * Private constructor to prevent instantiation of utility class
+     */
+    private FastAggregation() {
+    }
+
+    /**
+     * Compute the AND aggregate.
+     * <p>
+     * In practice, calls {#link naive_and}
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap and(Iterator<RoaringBitmap> bitmaps) {
+        return naive_and(bitmaps);
+    }
+
+    /**
+     * Compute the AND aggregate.
+     * <p>
+     * In practice, calls {#link naive_and}
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap and(RoaringBitmap... bitmaps) {
+        return naive_and(bitmaps);
+    }
+
+    /**
+     * Calls naive_or.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    @Deprecated
+    public static RoaringBitmap horizontal_or(Iterator<RoaringBitmap> bitmaps) {
+        return naive_or(bitmaps);
+    }
+
+    /**
+     * Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #or(RoaringBitmap...)
+     */
+    public static RoaringBitmap horizontal_or(List<RoaringBitmap> bitmaps) {
+        RoaringBitmap answer = new RoaringBitmap();
+        if (bitmaps.isEmpty()) {
+            return answer;
+        }
+        PriorityQueue<ContainerPointer> pq = new PriorityQueue<ContainerPointer>(bitmaps.size());
+        for (int k = 0; k < bitmaps.size(); ++k) {
+            ContainerPointer x = bitmaps.get(k).highLowContainer.getContainerPointer();
+            if (x.getContainer() != null) {
+                pq.add(x);
+            }
+        }
+
+        while (!pq.isEmpty()) {
+            ContainerPointer x1 = pq.poll();
+            if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
+                answer.highLowContainer.append(x1.key(), x1.getContainer().clone());
+                x1.advance();
+                if (x1.getContainer() != null) {
+                    pq.add(x1);
+                }
+                continue;
+            }
+            ContainerPointer x2 = pq.poll();
+            Container newc = x1.getContainer().lazyOR(x2.getContainer());
+            while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
+
+                ContainerPointer x = pq.poll();
+                newc = newc.lazyIOR(x.getContainer());
+                x.advance();
+                if (x.getContainer() != null) {
+                    pq.add(x);
+                } else if (pq.isEmpty()) {
+                    break;
+                }
+            }
+            newc = newc.repairAfterLazy();
+            answer.highLowContainer.append(x1.key(), newc);
+            x1.advance();
+            if (x1.getContainer() != null) {
+                pq.add(x1);
+            }
+            x2.advance();
+            if (x2.getContainer() != null) {
+                pq.add(x2);
+            }
+        }
+        return answer;
+    }
+
+    /**
+     * Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #or(RoaringBitmap...)
+     */
+    public static RoaringBitmap horizontal_or(RoaringBitmap... bitmaps) {
+        RoaringBitmap answer = new RoaringBitmap();
+        if (bitmaps.length == 0) {
+            return answer;
+        }
+        PriorityQueue<ContainerPointer> pq = new PriorityQueue<ContainerPointer>(bitmaps.length);
+        for (int k = 0; k < bitmaps.length; ++k) {
+            ContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
+            if (x.getContainer() != null) {
+                pq.add(x);
+            }
+        }
+
+        while (!pq.isEmpty()) {
+            ContainerPointer x1 = pq.poll();
+            if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
+                answer.highLowContainer.append(x1.key(), x1.getContainer().clone());
+                x1.advance();
+                if (x1.getContainer() != null) {
+                    pq.add(x1);
+                }
+                continue;
+            }
+            ContainerPointer x2 = pq.poll();
+            Container newc = x1.getContainer().lazyOR(x2.getContainer());
+            while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
+
+                ContainerPointer x = pq.poll();
+                newc = newc.lazyIOR(x.getContainer());
+                x.advance();
+                if (x.getContainer() != null) {
+                    pq.add(x);
+                } else if (pq.isEmpty()) {
+                    break;
+                }
+            }
+            newc = newc.repairAfterLazy();
+            answer.highLowContainer.append(x1.key(), newc);
+            x1.advance();
+            if (x1.getContainer() != null) {
+                pq.add(x1);
+            }
+            x2.advance();
+            if (x2.getContainer() != null) {
+                pq.add(x2);
+            }
+        }
+        return answer;
+    }
+
+    /**
+     * Minimizes memory usage while computing the xor aggregate on a moderate number of bitmaps.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #xor(RoaringBitmap...)
+     */
+    public static RoaringBitmap horizontal_xor(RoaringBitmap... bitmaps) {
+        RoaringBitmap answer = new RoaringBitmap();
+        if (bitmaps.length == 0) {
+            return answer;
+        }
+        PriorityQueue<ContainerPointer> pq = new PriorityQueue<ContainerPointer>(bitmaps.length);
+        for (int k = 0; k < bitmaps.length; ++k) {
+            ContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
+            if (x.getContainer() != null) {
+                pq.add(x);
+            }
+        }
+
+        while (!pq.isEmpty()) {
+            ContainerPointer x1 = pq.poll();
+            if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
+                answer.highLowContainer.append(x1.key(), x1.getContainer().clone());
+                x1.advance();
+                if (x1.getContainer() != null) {
+                    pq.add(x1);
+                }
+                continue;
+            }
+            ContainerPointer x2 = pq.poll();
+            Container newc = x1.getContainer().xor(x2.getContainer());
+            while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
+                ContainerPointer x = pq.poll();
+                newc = newc.ixor(x.getContainer());
+                x.advance();
+                if (x.getContainer() != null) {
+                    pq.add(x);
+                } else if (pq.isEmpty()) {
+                    break;
+                }
+            }
+            answer.highLowContainer.append(x1.key(), newc);
+            x1.advance();
+            if (x1.getContainer() != null) {
+                pq.add(x1);
+            }
+            x2.advance();
+            if (x2.getContainer() != null) {
+                pq.add(x2);
+            }
+        }
+        return answer;
+    }
+
+    /**
+     * Compute overall AND between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap naive_and(Iterator<RoaringBitmap> bitmaps) {
+        if (!bitmaps.hasNext()) {
+            return new RoaringBitmap();
+        }
+        RoaringBitmap answer = bitmaps.next().clone();
+        while (bitmaps.hasNext()) {
+            answer.and(bitmaps.next());
+        }
+        return answer;
+    }
+
+    /**
+     * Compute overall AND between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap naive_and(RoaringBitmap... bitmaps) {
+        if (bitmaps.length == 0) {
+            return new RoaringBitmap();
+        }
+        RoaringBitmap answer = bitmaps[0].clone();
+        for (int k = 1; k < bitmaps.length; ++k) {
+            answer.and(bitmaps[k]);
+        }
+        return answer;
+    }
+
+    /**
+     * Compute overall OR between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap naive_or(Iterator<RoaringBitmap> bitmaps) {
+        RoaringBitmap answer = new RoaringBitmap();
+        while (bitmaps.hasNext()) {
+            answer.naivelazyor(bitmaps.next());
+        }
+        answer.repairAfterLazy();
+        return answer;
+    }
+
+    /**
+     * Compute overall OR between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap naive_or(RoaringBitmap... bitmaps) {
+        RoaringBitmap answer = new RoaringBitmap();
+        for (int k = 0; k < bitmaps.length; ++k) {
+            answer.naivelazyor(bitmaps[k]);
+        }
+        answer.repairAfterLazy();
+        return answer;
+    }
+
+    /**
+     * Compute overall XOR between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap naive_xor(Iterator<RoaringBitmap> bitmaps) {
+        RoaringBitmap answer = new RoaringBitmap();
+        while (bitmaps.hasNext()) {
+            answer.xor(bitmaps.next());
+        }
+        return answer;
+    }
+
+    /**
+     * Compute overall XOR between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap naive_xor(RoaringBitmap... bitmaps) {
+        RoaringBitmap answer = new RoaringBitmap();
+        for (int k = 0; k < bitmaps.length; ++k) {
+            answer.xor(bitmaps[k]);
+        }
+        return answer;
+    }
+
+    /**
+     * Compute overall OR between bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap or(Iterator<RoaringBitmap> bitmaps) {
+        return naive_or(bitmaps);
+    }
+
+    /**
+     * Compute overall OR between bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap or(RoaringBitmap... bitmaps) {
+        return naive_or(bitmaps);
+    }
+
+    /**
+     * Uses a priority queue to compute the or aggregate.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #horizontal_or(RoaringBitmap...)
+     */
+    public static RoaringBitmap priorityqueue_or(Iterator<RoaringBitmap> bitmaps) {
+        if (!bitmaps.hasNext()) {
+            return new RoaringBitmap();
+        }
+        // we buffer the call to getSizeInBytes(), hence the code complexity
+        ArrayList<RoaringBitmap> buffer = new ArrayList<RoaringBitmap>();
+        while (bitmaps.hasNext()) {
+            buffer.add(bitmaps.next());
+        }
+        final long[] sizes = new long[buffer.size()];
+        final boolean[] istmp = new boolean[buffer.size()];
+        for (int k = 0; k < sizes.length; ++k) {
+            sizes[k] = buffer.get(k).getLongSizeInBytes();
+        }
+        PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
+            @Override
+            public int compare(Integer a, Integer b) {
+                return (int) (sizes[a] - sizes[b]);
+            }
+        });
+        for (int k = 0; k < sizes.length; ++k) {
+            pq.add(k);
+        }
+        while (pq.size() > 1) {
+            Integer x1 = pq.poll();
+            Integer x2 = pq.poll();
+            if (istmp[x2] && istmp[x1]) {
+                buffer.set(x1, RoaringBitmap.lazyorfromlazyinputs(buffer.get(x1), buffer.get(x2)));
+                sizes[x1] = buffer.get(x1).getLongSizeInBytes();
+                istmp[x1] = true;
+                pq.add(x1);
+            } else if (istmp[x2]) {
+                buffer.get(x2).lazyor(buffer.get(x1));
+                sizes[x2] = buffer.get(x2).getLongSizeInBytes();
+                pq.add(x2);
+            } else if (istmp[x1]) {
+                buffer.get(x1).lazyor(buffer.get(x2));
+                sizes[x1] = buffer.get(x1).getLongSizeInBytes();
+                pq.add(x1);
+            } else {
+                buffer.set(x1, RoaringBitmap.lazyor(buffer.get(x1), buffer.get(x2)));
+                sizes[x1] = buffer.get(x1).getLongSizeInBytes();
+                istmp[x1] = true;
+                pq.add(x1);
+            }
+        }
+        RoaringBitmap answer = buffer.get(pq.poll());
+        answer.repairAfterLazy();
+        return answer;
+    }
+
+    /**
+     * Uses a priority queue to compute the or aggregate.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #horizontal_or(RoaringBitmap...)
+     */
+    public static RoaringBitmap priorityqueue_or(RoaringBitmap... bitmaps) {
+        if (bitmaps.length == 0) {
+            return new RoaringBitmap();
+        }
+        // we buffer the call to getSizeInBytes(), hence the code complexity
+        final RoaringBitmap[] buffer = Arrays.copyOf(bitmaps, bitmaps.length);
+        final long[] sizes = new long[buffer.length];
+        final boolean[] istmp = new boolean[buffer.length];
+        for (int k = 0; k < sizes.length; ++k) {
+            sizes[k] = buffer[k].getLongSizeInBytes();
+        }
+        PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
+            @Override
+            public int compare(Integer a, Integer b) {
+                return (int) (sizes[a] - sizes[b]);
+            }
+        });
+        for (int k = 0; k < sizes.length; ++k) {
+            pq.add(k);
+        }
+        while (pq.size() > 1) {
+            Integer x1 = pq.poll();
+            Integer x2 = pq.poll();
+            if (istmp[x2] && istmp[x1]) {
+                buffer[x1] = RoaringBitmap.lazyorfromlazyinputs(buffer[x1], buffer[x2]);
+                sizes[x1] = buffer[x1].getLongSizeInBytes();
+                istmp[x1] = true;
+                pq.add(x1);
+            } else if (istmp[x2]) {
+                buffer[x2].lazyor(buffer[x1]);
+                sizes[x2] = buffer[x2].getLongSizeInBytes();
+                pq.add(x2);
+            } else if (istmp[x1]) {
+                buffer[x1].lazyor(buffer[x2]);
+                sizes[x1] = buffer[x1].getLongSizeInBytes();
+                pq.add(x1);
+            } else {
+                buffer[x1] = RoaringBitmap.lazyor(buffer[x1], buffer[x2]);
+                sizes[x1] = buffer[x1].getLongSizeInBytes();
+                istmp[x1] = true;
+                pq.add(x1);
+            }
+        }
+        RoaringBitmap answer = buffer[pq.poll()];
+        answer.repairAfterLazy();
+        return answer;
+    }
+
+    /**
+     * Uses a priority queue to compute the xor aggregate.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #horizontal_xor(RoaringBitmap...)
+     */
+    public static RoaringBitmap priorityqueue_xor(RoaringBitmap... bitmaps) {
+        // TODO: This code could be faster, see priorityqueue_or
+        if (bitmaps.length == 0) {
+            return new RoaringBitmap();
+        }
+
+        PriorityQueue<RoaringBitmap> pq =
+                new PriorityQueue<RoaringBitmap>(bitmaps.length, new Comparator<RoaringBitmap>() {
+                    @Override
+                    public int compare(RoaringBitmap a, RoaringBitmap b) {
+                        return (int) (a.getLongSizeInBytes() - b.getLongSizeInBytes());
+                    }
+                });
+        Collections.addAll(pq, bitmaps);
+        while (pq.size() > 1) {
+            RoaringBitmap x1 = pq.poll();
+            RoaringBitmap x2 = pq.poll();
+            pq.add(RoaringBitmap.xor(x1, x2));
+        }
+        return pq.poll();
+    }
+
+    /**
+     * Compute overall XOR between bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap xor(Iterator<RoaringBitmap> bitmaps) {
+        return naive_xor(bitmaps);
+    }
+
+    /**
+     * Compute overall XOR between bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static RoaringBitmap xor(RoaringBitmap... bitmaps) {
+        return naive_xor(bitmaps);
+    }
+
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ImmutableBitmapDataProvider.java

@@ -0,0 +1,157 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Interface representing an immutable bitmap.
+ */
+public interface ImmutableBitmapDataProvider {
+    /**
+     * Checks whether the value in included, which is equivalent to checking if the corresponding bit
+     * is set (get in BitSet class).
+     *
+     * @param x integer value
+     * @return whether the integer value is included.
+     */
+    public boolean contains(int x);
+
+    /**
+     * Returns the number of distinct integers added to the bitmap (e.g., number of bits set).
+     * Internally, this is computed as a 64-bit number.
+     *
+     * @return the cardinality
+     */
+    public int getCardinality();
+
+    /**
+     * Returns the number of distinct integers added to the bitmap (e.g., number of bits set).
+     * This returns a full 64-bit result.
+     *
+     * @return the cardinality
+     */
+    public long getLongCardinality();
+
+    /**
+     * Visit all values in the bitmap and pass them to the consumer.
+     * <p>
+     * * Usage:
+     * <pre>
+     * {@code
+     *  bitmap.forEach(new IntConsumer() {
+     *
+     *    {@literal @}Override
+     *    public void accept(int value) {
+     *      // do something here
+     *
+     *    }});
+     *   }
+     * }
+     * </pre>
+     *
+     * @param ic the consumer
+     */
+    public void forEach(IntConsumer ic);
+
+    /**
+     * For better performance, consider the Use the {@link #forEach forEach} method.
+     *
+     * @return a custom iterator over set bits, the bits are traversed in ascending sorted order
+     */
+    public PeekableIntIterator getIntIterator();
+
+    /**
+     * @return a custom iterator over set bits, the bits are traversed in descending sorted order
+     */
+    public IntIterator getReverseIntIterator();
+
+    /**
+     * Estimate of the memory usage of this data structure.
+     * <p>
+     * Internally, this is computed as a 64-bit counter.
+     *
+     * @return estimated memory usage.
+     */
+    public int getSizeInBytes();
+
+    /**
+     * Estimate of the memory usage of this data structure. Provides
+     * full 64-bit number.
+     *
+     * @return estimated memory usage.
+     */
+    public long getLongSizeInBytes();
+
+    /**
+     * Checks whether the bitmap is empty.
+     *
+     * @return true if this bitmap contains no set bit
+     */
+    public boolean isEmpty();
+
+    /**
+     * Create a new bitmap of the same class, containing at most maxcardinality integers.
+     *
+     * @param x maximal cardinality
+     * @return a new bitmap with cardinality no more than maxcardinality
+     */
+    public ImmutableBitmapDataProvider limit(int x);
+
+    /**
+     * Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be
+     * GetCardinality()).
+     * <p>
+     * The value is internally computed as a 64-bit number.
+     *
+     * @param x upper limit
+     * @return the rank
+     */
+    public int rank(int x);
+
+    /**
+     * Same as "rank" but produces a full 64-bit value.
+     *
+     * @param x upper limit
+     * @return the rank
+     */
+    public long rankLong(int x);
+
+    /**
+     * Return the jth value stored in this bitmap.
+     *
+     * @param j index of the value
+     * @return the value
+     */
+    public int select(int j);
+
+    /**
+     * Serialize this bitmap.
+     * <p>
+     * The current bitmap is not modified.
+     *
+     * @param out the DataOutput stream
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    public void serialize(DataOutput out) throws IOException;
+
+    /**
+     * Report the number of bytes required to serialize this bitmap. This is the number of bytes
+     * written out when using the serialize method. When using the writeExternal method, the count
+     * will be higher due to the overhead of Java serialization.
+     *
+     * @return the size in bytes
+     */
+    public int serializedSizeInBytes();
+
+    /**
+     * Return the set values as an array. The integer values are in sorted order.
+     *
+     * @return array representing the set values.
+     */
+    public int[] toArray();
+
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/IntConsumer.java

@@ -0,0 +1,29 @@
+package com.fr.third.bitmap.roaringbitmap;
+
+/**
+ * An IntConsumer receives the int values contained in a data structure.
+ * Each value is visited once.
+ * <p>
+ * Usage:
+ * <p>
+ * <pre>
+ * {@code
+ *  bitmap.forEach(new IntConsumer() {
+ *
+ *    &#64;Override
+ *    public void accept(int value) {
+ *      // do something here
+ *
+ *    }});
+ *   }
+ * }
+ * </pre>
+ */
+public interface IntConsumer {
+    /**
+     * Receives the integer
+     *
+     * @param value the integer value
+     */
+    void accept(int value);
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/IntIterator.java

@@ -0,0 +1,28 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+/**
+ * A simple iterator over integer values
+ */
+public interface IntIterator extends Cloneable {
+    /**
+     * Creates a copy of the iterator.
+     *
+     * @return a clone of the current iterator
+     */
+    IntIterator clone();
+
+    /**
+     * @return whether there is another value
+     */
+    boolean hasNext();
+
+    /**
+     * @return next integer value
+     */
+    int next();
+
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/IntIteratorFlyweight.java

@@ -0,0 +1,124 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+/**
+ * Fast iterator minimizing the stress on the garbage collector. You can create one reusable
+ * instance of this class and then {@link #wrap(RoaringBitmap)}
+ * <p>
+ * For better performance, consider the {@link RoaringBitmap#forEach} method.
+ *
+ * @author Borislav Ivanov
+ **/
+public class IntIteratorFlyweight implements PeekableIntIterator {
+
+    private int hs;
+
+    private PeekableShortIterator iter;
+
+    private ArrayContainerShortIterator arrIter = new ArrayContainerShortIterator();
+
+    private BitmapContainerShortIterator bitmapIter = new BitmapContainerShortIterator();
+
+    private RunContainerShortIterator runIter = new RunContainerShortIterator();
+
+    private int pos;
+
+    private RoaringBitmap roaringBitmap = null;
+
+    /**
+     * Creates an instance that is not ready for iteration. You must first call
+     * {@link #wrap(RoaringBitmap)}.
+     */
+    public IntIteratorFlyweight() {
+
+    }
+
+    /**
+     * Creates an instance that is ready for iteration.
+     *
+     * @param r bitmap to be iterated over
+     */
+    public IntIteratorFlyweight(RoaringBitmap r) {
+        wrap(r);
+    }
+
+    @Override
+    public PeekableIntIterator clone() {
+        try {
+            IntIteratorFlyweight x = (IntIteratorFlyweight) super.clone();
+            x.iter = this.iter.clone();
+            return x;
+        } catch (CloneNotSupportedException e) {
+            return null;// will not happen
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        return pos < this.roaringBitmap.highLowContainer.size();
+    }
+
+    @Override
+    public int next() {
+        int x = iter.nextAsInt() | hs;
+        if (!iter.hasNext()) {
+            ++pos;
+            nextContainer();
+        }
+        return x;
+    }
+
+    private void nextContainer() {
+        if (pos < this.roaringBitmap.highLowContainer.size()) {
+
+            Container container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
+
+            if (container instanceof BitmapContainer) {
+                bitmapIter.wrap(((BitmapContainer) container).bitmap);
+                iter = bitmapIter;
+            } else if (container instanceof ArrayContainer) {
+                arrIter.wrap((ArrayContainer) container);
+                iter = arrIter;
+            } else {
+                runIter.wrap((RunContainer) container);
+                iter = runIter;
+            }
+            hs = Util.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
+        }
+    }
+
+    /**
+     * Prepares a bitmap for iteration
+     *
+     * @param r bitmap to be iterated over
+     */
+    public void wrap(RoaringBitmap r) {
+        this.hs = 0;
+        this.pos = 0;
+        this.roaringBitmap = r;
+        this.nextContainer();
+    }
+
+    @Override
+    public void advanceIfNeeded(final int minval) {
+        while (hasNext() && ((hs >>> 16) < (minval >>> 16))) {
+            ++pos;
+            nextContainer();
+        }
+        if (hasNext() && ((hs >>> 16) == (minval >>> 16))) {
+            iter.advanceIfNeeded(Util.lowbits(minval));
+            if (!iter.hasNext()) {
+                ++pos;
+                nextContainer();
+            }
+        }
+    }
+
+    @Override
+    public int peekNext() {
+        return Util.toIntUnsigned(iter.peekNext()) | hs;
+    }
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/PeekableIntIterator.java

@@ -0,0 +1,33 @@
+package com.fr.third.bitmap.roaringbitmap;
+
+
+/**
+ * Simple extension to the IntIterator interface.
+ * It allows you to "skip" values using the advanceIfNeeded
+ * method, and to look at the value without advancing (peekNext).
+ */
+public interface PeekableIntIterator extends IntIterator {
+    /**
+     * If needed, advance as long as the next value is smaller than minval
+     *
+     * @param minval threshold
+     */
+    public void advanceIfNeeded(int minval);
+
+    /**
+     * Look at the next value without advancing
+     *
+     * @return next value
+     */
+    public int peekNext();
+
+    /**
+     * Creates a copy of the iterator.
+     *
+     * @return a clone of the current iterator
+     */
+    @Override
+    PeekableIntIterator clone();
+}
+
+

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/PeekableShortIterator.java

@@ -0,0 +1,34 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+package com.fr.third.bitmap.roaringbitmap;
+
+
+/**
+ * Simple extension to the ShortIterator interface
+ */
+public interface PeekableShortIterator extends ShortIterator {
+    /**
+     * If needed, advance as long as the next value is smaller than minval (as an unsigned
+     * short)
+     *
+     * @param minval threshold
+     */
+    public void advanceIfNeeded(short minval);
+
+    /**
+     * Look at the next value without advancing
+     *
+     * @return next value
+     */
+    public short peekNext();
+
+    /**
+     * Creates a copy of the iterator.
+     *
+     * @return a clone of the current iterator
+     */
+    @Override
+    PeekableShortIterator clone();
+}
+

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ReverseIntIteratorFlyweight.java

@@ -0,0 +1,110 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+/**
+ * Fast iterator minimizing the stress on the garbage collector. You can create one reusable
+ * instance of this class and then {@link #wrap(RoaringBitmap)}
+ * <p>
+ * This iterator enumerates the stored values in reverse (starting from the end).
+ *
+ * @author Borislav Ivanov
+ **/
+public class ReverseIntIteratorFlyweight implements IntIterator {
+
+    private int hs;
+
+    private ShortIterator iter;
+
+    private ReverseArrayContainerShortIterator arrIter = new ReverseArrayContainerShortIterator();
+
+    private ReverseBitmapContainerShortIterator bitmapIter =
+            new ReverseBitmapContainerShortIterator();
+
+    private ReverseRunContainerShortIterator runIter = new ReverseRunContainerShortIterator();
+
+    private short pos;
+
+    private RoaringBitmap roaringBitmap = null;
+
+
+    /**
+     * Creates an instance that is not ready for iteration. You must first call
+     * {@link #wrap(RoaringBitmap)}.
+     */
+    public ReverseIntIteratorFlyweight() {
+
+    }
+
+    /**
+     * Creates an instance that is ready for iteration.
+     *
+     * @param r bitmap to be iterated over
+     */
+    public ReverseIntIteratorFlyweight(RoaringBitmap r) {
+        wrap(r);
+    }
+
+    @Override
+    public IntIterator clone() {
+        try {
+            ReverseIntIteratorFlyweight x = (ReverseIntIteratorFlyweight) super.clone();
+            x.iter = this.iter.clone();
+            return x;
+        } catch (CloneNotSupportedException e) {
+            return null;// will not happen
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        return pos >= 0;
+    }
+
+
+    @Override
+    public int next() {
+        final int x = iter.nextAsInt() | hs;
+        if (!iter.hasNext()) {
+            --pos;
+            nextContainer();
+        }
+        return x;
+    }
+
+    private void nextContainer() {
+
+
+        if (pos >= 0) {
+
+            Container container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
+            if (container instanceof BitmapContainer) {
+                bitmapIter.wrap(((BitmapContainer) container).bitmap);
+                iter = bitmapIter;
+            } else if (container instanceof ArrayContainer) {
+                arrIter.wrap((ArrayContainer) container);
+                iter = arrIter;
+            } else {
+                runIter.wrap((RunContainer) container);
+                iter = runIter;
+            }
+            hs = Util.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
+        }
+    }
+
+    /**
+     * Prepares a bitmap for iteration
+     *
+     * @param r bitmap to be iterated over
+     */
+    public void wrap(RoaringBitmap r) {
+        this.roaringBitmap = r;
+        this.hs = 0;
+        this.pos = (short) (this.roaringBitmap.highLowContainer.size() - 1);
+        this.nextContainer();
+    }
+
+}
+

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/RoaringArray.java

@@ -0,0 +1,582 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Arrays;
+
+
+/**
+ * Specialized array to store the containers used by a RoaringBitmap. This is not meant to be used
+ * by end users.
+ */
+public final class RoaringArray implements Cloneable, Externalizable {
+    protected static final short SERIAL_COOKIE_NO_RUNCONTAINER = 12346;
+    protected static final short SERIAL_COOKIE = 12347;
+    protected static final int NO_OFFSET_THRESHOLD = 4;
+    static final int INITIAL_CAPACITY = 4;
+    // bumped serialVersionUID with runcontainers, so default serialization
+    // will not work...
+    private static final long serialVersionUID = 8L;
+    short[] keys = null;
+
+    Container[] values = null;
+
+    int size = 0;
+
+    protected RoaringArray() {
+        this.keys = new short[INITIAL_CAPACITY];
+        this.values = new Container[INITIAL_CAPACITY];
+    }
+
+    /**
+     * Find the smallest integer index larger than pos such that array[index].key>=x. If none can
+     * be found, return size. Based on code by O. Kaser.
+     *
+     * @param x   minimal value
+     * @param pos index to exceed
+     * @return the smallest index greater than pos such that array[index].key is at least as large as
+     * min, or size if it is not possible.
+     */
+    protected int advanceUntil(short x, int pos) {
+        int lower = pos + 1;
+
+        // special handling for a possibly common sequential case
+        if (lower >= size || Util.toIntUnsigned(keys[lower]) >= Util.toIntUnsigned(x)) {
+            return lower;
+        }
+
+        int spansize = 1; // could set larger
+        // bootstrap an upper limit
+
+        while (lower + spansize < size
+                && Util.toIntUnsigned(keys[lower + spansize]) < Util.toIntUnsigned(x)) {
+            spansize *= 2; // hoping for compiler will reduce to shift
+        }
+        int upper = (lower + spansize < size) ? lower + spansize : size - 1;
+
+        // maybe we are lucky (could be common case when the seek ahead
+        // expected to be small and sequential will otherwise make us look bad)
+        if (keys[upper] == x) {
+            return upper;
+        }
+
+        if (Util.toIntUnsigned(keys[upper]) < Util.toIntUnsigned(x)) {// means array has no item key >=
+            // x
+            return size;
+        }
+
+        // we know that the next-smallest span was too small
+        lower += (spansize / 2);
+
+        // else begin binary search
+        // invariant: array[lower]<x && array[upper]>x
+        while (lower + 1 != upper) {
+            int mid = (lower + upper) / 2;
+            if (keys[mid] == x) {
+                return mid;
+            } else if (Util.toIntUnsigned(keys[mid]) < Util.toIntUnsigned(x)) {
+                lower = mid;
+            } else {
+                upper = mid;
+            }
+        }
+        return upper;
+    }
+
+    protected void append(short key, Container value) {
+        extendArray(1);
+        this.keys[this.size] = key;
+        this.values[this.size] = value;
+        this.size++;
+    }
+
+    /**
+     * Append copies of the values AFTER a specified key (may or may not be present) to end.
+     *
+     * @param sa          other array
+     * @param beforeStart given key is the largest key that we won't copy
+     */
+    protected void appendCopiesAfter(RoaringArray sa, short beforeStart) {
+        int startLocation = sa.getIndex(beforeStart);
+        if (startLocation >= 0) {
+            startLocation++;
+        } else {
+            startLocation = -startLocation - 1;
+        }
+        extendArray(sa.size - startLocation);
+
+        for (int i = startLocation; i < sa.size; ++i) {
+            this.keys[this.size] = sa.keys[i];
+            this.values[this.size] = sa.values[i].clone();
+            this.size++;
+        }
+    }
+
+    /**
+     * Append copies of the values from another array, from the start
+     *
+     * @param sourceArray The array to copy from
+     * @param stoppingKey any equal or larger key in other array will terminate copying
+     */
+    protected void appendCopiesUntil(RoaringArray sourceArray, short stoppingKey) {
+        int stopKey = Util.toIntUnsigned(stoppingKey);
+        for (int i = 0; i < sourceArray.size; ++i) {
+            if (Util.toIntUnsigned(sourceArray.keys[i]) >= stopKey) {
+                break;
+            }
+            extendArray(1);
+            this.keys[this.size] = sourceArray.keys[i];
+            this.values[this.size] = sourceArray.values[i].clone();
+            this.size++;
+        }
+    }
+
+    /**
+     * Append copy of the one value from another array
+     *
+     * @param sa    other array
+     * @param index index in the other array
+     */
+    protected void appendCopy(RoaringArray sa, int index) {
+        extendArray(1);
+        this.keys[this.size] = sa.keys[index];
+        this.values[this.size] = sa.values[index].clone();
+        this.size++;
+    }
+
+    /**
+     * Append copies of the values from another array
+     *
+     * @param sa            other array
+     * @param startingIndex starting index in the other array
+     * @param end           endingIndex (exclusive) in the other array
+     */
+    protected void appendCopy(RoaringArray sa, int startingIndex, int end) {
+        extendArray(end - startingIndex);
+        for (int i = startingIndex; i < end; ++i) {
+            this.keys[this.size] = sa.keys[i];
+            this.values[this.size] = sa.values[i].clone();
+            this.size++;
+        }
+    }
+
+
+    /**
+     * Append the values from another array, no copy is made (use with care)
+     *
+     * @param sa            other array
+     * @param startingIndex starting index in the other array
+     * @param end           endingIndex (exclusive) in the other array
+     */
+    protected void append(RoaringArray sa, int startingIndex, int end) {
+        extendArray(end - startingIndex);
+        for (int i = startingIndex; i < end; ++i) {
+            this.keys[this.size] = sa.keys[i];
+            this.values[this.size] = sa.values[i];
+            this.size++;
+        }
+    }
+
+
+    private int binarySearch(int begin, int end, short key) {
+        return Util.unsignedBinarySearch(keys, begin, end, key);
+    }
+
+    protected void clear() {
+        this.keys = null;
+        this.values = null;
+        this.size = 0;
+    }
+
+    @Override
+    public RoaringArray clone() throws CloneNotSupportedException {
+        RoaringArray sa;
+        sa = (RoaringArray) super.clone();
+        sa.keys = Arrays.copyOf(this.keys, this.size);
+        sa.values = Arrays.copyOf(this.values, this.size);
+        for (int k = 0; k < this.size; ++k) {
+            sa.values[k] = sa.values[k].clone();
+        }
+        sa.size = this.size;
+        return sa;
+    }
+
+    protected void copyRange(int begin, int end, int newBegin) {
+        // assuming begin <= end and newBegin < begin
+        final int range = end - begin;
+        System.arraycopy(this.keys, begin, this.keys, newBegin, range);
+        System.arraycopy(this.values, begin, this.values, newBegin, range);
+    }
+
+    /**
+     * Deserialize.
+     *
+     * @param in the DataInput stream
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    public void deserialize(DataInput in) throws IOException {
+        this.clear();
+        // little endian
+        final int cookie = Integer.reverseBytes(in.readInt());
+        if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) {
+            throw new IOException("I failed to find one of the right cookies.");
+        }
+        this.size = ((cookie & 0xFFFF) == SERIAL_COOKIE) ? (cookie >>> 16) + 1
+                : Integer.reverseBytes(in.readInt());
+
+        if ((this.keys == null) || (this.keys.length < this.size)) {
+            this.keys = new short[this.size];
+            this.values = new Container[this.size];
+        }
+
+
+        byte[] bitmapOfRunContainers = null;
+        boolean hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE;
+        if (hasrun) {
+            bitmapOfRunContainers = new byte[(size + 7) / 8];
+            in.readFully(bitmapOfRunContainers);
+        }
+
+        final short keys[] = new short[this.size];
+        final int cardinalities[] = new int[this.size];
+        final boolean isBitmap[] = new boolean[this.size];
+        for (int k = 0; k < this.size; ++k) {
+            keys[k] = Short.reverseBytes(in.readShort());
+            cardinalities[k] = 1 + (0xFFFF & Short.reverseBytes(in.readShort()));
+
+            isBitmap[k] = cardinalities[k] > ArrayContainer.DEFAULT_MAX_SIZE;
+            if (bitmapOfRunContainers != null && (bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) {
+                isBitmap[k] = false;
+            }
+        }
+        if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
+            // skipping the offsets
+            in.skipBytes(this.size * 4);
+        }
+        // Reading the containers
+        for (int k = 0; k < this.size; ++k) {
+            Container val;
+            if (isBitmap[k]) {
+                final long[] bitmapArray = new long[BitmapContainer.MAX_CAPACITY / 64];
+                // little endian
+                for (int l = 0; l < bitmapArray.length; ++l) {
+                    bitmapArray[l] = Long.reverseBytes(in.readLong());
+                }
+                val = new BitmapContainer(bitmapArray, cardinalities[k]);
+            } else if (bitmapOfRunContainers != null
+                    && ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0)) {
+                // cf RunContainer.writeArray()
+                int nbrruns = Util.toIntUnsigned(Short.reverseBytes(in.readShort()));
+                final short lengthsAndValues[] = new short[2 * nbrruns];
+
+                for (int j = 0; j < 2 * nbrruns; ++j) {
+                    lengthsAndValues[j] = Short.reverseBytes(in.readShort());
+                }
+                val = new RunContainer(lengthsAndValues, nbrruns);
+            } else {
+                final short[] shortArray = new short[cardinalities[k]];
+                for (int l = 0; l < shortArray.length; ++l) {
+                    shortArray[l] = Short.reverseBytes(in.readShort());
+                }
+                val = new ArrayContainer(shortArray);
+            }
+            this.keys[k] = keys[k];
+            this.values[k] = val;
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o instanceof RoaringArray) {
+            RoaringArray srb = (RoaringArray) o;
+            if (srb.size != this.size) {
+                return false;
+            }
+            for (int i = 0; i < srb.size; ++i) {
+                if (this.keys[i] != srb.keys[i] || !this.values[i].equals(srb.values[i])) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return false;
+    }
+
+    // make sure there is capacity for at least k more elements
+    protected void extendArray(int k) {
+        // size + 1 could overflow
+        if (this.size + k >= this.keys.length) {
+            int newCapacity;
+            if (this.keys.length < 1024) {
+                newCapacity = 2 * (this.size + k);
+            } else {
+                newCapacity = 5 * (this.size + k) / 4;
+            }
+            this.keys = Arrays.copyOf(this.keys, newCapacity);
+            this.values = Arrays.copyOf(this.values, newCapacity);
+        }
+    }
+
+    // involves a binary search
+    protected Container getContainer(short x) {
+        int i = this.binarySearch(0, size, x);
+        if (i < 0) {
+            return null;
+        }
+        return this.values[i];
+    }
+
+    protected Container getContainerAtIndex(int i) {
+        return this.values[i];
+    }
+
+    /**
+     * Create a ContainerPointer for this RoaringArray
+     *
+     * @return a ContainerPointer
+     */
+    public ContainerPointer getContainerPointer() {
+        return getContainerPointer(0);
+    }
+
+    /**
+     * Create a ContainerPointer for this RoaringArray
+     *
+     * @param startIndex starting index in the container list
+     * @return a ContainerPointer
+     */
+    public ContainerPointer getContainerPointer(final int startIndex) {
+        return new ContainerPointer() {
+            int k = startIndex;
+
+            @Override
+            public void advance() {
+                ++k;
+
+            }
+
+            @Override
+            public ContainerPointer clone() {
+                try {
+                    return (ContainerPointer) super.clone();
+                } catch (CloneNotSupportedException e) {
+                    return null;// will not happen
+                }
+            }
+
+            @Override
+            public int compareTo(ContainerPointer o) {
+                if (key() != o.key()) {
+                    return Util.toIntUnsigned(key()) - Util.toIntUnsigned(o.key());
+                }
+                return o.getCardinality() - getCardinality();
+            }
+
+            @Override
+            public int getCardinality() {
+                return getContainer().getCardinality();
+            }
+
+            @Override
+            public Container getContainer() {
+                if (k >= RoaringArray.this.size) {
+                    return null;
+                }
+                return RoaringArray.this.values[k];
+            }
+
+
+            @Override
+            public boolean isBitmapContainer() {
+                return getContainer() instanceof BitmapContainer;
+            }
+
+            @Override
+            public boolean isRunContainer() {
+                return getContainer() instanceof RunContainer;
+            }
+
+
+            @Override
+            public short key() {
+                return RoaringArray.this.keys[k];
+
+            }
+        };
+    }
+
+    // involves a binary search
+    protected int getIndex(short x) {
+        // before the binary search, we optimize for frequent cases
+        if ((size == 0) || (keys[size - 1] == x)) {
+            return size - 1;
+        }
+        // no luck we have to go through the list
+        return this.binarySearch(0, size, x);
+    }
+
+    protected short getKeyAtIndex(int i) {
+        return this.keys[i];
+    }
+
+    @Override
+    public int hashCode() {
+        int hashvalue = 0;
+        for (int k = 0; k < this.size; ++k) {
+            hashvalue = 31 * hashvalue + keys[k] * 0xF0F0F0 + values[k].hashCode();
+        }
+        return hashvalue;
+    }
+
+    boolean hasRunContainer() {
+        for (int k = 0; k < size; ++k) {
+            Container ck = values[k];
+            if (ck instanceof RunContainer) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    protected int headerSize() {
+        if (hasRunContainer()) {
+            if (size < NO_OFFSET_THRESHOLD) {// for small bitmaps, we omit the offsets
+                return 4 + (size + 7) / 8 + 4 * size;
+            }
+            return 4 + (size + 7) / 8 + 8 * size;// - 4 because we pack the size with the cookie
+        } else {
+            return 4 + 4 + 8 * size;
+        }
+    }
+
+
+    // insert a new key, it is assumed that it does not exist
+    protected void insertNewKeyValueAt(int i, short key, Container value) {
+        extendArray(1);
+        System.arraycopy(keys, i, keys, i + 1, size - i);
+        keys[i] = key;
+        System.arraycopy(values, i, values, i + 1, size - i);
+        values[i] = value;
+        size++;
+    }
+
+    @Override
+    public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        deserialize(in);
+    }
+
+    protected void removeAtIndex(int i) {
+        System.arraycopy(keys, i + 1, keys, i, size - i - 1);
+        keys[size - 1] = 0;
+        System.arraycopy(values, i + 1, values, i, size - i - 1);
+        values[size - 1] = null;
+        size--;
+    }
+
+    protected void removeIndexRange(int begin, int end) {
+        if (end <= begin) {
+            return;
+        }
+        final int range = end - begin;
+        System.arraycopy(keys, end, keys, begin, size - end);
+        System.arraycopy(values, end, values, begin, size - end);
+        for (int i = 1; i <= range; ++i) {
+            keys[size - i] = 0;
+            values[size - i] = null;
+        }
+        size -= range;
+    }
+
+    protected void replaceKeyAndContainerAtIndex(int i, short key, Container c) {
+        this.keys[i] = key;
+        this.values[i] = c;
+    }
+
+    protected void resize(int newLength) {
+        Arrays.fill(this.keys, newLength, this.size, (short) 0);
+        Arrays.fill(this.values, newLength, this.size, null);
+        this.size = newLength;
+    }
+
+
+    /**
+     * Serialize.
+     * <p>
+     * The current bitmap is not modified.
+     *
+     * @param out the DataOutput stream
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    public void serialize(DataOutput out) throws IOException {
+        int startOffset = 0;
+        boolean hasrun = hasRunContainer();
+        if (hasrun) {
+            out.writeInt(Integer.reverseBytes(SERIAL_COOKIE | ((size - 1) << 16)));
+            byte[] bitmapOfRunContainers = new byte[(size + 7) / 8];
+            for (int i = 0; i < size; ++i) {
+                if (this.values[i] instanceof RunContainer) {
+                    bitmapOfRunContainers[i / 8] |= (1 << (i % 8));
+                }
+            }
+            out.write(bitmapOfRunContainers);
+            if (this.size < NO_OFFSET_THRESHOLD) {
+                startOffset = 4 + 4 * this.size + bitmapOfRunContainers.length;
+            } else {
+                startOffset = 4 + 8 * this.size + bitmapOfRunContainers.length;
+            }
+        } else { // backwards compatibility
+            out.writeInt(Integer.reverseBytes(SERIAL_COOKIE_NO_RUNCONTAINER));
+            out.writeInt(Integer.reverseBytes(size));
+            startOffset = 4 + 4 + 4 * this.size + 4 * this.size;
+        }
+        for (int k = 0; k < size; ++k) {
+            out.writeShort(Short.reverseBytes(this.keys[k]));
+            out.writeShort(Short.reverseBytes((short) (this.values[k].getCardinality() - 1)));
+        }
+        if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
+            // writing the containers offsets
+            for (int k = 0; k < this.size; k++) {
+                out.writeInt(Integer.reverseBytes(startOffset));
+                startOffset = startOffset + this.values[k].getArraySizeInBytes();
+            }
+        }
+        for (int k = 0; k < size; ++k) {
+            values[k].writeArray(out);
+        }
+    }
+
+    /**
+     * Report the number of bytes required for serialization.
+     *
+     * @return the size in bytes
+     */
+    public int serializedSizeInBytes() {
+        int count = headerSize();
+        for (int k = 0; k < size; ++k) {
+            count += values[k].getArraySizeInBytes();
+        }
+        return count;
+    }
+
+    protected void setContainerAtIndex(int i, Container c) {
+        this.values[i] = c;
+    }
+
+    protected int size() {
+        return this.size;
+    }
+
+    @Override
+    public void writeExternal(ObjectOutput out) throws IOException {
+        serialize(out);
+    }
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ShortIterator.java

@@ -0,0 +1,39 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+/**
+ * Iterator over short values.
+ */
+public interface ShortIterator extends Cloneable {
+    /**
+     * Creates a copy of the iterator.
+     *
+     * @return a clone of the current iterator
+     */
+    ShortIterator clone();
+
+    /**
+     * @return whether there is another value
+     */
+    boolean hasNext();
+
+
+    /**
+     * @return next short value
+     */
+    short next();
+
+    /**
+     * @return next short value as int value (using the least significant 16 bits)
+     */
+    int nextAsInt();
+
+    /**
+     * If possible, remove the current value
+     */
+    void remove();
+
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/Util.java

@@ -0,0 +1,931 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap;
+
+/**
+ * Various useful methods for roaring bitmaps.
+ */
+public final class Util {
+
+    /**
+     * optimization flag: whether to use hybrid binary search: hybrid formats
+     * combine a binary search with a sequential search
+     */
+    public static boolean USE_HYBRID_BINSEARCH = true;
+
+
+    /**
+     * Private constructor to prevent instantiation of utility class
+     */
+    private Util() {
+    }
+
+    /**
+     * Find the smallest integer larger than pos such that array[pos]>= min. If none can be found,
+     * return length. Based on code by O. Kaser.
+     *
+     * @param array  array to search within
+     * @param pos    starting position of the search
+     * @param length length of the array to search
+     * @param min    minimum value
+     * @return x greater than pos such that array[pos] is at least as large as min, pos is is equal to
+     * length if it is not possible.
+     */
+    public static int advanceUntil(short[] array, int pos, int length, short min) {
+        int lower = pos + 1;
+
+        // special handling for a possibly common sequential case
+        if (lower >= length || toIntUnsigned(array[lower]) >= toIntUnsigned(min)) {
+            return lower;
+        }
+
+        int spansize = 1; // could set larger
+        // bootstrap an upper limit
+
+        while (lower + spansize < length
+                && toIntUnsigned(array[lower + spansize]) < toIntUnsigned(min)) {
+            spansize *= 2; // hoping for compiler will reduce to
+        }
+        // shift
+        int upper = (lower + spansize < length) ? lower + spansize : length - 1;
+
+        // maybe we are lucky (could be common case when the seek ahead
+        // expected
+        // to be small and sequential will otherwise make us look bad)
+        if (array[upper] == min) {
+            return upper;
+        }
+
+        if (toIntUnsigned(array[upper]) < toIntUnsigned(min)) {// means
+            // array
+            // has no
+            // item
+            // >= min
+            // pos = array.length;
+            return length;
+        }
+
+        // we know that the next-smallest span was too small
+        lower += (spansize / 2);
+
+        // else begin binary search
+        // invariant: array[lower]<min && array[upper]>min
+        while (lower + 1 != upper) {
+            int mid = (lower + upper) / 2;
+            short arraymid = array[mid];
+            if (arraymid == min) {
+                return mid;
+            } else if (toIntUnsigned(arraymid) < toIntUnsigned(min)) {
+                lower = mid;
+            } else {
+                upper = mid;
+            }
+        }
+        return upper;
+
+    }
+
+    protected static int branchyUnsignedBinarySearch(final short[] array, final int begin,
+                                                     final int end, final short k) {
+        int ikey = toIntUnsigned(k);
+        // next line accelerates the possibly common case where the value would
+        // be inserted at the end
+        if ((end > 0) && (toIntUnsigned(array[end - 1]) < ikey)) {
+            return -end - 1;
+        }
+        int low = begin;
+        int high = end - 1;
+        while (low <= high) {
+            final int middleIndex = (low + high) >>> 1;
+            final int middleValue = toIntUnsigned(array[middleIndex]);
+
+            if (middleValue < ikey) {
+                low = middleIndex + 1;
+            } else if (middleValue > ikey) {
+                high = middleIndex - 1;
+            } else {
+                return middleIndex;
+            }
+        }
+        return -(low + 1);
+    }
+
+    /**
+     * Compares the two specified {@code short} values, treating them as unsigned values between
+     * {@code 0} and {@code 2^16 - 1} inclusive.
+     *
+     * @param a the first unsigned {@code short} to compare
+     * @param b the second unsigned {@code short} to compare
+     * @return a negative value if {@code a} is less than {@code b}; a positive value if {@code a} is
+     * greater than {@code b}; or zero if they are equal
+     */
+    public static int compareUnsigned(short a, short b) {
+        return toIntUnsigned(a) - toIntUnsigned(b);
+    }
+
+    /**
+     * Compute the bitwise AND between two long arrays and write the set bits in the container.
+     *
+     * @param container where we write
+     * @param bitmap1   first bitmap
+     * @param bitmap2   second bitmap
+     */
+    public static void fillArrayAND(final short[] container, final long[] bitmap1,
+                                    final long[] bitmap2) {
+        int pos = 0;
+        if (bitmap1.length != bitmap2.length) {
+            throw new IllegalArgumentException("not supported");
+        }
+        for (int k = 0; k < bitmap1.length; ++k) {
+            long bitset = bitmap1[k] & bitmap2[k];
+            while (bitset != 0) {
+                long t = bitset & -bitset;
+                container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
+                bitset ^= t;
+            }
+        }
+    }
+
+    /**
+     * Compute the bitwise ANDNOT between two long arrays and write the set bits in the container.
+     *
+     * @param container where we write
+     * @param bitmap1   first bitmap
+     * @param bitmap2   second bitmap
+     */
+    public static void fillArrayANDNOT(final short[] container, final long[] bitmap1,
+                                       final long[] bitmap2) {
+        int pos = 0;
+        if (bitmap1.length != bitmap2.length) {
+            throw new IllegalArgumentException("not supported");
+        }
+        for (int k = 0; k < bitmap1.length; ++k) {
+            long bitset = bitmap1[k] & (~bitmap2[k]);
+            while (bitset != 0) {
+                long t = bitset & -bitset;
+                container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
+                bitset ^= t;
+            }
+        }
+    }
+
+    /**
+     * Compute the bitwise XOR between two long arrays and write the set bits in the container.
+     *
+     * @param container where we write
+     * @param bitmap1   first bitmap
+     * @param bitmap2   second bitmap
+     */
+    public static void fillArrayXOR(final short[] container, final long[] bitmap1,
+                                    final long[] bitmap2) {
+        int pos = 0;
+        if (bitmap1.length != bitmap2.length) {
+            throw new IllegalArgumentException("not supported");
+        }
+        for (int k = 0; k < bitmap1.length; ++k) {
+            long bitset = bitmap1[k] ^ bitmap2[k];
+            while (bitset != 0) {
+                long t = bitset & -bitset;
+                container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
+                bitset ^= t;
+            }
+        }
+    }
+
+    /**
+     * flip bits at start, start+1,..., end-1
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     */
+    public static void flipBitmapRange(long[] bitmap, int start, int end) {
+        if (start == end) {
+            return;
+        }
+        int firstword = start / 64;
+        int endword = (end - 1) / 64;
+        bitmap[firstword] ^= ~(~0L << start);
+        for (int i = firstword; i < endword; i++) {
+            bitmap[i] = ~bitmap[i];
+        }
+        bitmap[endword] ^= ~0L >>> -end;
+    }
+
+    /**
+     * Hamming weight of the 64-bit words involved in the range
+     * start, start+1,..., end-1
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     * @return the hamming weight
+     */
+    public static int cardinalityInBitmapWordRange(long[] bitmap, int start, int end) {
+        if (start == end) {
+            return 0;
+        }
+        int firstword = start / 64;
+        int endword = (end - 1) / 64;
+        int answer = 0;
+        for (int i = firstword; i <= endword; i++) {
+            answer += Long.bitCount(bitmap[i]);
+        }
+        return answer;
+    }
+
+    protected static short highbits(int x) {
+        return (short) (x >>> 16);
+    }
+
+    protected static short highbits(long x) {
+        return (short) (x >>> 16);
+    }
+
+    // starts with binary search and finishes with a sequential search
+    protected static int hybridUnsignedBinarySearch(final short[] array, final int begin,
+                                                    final int end, final short k) {
+        int ikey = toIntUnsigned(k);
+        // next line accelerates the possibly common case where the value would
+        // be inserted at the end
+        if ((end > 0) && (toIntUnsigned(array[end - 1]) < ikey)) {
+            return -end - 1;
+        }
+        int low = begin;
+        int high = end - 1;
+        // 32 in the next line matches the size of a cache line
+        while (low + 32 <= high) {
+            final int middleIndex = (low + high) >>> 1;
+            final int middleValue = toIntUnsigned(array[middleIndex]);
+
+            if (middleValue < ikey) {
+                low = middleIndex + 1;
+            } else if (middleValue > ikey) {
+                high = middleIndex - 1;
+            } else {
+                return middleIndex;
+            }
+        }
+        // we finish the job with a sequential search
+        int x = low;
+        for (; x <= high; ++x) {
+            final int val = toIntUnsigned(array[x]);
+            if (val >= ikey) {
+                if (val == ikey) {
+                    return x;
+                }
+                break;
+            }
+        }
+        return -(x + 1);
+    }
+
+    protected static short lowbits(int x) {
+        return (short) (x & 0xFFFF);
+    }
+
+    protected static short lowbits(long x) {
+        return (short) (x & 0xFFFF);
+    }
+
+    protected static short maxLowBit() {
+        return (short) 0xFFFF;
+    }
+
+    protected static int maxLowBitAsInteger() {
+        return 0xFFFF;
+    }
+
+    /**
+     * clear bits at start, start+1,..., end-1
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     */
+    public static void resetBitmapRange(long[] bitmap, int start, int end) {
+        if (start == end) {
+            return;
+        }
+        int firstword = start / 64;
+        int endword = (end - 1) / 64;
+
+        if (firstword == endword) {
+            bitmap[firstword] &= ~((~0L << start) & (~0L >>> -end));
+            return;
+        }
+        bitmap[firstword] &= ~(~0L << start);
+        for (int i = firstword + 1; i < endword; i++) {
+            bitmap[i] = 0;
+        }
+        bitmap[endword] &= ~(~0L >>> -end);
+
+    }
+
+    /**
+     * Given a word w, return the position of the jth true bit.
+     *
+     * @param w word
+     * @param j index
+     * @return position of jth true bit in w
+     */
+    public static int select(long w, int j) {
+        int seen = 0;
+        // Divide 64bit
+        int part = (int) (w & 0xFFFFFFFF);
+        int n = Integer.bitCount(part);
+        if (n <= j) {
+            part = (int) (w >>> 32);
+            seen += 32;
+            j -= n;
+        }
+        int ww = part;
+
+        // Divide 32bit
+        part = ww & 0xFFFF;
+
+        n = Integer.bitCount(part);
+        if (n <= j) {
+
+            part = ww >>> 16;
+            seen += 16;
+            j -= n;
+        }
+        ww = part;
+
+        // Divide 16bit
+        part = ww & 0xFF;
+        n = Integer.bitCount(part);
+        if (n <= j) {
+            part = ww >>> 8;
+            seen += 8;
+            j -= n;
+        }
+        ww = part;
+
+        // Lookup in final byte
+        int counter;
+        for (counter = 0; counter < 8; counter++) {
+            j -= (ww >>> counter) & 1;
+            if (j < 0) {
+                break;
+            }
+        }
+        return seen + counter;
+    }
+
+    /**
+     * set bits at start, start+1,..., end-1
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     */
+    public static void setBitmapRange(long[] bitmap, int start, int end) {
+        if (start == end) {
+            return;
+        }
+        int firstword = start / 64;
+        int endword = (end - 1) / 64;
+        if (firstword == endword) {
+            bitmap[firstword] |= (~0L << start) & (~0L >>> -end);
+            return;
+        }
+        bitmap[firstword] |= ~0L << start;
+        for (int i = firstword + 1; i < endword; i++) {
+            bitmap[i] = ~0L;
+        }
+        bitmap[endword] |= ~0L >>> -end;
+    }
+
+    /**
+     * set bits at start, start+1,..., end-1 and report the
+     * cardinality change
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     * @return cardinality change
+     */
+    public static int setBitmapRangeAndCardinalityChange(long[] bitmap, int start, int end) {
+        int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
+        setBitmapRange(bitmap, start, end);
+        int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
+        return cardafter - cardbefore;
+    }
+
+    /**
+     * flip  bits at start, start+1,..., end-1 and report the
+     * cardinality change
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     * @return cardinality change
+     */
+    public static int flipBitmapRangeAndCardinalityChange(long[] bitmap, int start, int end) {
+        int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
+        flipBitmapRange(bitmap, start, end);
+        int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
+        return cardafter - cardbefore;
+    }
+
+    /**
+     * reset  bits at start, start+1,..., end-1 and report the
+     * cardinality change
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     * @return cardinality change
+     */
+    public static int resetBitmapRangeAndCardinalityChange(long[] bitmap, int start, int end) {
+        int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
+        resetBitmapRange(bitmap, start, end);
+        int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
+        return cardafter - cardbefore;
+    }
+
+    protected static int toIntUnsigned(short x) {
+        return x & 0xFFFF;
+    }
+
+    /**
+     * Look for value k in array in the range [begin,end). If the value is found, return its index. If
+     * not, return -(i+1) where i is the index where the value would be inserted. The array is assumed
+     * to contain sorted values where shorts are interpreted as unsigned integers.
+     *
+     * @param array array where we search
+     * @param begin first index (inclusive)
+     * @param end   last index (exclusive)
+     * @param k     value we search for
+     * @return count
+     */
+    public static int unsignedBinarySearch(final short[] array, final int begin, final int end,
+                                           final short k) {
+        if (USE_HYBRID_BINSEARCH) {
+            return hybridUnsignedBinarySearch(array, begin, end, k);
+        } else {
+            return branchyUnsignedBinarySearch(array, begin, end, k);
+        }
+    }
+
+    /**
+     * Compute the difference between two sorted lists and write the result to the provided output
+     * array
+     *
+     * @param set1    first array
+     * @param length1 length of first array
+     * @param set2    second array
+     * @param length2 length of second array
+     * @param buffer  output array
+     * @return cardinality of the difference
+     */
+    public static int unsignedDifference(final short[] set1, final int length1, final short[] set2,
+                                         final int length2, final short[] buffer) {
+        int pos = 0;
+        int k1 = 0, k2 = 0;
+        if (0 == length2) {
+            System.arraycopy(set1, 0, buffer, 0, length1);
+            return length1;
+        }
+        if (0 == length1) {
+            return 0;
+        }
+        short s1 = set1[k1];
+        short s2 = set2[k2];
+        while (true) {
+            if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
+                buffer[pos++] = s1;
+                ++k1;
+                if (k1 >= length1) {
+                    break;
+                }
+                s1 = set1[k1];
+            } else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
+                ++k1;
+                ++k2;
+                if (k1 >= length1) {
+                    break;
+                }
+                if (k2 >= length2) {
+                    System.arraycopy(set1, k1, buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s1 = set1[k1];
+                s2 = set2[k2];
+            } else {// if (val1>val2)
+                ++k2;
+                if (k2 >= length2) {
+                    System.arraycopy(set1, k1, buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s2 = set2[k2];
+            }
+        }
+        return pos;
+    }
+
+    /**
+     * Compute the difference between two sorted lists and write the result to the provided output
+     * array
+     *
+     * @param set1   first array
+     * @param set2   second array
+     * @param buffer output array
+     * @return cardinality of the difference
+     */
+    public static int unsignedDifference(ShortIterator set1, ShortIterator set2,
+                                         final short[] buffer) {
+        int pos = 0;
+        if (!set2.hasNext()) {
+            while (set1.hasNext()) {
+                buffer[pos++] = set1.next();
+            }
+            return pos;
+        }
+        if (!set1.hasNext()) {
+            return 0;
+        }
+        short v1 = set1.next();
+        short v2 = set2.next();
+        while (true) {
+            if (toIntUnsigned(v1) < toIntUnsigned(v2)) {
+                buffer[pos++] = v1;
+                if (!set1.hasNext()) {
+                    return pos;
+                }
+                v1 = set1.next();
+            } else if (v1 == v2) {
+                if (!set1.hasNext()) {
+                    break;
+                }
+                if (!set2.hasNext()) {
+                    while (set1.hasNext()) {
+                        buffer[pos++] = set1.next();
+                    }
+                    return pos;
+                }
+                v1 = set1.next();
+                v2 = set2.next();
+            } else {// if (val1>val2)
+                if (!set2.hasNext()) {
+                    buffer[pos++] = v1;
+                    while (set1.hasNext()) {
+                        buffer[pos++] = set1.next();
+                    }
+                    return pos;
+                }
+                v2 = set2.next();
+            }
+        }
+        return pos;
+    }
+
+    /**
+     * Compute the exclusive union of two sorted lists and write the result to the provided output
+     * array
+     *
+     * @param set1    first array
+     * @param length1 length of first array
+     * @param set2    second array
+     * @param length2 length of second array
+     * @param buffer  output array
+     * @return cardinality of the exclusive union
+     */
+    public static int unsignedExclusiveUnion2by2(final short[] set1, final int length1,
+                                                 final short[] set2, final int length2, final short[] buffer) {
+        int pos = 0;
+        int k1 = 0, k2 = 0;
+        if (0 == length2) {
+            System.arraycopy(set1, 0, buffer, 0, length1);
+            return length1;
+        }
+        if (0 == length1) {
+            System.arraycopy(set2, 0, buffer, 0, length2);
+            return length2;
+        }
+        short s1 = set1[k1];
+        short s2 = set2[k2];
+        while (true) {
+            if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
+                buffer[pos++] = s1;
+                ++k1;
+                if (k1 >= length1) {
+                    System.arraycopy(set2, k2, buffer, pos, length2 - k2);
+                    return pos + length2 - k2;
+                }
+                s1 = set1[k1];
+            } else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
+                ++k1;
+                ++k2;
+                if (k1 >= length1) {
+                    System.arraycopy(set2, k2, buffer, pos, length2 - k2);
+                    return pos + length2 - k2;
+                }
+                if (k2 >= length2) {
+                    System.arraycopy(set1, k1, buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s1 = set1[k1];
+                s2 = set2[k2];
+            } else {// if (val1>val2)
+                buffer[pos++] = s2;
+                ++k2;
+                if (k2 >= length2) {
+                    System.arraycopy(set1, k1, buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s2 = set2[k2];
+            }
+        }
+        // return pos;
+    }
+
+    /**
+     * Intersect two sorted lists and write the result to the provided output array
+     *
+     * @param set1    first array
+     * @param length1 length of first array
+     * @param set2    second array
+     * @param length2 length of second array
+     * @param buffer  output array
+     * @return cardinality of the intersection
+     */
+    public static int unsignedIntersect2by2(final short[] set1, final int length1, final short[] set2,
+                                            final int length2, final short[] buffer) {
+        if (set1.length * 64 < set2.length) {
+            return unsignedOneSidedGallopingIntersect2by2(set1, length1, set2, length2, buffer);
+        } else if (set2.length * 64 < set1.length) {
+            return unsignedOneSidedGallopingIntersect2by2(set2, length2, set1, length1, buffer);
+        } else {
+            return unsignedLocalIntersect2by2(set1, length1, set2, length2, buffer);
+        }
+    }
+
+    /**
+     * Checks if two arrays intersect
+     *
+     * @param set1    first array
+     * @param length1 length of first array
+     * @param set2    second array
+     * @param length2 length of second array
+     * @return true if they intersect
+     */
+    public static boolean unsignedIntersects(short[] set1, int length1, short[] set2, int length2) {
+        // galloping might be faster, but we do not expect this function to be slow
+        if ((0 == length1) || (0 == length2)) {
+            return false;
+        }
+        int k1 = 0;
+        int k2 = 0;
+        short s1 = set1[k1];
+        short s2 = set2[k2];
+        mainwhile:
+        while (true) {
+            if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
+                do {
+                    ++k2;
+                    if (k2 == length2) {
+                        break mainwhile;
+                    }
+                    s2 = set2[k2];
+                } while (toIntUnsigned(s2) < toIntUnsigned(s1));
+            }
+            if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
+                do {
+                    ++k1;
+                    if (k1 == length1) {
+                        break mainwhile;
+                    }
+                    s1 = set1[k1];
+                } while (toIntUnsigned(s1) < toIntUnsigned(s2));
+            } else {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    protected static int unsignedLocalIntersect2by2(final short[] set1, final int length1,
+                                                    final short[] set2, final int length2, final short[] buffer) {
+        if ((0 == length1) || (0 == length2)) {
+            return 0;
+        }
+        int k1 = 0;
+        int k2 = 0;
+        int pos = 0;
+        short s1 = set1[k1];
+        short s2 = set2[k2];
+
+        mainwhile:
+        while (true) {
+            int v1 = toIntUnsigned(s1);
+            int v2 = toIntUnsigned(s2);
+            if (v2 < v1) {
+                do {
+                    ++k2;
+                    if (k2 == length2) {
+                        break mainwhile;
+                    }
+                    s2 = set2[k2];
+                    v2 = toIntUnsigned(s2);
+                } while (v2 < v1);
+            }
+            if (v1 < v2) {
+                do {
+                    ++k1;
+                    if (k1 == length1) {
+                        break mainwhile;
+                    }
+                    s1 = set1[k1];
+                    v1 = toIntUnsigned(s1);
+                } while (v1 < v2);
+            } else {
+                // (set2[k2] == set1[k1])
+                buffer[pos++] = s1;
+                ++k1;
+                if (k1 == length1) {
+                    break;
+                }
+                ++k2;
+                if (k2 == length2) {
+                    break;
+                }
+                s1 = set1[k1];
+                s2 = set2[k2];
+            }
+        }
+        return pos;
+    }
+
+    /**
+     * Compute the cardinality of the intersection
+     *
+     * @param set1    first set
+     * @param length1 how many values to consider in the first set
+     * @param set2    second set
+     * @param length2 how many values to consider in the second set
+     * @return cardinality of the intersection
+     */
+    public static int unsignedLocalIntersect2by2Cardinality(final short[] set1, final int length1,
+                                                            final short[] set2, final int length2) {
+        if ((0 == length1) || (0 == length2)) {
+            return 0;
+        }
+        int k1 = 0;
+        int k2 = 0;
+        int pos = 0;
+        short s1 = set1[k1];
+        short s2 = set2[k2];
+
+        mainwhile:
+        while (true) {
+            int v1 = toIntUnsigned(s1);
+            int v2 = toIntUnsigned(s2);
+            if (v2 < v1) {
+                do {
+                    ++k2;
+                    if (k2 == length2) {
+                        break mainwhile;
+                    }
+                    s2 = set2[k2];
+                    v2 = toIntUnsigned(s2);
+                } while (v2 < v1);
+            }
+            if (v1 < v2) {
+                do {
+                    ++k1;
+                    if (k1 == length1) {
+                        break mainwhile;
+                    }
+                    s1 = set1[k1];
+                    v1 = toIntUnsigned(s1);
+                } while (v1 < v2);
+            } else {
+                // (set2[k2] == set1[k1])
+                pos++;
+                ++k1;
+                if (k1 == length1) {
+                    break;
+                }
+                ++k2;
+                if (k2 == length2) {
+                    break;
+                }
+                s1 = set1[k1];
+                s2 = set2[k2];
+            }
+        }
+        return pos;
+    }
+
+    protected static int unsignedOneSidedGallopingIntersect2by2(final short[] smallSet,
+                                                                final int smallLength, final short[] largeSet, final int largeLength, final short[] buffer) {
+        if (0 == smallLength) {
+            return 0;
+        }
+        int k1 = 0;
+        int k2 = 0;
+        int pos = 0;
+        short s1 = largeSet[k1];
+        short s2 = smallSet[k2];
+        while (true) {
+            if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
+                k1 = advanceUntil(largeSet, k1, largeLength, s2);
+                if (k1 == largeLength) {
+                    break;
+                }
+                s1 = largeSet[k1];
+            }
+            if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
+                ++k2;
+                if (k2 == smallLength) {
+                    break;
+                }
+                s2 = smallSet[k2];
+            } else {
+                // (set2[k2] == set1[k1])
+                buffer[pos++] = s2;
+                ++k2;
+                if (k2 == smallLength) {
+                    break;
+                }
+                s2 = smallSet[k2];
+                k1 = advanceUntil(largeSet, k1, largeLength, s2);
+                if (k1 == largeLength) {
+                    break;
+                }
+                s1 = largeSet[k1];
+            }
+
+        }
+        return pos;
+
+    }
+
+    /**
+     * Unite two sorted lists and write the result to the provided output array
+     *
+     * @param set1    first array
+     * @param length1 length of first array
+     * @param set2    second array
+     * @param length2 length of second array
+     * @param buffer  output array
+     * @return cardinality of the union
+     */
+    public static int unsignedUnion2by2(final short[] set1, final int length1, final short[] set2,
+                                        final int length2, final short[] buffer) {
+        int pos = 0;
+        int k1 = 0, k2 = 0;
+        if (0 == length2) {
+            System.arraycopy(set1, 0, buffer, 0, length1);
+            return length1;
+        }
+        if (0 == length1) {
+            System.arraycopy(set2, 0, buffer, 0, length2);
+            return length2;
+        }
+        short s1 = set1[k1];
+        short s2 = set2[k2];
+        while (true) {
+            int v1 = toIntUnsigned(s1);
+            int v2 = toIntUnsigned(s2);
+            if (v1 < v2) {
+                buffer[pos++] = s1;
+                ++k1;
+                if (k1 >= length1) {
+                    System.arraycopy(set2, k2, buffer, pos, length2 - k2);
+                    return pos + length2 - k2;
+                }
+                s1 = set1[k1];
+            } else if (v1 == v2) {
+                buffer[pos++] = s1;
+                ++k1;
+                ++k2;
+                if (k1 >= length1) {
+                    System.arraycopy(set2, k2, buffer, pos, length2 - k2);
+                    return pos + length2 - k2;
+                }
+                if (k2 >= length2) {
+                    System.arraycopy(set1, k1, buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s1 = set1[k1];
+                s2 = set2[k2];
+            } else {// if (set1[k1]>set2[k2])
+                buffer[pos++] = s2;
+                ++k2;
+                if (k2 >= length2) {
+                    System.arraycopy(set1, k1, buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s2 = set2[k2];
+            }
+        }
+        // return pos;
+    }
+
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferBitSetUtil.java

@@ -0,0 +1,113 @@
+package com.fr.third.bitmap.roaringbitmap.buffer;
+
+
+import com.fr.third.bitmap.roaringbitmap.IntIterator;
+
+import java.nio.LongBuffer;
+import java.nio.ShortBuffer;
+import java.util.Arrays;
+import java.util.BitSet;
+
+
+/***
+ *
+ * This class provides convenience functions to manipulate BitSet and MutableRoaringBitmap objects.
+ *
+ */
+public class BufferBitSetUtil {
+    // todo: add a method to convert an ImmutableRoaringBitmap to a BitSet using BitSet.valueOf
+
+    // a block consists has a maximum of 1024 words, each representing 64 bits,
+    // thus representing at maximum 65536 bits
+    static final private int BLOCK_LENGTH = MappeableBitmapContainer.MAX_CAPACITY / Long.SIZE; //
+    // 64-bit
+    // word
+
+    private static MappeableArrayContainer arrayContainerOf(final int from, final int to,
+                                                            final int cardinality, final long[] words) {
+        // precondition: cardinality is max 4096
+        final short[] content = new short[cardinality];
+        int index = 0;
+
+        for (int i = from, socket = 0; i < to; ++i, socket += Long.SIZE) {
+            long word = words[i];
+            while (word != 0) {
+                long t = word & -word;
+                content[index++] = (short) (socket + Long.bitCount(t - 1));
+                word ^= t;
+            }
+        }
+        return new MappeableArrayContainer(ShortBuffer.wrap(content), cardinality);
+    }
+
+    /**
+     * Generate a MutableRoaringBitmap out of a long[], each long using little-endian representation
+     * of its bits
+     *
+     * @param words array of longs (will not be modified)
+     * @return roaring bitmap
+     * @see BitSet#toLongArray() for an equivalent
+     */
+    public static MutableRoaringBitmap bitmapOf(final long[] words) {
+        // split long[] into blocks.
+        // each block becomes a single container, if any bit is set
+        final MutableRoaringBitmap ans = new MutableRoaringBitmap();
+        int containerIndex = 0;
+        for (int from = 0; from < words.length; from += BLOCK_LENGTH) {
+            final int to = Math.min(from + BLOCK_LENGTH, words.length);
+            final int blockCardinality = cardinality(from, to, words);
+            if (blockCardinality > 0) {
+                ((MutableRoaringArray) ans.highLowContainer).insertNewKeyValueAt(containerIndex++,
+                        BufferUtil.highbits(from * Long.SIZE),
+                        BufferBitSetUtil.containerOf(from, to, blockCardinality, words));
+            }
+        }
+        return ans;
+    }
+
+    private static int cardinality(final int from, final int to, final long[] words) {
+        int sum = 0;
+        for (int i = from; i < to; i++) {
+            sum += Long.bitCount(words[i]);
+        }
+        return sum;
+    }
+
+
+    private static MappeableContainer containerOf(final int from, final int to,
+                                                  final int blockCardinality, final long[] words) {
+        // find the best container available
+        if (blockCardinality <= MappeableArrayContainer.DEFAULT_MAX_SIZE) {
+            // containers with DEFAULT_MAX_SIZE or less integers should be
+            // ArrayContainers
+            return arrayContainerOf(from, to, blockCardinality, words);
+        } else {
+            // otherwise use bitmap container
+            return new MappeableBitmapContainer(
+                    LongBuffer.wrap(Arrays.copyOfRange(words, from, from + BLOCK_LENGTH)), blockCardinality);
+        }
+    }
+
+
+    /**
+     * Compares a RoaringBitmap and a BitSet. They are equal if and only if they contain the same set
+     * of integers.
+     *
+     * @param bitset first object to be compared
+     * @param bitmap second object to be compared
+     * @return whether they are equal
+     */
+    public static boolean equals(final BitSet bitset, final ImmutableRoaringBitmap bitmap) {
+        if (bitset.cardinality() != bitmap.getCardinality()) {
+            return false;
+        }
+        final IntIterator it = bitmap.getIntIterator();
+        while (it.hasNext()) {
+            int val = it.next();
+            if (!bitset.get(val)) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferFastAggregation.java

@@ -0,0 +1,631 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap.buffer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.PriorityQueue;
+
+
+/**
+ * Fast algorithms to aggregate many bitmaps.
+ *
+ * @author Daniel Lemire
+ */
+public final class BufferFastAggregation {
+
+
+    /**
+     * Private constructor to prevent instantiation of utility class
+     */
+    private BufferFastAggregation() {
+    }
+
+    /**
+     * Compute the AND aggregate.
+     * <p>
+     * In practice, calls {#link naive_and}
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap and(ImmutableRoaringBitmap... bitmaps) {
+        return naive_and(bitmaps);
+    }
+
+    /**
+     * Compute the AND aggregate.
+     * <p>
+     * In practice, calls {#link naive_and}
+     *
+     * @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap and(@SuppressWarnings("rawtypes") Iterator bitmaps) {
+        return naive_and(bitmaps);
+    }
+
+    /**
+     * Compute the AND aggregate.
+     * <p>
+     * In practice, calls {#link naive_and}
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap and(MutableRoaringBitmap... bitmaps) {
+        return and(convertToImmutable(bitmaps));
+    }
+
+    /**
+     * Convenience method converting one type of iterator into another, to avoid unnecessary warnings.
+     *
+     * @param i input bitmaps
+     * @return an iterator over the provided iterator, with a different type
+     */
+    public static Iterator<ImmutableRoaringBitmap> convertToImmutable(
+            final Iterator<MutableRoaringBitmap> i) {
+        return new Iterator<ImmutableRoaringBitmap>() {
+
+            @Override
+            public boolean hasNext() {
+                return i.hasNext();
+            }
+
+            @Override
+            public ImmutableRoaringBitmap next() {
+                return i.next();
+            }
+
+            @Override
+            public void remove() {
+            }
+
+            ;
+
+        };
+
+    }
+
+    private static ImmutableRoaringBitmap[] convertToImmutable(MutableRoaringBitmap[] array) {
+        ImmutableRoaringBitmap[] answer = new ImmutableRoaringBitmap[array.length];
+        for (int k = 0; k < answer.length; ++k) {
+            answer[k] = array[k];
+        }
+        return answer;
+    }
+
+    /**
+     * Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #or(ImmutableRoaringBitmap...)
+     */
+    public static MutableRoaringBitmap horizontal_or(ImmutableRoaringBitmap... bitmaps) {
+        MutableRoaringBitmap answer = new MutableRoaringBitmap();
+        if (bitmaps.length == 0) {
+            return answer;
+        }
+        PriorityQueue<MappeableContainerPointer> pq = new PriorityQueue<MappeableContainerPointer>(bitmaps.length);
+        for (int k = 0; k < bitmaps.length; ++k) {
+            MappeableContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
+            if (x.getContainer() != null) {
+                pq.add(x);
+            }
+        }
+
+        while (!pq.isEmpty()) {
+            MappeableContainerPointer x1 = pq.poll();
+            if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
+                answer.getMappeableRoaringArray().append(x1.key(), x1.getContainer().clone());
+                x1.advance();
+                if (x1.getContainer() != null) {
+                    pq.add(x1);
+                }
+                continue;
+            }
+            MappeableContainerPointer x2 = pq.poll();
+            MappeableContainer newc = x1.getContainer().lazyOR(x2.getContainer());
+            while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
+
+                MappeableContainerPointer x = pq.poll();
+                newc = newc.lazyIOR(x.getContainer());
+                x.advance();
+                if (x.getContainer() != null) {
+                    pq.add(x);
+                } else if (pq.isEmpty()) {
+                    break;
+                }
+            }
+            newc = newc.repairAfterLazy();
+            answer.getMappeableRoaringArray().append(x1.key(), newc);
+            x1.advance();
+            if (x1.getContainer() != null) {
+                pq.add(x1);
+            }
+            x2.advance();
+            if (x2.getContainer() != null) {
+                pq.add(x2);
+            }
+        }
+        return answer;
+    }
+
+    /**
+     * Calls naive_or.
+     *
+     * @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
+     * @return aggregated bitmap
+     */
+    @Deprecated
+    public static MutableRoaringBitmap horizontal_or(@SuppressWarnings("rawtypes") Iterator bitmaps) {
+        return naive_or(bitmaps);
+    }
+
+    /**
+     * Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #or(ImmutableRoaringBitmap...)
+     */
+    public static MutableRoaringBitmap horizontal_or(MutableRoaringBitmap... bitmaps) {
+        return horizontal_or(convertToImmutable(bitmaps));
+    }
+
+    /**
+     * Minimizes memory usage while computing the xor aggregate on a moderate number of bitmaps.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #xor(ImmutableRoaringBitmap...)
+     */
+    public static MutableRoaringBitmap horizontal_xor(ImmutableRoaringBitmap... bitmaps) {
+        MutableRoaringBitmap answer = new MutableRoaringBitmap();
+        if (bitmaps.length == 0) {
+            return answer;
+        }
+        PriorityQueue<MappeableContainerPointer> pq = new PriorityQueue<MappeableContainerPointer>(bitmaps.length);
+        for (int k = 0; k < bitmaps.length; ++k) {
+            MappeableContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
+            if (x.getContainer() != null) {
+                pq.add(x);
+            }
+        }
+
+        while (!pq.isEmpty()) {
+            MappeableContainerPointer x1 = pq.poll();
+            if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
+                answer.getMappeableRoaringArray().append(x1.key(), x1.getContainer().clone());
+                x1.advance();
+                if (x1.getContainer() != null) {
+                    pq.add(x1);
+                }
+                continue;
+            }
+            MappeableContainerPointer x2 = pq.poll();
+            MappeableContainer newc = x1.getContainer().xor(x2.getContainer());
+            while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
+
+                MappeableContainerPointer x = pq.poll();
+                newc = newc.ixor(x.getContainer());
+                x.advance();
+                if (x.getContainer() != null) {
+                    pq.add(x);
+                } else if (pq.isEmpty()) {
+                    break;
+                }
+            }
+            answer.getMappeableRoaringArray().append(x1.key(), newc);
+            x1.advance();
+            if (x1.getContainer() != null) {
+                pq.add(x1);
+            }
+            x2.advance();
+            if (x2.getContainer() != null) {
+                pq.add(x2);
+            }
+        }
+        return answer;
+    }
+
+    /**
+     * Minimizes memory usage while computing the xor aggregate on a moderate number of bitmaps.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #xor(ImmutableRoaringBitmap...)
+     */
+    public static MutableRoaringBitmap horizontal_xor(MutableRoaringBitmap... bitmaps) {
+        return horizontal_xor(convertToImmutable(bitmaps));
+    }
+
+    /**
+     * Compute overall AND between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap naive_and(ImmutableRoaringBitmap... bitmaps) {
+        MutableRoaringBitmap answer;
+
+        if (bitmaps.length > 0) {
+            answer = (bitmaps[0]).toMutableRoaringBitmap();
+            for (int k = 1; k < bitmaps.length; ++k) {
+                answer = ImmutableRoaringBitmap.and(answer, bitmaps[k]);
+            }
+        } else {
+            answer = new MutableRoaringBitmap();
+        }
+
+        return answer;
+    }
+
+    /**
+     * Compute overall AND between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap naive_and(@SuppressWarnings("rawtypes") Iterator bitmaps) {
+        if (!bitmaps.hasNext()) {
+            return new MutableRoaringBitmap();
+        }
+        MutableRoaringBitmap answer =
+                ((ImmutableRoaringBitmap) bitmaps.next()).toMutableRoaringBitmap();
+        while (bitmaps.hasNext()) {
+            answer.and((ImmutableRoaringBitmap) bitmaps.next());
+        }
+        return answer;
+    }
+
+    /**
+     * Compute overall AND between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap naive_and(MutableRoaringBitmap... bitmaps) {
+        if (bitmaps.length == 0) {
+            return new MutableRoaringBitmap();
+        }
+        MutableRoaringBitmap answer = bitmaps[0].clone();
+        for (int k = 1; k < bitmaps.length; ++k) {
+            answer.and(bitmaps[k]);
+        }
+        return answer;
+    }
+
+    /**
+     * Compute overall OR between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap naive_or(ImmutableRoaringBitmap... bitmaps) {
+        MutableRoaringBitmap answer = new MutableRoaringBitmap();
+        for (int k = 0; k < bitmaps.length; ++k) {
+            answer.naivelazyor(bitmaps[k]);
+        }
+        answer.repairAfterLazy();
+        return answer;
+    }
+
+    /**
+     * Compute overall OR between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap naive_or(@SuppressWarnings("rawtypes") Iterator bitmaps) {
+        MutableRoaringBitmap answer = new MutableRoaringBitmap();
+        while (bitmaps.hasNext()) {
+            answer.naivelazyor((ImmutableRoaringBitmap) bitmaps.next());
+        }
+        answer.repairAfterLazy();
+        return answer;
+    }
+
+    /**
+     * Compute overall OR between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap naive_or(MutableRoaringBitmap... bitmaps) {
+        MutableRoaringBitmap answer = new MutableRoaringBitmap();
+        for (int k = 0; k < bitmaps.length; ++k) {
+            answer.lazyor(bitmaps[k]);
+        }
+        answer.repairAfterLazy();
+        return answer;
+    }
+
+    /**
+     * Compute overall XOR between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap naive_xor(ImmutableRoaringBitmap... bitmaps) {
+        MutableRoaringBitmap answer = new MutableRoaringBitmap();
+        for (int k = 0; k < bitmaps.length; ++k) {
+            answer.xor(bitmaps[k]);
+        }
+        return answer;
+    }
+
+    /**
+     * Compute overall XOR between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap naive_xor(@SuppressWarnings("rawtypes") Iterator bitmaps) {
+        MutableRoaringBitmap answer = new MutableRoaringBitmap();
+        while (bitmaps.hasNext()) {
+            answer.xor((ImmutableRoaringBitmap) bitmaps.next());
+        }
+        return answer;
+    }
+
+    /**
+     * Compute overall XOR between bitmaps two-by-two.
+     * <p>
+     * This function runs in linear time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap naive_xor(MutableRoaringBitmap... bitmaps) {
+        MutableRoaringBitmap answer = new MutableRoaringBitmap();
+        for (int k = 0; k < bitmaps.length; ++k) {
+            answer.xor(bitmaps[k]);
+        }
+        return answer;
+    }
+
+    /**
+     * Compute overall OR between bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap or(ImmutableRoaringBitmap... bitmaps) {
+        return naive_or(bitmaps);
+    }
+
+    /**
+     * Compute overall OR between bitmaps.
+     *
+     * @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap or(@SuppressWarnings("rawtypes") Iterator bitmaps) {
+        return naive_or(bitmaps);
+    }
+
+    /**
+     * Compute overall OR between bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap or(MutableRoaringBitmap... bitmaps) {
+        return naive_or(bitmaps);
+    }
+
+    /**
+     * Uses a priority queue to compute the or aggregate.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #horizontal_or(ImmutableRoaringBitmap...)
+     */
+    public static MutableRoaringBitmap priorityqueue_or(ImmutableRoaringBitmap... bitmaps) {
+        if (bitmaps.length == 0) {
+            return new MutableRoaringBitmap();
+        } else if (bitmaps.length == 1) {
+            return bitmaps[0].toMutableRoaringBitmap();
+        }
+        // we buffer the call to getLongSizeInBytes(), hence the code complexity
+        final ImmutableRoaringBitmap[] buffer = Arrays.copyOf(bitmaps, bitmaps.length);
+        final int[] sizes = new int[buffer.length];
+        final boolean[] istmp = new boolean[buffer.length];
+        for (int k = 0; k < sizes.length; ++k) {
+            sizes[k] = buffer[k].serializedSizeInBytes();
+        }
+        PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
+            @Override
+            public int compare(Integer a, Integer b) {
+                return sizes[a] - sizes[b];
+            }
+        });
+        for (int k = 0; k < sizes.length; ++k) {
+            pq.add(k);
+        }
+        while (pq.size() > 1) {
+            Integer x1 = pq.poll();
+            Integer x2 = pq.poll();
+            if (istmp[x1] && istmp[x2]) {
+                buffer[x1] = MutableRoaringBitmap.lazyorfromlazyinputs((MutableRoaringBitmap) buffer[x1],
+                        (MutableRoaringBitmap) buffer[x2]);
+                sizes[x1] = buffer[x1].serializedSizeInBytes();
+                pq.add(x1);
+            } else if (istmp[x2]) {
+                ((MutableRoaringBitmap) buffer[x2]).lazyor(buffer[x1]);
+                sizes[x2] = buffer[x2].serializedSizeInBytes();
+                pq.add(x2);
+            } else if (istmp[x1]) {
+                ((MutableRoaringBitmap) buffer[x1]).lazyor(buffer[x2]);
+                sizes[x1] = buffer[x1].serializedSizeInBytes();
+                pq.add(x1);
+            } else {
+                buffer[x1] = ImmutableRoaringBitmap.lazyor(buffer[x1], buffer[x2]);
+                sizes[x1] = buffer[x1].serializedSizeInBytes();
+                istmp[x1] = true;
+                pq.add(x1);
+            }
+        }
+        MutableRoaringBitmap answer = (MutableRoaringBitmap) buffer[pq.poll()];
+        answer.repairAfterLazy();
+        return answer;
+    }
+
+    /**
+     * Uses a priority queue to compute the or aggregate.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #horizontal_or(ImmutableRoaringBitmap...)
+     */
+    public static MutableRoaringBitmap priorityqueue_or(
+            @SuppressWarnings("rawtypes") Iterator bitmaps) {
+        if (!bitmaps.hasNext()) {
+            return new MutableRoaringBitmap();
+        }
+        // we buffer the call to getLongSizeInBytes(), hence the code complexity
+        ArrayList<ImmutableRoaringBitmap> buffer = new ArrayList<ImmutableRoaringBitmap>();
+        while (bitmaps.hasNext()) {
+            buffer.add((ImmutableRoaringBitmap) bitmaps.next());
+        }
+        final long[] sizes = new long[buffer.size()];
+        final boolean[] istmp = new boolean[buffer.size()];
+        for (int k = 0; k < sizes.length; ++k) {
+            sizes[k] = buffer.get(k).getLongSizeInBytes();
+        }
+        PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
+            @Override
+            public int compare(Integer a, Integer b) {
+                return (int) (sizes[a] - sizes[b]);
+            }
+        });
+        for (int k = 0; k < sizes.length; ++k) {
+            pq.add(k);
+        }
+        if (pq.size() == 1) {
+            return buffer.get(pq.poll()).toMutableRoaringBitmap();
+        }
+        while (pq.size() > 1) {
+            Integer x1 = pq.poll();
+            Integer x2 = pq.poll();
+            if (istmp[x1] && istmp[x2]) {
+                buffer.set(x1, MutableRoaringBitmap.lazyorfromlazyinputs(
+                        (MutableRoaringBitmap) buffer.get(x1), (MutableRoaringBitmap) buffer.get(x2)));
+                sizes[x1] = buffer.get(x1).getLongSizeInBytes();
+                pq.add(x1);
+            } else if (istmp[x2]) {
+                ((MutableRoaringBitmap) buffer.get(x2)).lazyor(buffer.get(x1));
+                sizes[x2] = buffer.get(x2).getLongSizeInBytes();
+                pq.add(x2);
+            } else if (istmp[x1]) {
+                ((MutableRoaringBitmap) buffer.get(x1)).lazyor(buffer.get(x2));
+                sizes[x1] = buffer.get(x1).getLongSizeInBytes();
+                pq.add(x1);
+            } else {
+                buffer.set(x1, ImmutableRoaringBitmap.lazyor(buffer.get(x1), buffer.get(x2)));
+                sizes[x1] = buffer.get(x1).getLongSizeInBytes();
+                istmp[x1] = true;
+                pq.add(x1);
+            }
+        }
+        MutableRoaringBitmap answer = (MutableRoaringBitmap) buffer.get(pq.poll());
+        answer.repairAfterLazy();
+        return answer;
+    }
+
+    /**
+     * Uses a priority queue to compute the xor aggregate.
+     * <p>
+     * This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     * @see #horizontal_xor(ImmutableRoaringBitmap...)
+     */
+    public static MutableRoaringBitmap priorityqueue_xor(ImmutableRoaringBitmap... bitmaps) {
+        // code could be faster, see priorityqueue_or
+        if (bitmaps.length < 2) {
+            throw new IllegalArgumentException("Expecting at least 2 bitmaps");
+        }
+        final PriorityQueue<ImmutableRoaringBitmap> pq =
+                new PriorityQueue<ImmutableRoaringBitmap>(bitmaps.length, new Comparator<ImmutableRoaringBitmap>() {
+                    @Override
+                    public int compare(ImmutableRoaringBitmap a, ImmutableRoaringBitmap b) {
+                        return (int) (a.getLongSizeInBytes() - b.getLongSizeInBytes());
+                    }
+                });
+        Collections.addAll(pq, bitmaps);
+        while (pq.size() > 1) {
+            final ImmutableRoaringBitmap x1 = pq.poll();
+            final ImmutableRoaringBitmap x2 = pq.poll();
+            pq.add(ImmutableRoaringBitmap.xor(x1, x2));
+        }
+        return (MutableRoaringBitmap) pq.poll();
+    }
+
+    /**
+     * Compute overall XOR between bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap xor(ImmutableRoaringBitmap... bitmaps) {
+        return naive_xor(bitmaps);
+    }
+
+    /**
+     * Compute overall XOR between bitmaps.
+     *
+     * @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap xor(@SuppressWarnings("rawtypes") Iterator bitmaps) {
+        return naive_xor(bitmaps);
+    }
+
+    /**
+     * Compute overall XOR between bitmaps.
+     *
+     * @param bitmaps input bitmaps
+     * @return aggregated bitmap
+     */
+    public static MutableRoaringBitmap xor(MutableRoaringBitmap... bitmaps) {
+        return naive_xor(bitmaps);
+    }
+
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferIntIteratorFlyweight.java

@@ -0,0 +1,132 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap.buffer;
+
+import com.fr.third.bitmap.roaringbitmap.PeekableIntIterator;
+import com.fr.third.bitmap.roaringbitmap.PeekableShortIterator;
+
+/**
+ * Fast iterator minimizing the stress on the garbage collector. You can create one reusable
+ * instance of this class and then {@link #wrap(ImmutableRoaringBitmap)}
+ * <p>
+ * For better performance, consider the {@link ImmutableRoaringBitmap#forEach} method.
+ *
+ * @author Borislav Ivanov
+ **/
+public class BufferIntIteratorFlyweight implements PeekableIntIterator {
+
+    private int hs;
+
+    private PeekableShortIterator iter;
+
+    private MappeableArrayContainerShortIterator arrIter = new MappeableArrayContainerShortIterator();
+
+    private MappeableBitmapContainerShortIterator bitmapIter =
+            new MappeableBitmapContainerShortIterator();
+
+    private MappeableRunContainerShortIterator runIter = new MappeableRunContainerShortIterator();
+
+
+    private int pos;
+
+    private ImmutableRoaringBitmap roaringBitmap = null;
+
+    /**
+     * Creates an instance that is not ready for iteration. You must first call
+     * {@link #wrap(ImmutableRoaringBitmap)}.
+     */
+    public BufferIntIteratorFlyweight() {
+
+    }
+
+    /**
+     * Creates an instance that is ready for iteration.
+     *
+     * @param r bitmap to be iterated over
+     */
+    public BufferIntIteratorFlyweight(ImmutableRoaringBitmap r) {
+        wrap(r);
+    }
+
+    @Override
+    public PeekableIntIterator clone() {
+        try {
+            BufferIntIteratorFlyweight x = (BufferIntIteratorFlyweight) super.clone();
+            x.iter = this.iter.clone();
+            return x;
+        } catch (CloneNotSupportedException e) {
+            return null;// will not happen
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        return pos < this.roaringBitmap.highLowContainer.size();
+    }
+
+    @Override
+    public int next() {
+        int x = iter.nextAsInt() | hs;
+        if (!iter.hasNext()) {
+            ++pos;
+            nextContainer();
+        }
+        return x;
+    }
+
+    private void nextContainer() {
+        if (pos < this.roaringBitmap.highLowContainer.size()) {
+
+            MappeableContainer container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
+
+            if (container instanceof MappeableBitmapContainer) {
+                bitmapIter.wrap((MappeableBitmapContainer) container);
+                iter = bitmapIter;
+            } else if (container instanceof MappeableRunContainer) {
+                runIter.wrap((MappeableRunContainer) container);
+                iter = runIter;
+            } else {
+                arrIter.wrap((MappeableArrayContainer) container);
+                iter = arrIter;
+            }
+
+            hs = BufferUtil.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
+        }
+    }
+
+    /**
+     * Prepares a bitmap for iteration
+     *
+     * @param r bitmap to be iterated over
+     */
+    public void wrap(ImmutableRoaringBitmap r) {
+        this.hs = 0;
+        this.pos = 0;
+        this.roaringBitmap = r;
+        this.nextContainer();
+    }
+
+    @Override
+    public void advanceIfNeeded(int minval) {
+        while (hasNext() && ((hs >>> 16) < (minval >>> 16))) {
+            ++pos;
+            nextContainer();
+        }
+        if (hasNext() && ((hs >>> 16) == (minval >>> 16))) {
+            iter.advanceIfNeeded(BufferUtil.lowbits(minval));
+            if (!iter.hasNext()) {
+                ++pos;
+                nextContainer();
+            }
+        }
+    }
+
+    @Override
+    public int peekNext() {
+        return BufferUtil.toIntUnsigned(iter.peekNext()) | hs;
+    }
+
+
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferReverseIntIteratorFlyweight.java

@@ -0,0 +1,116 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap.buffer;
+
+import com.fr.third.bitmap.roaringbitmap.IntIterator;
+import com.fr.third.bitmap.roaringbitmap.ShortIterator;
+
+/**
+ * Fast iterator minimizing the stress on the garbage collector. You can create one reusable
+ * instance of this class and then {@link #wrap(ImmutableRoaringBitmap)}
+ * <p>
+ * This iterator enumerates the stored values in reverse (starting from the end).
+ *
+ * @author Borislav Ivanov
+ **/
+public class BufferReverseIntIteratorFlyweight implements IntIterator {
+
+    private int hs;
+
+    private ShortIterator iter;
+
+    private ReverseMappeableArrayContainerShortIterator arrIter =
+            new ReverseMappeableArrayContainerShortIterator();
+
+    private ReverseMappeableBitmapContainerShortIterator bitmapIter =
+            new ReverseMappeableBitmapContainerShortIterator();
+
+    private ReverseMappeableRunContainerShortIterator runIter =
+            new ReverseMappeableRunContainerShortIterator();
+
+    private short pos;
+
+    private ImmutableRoaringBitmap roaringBitmap = null;
+
+
+    /**
+     * Creates an instance that is not ready for iteration. You must first call
+     * {@link #wrap(ImmutableRoaringBitmap)}.
+     */
+    public BufferReverseIntIteratorFlyweight() {
+
+    }
+
+    /**
+     * Creates an instance that is ready for iteration.
+     *
+     * @param r bitmap to be iterated over
+     */
+    public BufferReverseIntIteratorFlyweight(ImmutableRoaringBitmap r) {
+        wrap(r);
+    }
+
+    @Override
+    public IntIterator clone() {
+        try {
+            BufferReverseIntIteratorFlyweight x = (BufferReverseIntIteratorFlyweight) super.clone();
+            x.iter = this.iter.clone();
+            return x;
+        } catch (CloneNotSupportedException e) {
+            return null;// will not happen
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        return pos >= 0;
+    }
+
+
+    @Override
+    public int next() {
+        final int x = iter.nextAsInt() | hs;
+        if (!iter.hasNext()) {
+            --pos;
+            nextContainer();
+        }
+        return x;
+    }
+
+    private void nextContainer() {
+
+        if (pos >= 0) {
+
+            MappeableContainer container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
+
+            if (container instanceof MappeableBitmapContainer) {
+                bitmapIter.wrap((MappeableBitmapContainer) container);
+                iter = bitmapIter;
+            } else if (container instanceof MappeableRunContainer) {
+                runIter.wrap((MappeableRunContainer) container);
+                iter = runIter;
+            } else {
+                arrIter.wrap((MappeableArrayContainer) container);
+                iter = arrIter;
+            }
+
+            hs = BufferUtil.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
+        }
+    }
+
+    /**
+     * Prepares a bitmap for iteration
+     *
+     * @param r bitmap to be iterated over
+     */
+    public void wrap(ImmutableRoaringBitmap r) {
+        this.roaringBitmap = r;
+        this.hs = 0;
+        this.pos = (short) (this.roaringBitmap.highLowContainer.size() - 1);
+        this.nextContainer();
+    }
+
+}
+

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferUtil.java

@@ -0,0 +1,825 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap.buffer;
+
+import com.fr.third.bitmap.roaringbitmap.Util;
+
+import java.nio.Buffer;
+import java.nio.LongBuffer;
+import java.nio.ShortBuffer;
+
+/**
+ * Various useful methods for roaring bitmaps.
+ * <p>
+ * This class is similar to Util but meant to be used with memory mapping.
+ */
+public final class BufferUtil {
+
+
+    /**
+     * Private constructor to prevent instantiation of utility class
+     */
+    private BufferUtil() {
+    }
+
+    /**
+     * Find the smallest integer larger than pos such that array[pos]&gt;= min. If none can be found,
+     * return length. Based on code by O. Kaser.
+     *
+     * @param array  container where we search
+     * @param pos    initial position
+     * @param min    minimal threshold
+     * @param length how big should the array consider to be
+     * @return x greater than pos such that array[pos] is at least as large as min, pos is is equal to
+     * length if it is not possible.
+     */
+    protected static int advanceUntil(ShortBuffer array, int pos, int length, short min) {
+        int lower = pos + 1;
+
+        // special handling for a possibly common sequential case
+        if (lower >= length || toIntUnsigned(array.get(lower)) >= toIntUnsigned(min)) {
+            return lower;
+        }
+
+        int spansize = 1; // could set larger
+        // bootstrap an upper limit
+
+        while (lower + spansize < length
+                && toIntUnsigned(array.get(lower + spansize)) < toIntUnsigned(min)) {
+            spansize *= 2; // hoping for compiler will reduce to
+        }
+        // shift
+        int upper = (lower + spansize < length) ? lower + spansize : length - 1;
+
+        // maybe we are lucky (could be common case when the seek ahead
+        // expected
+        // to be small and sequential will otherwise make us look bad)
+        if (array.get(upper) == min) {
+            return upper;
+        }
+
+        if (toIntUnsigned(array.get(upper)) < toIntUnsigned(min)) {// means
+            // array
+            // has no
+            // item
+            // >= min
+            // pos = array.length;
+            return length;
+        }
+
+        // we know that the next-smallest span was too small
+        lower += (spansize / 2);
+
+        // else begin binary search
+        // invariant: array[lower]<min && array[upper]>min
+        while (lower + 1 != upper) {
+            int mid = (lower + upper) / 2;
+            short arraymid = array.get(mid);
+            if (arraymid == min) {
+                return mid;
+            } else if (toIntUnsigned(arraymid) < toIntUnsigned(min)) {
+                lower = mid;
+            } else {
+                upper = mid;
+            }
+        }
+        return upper;
+
+    }
+
+    protected static void arraycopy(ShortBuffer src, int srcPos, ShortBuffer dest, int destPos,
+                                    int length) {
+        if (BufferUtil.isBackedBySimpleArray(src) && BufferUtil.isBackedBySimpleArray(dest)) {
+            System.arraycopy(src.array(), srcPos, dest.array(), destPos, length);
+        } else {
+            if (srcPos < destPos) {
+                for (int k = length - 1; k >= 0; --k) {
+                    dest.put(destPos + k, src.get(k + srcPos));
+                }
+            } else {
+                for (int k = 0; k < length; ++k) {
+                    dest.put(destPos + k, src.get(k + srcPos));
+                }
+            }
+        }
+    }
+
+    protected static int branchyUnsignedBinarySearch(final ShortBuffer array, final int begin,
+                                                     final int end, final short k) {
+        final int ikey = toIntUnsigned(k);
+        // next line accelerates the possibly common case where the value would be inserted at the end
+        if ((end > 0) && (toIntUnsigned(array.get(end - 1)) < ikey)) {
+            return -end - 1;
+        }
+        int low = begin;
+        int high = end - 1;
+        while (low <= high) {
+            final int middleIndex = (low + high) >>> 1;
+            final int middleValue = toIntUnsigned(array.get(middleIndex));
+
+            if (middleValue < ikey) {
+                low = middleIndex + 1;
+            } else if (middleValue > ikey) {
+                high = middleIndex - 1;
+            } else {
+                return middleIndex;
+            }
+        }
+        return -(low + 1);
+    }
+
+    /**
+     * Compares the two specified {@code short} values, treating them as unsigned values between
+     * {@code 0} and {@code 2^16 - 1} inclusive.
+     *
+     * @param a the first unsigned {@code short} to compare
+     * @param b the second unsigned {@code short} to compare
+     * @return a negative value if {@code a} is less than {@code b}; a positive value if {@code a} is
+     * greater than {@code b}; or zero if they are equal
+     */
+    public static int compareUnsigned(short a, short b) {
+        return toIntUnsigned(a) - toIntUnsigned(b);
+    }
+
+    protected static void fillArrayAND(short[] container, LongBuffer bitmap1, LongBuffer bitmap2) {
+        int pos = 0;
+        if (bitmap1.limit() != bitmap2.limit()) {
+            throw new IllegalArgumentException("not supported");
+        }
+        if (BufferUtil.isBackedBySimpleArray(bitmap1) && BufferUtil.isBackedBySimpleArray(bitmap2)) {
+            int len = bitmap1.limit();
+            long[] b1 = bitmap1.array();
+            long[] b2 = bitmap2.array();
+            for (int k = 0; k < len; ++k) {
+                long bitset = b1[k] & b2[k];
+                while (bitset != 0) {
+                    final long t = bitset & -bitset;
+                    container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
+                    bitset ^= t;
+                }
+            }
+        } else {
+            int len = bitmap1.limit();
+            for (int k = 0; k < len; ++k) {
+                long bitset = bitmap1.get(k) & bitmap2.get(k);
+                while (bitset != 0) {
+                    final long t = bitset & -bitset;
+                    container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
+                    bitset ^= t;
+                }
+            }
+        }
+    }
+
+    protected static void fillArrayANDNOT(short[] container, LongBuffer bitmap1, LongBuffer bitmap2) {
+        int pos = 0;
+        if (bitmap1.limit() != bitmap2.limit()) {
+            throw new IllegalArgumentException("not supported");
+        }
+        if (BufferUtil.isBackedBySimpleArray(bitmap1) && BufferUtil.isBackedBySimpleArray(bitmap2)) {
+            int len = bitmap1.limit();
+            long[] b1 = bitmap1.array();
+            long[] b2 = bitmap2.array();
+            for (int k = 0; k < len; ++k) {
+                long bitset = b1[k] & (~b2[k]);
+                while (bitset != 0) {
+                    final long t = bitset & -bitset;
+                    container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
+                    bitset ^= t;
+                }
+            }
+        } else {
+            int len = bitmap1.limit();
+            for (int k = 0; k < len; ++k) {
+                long bitset = bitmap1.get(k) & (~bitmap2.get(k));
+                while (bitset != 0) {
+                    final long t = bitset & -bitset;
+                    container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
+                    bitset ^= t;
+                }
+            }
+        }
+    }
+
+    protected static void fillArrayXOR(short[] container, LongBuffer bitmap1, LongBuffer bitmap2) {
+        int pos = 0;
+        if (bitmap1.limit() != bitmap2.limit()) {
+            throw new IllegalArgumentException("not supported");
+        }
+        if (BufferUtil.isBackedBySimpleArray(bitmap1) && BufferUtil.isBackedBySimpleArray(bitmap2)) {
+            Util.fillArrayXOR(container, bitmap1.array(), bitmap2.array());
+        } else {
+            int len = bitmap1.limit();
+            for (int k = 0; k < len; ++k) {
+                long bitset = bitmap1.get(k) ^ bitmap2.get(k);
+                while (bitset != 0) {
+                    final long t = bitset & -bitset;
+                    container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
+                    bitset ^= t;
+                }
+            }
+        }
+    }
+
+    /**
+     * flip bits at start, start+1,..., end-1
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     */
+    public static void flipBitmapRange(LongBuffer bitmap, int start, int end) {
+        if (isBackedBySimpleArray(bitmap)) {
+            Util.flipBitmapRange(bitmap.array(), start, end);
+            return;
+        }
+        if (start == end) {
+            return;
+        }
+        int firstword = start / 64;
+        int endword = (end - 1) / 64;
+        bitmap.put(firstword, bitmap.get(firstword) ^ ~(~0L << start));
+        for (int i = firstword; i < endword; i++) {
+            bitmap.put(i, ~bitmap.get(i));
+        }
+        bitmap.put(endword, bitmap.get(endword) ^ (~0L >>> -end));
+    }
+
+    /**
+     * Hamming weight of the 64-bit words involved in the range start, start+1,..., end-1
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     */
+    private static int cardinalityInBitmapWordRange(LongBuffer bitmap, int start, int end) {
+        if (isBackedBySimpleArray(bitmap)) {
+            return Util.cardinalityInBitmapWordRange(bitmap.array(), start, end);
+        }
+        if (start == end) {
+            return 0;
+        }
+        int firstword = start / 64;
+        int endword = (end - 1) / 64;
+        int answer = 0;
+        for (int i = firstword; i <= endword; i++) {
+            answer += Long.bitCount(bitmap.get(i));
+        }
+        return answer;
+    }
+
+    /**
+     * set bits at start, start+1,..., end-1 and report the cardinality change
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     * @return cardinality change
+     */
+    public static int setBitmapRangeAndCardinalityChange(LongBuffer bitmap, int start, int end) {
+        if (BufferUtil.isBackedBySimpleArray(bitmap)) {
+            return Util.setBitmapRangeAndCardinalityChange(bitmap.array(), start, end);
+        }
+        int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
+        setBitmapRange(bitmap, start, end);
+        int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
+        return cardafter - cardbefore;
+    }
+
+    /**
+     * flip bits at start, start+1,..., end-1 and report the cardinality change
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     * @return cardinality change
+     */
+    public static int flipBitmapRangeAndCardinalityChange(LongBuffer bitmap, int start, int end) {
+        if (BufferUtil.isBackedBySimpleArray(bitmap)) {
+            return Util.flipBitmapRangeAndCardinalityChange(bitmap.array(), start, end);
+        }
+        int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
+        flipBitmapRange(bitmap, start, end);
+        int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
+        return cardafter - cardbefore;
+    }
+
+    /**
+     * reset bits at start, start+1,..., end-1 and report the cardinality change
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     * @return cardinality change
+     */
+    public static int resetBitmapRangeAndCardinalityChange(LongBuffer bitmap, int start, int end) {
+        if (BufferUtil.isBackedBySimpleArray(bitmap)) {
+            return Util.resetBitmapRangeAndCardinalityChange(bitmap.array(), start, end);
+        }
+        int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
+        resetBitmapRange(bitmap, start, end);
+        int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
+        return cardafter - cardbefore;
+    }
+
+    /**
+     * From the cardinality of a container, compute the corresponding size in bytes of the container.
+     * Additional information is required if the container is run encoded.
+     *
+     * @param card         the cardinality if this is not run encoded, otherwise ignored
+     * @param numRuns      number of runs if run encoded, othewise ignored
+     * @param isRunEncoded boolean
+     * @return the size in bytes
+     */
+    protected static int getSizeInBytesFromCardinalityEtc(int card, int numRuns,
+                                                          boolean isRunEncoded) {
+        if (isRunEncoded) {
+            return 2 + numRuns * 2 * 2; // each run uses 2 shorts, plus the initial short giving num runs
+        }
+        boolean isBitmap = card > MappeableArrayContainer.DEFAULT_MAX_SIZE;
+        if (isBitmap) {
+            return MappeableBitmapContainer.MAX_CAPACITY / 8;
+        } else {
+            return card * 2;
+        }
+
+    }
+
+    protected static short highbits(int x) {
+        return (short) (x >>> 16);
+    }
+
+    protected static short highbits(long x) {
+        return (short) (x >>> 16);
+    }
+
+    /**
+     * Checks whether the Buffer is backed by a simple array. In java, a Buffer is an abstraction that
+     * can represent various data, from data on disk all the way to native Java arrays. Like all
+     * abstractions, a Buffer might carry a performance penalty. Thus, we sometimes check whether the
+     * Buffer is simply a wrapper around a Java array. In these instances, it might be best, from a
+     * performance point of view, to access the underlying array (using the array()) method.
+     *
+     * @param b the provided Buffer
+     * @return whether the Buffer is backed by a simple array
+     */
+    protected static boolean isBackedBySimpleArray(Buffer b) {
+        return b.hasArray() && (b.arrayOffset() == 0);
+    }
+
+    protected static short lowbits(int x) {
+        return (short) (x & 0xFFFF);
+    }
+
+    protected static short lowbits(long x) {
+        return (short) (x & 0xFFFF);
+    }
+
+    protected static short maxLowBit() {
+        return (short) 0xFFFF;
+    }
+
+    protected static int maxLowBitAsInteger() {
+        return 0xFFFF;
+    }
+
+    /**
+     * clear bits at start, start+1,..., end-1
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     */
+    public static void resetBitmapRange(LongBuffer bitmap, int start, int end) {
+        if (isBackedBySimpleArray(bitmap)) {
+            Util.resetBitmapRange(bitmap.array(), start, end);
+            return;
+        }
+        if (start == end) {
+            return;
+        }
+        int firstword = start / 64;
+        int endword = (end - 1) / 64;
+        if (firstword == endword) {
+            bitmap.put(firstword, bitmap.get(firstword) & ~((~0L << start) & (~0L >>> -end)));
+            return;
+        }
+        bitmap.put(firstword, bitmap.get(firstword) & (~(~0L << start)));
+        for (int i = firstword + 1; i < endword; i++) {
+            bitmap.put(i, 0L);
+        }
+        bitmap.put(endword, bitmap.get(endword) & (~(~0L >>> -end)));
+    }
+
+    /**
+     * set bits at start, start+1,..., end-1
+     *
+     * @param bitmap array of words to be modified
+     * @param start  first index to be modified (inclusive)
+     * @param end    last index to be modified (exclusive)
+     */
+    public static void setBitmapRange(LongBuffer bitmap, int start, int end) {
+        if (isBackedBySimpleArray(bitmap)) {
+            Util.setBitmapRange(bitmap.array(), start, end);
+            return;
+        }
+        if (start == end) {
+            return;
+        }
+        int firstword = start / 64;
+        int endword = (end - 1) / 64;
+        if (firstword == endword) {
+            bitmap.put(firstword, bitmap.get(firstword) | ((~0L << start) & (~0L >>> -end)));
+
+            return;
+        }
+        bitmap.put(firstword, bitmap.get(firstword) | (~0L << start));
+        for (int i = firstword + 1; i < endword; i++) {
+            bitmap.put(i, ~0L);
+        }
+        bitmap.put(endword, bitmap.get(endword) | (~0L >>> -end));
+    }
+
+    protected static int toIntUnsigned(short x) {
+        return x & 0xFFFF;
+    }
+
+    /**
+     * Look for value k in buffer in the range [begin,end). If the value is found, return its index.
+     * If not, return -(i+1) where i is the index where the value would be inserted. The buffer is
+     * assumed to contain sorted values where shorts are interpreted as unsigned integers.
+     *
+     * @param array buffer where we search
+     * @param begin first index (inclusive)
+     * @param end   last index (exclusive)
+     * @param k     value we search for
+     * @return count
+     */
+    public static int unsignedBinarySearch(final ShortBuffer array, final int begin, final int end,
+                                           final short k) {
+        return branchyUnsignedBinarySearch(array, begin, end, k);
+    }
+
+    protected static int unsignedDifference(final ShortBuffer set1, final int length1,
+                                            final ShortBuffer set2, final int length2, final short[] buffer) {
+        int pos = 0;
+        int k1 = 0, k2 = 0;
+        if (0 == length2) {
+            set1.get(buffer, 0, length1);
+            return length1;
+        }
+        if (0 == length1) {
+            return 0;
+        }
+        short s1 = set1.get(k1);
+        short s2 = set2.get(k2);
+        while (true) {
+            if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
+                buffer[pos++] = s1;
+                ++k1;
+                if (k1 >= length1) {
+                    break;
+                }
+                s1 = set1.get(k1);
+            } else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
+                ++k1;
+                ++k2;
+                if (k1 >= length1) {
+                    break;
+                }
+                if (k2 >= length2) {
+                    set1.position(k1);
+                    set1.get(buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s1 = set1.get(k1);
+                s2 = set2.get(k2);
+            } else {// if (val1>val2)
+                ++k2;
+                if (k2 >= length2) {
+                    set1.position(k1);
+                    set1.get(buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s2 = set2.get(k2);
+            }
+        }
+        return pos;
+    }
+
+    protected static int unsignedExclusiveUnion2by2(final ShortBuffer set1, final int length1,
+                                                    final ShortBuffer set2, final int length2, final short[] buffer) {
+        int pos = 0;
+        int k1 = 0, k2 = 0;
+        if (0 == length2) {
+            set1.get(buffer, 0, length1);
+            return length1;
+        }
+        if (0 == length1) {
+            set2.get(buffer, 0, length2);
+            return length2;
+        }
+        short s1 = set1.get(k1);
+        short s2 = set2.get(k2);
+        while (true) {
+            if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
+                buffer[pos++] = s1;
+                ++k1;
+                if (k1 >= length1) {
+                    set2.position(k2);
+                    set2.get(buffer, pos, length2 - k2);
+                    return pos + length2 - k2;
+                }
+                s1 = set1.get(k1);
+            } else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
+                ++k1;
+                ++k2;
+                if (k1 >= length1) {
+                    set2.position(k2);
+                    set2.get(buffer, pos, length2 - k2);
+                    return pos + length2 - k2;
+                }
+                if (k2 >= length2) {
+                    set1.position(k1);
+                    set1.get(buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s1 = set1.get(k1);
+                s2 = set2.get(k2);
+            } else {// if (val1>val2)
+                buffer[pos++] = s2;
+                ++k2;
+                if (k2 >= length2) {
+                    set1.position(k1);
+                    set1.get(buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s2 = set2.get(k2);
+            }
+        }
+        // return pos;
+    }
+
+    protected static int unsignedIntersect2by2(final ShortBuffer set1, final int length1,
+                                               final ShortBuffer set2, final int length2, final short[] buffer) {
+        if (length1 * 64 < length2) {
+            return unsignedOneSidedGallopingIntersect2by2(set1, length1, set2, length2, buffer);
+        } else if (length2 * 64 < length1) {
+            return unsignedOneSidedGallopingIntersect2by2(set2, length2, set1, length1, buffer);
+        } else {
+            return unsignedLocalIntersect2by2(set1, length1, set2, length2, buffer);
+        }
+    }
+
+    /**
+     * Checks if two arrays intersect
+     *
+     * @param set1    first array
+     * @param length1 length of first array
+     * @param set2    second array
+     * @param length2 length of second array
+     * @return true if they intersect
+     */
+    public static boolean unsignedIntersects(ShortBuffer set1, int length1, ShortBuffer set2,
+                                             int length2) {
+        if ((0 == length1) || (0 == length2)) {
+            return false;
+        }
+        int k1 = 0;
+        int k2 = 0;
+
+        // could be more efficient with galloping
+        short s1 = set1.get(k1);
+        short s2 = set2.get(k2);
+
+        mainwhile:
+        while (true) {
+            if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
+                do {
+                    ++k2;
+                    if (k2 == length2) {
+                        break mainwhile;
+                    }
+                    s2 = set2.get(k2);
+                } while (toIntUnsigned(s2) < toIntUnsigned(s1));
+            }
+            if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
+                do {
+                    ++k1;
+                    if (k1 == length1) {
+                        break mainwhile;
+                    }
+                    s1 = set1.get(k1);
+                } while (toIntUnsigned(s1) < toIntUnsigned(s2));
+            } else {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    protected static int unsignedLocalIntersect2by2(final ShortBuffer set1, final int length1,
+                                                    final ShortBuffer set2, final int length2, final short[] buffer) {
+        if ((0 == length1) || (0 == length2)) {
+            return 0;
+        }
+        int k1 = 0;
+        int k2 = 0;
+        int pos = 0;
+        short s1 = set1.get(k1);
+        short s2 = set2.get(k2);
+
+        mainwhile:
+        while (true) {
+            if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
+                do {
+                    ++k2;
+                    if (k2 == length2) {
+                        break mainwhile;
+                    }
+                    s2 = set2.get(k2);
+
+                } while (toIntUnsigned(s2) < toIntUnsigned(s1));
+            }
+            if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
+                do {
+                    ++k1;
+                    if (k1 == length1) {
+                        break mainwhile;
+                    }
+                    s1 = set1.get(k1);
+
+                } while (toIntUnsigned(s1) < toIntUnsigned(s2));
+            } else {
+                // (set2.get(k2) == set1.get(k1))
+                buffer[pos++] = s1;
+                ++k1;
+                if (k1 == length1) {
+                    break;
+                }
+                s1 = set1.get(k1);
+                ++k2;
+                if (k2 == length2) {
+                    break;
+                }
+                s2 = set2.get(k2);
+
+            }
+        }
+        return pos;
+    }
+
+    protected static int unsignedLocalIntersect2by2Cardinality(final ShortBuffer set1,
+                                                               final int length1, final ShortBuffer set2, final int length2) {
+        if ((0 == length1) || (0 == length2)) {
+            return 0;
+        }
+        int k1 = 0;
+        int k2 = 0;
+        int pos = 0;
+        short s1 = set1.get(k1);
+        short s2 = set2.get(k2);
+
+        mainwhile:
+        while (true) {
+            if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
+                do {
+                    ++k2;
+                    if (k2 == length2) {
+                        break mainwhile;
+                    }
+                    s2 = set2.get(k2);
+
+                } while (toIntUnsigned(s2) < toIntUnsigned(s1));
+            }
+            if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
+                do {
+                    ++k1;
+                    if (k1 == length1) {
+                        break mainwhile;
+                    }
+                    s1 = set1.get(k1);
+
+                } while (toIntUnsigned(s1) < toIntUnsigned(s2));
+            } else {
+                ++pos;
+                ++k1;
+                if (k1 == length1) {
+                    break;
+                }
+                s1 = set1.get(k1);
+                ++k2;
+                if (k2 == length2) {
+                    break;
+                }
+                s2 = set2.get(k2);
+
+            }
+        }
+        return pos;
+    }
+
+    protected static int unsignedOneSidedGallopingIntersect2by2(final ShortBuffer smallSet,
+                                                                final int smallLength, final ShortBuffer largeSet, final int largeLength,
+                                                                final short[] buffer) {
+        if (0 == smallLength) {
+            return 0;
+        }
+        int k1 = 0;
+        int k2 = 0;
+        int pos = 0;
+
+        short s1 = largeSet.get(k1);
+        short s2 = smallSet.get(k2);
+        while (true) {
+            if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
+                k1 = advanceUntil(largeSet, k1, largeLength, s2);
+                if (k1 == largeLength) {
+                    break;
+                }
+                s1 = largeSet.get(k1);
+            }
+            if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
+                ++k2;
+                if (k2 == smallLength) {
+                    break;
+                }
+                s2 = smallSet.get(k2);
+            } else {
+                // (set2.get(k2) == set1.get(k1))
+                buffer[pos++] = s2;
+                ++k2;
+                if (k2 == smallLength) {
+                    break;
+                }
+                s2 = smallSet.get(k2);
+                k1 = advanceUntil(largeSet, k1, largeLength, s2);
+                if (k1 == largeLength) {
+                    break;
+                }
+                s1 = largeSet.get(k1);
+            }
+
+        }
+        return pos;
+
+    }
+
+    protected static int unsignedUnion2by2(final ShortBuffer set1, final int length1,
+                                           final ShortBuffer set2, final int length2, final short[] buffer) {
+        int pos = 0;
+        int k1 = 0, k2 = 0;
+        if (0 == length2) {
+            set1.get(buffer, 0, length1);
+            return length1;
+        }
+        if (0 == length1) {
+            set2.get(buffer, 0, length2);
+            return length2;
+        }
+        short s1 = set1.get(k1);
+        short s2 = set2.get(k2);
+        while (true) {
+            int v1 = toIntUnsigned(s1);
+            int v2 = toIntUnsigned(s2);
+            if (v1 < v2) {
+                buffer[pos++] = s1;
+                ++k1;
+                if (k1 >= length1) {
+                    set2.position(k2);
+                    set2.get(buffer, pos, length2 - k2);
+                    return pos + length2 - k2;
+                }
+                s1 = set1.get(k1);
+            } else if (v1 == v2) {
+                buffer[pos++] = s1;
+                ++k1;
+                ++k2;
+                if (k1 >= length1) {
+                    set2.position(k2);
+                    set2.get(buffer, pos, length2 - k2);
+                    return pos + length2 - k2;
+                }
+                if (k2 >= length2) {
+                    set1.position(k1);
+                    set1.get(buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s1 = set1.get(k1);
+                s2 = set2.get(k2);
+            } else {// if (set1.get(k1)>set2.get(k2))
+                buffer[pos++] = s2;
+                ++k2;
+                if (k2 >= length2) {
+                    set1.position(k1);
+                    set1.get(buffer, pos, length1 - k1);
+                    return pos + length1 - k1;
+                }
+                s2 = set2.get(k2);
+            }
+        }
+        // return pos;
+    }
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/ImmutableRoaringArray.java

@@ -0,0 +1,463 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap.buffer;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.LongBuffer;
+import java.nio.ShortBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+
+
+/**
+ * This is the underlying data structure for an ImmutableRoaringBitmap. This class is not meant for
+ * end-users.
+ */
+public final class ImmutableRoaringArray implements PointableRoaringArray {
+
+    protected static final short SERIAL_COOKIE = MutableRoaringArray.SERIAL_COOKIE;
+    protected static final short SERIAL_COOKIE_NO_RUNCONTAINER =
+            MutableRoaringArray.SERIAL_COOKIE_NO_RUNCONTAINER;
+    private final static int startofrunbitmap = 4; // if there is a runcontainer bitmap
+
+    ByteBuffer buffer;
+    int size;
+
+    /**
+     * Create an array based on a previously serialized ByteBuffer. The input ByteBuffer is
+     * effectively copied (with the slice operation) so you should expect the provided ByteBuffer to
+     * remain unchanged.
+     *
+     * @param bbf The source ByteBuffer
+     */
+    protected ImmutableRoaringArray(ByteBuffer bbf) {
+        buffer = bbf.slice();
+        buffer.order(ByteOrder.LITTLE_ENDIAN);
+        final int cookie = buffer.getInt(0);
+        if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) {
+            throw new RuntimeException("I failed to find one of the right cookies. " + cookie);
+        }
+        boolean hasRunContainers = (cookie & 0xFFFF) == SERIAL_COOKIE;
+        this.size = hasRunContainers ? (cookie >>> 16) + 1 : buffer.getInt(4);
+        int theLimit = size > 0 ? computeSerializedSizeInBytes() : headerSize(hasRunContainers);
+        buffer.limit(theLimit);
+    }
+
+    @Override
+    public int advanceUntil(short x, int pos) {
+        int lower = pos + 1;
+
+        // special handling for a possibly common sequential case
+        if (lower >= size || getKey(lower) >= BufferUtil.toIntUnsigned(x)) {
+            return lower;
+        }
+
+        int spansize = 1; // could set larger
+        // bootstrap an upper limit
+
+        while (lower + spansize < size && getKey(lower + spansize) < BufferUtil.toIntUnsigned(x)) {
+            spansize *= 2; // hoping for compiler will reduce to shift
+        }
+        int upper = (lower + spansize < size) ? lower + spansize : size - 1;
+
+        // maybe we are lucky (could be common case when the seek ahead
+        // expected to be small and sequential will otherwise make us look bad)
+        if (getKey(upper) == BufferUtil.toIntUnsigned(x)) {
+            return upper;
+        }
+
+        if (getKey(upper) < BufferUtil.toIntUnsigned(x)) {// means array has no item key >= x
+            return size;
+        }
+
+        // we know that the next-smallest span was too small
+        lower += (spansize / 2);
+
+        // else begin binary search
+        // invariant: array[lower]<x && array[upper]>x
+        while (lower + 1 != upper) {
+            int mid = (lower + upper) / 2;
+            if (getKey(mid) == BufferUtil.toIntUnsigned(x)) {
+                return mid;
+            } else if (getKey(mid) < BufferUtil.toIntUnsigned(x)) {
+                lower = mid;
+            } else {
+                upper = mid;
+            }
+        }
+        return upper;
+    }
+
+    private int branchyUnsignedBinarySearch(final short k) {
+        int low = 0;
+        int high = this.size - 1;
+        final int ikey = BufferUtil.toIntUnsigned(k);
+        while (low <= high) {
+            final int middleIndex = (low + high) >>> 1;
+            final int middleValue = getKey(middleIndex);
+            if (middleValue < ikey) {
+                low = middleIndex + 1;
+            } else if (middleValue > ikey) {
+                high = middleIndex - 1;
+            } else {
+                return middleIndex;
+            }
+        }
+        return -(low + 1);
+    }
+
+
+    @Override
+    public ImmutableRoaringArray clone() {
+        ImmutableRoaringArray sa;
+        try {
+            sa = (ImmutableRoaringArray) super.clone();
+        } catch (CloneNotSupportedException e) {
+            return null;// should never happen
+        }
+        return sa;
+    }
+
+
+    private int computeSerializedSizeInBytes() {
+        if (this.size == 0) {
+            return headerSize(hasRunCompression());
+        }
+        int CardinalityOfLastContainer = getCardinality(this.size - 1);
+        int PositionOfLastContainer = getOffsetContainer(this.size - 1);
+        int SizeOfLastContainer;
+        boolean hasrun = hasRunCompression();
+        if (isRunContainer(this.size - 1, hasrun)) {
+            int nbrruns = BufferUtil.toIntUnsigned(buffer.getShort(PositionOfLastContainer));
+            SizeOfLastContainer = BufferUtil.getSizeInBytesFromCardinalityEtc(0, nbrruns, true);
+        } else {
+            SizeOfLastContainer =
+                    BufferUtil.getSizeInBytesFromCardinalityEtc(CardinalityOfLastContainer, 0, false);
+        }
+        return SizeOfLastContainer + PositionOfLastContainer;
+    }
+
+    @Override
+    public int getCardinality(int k) {
+        if ((k < 0) || (k >= this.size)) {
+            throw new IllegalArgumentException(
+                    "out of range container index: " + k + " (report as a bug)");
+        }
+        return BufferUtil.toIntUnsigned(buffer.getShort(this.getStartOfKeys() + 4 * k + 2)) + 1;
+    }
+
+
+    // involves a binary search
+    @Override
+    public MappeableContainer getContainer(short x) {
+        final int i = unsignedBinarySearch(x);
+        if (i < 0) {
+            return null;
+        }
+        return getContainerAtIndex(i);
+    }
+
+    @Override
+    public MappeableContainer getContainerAtIndex(int i) {
+        int cardinality = getCardinality(i);
+        final boolean isBitmap = cardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE; // if not a
+        // runcontainer
+        ByteBuffer tmp = buffer.duplicate();// sad but ByteBuffer is not thread-safe so it is either a
+        // duplicate or a lock
+        // note that tmp will indeed be garbage-collected some time after the end of this function
+        tmp.order(buffer.order());
+        tmp.position(getOffsetContainer(i));
+        boolean hasrun = hasRunCompression();
+        if (isRunContainer(i, hasrun)) {
+            // first, we have a short giving the number of runs
+            int nbrruns = BufferUtil.toIntUnsigned(tmp.getShort());
+            final ShortBuffer shortArray = tmp.asShortBuffer();
+            shortArray.limit(2 * nbrruns);
+            return new MappeableRunContainer(shortArray, nbrruns);
+        }
+        if (isBitmap) {
+            final LongBuffer bitmapArray = tmp.asLongBuffer();
+            bitmapArray.limit(MappeableBitmapContainer.MAX_CAPACITY / 64);
+            return new MappeableBitmapContainer(bitmapArray, cardinality);
+        } else {
+            final ShortBuffer shortArray = tmp.asShortBuffer();
+            shortArray.limit(cardinality);
+            return new MappeableArrayContainer(shortArray, cardinality);
+        }
+    }
+
+
+    @Override
+    public MappeableContainerPointer getContainerPointer() {
+        return getContainerPointer(0);
+    }
+
+    @Override
+    public MappeableContainerPointer getContainerPointer(final int startIndex) {
+        final boolean hasrun = isEmpty() ? false : hasRunCompression();
+        return new MappeableContainerPointer() {
+            int k = startIndex;
+
+            @Override
+            public void advance() {
+                ++k;
+            }
+
+
+            @Override
+            public MappeableContainerPointer clone() {
+                try {
+                    return (MappeableContainerPointer) super.clone();
+                } catch (CloneNotSupportedException e) {
+                    return null;// will not happen
+                }
+            }
+
+            @Override
+            public int compareTo(MappeableContainerPointer o) {
+                if (key() != o.key()) {
+                    return BufferUtil.toIntUnsigned(key()) - BufferUtil.toIntUnsigned(o.key());
+                }
+                return o.getCardinality() - this.getCardinality();
+            }
+
+            @Override
+            public int getCardinality() {
+                return ImmutableRoaringArray.this.getCardinality(k);
+            }
+
+            @Override
+            public MappeableContainer getContainer() {
+                if (k >= ImmutableRoaringArray.this.size) {
+                    return null;
+                }
+                return ImmutableRoaringArray.this.getContainerAtIndex(k);
+            }
+
+
+            @Override
+            public int getSizeInBytes() {
+                // might be a tad expensive
+                if (ImmutableRoaringArray.this.isRunContainer(k, hasrun)) {
+                    int pos = getOffsetContainer(k);
+                    int nbrruns = BufferUtil.toIntUnsigned(buffer.getShort(pos));
+                    return BufferUtil.getSizeInBytesFromCardinalityEtc(0, nbrruns, true);
+                } else {
+                    int CardinalityOfLastContainer = getCardinality();
+                    return BufferUtil.getSizeInBytesFromCardinalityEtc(CardinalityOfLastContainer, 0, false);
+                }
+            }
+
+            @Override
+            public boolean hasContainer() {
+                return 0 <= k & k < ImmutableRoaringArray.this.size;
+            }
+
+            @Override
+            public boolean isBitmapContainer() {
+                if (ImmutableRoaringArray.this.isRunContainer(k, hasrun)) {
+                    return false;
+                }
+                return getCardinality() > MappeableArrayContainer.DEFAULT_MAX_SIZE;
+            }
+
+            @Override
+            public boolean isRunContainer() {
+                return ImmutableRoaringArray.this.isRunContainer(k, hasrun);
+            }
+
+            @Override
+            public short key() {
+                return ImmutableRoaringArray.this.getKeyAtIndex(k);
+
+            }
+
+
+            @Override
+            public void previous() {
+                --k;
+            }
+        };
+    }
+
+    // involves a binary search
+    @Override
+    public int getIndex(short x) {
+        return unsignedBinarySearch(x);
+    }
+
+    private int getKey(int k) {
+        return BufferUtil.toIntUnsigned(buffer.getShort(getStartOfKeys() + 4 * k));
+    }
+
+    @Override
+    public short getKeyAtIndex(int i) {
+        return buffer.getShort(4 * i + getStartOfKeys());
+    }
+
+    private int getOffsetContainer(int k) {
+        if ((k < 0) || (k >= this.size)) {
+            throw new IllegalArgumentException(
+                    "out of range container index: " + k + " (report as a bug)");
+        }
+        if (hasRunCompression()) { // account for size of runcontainer bitmap
+            if (this.size < MutableRoaringArray.NO_OFFSET_THRESHOLD) {
+                // we do it the hard way
+                return getOffsetContainerSlow(k);
+            }
+            return buffer.getInt(4 + 4 * this.size + ((this.size + 7) / 8) + 4 * k);
+        } else {
+            return buffer.getInt(4 + 4 + 4 * this.size + 4 * k);
+        }
+    }
+
+
+    private int getOffsetContainerSlow(int k) {
+        boolean hasrun = hasRunCompression();
+        int pos = this.headerSize(hasrun);
+        for (int z = 0; z < k; ++z) {
+            if (isRunContainer(z, hasrun)) {
+                int nbrruns = BufferUtil.toIntUnsigned(buffer.getShort(pos));
+                int SizeOfLastContainer = BufferUtil.getSizeInBytesFromCardinalityEtc(0, nbrruns, true);
+                pos += SizeOfLastContainer;
+            } else {
+                int CardinalityOfLastContainer = this.getCardinality(z);
+                int SizeOfLastContainer =
+                        BufferUtil.getSizeInBytesFromCardinalityEtc(CardinalityOfLastContainer, 0, false);
+                pos += SizeOfLastContainer;
+            }
+        }
+        return pos;
+    }
+
+    private int getStartOfKeys() {
+        if (hasRunCompression()) { // info is in the buffer
+            return 4 + ((this.size + 7) / 8);
+        } else {
+            return 8;
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        MappeableContainerPointer cp = this.getContainerPointer();
+        int hashvalue = 0;
+        while (cp.hasContainer()) {
+            int th = cp.key() * 0xF0F0F0 + cp.getContainer().hashCode();
+            hashvalue = 31 * hashvalue + th;
+            cp.advance();
+        }
+        return hashvalue;
+    }
+
+    @Override
+    public boolean hasRunCompression() {
+        return (buffer.getInt(0) & 0xFFFF) == SERIAL_COOKIE;
+    }
+
+    // hasrun should be equal to hasRunCompression()
+    protected int headerSize(boolean hasrun) {
+        if (hasrun) {
+            if (size < MutableRoaringArray.NO_OFFSET_THRESHOLD) {// for small bitmaps, we omit the offsets
+                return 4 + (size + 7) / 8 + 4 * size;
+            }
+            return 4 + (size + 7) / 8 + 8 * size;// - 4 because we pack the size with the cookie
+        } else {
+            return 4 + 4 + 8 * size;
+        }
+    }
+
+    // starts with binary search and finishes with a sequential search
+  /*private int hybridUnsignedBinarySearch(final short k) {
+    int low = 0;
+    int high = this.size - 1;
+    final int ikey = BufferUtil.toIntUnsigned(k);
+    // 32 in the next line matches the size of a cache line
+    while (low + 16 <= high) {
+      final int middleIndex = (low + high) >>> 1;
+      final int middleValue = getKey(middleIndex);
+      if (middleValue < ikey) {
+        low = middleIndex + 1;
+      } else if (middleValue > ikey) {
+        high = middleIndex - 1;
+      } else {
+        return middleIndex;
+      }
+    }
+    // we finish the job with a sequential search
+    int x = low;
+    for (; x <= high; ++x) {
+      final int val = getKey(x);
+      if (val >= ikey) {
+        if (val == ikey) {
+          return x;
+        }
+        break;
+      }
+    }
+    return -(x + 1);
+  }*/
+
+    /**
+     * Returns true if this bitmap is empty.
+     *
+     * @return true if empty
+     */
+    public boolean isEmpty() {
+        return this.size == 0;
+    }
+
+    // hasrun should be initialized with hasRunCompression()
+    private boolean isRunContainer(int i, boolean hasrun) {
+        if (hasrun) { // info is in the buffer
+            int j = buffer.get(startofrunbitmap + i / 8);
+            int mask = 1 << (i % 8);
+            return (j & mask) != 0;
+        } else {
+            return false;
+        }
+    }
+
+    /**
+     * Serialize.
+     * <p>
+     * The current bitmap is not modified.
+     *
+     * @param out the DataOutput stream
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    @Override
+    public void serialize(DataOutput out) throws IOException {
+        if (buffer.hasArray()) {
+            out.write(buffer.array(), buffer.arrayOffset(), buffer.limit());
+        } else {
+            ByteBuffer tmp = buffer.duplicate();
+            tmp.position(0);
+            WritableByteChannel channel = Channels.newChannel((OutputStream) out);
+            channel.write(tmp);
+        }
+    }
+
+    /**
+     * @return the size that the data structure occupies on disk
+     */
+    @Override
+    public int serializedSizeInBytes() {
+        return buffer.limit();
+    }
+
+    @Override
+    public int size() {
+        return this.size;
+    }
+
+    private int unsignedBinarySearch(short k) {
+        return branchyUnsignedBinarySearch(k);
+    }
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableContainer.java

@@ -0,0 +1,759 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap.buffer;
+
+import com.fr.third.bitmap.roaringbitmap.Container;
+import com.fr.third.bitmap.roaringbitmap.IntConsumer;
+import com.fr.third.bitmap.roaringbitmap.PeekableShortIterator;
+import com.fr.third.bitmap.roaringbitmap.ShortIterator;
+
+import java.io.DataOutput;
+import java.io.Externalizable;
+import java.io.IOException;
+
+/**
+ * Base container class. This class is similar to Container but meant to be used
+ * with memory mapping.
+ */
+public abstract class MappeableContainer implements Iterable<Short>, Cloneable, Externalizable {
+    /**
+     * Name of the various possible containers
+     */
+    public static String ContainerNames[] = {"mappeablebitmap", "mappeablearray", "mappeablerun"};
+
+    /**
+     * Create a container initialized with a range of consecutive values
+     *
+     * @param start first index
+     * @param last  last index (range is exclusive)
+     * @return a new container initialized with the specified values
+     */
+    public static MappeableContainer rangeOfOnes(final int start, final int last) {
+        final int sizeAsArrayContainer = MappeableArrayContainer.serializedSizeInBytes(last - start);
+        final int sizeAsRunContainer = MappeableRunContainer.serializedSizeInBytes(1);
+        MappeableContainer answer = sizeAsRunContainer < sizeAsArrayContainer
+                ? new MappeableRunContainer() : new MappeableArrayContainer();
+        answer = answer.iadd(start, last);
+        return answer;
+    }
+
+    /**
+     * Return a new container with all shorts in [begin,end) added using an unsigned interpretation.
+     *
+     * @param begin start of range (inclusive)
+     * @param end   end of range (exclusive)
+     * @return the new container
+     */
+    public abstract MappeableContainer add(int begin, int end);
+
+    /**
+     * Add a short to the container. May generate a new container.
+     *
+     * @param x short to be added
+     * @return the new container
+     */
+    public abstract MappeableContainer add(short x);
+
+    /**
+     * Computes the bitwise AND of this container with another (intersection). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer and(MappeableArrayContainer x);
+
+    /**
+     * Computes the bitwise AND of this container with another (intersection). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer and(MappeableBitmapContainer x);
+
+    protected MappeableContainer and(MappeableContainer x) {
+        if (x instanceof MappeableArrayContainer) {
+            return and((MappeableArrayContainer) x);
+        } else if (x instanceof MappeableRunContainer) {
+            return and((MappeableRunContainer) x);
+        }
+        return and((MappeableBitmapContainer) x);
+
+    }
+
+    protected abstract int andCardinality(MappeableArrayContainer x);
+
+    protected abstract int andCardinality(MappeableBitmapContainer x);
+
+    protected abstract int andCardinality(MappeableRunContainer x);
+
+    /**
+     * Computes the bitwise AND of this container with another (intersection). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public int andCardinality(MappeableContainer x) {
+        if (this.getCardinality() == 0) {
+            return 0;
+        } else if (x.getCardinality() == 0) {
+            return 0;
+        } else {
+            if (x instanceof MappeableArrayContainer) {
+                return andCardinality((MappeableArrayContainer) x);
+            } else if (x instanceof MappeableBitmapContainer) {
+                return andCardinality((MappeableBitmapContainer) x);
+            }
+            return andCardinality((MappeableRunContainer) x);
+        }
+    }
+
+    /**
+     * Computes the bitwise AND of this container with another (intersection). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+
+    public abstract MappeableContainer and(MappeableRunContainer x);
+
+    /**
+     * Computes the bitwise ANDNOT of this container with another (difference). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer andNot(MappeableArrayContainer x);
+
+    /**
+     * Computes the bitwise ANDNOT of this container with another (difference). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer andNot(MappeableBitmapContainer x);
+
+    protected MappeableContainer andNot(MappeableContainer x) {
+        if (x instanceof MappeableArrayContainer) {
+            return andNot((MappeableArrayContainer) x);
+        } else if (x instanceof MappeableRunContainer) {
+            return andNot((MappeableRunContainer) x);
+        }
+
+        return andNot((MappeableBitmapContainer) x);
+    }
+
+    /**
+     * Computes the bitwise ANDNOT of this container with another (difference). This container as well
+     * as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+
+    public abstract MappeableContainer andNot(MappeableRunContainer x);
+
+    /**
+     * Empties the container
+     */
+    public abstract void clear();
+
+    @Override
+    public abstract MappeableContainer clone();
+
+    /**
+     * Checks whether the contain contains the provided value
+     *
+     * @param x value to check
+     * @return whether the value is in the container
+     */
+    public abstract boolean contains(short x);
+
+    /**
+     * Fill the least significant 16 bits of the integer array, starting at index index, with the
+     * short values from this container. The caller is responsible to allocate enough room. The most
+     * significant 16 bits of each integer are given by the most significant bits of the provided
+     * mask.
+     *
+     * @param x    provided array
+     * @param i    starting index
+     * @param mask indicates most significant bits
+     */
+    public abstract void fillLeastSignificant16bits(int[] x, int i, int mask);
+
+    /**
+     * Add a short to the container if it is not present, otherwise remove it. May generate a new
+     * container.
+     *
+     * @param x short to be added
+     * @return the new container
+     */
+    public abstract MappeableContainer flip(short x);
+
+    /**
+     * Size of the underlying array
+     *
+     * @return size in bytes
+     */
+    protected abstract int getArraySizeInBytes();
+
+    /**
+     * Computes the distinct number of short values in the container. Can be expected to run in
+     * constant time.
+     *
+     * @return the cardinality
+     */
+    public abstract int getCardinality();
+
+    /**
+     * Get the name of this container.
+     *
+     * @return name of the container
+     */
+    public String getContainerName() {
+        if (this instanceof MappeableBitmapContainer) {
+            return ContainerNames[0];
+        } else if (this instanceof MappeableArrayContainer) {
+            return ContainerNames[1];
+        } else {
+            return ContainerNames[2];
+        }
+    }
+
+    /**
+     * Iterator to visit the short values in the container in descending order.
+     *
+     * @return iterator
+     */
+    public abstract ShortIterator getReverseShortIterator();
+
+
+    /**
+     * Iterator to visit the short values in the container in ascending order.
+     *
+     * @return iterator
+     */
+    public abstract PeekableShortIterator getShortIterator();
+
+
+    /**
+     * Iterate through the values of this container and pass them
+     * along to the IntConsumer, using msb as the 16 most significant bits.
+     *
+     * @param msb 16 most significant bits
+     * @param ic  consumer
+     */
+    public abstract void forEach(short msb, IntConsumer ic);
+
+    /**
+     * Computes an estimate of the memory usage of this container. The estimate is not meant to be
+     * exact.
+     *
+     * @return estimated memory usage in bytes
+     */
+    public abstract int getSizeInBytes();
+
+    /**
+     * Add all shorts in [begin,end) using an unsigned interpretation. May generate a new container.
+     *
+     * @param begin start of range (inclusive)
+     * @param end   end of range (exclusive)
+     * @return the new container
+     */
+    public abstract MappeableContainer iadd(int begin, int end);
+
+    /**
+     * Computes the in-place bitwise AND of this container with another (intersection). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer iand(MappeableArrayContainer x);
+
+    /**
+     * Computes the in-place bitwise AND of this container with another (intersection). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer iand(MappeableBitmapContainer x);
+
+
+    protected MappeableContainer iand(MappeableContainer x) {
+        if (x instanceof MappeableArrayContainer) {
+            return iand((MappeableArrayContainer) x);
+        } else if (x instanceof MappeableRunContainer) {
+            return iand((MappeableRunContainer) x);
+        }
+
+        return iand((MappeableBitmapContainer) x);
+
+    }
+
+    /**
+     * Computes the in-place bitwise AND of this container with another (intersection). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+
+    public abstract MappeableContainer iand(MappeableRunContainer x);
+
+    /**
+     * Computes the in-place bitwise ANDNOT of this container with another (difference). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer iandNot(MappeableArrayContainer x);
+
+    /**
+     * Computes the in-place bitwise ANDNOT of this container with another (difference). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer iandNot(MappeableBitmapContainer x);
+
+    protected MappeableContainer iandNot(MappeableContainer x) {
+        if (x instanceof MappeableArrayContainer) {
+            return iandNot((MappeableArrayContainer) x);
+        } else if (x instanceof MappeableRunContainer) {
+            return iandNot((MappeableRunContainer) x);
+        }
+
+        return iandNot((MappeableBitmapContainer) x);
+    }
+
+    /**
+     * Computes the in-place bitwise ANDNOT of this container with another (difference). The current
+     * container is generally modified, whereas the provided container (x) is unaffected. May generate
+     * a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+
+    public abstract MappeableContainer iandNot(MappeableRunContainer x);
+
+
+    /**
+     * Computes the in-place bitwise NOT of this container (complement). Only those bits within the
+     * range are affected. The current container is generally modified. May generate a new container.
+     *
+     * @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
+     * @param rangeEnd   ending of range (exclusive)
+     * @return (partially) completmented container
+     */
+    public abstract MappeableContainer inot(int rangeStart, int rangeEnd);
+
+
+    /**
+     * Returns true if the current container intersects the other container.
+     *
+     * @param x other container
+     * @return whether they intersect
+     */
+    public abstract boolean intersects(MappeableArrayContainer x);
+
+    /**
+     * Returns true if the current container intersects the other container.
+     *
+     * @param x other container
+     * @return whether they intersect
+     */
+    public abstract boolean intersects(MappeableBitmapContainer x);
+
+    /**
+     * Returns true if the current container intersects the other container.
+     *
+     * @param x other container
+     * @return whether they intersect
+     */
+    public boolean intersects(MappeableContainer x) {
+        if (x instanceof MappeableArrayContainer) {
+            return intersects((MappeableArrayContainer) x);
+        } else if (x instanceof MappeableBitmapContainer) {
+            return intersects((MappeableBitmapContainer) x);
+        }
+        return intersects((MappeableRunContainer) x);
+    }
+
+    /**
+     * Returns true if the current container intersects the other container.
+     *
+     * @param x other container
+     * @return whether they intersect
+     */
+    public abstract boolean intersects(MappeableRunContainer x);
+
+    /**
+     * Computes the in-place bitwise OR of this container with another (union). The current container
+     * is generally modified, whereas the provided container (x) is unaffected. May generate a new
+     * container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer ior(MappeableArrayContainer x);
+
+    /**
+     * Computes the in-place bitwise OR of this container with another (union). The current container
+     * is generally modified, whereas the provided container (x) is unaffected. May generate a new
+     * container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer ior(MappeableBitmapContainer x);
+
+    protected MappeableContainer ior(MappeableContainer x) {
+        if (x instanceof MappeableArrayContainer) {
+            return ior((MappeableArrayContainer) x);
+        } else if (x instanceof MappeableRunContainer) {
+            return ior((MappeableRunContainer) x);
+        }
+
+        return ior((MappeableBitmapContainer) x);
+    }
+
+    /**
+     * Computes the in-place bitwise OR of this container with another (union). The current container
+     * is generally modified, whereas the provided container (x) is unaffected. May generate a new
+     * container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+
+    public abstract MappeableContainer ior(MappeableRunContainer x);
+
+    /**
+     * Remove shorts in [begin,end) using an unsigned interpretation. May generate a new container.
+     *
+     * @param begin start of range (inclusive)
+     * @param end   end of range (exclusive)
+     * @return the new container
+     */
+    public abstract MappeableContainer iremove(int begin, int end);
+
+    protected abstract boolean isArrayBacked();
+
+
+    /**
+     * Computes the in-place bitwise XOR of this container with another (symmetric difference). The
+     * current container is generally modified, whereas the provided container (x) is unaffected. May
+     * generate a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer ixor(MappeableArrayContainer x);
+
+    /**
+     * Computes the in-place bitwise XOR of this container with another (symmetric difference). The
+     * current container is generally modified, whereas the provided container (x) is unaffected. May
+     * generate a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer ixor(MappeableBitmapContainer x);
+
+    protected MappeableContainer ixor(MappeableContainer x) {
+        if (x instanceof MappeableArrayContainer) {
+            return ixor((MappeableArrayContainer) x);
+        } else if (x instanceof MappeableRunContainer) {
+            return ixor((MappeableRunContainer) x);
+        }
+
+        return ixor((MappeableBitmapContainer) x);
+
+    }
+
+
+    /**
+     * Computes the in-place bitwise XOR of this container with another (symmetric difference). The
+     * current container is generally modified, whereas the provided container (x) is unaffected. May
+     * generate a new container.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+
+    public abstract MappeableContainer ixor(MappeableRunContainer x);
+
+
+    /**
+     * Computes the in-place bitwise OR of this container with another (union). The current container
+     * is generally modified, whereas the provided container (x) is unaffected. May generate a new
+     * container. The resulting container may not track its cardinality correctly. The resulting
+     * container may not track its cardinality correctly. This can be fixed as follows:
+     * if(c.getCardinality()&lt;0) ((MappeableBitmapContainer)c).computeCardinality();
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public MappeableContainer lazyIOR(MappeableContainer x) {
+        if (this instanceof MappeableArrayContainer) {
+            if (x instanceof MappeableArrayContainer) {
+                return ((MappeableArrayContainer) this).lazyor((MappeableArrayContainer) x);
+            } else if (x instanceof MappeableBitmapContainer) {
+                return ((MappeableBitmapContainer) x).lazyor((MappeableArrayContainer) this);
+            }
+            return ((MappeableRunContainer) x).lazyor((MappeableArrayContainer) this);
+        } else if (this instanceof MappeableRunContainer) {
+            if (x instanceof MappeableArrayContainer) {
+                return ((MappeableRunContainer) this).ilazyor((MappeableArrayContainer) x);
+            } else if (x instanceof MappeableBitmapContainer) {
+                return ((MappeableBitmapContainer) x).lazyor((MappeableRunContainer) this);
+            }
+            return ior((MappeableRunContainer) x);
+        } else {
+            if (x instanceof MappeableArrayContainer) {
+                return ((MappeableBitmapContainer) this).ilazyor((MappeableArrayContainer) x);
+            } else if (x instanceof MappeableBitmapContainer) {
+                return ((MappeableBitmapContainer) this).ilazyor((MappeableBitmapContainer) x);
+            }
+            return ((MappeableBitmapContainer) this).ilazyor((MappeableRunContainer) x);
+        }
+    }
+
+    /**
+     * Computes the bitwise OR of this container with another (union). This container as well as the
+     * provided container are left unaffected. The resulting container may not track its cardinality
+     * correctly. This can be fixed as follows: if(c.getCardinality()&lt;0)
+     * ((MappeableBitmapContainer)c).computeCardinality();
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public MappeableContainer lazyOR(MappeableContainer x) {
+        if (this instanceof MappeableArrayContainer) {
+            if (x instanceof MappeableArrayContainer) {
+                return ((MappeableArrayContainer) this).lazyor((MappeableArrayContainer) x);
+            } else if (x instanceof MappeableBitmapContainer) {
+                return ((MappeableBitmapContainer) x).lazyor((MappeableArrayContainer) this);
+            }
+            return ((MappeableRunContainer) x).lazyor((MappeableArrayContainer) this);
+        } else if (this instanceof MappeableRunContainer) {
+            if (x instanceof MappeableArrayContainer) {
+                return ((MappeableRunContainer) this).lazyor((MappeableArrayContainer) x);
+            } else if (x instanceof MappeableBitmapContainer) {
+                return ((MappeableBitmapContainer) x).lazyor((MappeableRunContainer) this);
+            }
+            return or((MappeableRunContainer) x);
+        } else {
+            if (x instanceof MappeableArrayContainer) {
+                return ((MappeableBitmapContainer) this).lazyor((MappeableArrayContainer) x);
+            } else if (x instanceof MappeableBitmapContainer) {
+                return ((MappeableBitmapContainer) this).lazyor((MappeableBitmapContainer) x);
+            }
+            return ((MappeableBitmapContainer) this).lazyor((MappeableRunContainer) x);
+        }
+    }
+
+    /**
+     * Create a new MappeableContainer containing at most maxcardinality integers.
+     *
+     * @param maxcardinality maximal cardinality
+     * @return a new bitmap with cardinality no more than maxcardinality
+     */
+    public abstract MappeableContainer limit(int maxcardinality);
+
+    /**
+     * Computes the bitwise NOT of this container (complement). Only those bits within the range are
+     * affected. The current container is left unaffected.
+     *
+     * @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
+     * @param rangeEnd   ending of range (exclusive)
+     * @return (partially) completmented container
+     */
+    public abstract MappeableContainer not(int rangeStart, int rangeEnd);
+
+    abstract int numberOfRuns();
+
+    /**
+     * Computes the bitwise OR of this container with another (union). This container as well as the
+     * provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer or(MappeableArrayContainer x);
+
+
+    /**
+     * Computes the bitwise OR of this container with another (union). This container as well as the
+     * provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer or(MappeableBitmapContainer x);
+
+
+    protected MappeableContainer or(MappeableContainer x) {
+        if (x instanceof MappeableArrayContainer) {
+            return or((MappeableArrayContainer) x);
+        } else if (x instanceof MappeableRunContainer) {
+            return or((MappeableRunContainer) x);
+        }
+
+        return or((MappeableBitmapContainer) x);
+    }
+
+    /**
+     * Computes the bitwise OR of this container with another (union). This container as well as the
+     * provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+
+    public abstract MappeableContainer or(MappeableRunContainer x);
+
+    /**
+     * Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be
+     * GetCardinality()).
+     *
+     * @param lowbits upper limit
+     * @return the rank
+     */
+    public abstract int rank(short lowbits);
+
+    /**
+     * Return a new container with all shorts in [begin,end) remove using an unsigned interpretation.
+     *
+     * @param begin start of range (inclusive)
+     * @param end   end of range (exclusive)
+     * @return the new container
+     */
+    public abstract MappeableContainer remove(int begin, int end);
+
+    /**
+     * Remove the short from this container. May create a new container.
+     *
+     * @param x to be removed
+     * @return New container
+     */
+    public abstract MappeableContainer remove(short x);
+
+    /**
+     * The output of a lazyOR or lazyIOR might be an invalid container, this should be called on it.
+     *
+     * @return a new valid container
+     */
+    public abstract MappeableContainer repairAfterLazy();
+
+    /**
+     * Convert to MappeableRunContainers, when the result is smaller. Overridden by
+     * MappeableRunContainer to possibly switch from MappeableRunContainer to a smaller alternative.
+     *
+     * @return the new container
+     */
+    public abstract MappeableContainer runOptimize();
+
+
+    /**
+     * Return the jth value
+     *
+     * @param j index of the value
+     * @return the value
+     */
+    public abstract short select(int j);
+
+
+    /**
+     * Report the number of bytes required to serialize this container.
+     *
+     * @return the size in bytes
+     */
+    public abstract int serializedSizeInBytes();
+
+
+    /**
+     * Convert to a non-mappeable container.
+     *
+     * @return the non-mappeable container
+     */
+    public abstract Container toContainer();
+
+    /**
+     * If possible, recover wasted memory.
+     */
+    public abstract void trim();
+
+
+    /**
+     * Write just the underlying array.
+     *
+     * @param out output stream
+     * @throws IOException in case of failure
+     */
+    protected abstract void writeArray(DataOutput out) throws IOException;
+
+    /**
+     * Computes the bitwise XOR of this container with another (symmetric difference). This container
+     * as well as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer xor(MappeableArrayContainer x);
+
+    /**
+     * Computes the bitwise XOR of this container with another (symmetric difference). This container
+     * as well as the provided container are left unaffected.
+     *
+     * @param x other container
+     * @return aggregated container
+     */
+    public abstract MappeableContainer xor(MappeableBitmapContainer x);
+
+    protected MappeableContainer xor(MappeableContainer x) {
+        if (x instanceof MappeableArrayContainer) {
+            return xor((MappeableArrayContainer) x);
+        } else if (x instanceof MappeableRunContainer) {
+            return xor((MappeableRunContainer) x);
+        }
+
+        return xor((MappeableBitmapContainer) x);
+
+    }
+
+    /**
+     * Computes the bitwise XOR of this container with another (symmetric difference). This container
+     * as well as the provided container are left unaffected.
+     *
+     * @param x other parameter
+     * @return aggregated container
+     */
+    public abstract MappeableContainer xor(MappeableRunContainer x);
+
+    /**
+     * Convert the current container to a BitmapContainer, if a conversion is needed.
+     * If the container is already a bitmap, the container is returned unchanged.
+     *
+     * @return a bitmap container
+     */
+    public abstract MappeableBitmapContainer toBitmapContainer();
+
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableContainerPointer.java

@@ -0,0 +1,80 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap.buffer;
+
+/**
+ * This interface allows you to iterate over the containers in a roaring bitmap.
+ */
+public interface MappeableContainerPointer
+        extends Comparable<MappeableContainerPointer>, Cloneable {
+    /**
+     * Move to the next container
+     */
+    void advance();
+
+    /**
+     * Create a copy
+     *
+     * @return return a clone of this pointer
+     */
+    MappeableContainerPointer clone();
+
+    /**
+     * Returns the cardinality of the current container. Can be faster than loading the container
+     * first.
+     *
+     * @return cardinality of the current container
+     */
+    int getCardinality();
+
+    /**
+     * This method can be used to check whether there is current a valid container as it returns null
+     * when there is not.
+     *
+     * @return null or the current container
+     */
+    MappeableContainer getContainer();
+
+    /**
+     * Get the size in bytes of the container. Used for sorting.
+     *
+     * @return the size in bytes
+     */
+    int getSizeInBytes();
+
+    /**
+     * @return whether there is a container at the current position
+     */
+    boolean hasContainer();
+
+    /**
+     * Returns true if it is a bitmap container (MappeableBitmapContainer).
+     *
+     * @return boolean indicated if it is a bitmap container
+     */
+    public boolean isBitmapContainer();
+
+    /**
+     * Returns true if it is a run container (MappeableRunContainer).
+     *
+     * @return boolean indicated if it is a run container
+     */
+    boolean isRunContainer();
+
+    /**
+     * The key is a 16-bit integer that indicates the position of the container in the roaring bitmap.
+     * To be interpreted as an unsigned integer.
+     *
+     * @return the key
+     */
+    short key();
+
+    /**
+     * Move to the previous container
+     */
+    void previous();
+
+
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MutableRoaringArray.java

@@ -0,0 +1,573 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap.buffer;
+
+
+import com.fr.third.bitmap.roaringbitmap.Util;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.LongBuffer;
+import java.nio.ShortBuffer;
+import java.util.Arrays;
+
+
+/**
+ * Specialized array to store the containers used by a RoaringBitmap. This class is similar to
+ * RoaringArray but meant to be used with memory mapping. This is not meant to be
+ * used by end users.
+ * <p>
+ * Objects of this class reside in RAM.
+ */
+public final class MutableRoaringArray implements Cloneable, Externalizable, PointableRoaringArray {
+
+    protected static final int INITIAL_CAPACITY = 4;
+
+    protected static final short SERIAL_COOKIE_NO_RUNCONTAINER = 12346;
+    protected static final short SERIAL_COOKIE = 12347;
+
+    protected static final int NO_OFFSET_THRESHOLD = 4;
+
+    private static final long serialVersionUID = 5L; // TODO: OFK was 4L, not sure
+    protected boolean mayHaveRunContainers = false; // does not necessarily have them, after
+    // optimization
+
+
+    short[] keys = null;
+    MappeableContainer[] values = null;
+
+    int size = 0;
+
+    protected MutableRoaringArray() {
+        this.keys = new short[INITIAL_CAPACITY];
+        this.values = new MappeableContainer[INITIAL_CAPACITY];
+    }
+
+
+    @Override
+    public int advanceUntil(short x, int pos) {
+        int lower = pos + 1;
+
+        // special handling for a possibly common sequential case
+        if (lower >= size || BufferUtil.toIntUnsigned(keys[lower]) >= BufferUtil.toIntUnsigned(x)) {
+            return lower;
+        }
+
+        int spansize = 1; // could set larger
+        // bootstrap an upper limit
+
+        while (lower + spansize < size
+                && BufferUtil.toIntUnsigned(keys[lower + spansize]) < BufferUtil.toIntUnsigned(x)) {
+            spansize *= 2; // hoping for compiler will reduce to shift
+        }
+        int upper = (lower + spansize < size) ? lower + spansize : size - 1;
+
+        // maybe we are lucky (could be common case when the seek ahead
+        // expected to be small and sequential will otherwise make us look bad)
+        if (keys[upper] == x) {
+            return upper;
+        }
+
+        if (BufferUtil.toIntUnsigned(keys[upper]) < BufferUtil.toIntUnsigned(x)) {// means array has no
+            // item key >= x
+            return size;
+        }
+
+        // we know that the next-smallest span was too small
+        lower += (spansize / 2);
+
+        // else begin binary search
+        // invariant: array[lower]<x && array[upper]>x
+        while (lower + 1 != upper) {
+            int mid = (lower + upper) / 2;
+            if (keys[mid] == x) {
+                return mid;
+            } else if (BufferUtil.toIntUnsigned(keys[mid]) < BufferUtil.toIntUnsigned(x)) {
+                lower = mid;
+            } else {
+                upper = mid;
+            }
+        }
+        return upper;
+    }
+
+    protected void append(short key, MappeableContainer value) {
+        extendArray(1);
+        this.keys[this.size] = key;
+        this.values[this.size] = value;
+        this.size++;
+    }
+
+    /**
+     * Append copies of the values AFTER a specified key (may or may not be present) to end.
+     *
+     * @param highLowContainer the other array
+     * @param beforeStart      given key is the largest key that we won't copy
+     */
+    protected void appendCopiesAfter(PointableRoaringArray highLowContainer, short beforeStart) {
+
+        int startLocation = highLowContainer.getIndex(beforeStart);
+        if (startLocation >= 0) {
+            startLocation++;
+        } else {
+            startLocation = -startLocation - 1;
+        }
+        extendArray(highLowContainer.size() - startLocation);
+
+        for (int i = startLocation; i < highLowContainer.size(); ++i) {
+            this.keys[this.size] = highLowContainer.getKeyAtIndex(i);
+            this.values[this.size] = highLowContainer.getContainerAtIndex(i).clone();
+            this.size++;
+        }
+    }
+
+    /**
+     * Append copies of the values from another array, from the start
+     *
+     * @param highLowContainer the other array
+     * @param stoppingKey      any equal or larger key in other array will terminate copying
+     */
+    protected void appendCopiesUntil(PointableRoaringArray highLowContainer, short stoppingKey) {
+        final int stopKey = BufferUtil.toIntUnsigned(stoppingKey);
+        MappeableContainerPointer cp = highLowContainer.getContainerPointer();
+        while (cp.hasContainer()) {
+            if (BufferUtil.toIntUnsigned(cp.key()) >= stopKey) {
+                break;
+            }
+            extendArray(1);
+            this.keys[this.size] = cp.key();
+            this.values[this.size] = cp.getContainer().clone();
+            this.size++;
+            cp.advance();
+        }
+    }
+
+    /**
+     * Append copies of the values from another array
+     *
+     * @param highLowContainer other array
+     * @param startingIndex    starting index in the other array
+     * @param end              last index array in the other array
+     */
+    protected void appendCopy(PointableRoaringArray highLowContainer, int startingIndex, int end) {
+        extendArray(end - startingIndex);
+        for (int i = startingIndex; i < end; ++i) {
+            this.keys[this.size] = highLowContainer.getKeyAtIndex(i);
+            this.values[this.size] = highLowContainer.getContainerAtIndex(i).clone();
+            this.size++;
+        }
+    }
+
+    protected void appendCopy(short key, MappeableContainer value) {
+        extendArray(1);
+        this.keys[this.size] = key;
+        this.values[this.size] = value.clone();
+        this.size++;
+    }
+
+    private int binarySearch(int begin, int end, short key) {
+        return Util.unsignedBinarySearch(keys, begin, end, key);
+    }
+
+    protected void clear() {
+        this.keys = null;
+        this.values = null;
+        this.size = 0;
+    }
+
+    @Override
+    public MutableRoaringArray clone() {
+        MutableRoaringArray sa;
+        try {
+            sa = (MutableRoaringArray) super.clone();
+
+            // OFK: do we need runcontainer bitmap? Guess not, this is just a directory
+            // and each container knows what kind it is.
+            sa.keys = Arrays.copyOf(this.keys, this.size);
+            sa.values = Arrays.copyOf(this.values, this.size);
+            for (int k = 0; k < this.size; ++k) {
+                sa.values[k] = sa.values[k].clone();
+            }
+            sa.size = this.size;
+            return sa;
+
+        } catch (CloneNotSupportedException e) {
+            return null;
+        }
+    }
+
+    protected void copyRange(int begin, int end, int newBegin) {
+        // assuming begin <= end and newBegin < begin
+        final int range = end - begin;
+        System.arraycopy(this.keys, begin, this.keys, newBegin, range);
+        System.arraycopy(this.values, begin, this.values, newBegin, range);
+    }
+
+    /**
+     * Deserialize.
+     *
+     * @param in the DataInput stream
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    public void deserialize(DataInput in) throws IOException {
+        this.clear();
+        // little endian
+        final int cookie = Integer.reverseBytes(in.readInt());
+        if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) {
+            throw new IOException("I failed to find the one of the right cookies.");
+        }
+        this.size = ((cookie & 0xFFFF) == SERIAL_COOKIE) ? (cookie >>> 16) + 1
+                : Integer.reverseBytes(in.readInt());
+        if ((this.keys == null) || (this.keys.length < this.size)) {
+            this.keys = new short[this.size];
+            this.values = new MappeableContainer[this.size];
+        }
+
+        byte[] bitmapOfRunContainers = null;
+        boolean hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE;
+        if (hasrun) {
+            bitmapOfRunContainers = new byte[(size + 7) / 8];
+            in.readFully(bitmapOfRunContainers);
+        }
+
+        final short keys[] = new short[this.size];
+        final int cardinalities[] = new int[this.size];
+        final boolean isBitmap[] = new boolean[this.size];
+        for (int k = 0; k < this.size; ++k) {
+            keys[k] = Short.reverseBytes(in.readShort());
+            cardinalities[k] = 1 + (0xFFFF & Short.reverseBytes(in.readShort()));
+            isBitmap[k] = cardinalities[k] > MappeableArrayContainer.DEFAULT_MAX_SIZE;
+            if (bitmapOfRunContainers != null && (bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) {
+                isBitmap[k] = false;
+            }
+        }
+        if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
+            // skipping the offsets
+            in.skipBytes(this.size * 4);
+        }
+        // Reading the containers
+        for (int k = 0; k < this.size; ++k) {
+            MappeableContainer val;
+            if (isBitmap[k]) {
+                final LongBuffer bitmapArray =
+                        LongBuffer.allocate(MappeableBitmapContainer.MAX_CAPACITY / 64);
+                // little endian
+                for (int l = 0; l < bitmapArray.limit(); ++l) {
+                    bitmapArray.put(l, Long.reverseBytes(in.readLong()));
+                }
+                val = new MappeableBitmapContainer(bitmapArray, cardinalities[k]);
+            } else if (bitmapOfRunContainers != null
+                    && ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0)) {
+                int nbrruns = BufferUtil.toIntUnsigned(Short.reverseBytes(in.readShort()));
+                final ShortBuffer shortArray = ShortBuffer.allocate(2 * nbrruns);
+                for (int l = 0; l < shortArray.limit(); ++l) {
+                    shortArray.put(l, Short.reverseBytes(in.readShort()));
+                }
+                val = new MappeableRunContainer(shortArray, nbrruns);
+            } else {
+                final ShortBuffer shortArray = ShortBuffer.allocate(cardinalities[k]);
+                for (int l = 0; l < shortArray.limit(); ++l) {
+                    shortArray.put(l, Short.reverseBytes(in.readShort()));
+                }
+                val = new MappeableArrayContainer(shortArray, cardinalities[k]);
+            }
+            this.keys[k] = keys[k];
+            this.values[k] = val;
+        }
+    }
+
+    // make sure there is capacity for at least k more elements
+    protected void extendArray(int k) {
+        // size + 1 could overflow
+        if (this.size + k >= this.keys.length) {
+            int newCapacity;
+            if (this.keys.length < 1024) {
+                newCapacity = 2 * (this.size + k);
+            } else {
+                newCapacity = 5 * (this.size + k) / 4;
+            }
+            this.keys = Arrays.copyOf(this.keys, newCapacity);
+            this.values = Arrays.copyOf(this.values, newCapacity);
+        }
+    }
+
+    @Override
+    public int getCardinality(int i) {
+        return getContainerAtIndex(i).getCardinality();
+    }
+
+    // involves a binary search
+    @Override
+    public MappeableContainer getContainer(short x) {
+        final int i = this.binarySearch(0, size, x);
+        if (i < 0) {
+            return null;
+        }
+        return this.values[i];
+    }
+
+    @Override
+    public MappeableContainer getContainerAtIndex(int i) {
+        return this.values[i];
+    }
+
+    @Override
+    public MappeableContainerPointer getContainerPointer() {
+        return getContainerPointer(0);
+    }
+
+    @Override
+    public MappeableContainerPointer getContainerPointer(final int startIndex) {
+        return new MappeableContainerPointer() {
+            int k = startIndex;
+
+            @Override
+            public void advance() {
+                ++k;
+            }
+
+            @Override
+            public MappeableContainerPointer clone() {
+                try {
+                    return (MappeableContainerPointer) super.clone();
+                } catch (CloneNotSupportedException e) {
+                    return null;// will not happen
+                }
+            }
+
+            @Override
+            public int compareTo(MappeableContainerPointer o) {
+                if (key() != o.key()) {
+                    return BufferUtil.toIntUnsigned(key()) - BufferUtil.toIntUnsigned(o.key());
+                }
+                return o.getCardinality() - this.getCardinality();
+            }
+
+            @Override
+            public int getCardinality() {
+                return getContainer().getCardinality();
+            }
+
+            @Override
+            public MappeableContainer getContainer() {
+                if (k >= MutableRoaringArray.this.size) {
+                    return null;
+                }
+                return MutableRoaringArray.this.values[k];
+            }
+
+            @Override
+            public int getSizeInBytes() {
+                return getContainer().getArraySizeInBytes();
+            }
+
+            @Override
+            public boolean hasContainer() {
+                return 0 <= k & k < MutableRoaringArray.this.size;
+            }
+
+            @Override
+            public boolean isBitmapContainer() {
+                return getContainer() instanceof MappeableBitmapContainer;
+            }
+
+            @Override
+            public boolean isRunContainer() {
+                return getContainer() instanceof MappeableRunContainer;
+            }
+
+            @Override
+            public short key() {
+                return MutableRoaringArray.this.keys[k];
+
+            }
+
+
+            @Override
+            public void previous() {
+                --k;
+            }
+        };
+
+    }
+
+    // involves a binary search
+    @Override
+    public int getIndex(short x) {
+        // before the binary search, we optimize for frequent cases
+        if ((size == 0) || (keys[size - 1] == x)) {
+            return size - 1;
+        }
+        // no luck we have to go through the list
+        return this.binarySearch(0, size, x);
+    }
+
+    @Override
+    public short getKeyAtIndex(int i) {
+        return this.keys[i];
+    }
+
+    @Override
+    public int hashCode() {
+        int hashvalue = 0;
+        for (int k = 0; k < this.size; ++k) {
+            hashvalue = 31 * hashvalue + keys[k] * 0xF0F0F0 + values[k].hashCode();
+        }
+        return hashvalue;
+    }
+
+    @Override
+    public boolean hasRunCompression() {
+        for (int k = 0; k < size; ++k) {
+            MappeableContainer ck = values[k];
+            if (ck instanceof MappeableRunContainer) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    protected int headerSize() {
+        if (hasRunCompression()) {
+            if (size < NO_OFFSET_THRESHOLD) {// for small bitmaps, we omit the offsets
+                return 4 + (size + 7) / 8 + 4 * size;
+            }
+            return 4 + (size + 7) / 8 + 8 * size;// - 4 because we pack the size with the cookie
+        } else {
+            return 4 + 4 + 8 * size;
+        }
+    }
+
+    // insert a new key, it is assumed that it does not exist
+    protected void insertNewKeyValueAt(int i, short key, MappeableContainer value) {
+        extendArray(1);
+        System.arraycopy(keys, i, keys, i + 1, size - i);
+        System.arraycopy(values, i, values, i + 1, size - i);
+        keys[i] = key;
+        values[i] = value;
+        size++;
+    }
+
+    @Override
+    public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        deserialize(in);
+    }
+
+    protected void removeAtIndex(int i) {
+        System.arraycopy(keys, i + 1, keys, i, size - i - 1);
+        keys[size - 1] = 0;
+        System.arraycopy(values, i + 1, values, i, size - i - 1);
+        values[size - 1] = null;
+        size--;
+    }
+
+
+    protected void removeIndexRange(int begin, int end) {
+        if (end <= begin) {
+            return;
+        }
+        final int range = end - begin;
+        System.arraycopy(keys, end, keys, begin, size - end);
+        System.arraycopy(values, end, values, begin, size - end);
+        for (int i = 1; i <= range; ++i) {
+            keys[size - i] = 0;
+            values[size - i] = null;
+        }
+        size -= range;
+    }
+
+    protected void replaceKeyAndContainerAtIndex(int i, short key, MappeableContainer c) {
+        this.keys[i] = key;
+        this.values[i] = c;
+    }
+
+
+    protected void resize(int newLength) {
+        Arrays.fill(this.keys, newLength, this.size, (short) 0);
+        Arrays.fill(this.values, newLength, this.size, null);
+        this.size = newLength;
+    }
+
+    /**
+     * Serialize.
+     * <p>
+     * The current bitmap is not modified.
+     *
+     * @param out the DataOutput stream
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    @Override
+    public void serialize(DataOutput out) throws IOException {
+        int startOffset = 0;
+        boolean hasrun = hasRunCompression();
+        if (hasrun) {
+            out.writeInt(Integer.reverseBytes(SERIAL_COOKIE | ((this.size - 1) << 16)));
+            byte[] bitmapOfRunContainers = new byte[(size + 7) / 8];
+            for (int i = 0; i < size; ++i) {
+                if (this.values[i] instanceof MappeableRunContainer) {
+                    bitmapOfRunContainers[i / 8] |= (1 << (i % 8));
+                }
+            }
+            out.write(bitmapOfRunContainers);
+            if (this.size < NO_OFFSET_THRESHOLD) {
+                startOffset = 4 + 4 * this.size + bitmapOfRunContainers.length;
+            } else {
+                startOffset = 4 + 8 * this.size + bitmapOfRunContainers.length;
+            }
+        } else { // backwards compatibilility
+            out.writeInt(Integer.reverseBytes(SERIAL_COOKIE_NO_RUNCONTAINER));
+            out.writeInt(Integer.reverseBytes(size));
+            startOffset = 4 + 4 + this.size * 4 + this.size * 4;
+        }
+        for (int k = 0; k < size; ++k) {
+            out.writeShort(Short.reverseBytes(this.keys[k]));
+            out.writeShort(Short.reverseBytes((short) (this.values[k].getCardinality() - 1)));
+        }
+        if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
+            for (int k = 0; k < this.size; k++) {
+                out.writeInt(Integer.reverseBytes(startOffset));
+                startOffset = startOffset + values[k].getArraySizeInBytes();
+            }
+        }
+        for (int k = 0; k < size; ++k) {
+            values[k].writeArray(out);
+        }
+
+    }
+
+    /**
+     * Report the number of bytes required for serialization.
+     *
+     * @return the size in bytes
+     */
+    @Override
+    public int serializedSizeInBytes() {
+        int count = headerSize();
+        // for each container, we store cardinality (16 bits), key (16 bits) and location offset (32
+        // bits).
+        for (int k = 0; k < this.size; ++k) {
+            count += values[k].getArraySizeInBytes();
+        }
+        return count;
+    }
+
+    protected void setContainerAtIndex(int i, MappeableContainer c) {
+        this.values[i] = c;
+    }
+
+    @Override
+    public int size() {
+        return this.size;
+    }
+
+    @Override
+    public void writeExternal(ObjectOutput out) throws IOException {
+        serialize(out);
+    }
+}

fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/PointableRoaringArray.java

@@ -0,0 +1,103 @@
+/*
+ * (c) the authors Licensed under the Apache License, Version 2.0.
+ */
+
+package com.fr.third.bitmap.roaringbitmap.buffer;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Generic interface for the array underlying roaring bitmap classes.
+ */
+public interface PointableRoaringArray extends Cloneable {
+    /**
+     * Find the smallest integer index larger than pos such that getKeyAtIndex(index)>=x. If none
+     * can be found, return size.
+     *
+     * @param x   minimal value
+     * @param pos index to exceed
+     * @return the smallest index greater than pos such that getKeyAtIndex(index) is at least as large
+     * as min, or size if it is not possible.
+     */
+    int advanceUntil(short x, int pos);
+
+    /**
+     * Create an independent copy of the underlying array
+     *
+     * @return a copy
+     */
+    PointableRoaringArray clone();
+
+    /**
+     * Returns the cardinality of the container at the given index. This method is expected to be
+     * fast.
+     *
+     * @param i index
+     * @return the cardinality
+     */
+    public int getCardinality(int i);
+
+    /**
+     * @param x 16-bit key
+     * @return matching container
+     */
+    MappeableContainer getContainer(short x);
+
+    /**
+     * @param i index
+     * @return matching container
+     */
+    MappeableContainer getContainerAtIndex(int i);
+
+    /**
+     * @return a ContainerPointer to iterator over the array
+     */
+    MappeableContainerPointer getContainerPointer();
+
+    /**
+     * @param startIndex starting index
+     * @return a ContainerPointer to iterator over the array initially positioned at startIndex
+     */
+    MappeableContainerPointer getContainerPointer(int startIndex);
+
+    /**
+     * @param x 16-bit key
+     * @return corresponding index
+     */
+    int getIndex(short x);
+
+    /**
+     * @param i the index
+     * @return 16-bit key at the index
+     */
+    short getKeyAtIndex(int i);
+
+    /**
+     * Check whether this bitmap has had its runs compressed.
+     *
+     * @return whether this bitmap has run compression
+     */
+    public boolean hasRunCompression();
+
+    /**
+     * Serialize.
+     * <p>
+     * The current bitmap is not modified.
+     *
+     * @param out the DataOutput stream
+     * @throws IOException Signals that an I/O exception has occurred.
+     */
+    public void serialize(DataOutput out) throws IOException;
+
+    /**
+     * @return the size that the data structure occupies on disk
+     */
+    public int serializedSizeInBytes();
+
+
+    /**
+     * @return number of keys
+     */
+    int size();
+}