Browse Source

Merge pull request #268 in CORE/base-third from final/10.0 to bugfix/10.0

* commit '49e3d226f913d90efe9ea3cdd417da2f40dfb6df':
  REPORT-20372 CVE-2017-5644 fix
  DEC-8889 jackson高危漏洞
  DEC-8889 jackson高危漏洞
  RoaringBitmap单独抽到third
  KERNER-724 j2v8升级
  DEC-8173 脏数据导致quartz启动失败
  DEC-7569 定时调度模块启动异常
  无JIRA任务 打包失败,job未初始化
  DEC-7376 定时调度任务表中存在脏数据的时候,不能影响整个scheduler
  DEC-7376 定时调度任务表中存在脏数据的时候,不能影响整个scheduler
  无 jira 任务, final --> persist 回复一下
  无 jira 任务, final --> persist 少了一个空行
  无 jira 任务, 解决打包冲突 还是有问题, 删掉 gradle
  无 jira 任务, 解决打包冲突
  升级apache工具类
  build.third_step0.gradle edited online with Bitbucket
bugfix/10.0
Harrison 5 years ago
parent
commit
0b3a3efd7b
  1. 2
      build.third_step0.gradle
  2. 4
      build.third_step6.gradle
  3. 10
      fine-jackson/src/com/fr/third/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java
  4. 51
      fine-poi/src/com/fr/third/v2/org/apache/poi/POIXMLTypeLoader.java
  5. 61
      fine-poi/src/com/fr/third/v2/org/apache/poi/util/DocumentHelper.java
  6. 1240
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ArrayContainer.java
  7. 108
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/BitSetUtil.java
  8. 1392
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/BitmapContainer.java
  9. 38
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/BitmapDataProvider.java
  10. 808
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/Container.java
  11. 60
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ContainerPointer.java
  12. 532
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/FastAggregation.java
  13. 157
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ImmutableBitmapDataProvider.java
  14. 29
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/IntConsumer.java
  15. 28
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/IntIterator.java
  16. 124
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/IntIteratorFlyweight.java
  17. 33
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/PeekableIntIterator.java
  18. 34
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/PeekableShortIterator.java
  19. 110
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ReverseIntIteratorFlyweight.java
  20. 582
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/RoaringArray.java
  21. 2205
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/RoaringBitmap.java
  22. 2518
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/RunContainer.java
  23. 39
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ShortIterator.java
  24. 931
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/Util.java
  25. 113
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferBitSetUtil.java
  26. 631
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferFastAggregation.java
  27. 132
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferIntIteratorFlyweight.java
  28. 116
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferReverseIntIteratorFlyweight.java
  29. 825
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferUtil.java
  30. 463
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/ImmutableRoaringArray.java
  31. 1320
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/ImmutableRoaringBitmap.java
  32. 1688
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableArrayContainer.java
  33. 2012
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableBitmapContainer.java
  34. 759
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableContainer.java
  35. 80
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableContainerPointer.java
  36. 2798
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableRunContainer.java
  37. 573
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MutableRoaringArray.java
  38. 1420
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MutableRoaringBitmap.java
  39. 103
      fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/PointableRoaringArray.java

2
build.third_step0.gradle

@ -41,7 +41,7 @@ def buildDir=files[0].path.substring(0,files[0].path.lastIndexOf (java.io.File.s
def branchName=buildDir.substring(buildDir.lastIndexOf (java.io.File.separator)+1)
def srcDir="."
def maven_version="${version}-RELEASE-SNAPSHOT"
def maven_version="${version}-FINAL-SNAPSHOT"
def jar_version = version
configurations {

4
build.third_step6.gradle

@ -46,7 +46,8 @@ sourceSets{
"${srcDir}/fine-kryo/src",
"${srcDir}/fine-lz4/src",
"${srcDir}/fine-log4j/src",
"${srcDir}/fine-jgit/src"
"${srcDir}/fine-jgit/src",
"${srcDir}/fine-roaringbitmap/src"
]
}
}
@ -121,6 +122,7 @@ task copyFiles(type:Copy,dependsOn:'compileJava'){
with dataContent.call("${srcDir}/fine-log4j/resources")
with dataContent.call("${srcDir}/fine-jgit/src")
with dataContent.call("${srcDir}/fine-jgit/resources")
with dataContent.call("${srcDir}/fine-roaringbitmap/src")
into "${classesDir}"
}
}

10
fine-jackson/src/com/fr/third/fasterxml/jackson/databind/jsontype/impl/SubTypeValidator.java

@ -78,6 +78,16 @@ public class SubTypeValidator
s.add("org.apache.openjpa.ee.RegistryManagedRuntime");
s.add("org.apache.openjpa.ee.JNDIManagedRuntime");
s.add("org.apache.axis2.transport.jms.JMSOutTransportInfo");
// [databind#2326] (2.9.9)
s.add("com.mysql.cj.jdbc.admin.MiniAdmin");
// [databind#2334]: logback-core (2.9.9.1)
s.add("ch.qos.logback.core.db.DriverManagerConnectionSource");
// [databind#2341]: jdom/jdom2 (2.9.9.1)
s.add("org.jdom.transform.XSLTransformer");
s.add("org.jdom2.transform.XSLTransformer");
DEFAULT_NO_DESER_CLASS_NAMES = Collections.unmodifiableSet(s);
}

51
fine-poi/src/com/fr/third/v2/org/apache/poi/POIXMLTypeLoader.java

@ -18,9 +18,11 @@
package com.fr.third.v2.org.apache.poi;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
@ -28,6 +30,7 @@ import java.util.Map;
import javax.xml.stream.XMLStreamReader;
import com.fr.third.v2.org.apache.poi.util.DocumentHelper;
import com.fr.third.v2.org.apache.xmlbeans.SchemaType;
import com.fr.third.v2.org.apache.xmlbeans.XmlBeans;
import com.fr.third.v2.org.apache.xmlbeans.XmlException;
@ -35,19 +38,26 @@ import com.fr.third.v2.org.apache.xmlbeans.XmlObject;
import com.fr.third.v2.org.apache.xmlbeans.XmlOptions;
import com.fr.third.v2.org.apache.xmlbeans.xml.stream.XMLInputStream;
import com.fr.third.v2.org.apache.xmlbeans.xml.stream.XMLStreamException;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
@SuppressWarnings("deprecation")
public class POIXMLTypeLoader {
public static final XmlOptions DEFAULT_XML_OPTIONS;
static {
DEFAULT_XML_OPTIONS = new XmlOptions();
DEFAULT_XML_OPTIONS.setSaveOuter();
DEFAULT_XML_OPTIONS.setUseDefaultNamespace();
DEFAULT_XML_OPTIONS.setSaveAggressiveNamespaces();
DEFAULT_XML_OPTIONS.setCharacterEncoding("UTF-8");
DEFAULT_XML_OPTIONS.setLoadEntityBytesLimit(4096);
// Piccolo is disabled for POI builts, i.e. JAXP is used for parsing
// so only user code using XmlObject/XmlToken.Factory.parse
// directly can bypass the entity check, which is probably unlikely (... and not within our responsibility :))
// DEFAULT_XML_OPTIONS.setLoadEntityBytesLimit(4096);
Map<String, String> map = new HashMap<String, String>();
map.put("http://schemas.openxmlformats.org/drawingml/2006/main", "a");
@ -68,8 +78,7 @@ public class POIXMLTypeLoader {
}
private static XmlOptions getXmlOptions(XmlOptions options) {
XmlOptions opt = (options == null) ? DEFAULT_XML_OPTIONS : options;
return opt;
return options == null ? DEFAULT_XML_OPTIONS : options;
}
public static XmlObject newInstance(SchemaType type, XmlOptions options) {
@ -77,19 +86,38 @@ public class POIXMLTypeLoader {
}
public static XmlObject parse(String xmlText, SchemaType type, XmlOptions options) throws XmlException {
return XmlBeans.getContextTypeLoader().parse(xmlText, type, getXmlOptions(options));
try {
return parse(new StringReader(xmlText), type, options);
} catch (IOException e) {
throw new XmlException("Unable to parse xml bean", e);
}
}
public static XmlObject parse(File file, SchemaType type, XmlOptions options) throws XmlException, IOException {
return XmlBeans.getContextTypeLoader().parse(file, type, getXmlOptions(options));
InputStream is = new FileInputStream(file);
try {
return parse(is, type, options);
} finally {
is.close();
}
}
public static XmlObject parse(URL file, SchemaType type, XmlOptions options) throws XmlException, IOException {
return XmlBeans.getContextTypeLoader().parse(file, type, getXmlOptions(options));
InputStream is = file.openStream();
try {
return parse(is, type, options);
} finally {
is.close();
}
}
public static XmlObject parse(InputStream jiois, SchemaType type, XmlOptions options) throws XmlException, IOException {
return XmlBeans.getContextTypeLoader().parse(jiois, type, getXmlOptions(options));
try {
Document doc = DocumentHelper.readDocument(jiois);
return XmlBeans.getContextTypeLoader().parse(doc.getDocumentElement(), type, getXmlOptions(options));
} catch (SAXException e) {
throw new IOException("Unable to parse xml bean", e);
}
}
public static XmlObject parse(XMLStreamReader xsr, SchemaType type, XmlOptions options) throws XmlException {
@ -97,7 +125,12 @@ public class POIXMLTypeLoader {
}
public static XmlObject parse(Reader jior, SchemaType type, XmlOptions options) throws XmlException, IOException {
return XmlBeans.getContextTypeLoader().parse(jior, type, getXmlOptions(options));
try {
Document doc = DocumentHelper.readDocument(new InputSource(jior));
return XmlBeans.getContextTypeLoader().parse(doc.getDocumentElement(), type, getXmlOptions(options));
} catch (SAXException e) {
throw new XmlException("Unable to parse xml bean", e);
}
}
public static XmlObject parse(Node node, SchemaType type, XmlOptions options) throws XmlException {
@ -108,7 +141,7 @@ public class POIXMLTypeLoader {
return XmlBeans.getContextTypeLoader().parse(xis, type, getXmlOptions(options));
}
public static XMLInputStream newValidatingXMLInputStream ( XMLInputStream xis, SchemaType type, XmlOptions options ) throws XmlException, XMLStreamException {
public static XMLInputStream newValidatingXMLInputStream(XMLInputStream xis, SchemaType type, XmlOptions options) throws XmlException, XMLStreamException {
return XmlBeans.getContextTypeLoader().newValidatingXMLInputStream(xis, type, getXmlOptions(options));
}
}

61
fine-poi/src/com/fr/third/v2/org/apache/poi/util/DocumentHelper.java

@ -29,12 +29,55 @@ import javax.xml.stream.events.Namespace;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.ErrorHandler;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
public final class DocumentHelper {
private static POILogger logger = POILogFactory.getLogger(DocumentHelper.class);
private DocumentHelper() {}
private DocumentHelper() {
}
private static class DocHelperErrorHandler implements ErrorHandler {
public void warning(SAXParseException exception) throws SAXException {
printError(POILogger.WARN, exception);
}
public void error(SAXParseException exception) throws SAXException {
printError(POILogger.ERROR, exception);
}
public void fatalError(SAXParseException exception) throws SAXException {
printError(POILogger.FATAL, exception);
throw exception;
}
/**
* Prints the error message.
*/
private void printError(int type, SAXParseException ex) {
StringBuilder sb = new StringBuilder();
String systemId = ex.getSystemId();
if (systemId != null) {
int index = systemId.lastIndexOf('/');
if (index != -1)
systemId = systemId.substring(index + 1);
sb.append(systemId);
}
sb.append(':');
sb.append(ex.getLineNumber());
sb.append(':');
sb.append(ex.getColumnNumber());
sb.append(": ");
sb.append(ex.getMessage());
logger.log(type, sb.toString(), ex);
}
}
/**
* Creates a new document builder, with sensible defaults
@ -43,6 +86,7 @@ public final class DocumentHelper {
try {
DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
documentBuilder.setEntityResolver(SAXHelper.IGNORING_ENTITY_RESOLVER);
documentBuilder.setErrorHandler(new DocHelperErrorHandler());
return documentBuilder;
} catch (ParserConfigurationException e) {
throw new IllegalStateException("cannot create a DocumentBuilder", e);
@ -50,6 +94,7 @@ public final class DocumentHelper {
}
private static final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
static {
documentBuilderFactory.setNamespaceAware(true);
documentBuilderFactory.setValidating(false);
@ -69,7 +114,7 @@ public final class DocumentHelper {
private static void trySetXercesSecurityManager(DocumentBuilderFactory documentBuilderFactory) {
// Try built-in JVM one first, standalone if not
for (String securityManagerClassName : new String[] {
for (String securityManagerClassName : new String[]{
"com.sun.org.apache.xerces.internal.util.SecurityManager",
"org.apache.xerces.util.SecurityManager"
}) {
@ -89,6 +134,7 @@ public final class DocumentHelper {
/**
* Parses the given stream via the default (sensible)
* DocumentBuilder
*
* @param inp Stream to read the XML data from
* @return the parsed Document
*/
@ -96,6 +142,17 @@ public final class DocumentHelper {
return newDocumentBuilder().parse(inp);
}
/**
* Parses the given stream via the default (sensible)
* DocumentBuilder
*
* @param inp sax source to read the XML data from
* @return the parsed Document
*/
public static Document readDocument(InputSource inp) throws IOException, SAXException {
return newDocumentBuilder().parse(inp);
}
// must only be used to create empty documents, do not use it for parsing!
private static final DocumentBuilder documentBuilderSingleton = newDocumentBuilder();

1240
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ArrayContainer.java

File diff suppressed because it is too large Load Diff

108
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/BitSetUtil.java

@ -0,0 +1,108 @@
package com.fr.third.bitmap.roaringbitmap;
import java.util.Arrays;
import java.util.BitSet;
/***
*
* This class provides convenience functions to manipulate BitSet and RoaringBitmap objects.
*
*/
public class BitSetUtil {
// todo: add a method to convert a RoaringBitmap to a BitSet using BitSet.valueOf
// a block consists has a maximum of 1024 words, each representing 64 bits,
// thus representing at maximum 65536 bits
static final private int BLOCK_LENGTH = BitmapContainer.MAX_CAPACITY / Long.SIZE; //
// 64-bit
// word
private static ArrayContainer arrayContainerOf(final int from, final int to,
final int cardinality, final long[] words) {
// precondition: cardinality is max 4096
final short[] content = new short[cardinality];
int index = 0;
for (int i = from, socket = 0; i < to; ++i, socket += Long.SIZE) {
long word = words[i];
while (word != 0) {
long t = word & -word;
content[index++] = (short) (socket + Long.bitCount(t - 1));
word ^= t;
}
}
return new ArrayContainer(content);
}
/**
* Generate a RoaringBitmap out of a long[], each long using little-endian representation of its
* bits
*
* @param words array of longs (will not be modified)
* @return roaring bitmap
* @see BitSet#toLongArray() for an equivalent
*/
public static RoaringBitmap bitmapOf(final long[] words) {
// split long[] into blocks.
// each block becomes a single container, if any bit is set
final RoaringBitmap ans = new RoaringBitmap();
int containerIndex = 0;
for (int from = 0; from < words.length; from += BLOCK_LENGTH) {
final int to = Math.min(from + BLOCK_LENGTH, words.length);
final int blockCardinality = cardinality(from, to, words);
if (blockCardinality > 0) {
ans.highLowContainer.insertNewKeyValueAt(containerIndex++, Util.highbits(from * Long.SIZE),
BitSetUtil.containerOf(from, to, blockCardinality, words));
}
}
return ans;
}
private static int cardinality(final int from, final int to, final long[] words) {
int sum = 0;
for (int i = from; i < to; i++) {
sum += Long.bitCount(words[i]);
}
return sum;
}
private static Container containerOf(final int from, final int to, final int blockCardinality,
final long[] words) {
// find the best container available
if (blockCardinality <= ArrayContainer.DEFAULT_MAX_SIZE) {
// containers with DEFAULT_MAX_SIZE or less integers should be
// ArrayContainers
return arrayContainerOf(from, to, blockCardinality, words);
} else {
// otherwise use bitmap container
return new BitmapContainer(Arrays.copyOfRange(words, from, from + BLOCK_LENGTH),
blockCardinality);
}
}
/**
* Compares a RoaringBitmap and a BitSet. They are equal if and only if they contain the same set
* of integers.
*
* @param bitset first object to be compared
* @param bitmap second object to be compared
* @return whether they are equals
*/
public static boolean equals(final BitSet bitset, final RoaringBitmap bitmap) {
if (bitset.cardinality() != bitmap.getCardinality()) {
return false;
}
final IntIterator it = bitmap.getIntIterator();
while (it.hasNext()) {
int val = it.next();
if (!bitset.get(val)) {
return false;
}
}
return true;
}
}

1392
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/BitmapContainer.java

File diff suppressed because it is too large Load Diff

38
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/BitmapDataProvider.java

@ -0,0 +1,38 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Representing a general bitmap interface.
*/
public interface BitmapDataProvider extends ImmutableBitmapDataProvider {
/**
* set the value to "true", whether it already appears or not.
*
* @param x integer value
*/
public void add(int x);
/**
* If present remove the specified integers (effectively, sets its bit value to false)
*
* @param x integer value representing the index in a bitmap
*/
public void remove(int x);
/**
* Return the jth value stored in this bitmap.
*
* @param j index of the value
* @return the value
*/
@Override
public int select(int j);
/**
* Recover allocated but unused memory.
*/
public void trim();
}

808
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/Container.java

@ -0,0 +1,808 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
import com.fr.third.bitmap.roaringbitmap.buffer.MappeableContainer;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
/**
* Base container class.
*/
public abstract class Container implements Iterable<Short>, Cloneable, Externalizable {
/**
* Name of the various possible containers
*/
public static String ContainerNames[] = {"bitmap", "array", "run"};
/**
* Create a container initialized with a range of consecutive values
*
* @param start first index
* @param last last index (range is exclusive)
* @return a new container initialized with the specified values
*/
public static Container rangeOfOnes(final int start, final int last) {
final int sizeAsArrayContainer = ArrayContainer.serializedSizeInBytes(last - start);
final int sizeAsRunContainer = RunContainer.serializedSizeInBytes(1);
Container answer =
sizeAsRunContainer < sizeAsArrayContainer ? new RunContainer() : new ArrayContainer();
answer = answer.iadd(start, last);
return answer;
}
/**
* Return a new container with all shorts in [begin,end) added using an unsigned interpretation.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract Container add(int begin, int end);
/**
* Add a short to the container. May generate a new container.
*
* @param x short to be added
* @return the new container
*/
public abstract Container add(short x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container and(ArrayContainer x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container and(BitmapContainer x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public Container and(Container x) {
if (x instanceof ArrayContainer) {
return and((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return and((BitmapContainer) x);
}
return and((RunContainer) x);
}
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container and(RunContainer x);
protected abstract int andCardinality(ArrayContainer x);
protected abstract int andCardinality(BitmapContainer x);
protected abstract int andCardinality(RunContainer x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public int andCardinality(Container x) {
if (this.getCardinality() == 0) {
return 0;
} else if (x.getCardinality() == 0) {
return 0;
} else {
if (x instanceof ArrayContainer) {
return andCardinality((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return andCardinality((BitmapContainer) x);
}
return andCardinality((RunContainer) x);
}
}
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container andNot(ArrayContainer x);
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container andNot(BitmapContainer x);
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public Container andNot(Container x) {
if (x instanceof ArrayContainer) {
return andNot((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return andNot((BitmapContainer) x);
}
return andNot((RunContainer) x);
}
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container andNot(RunContainer x);
/**
* Empties the container
*/
public abstract void clear();
@Override
public abstract Container clone();
/**
* Checks whether the contain contains the provided value
*
* @param x value to check
* @return whether the value is in the container
*/
public abstract boolean contains(short x);
/**
* Deserialize (recover) the container.
*
* @param in the DataInput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public abstract void deserialize(DataInput in) throws IOException;
/**
* Fill the least significant 16 bits of the integer array, starting at index i, with the short
* values from this container. The caller is responsible to allocate enough room. The most
* significant 16 bits of each integer are given by the most significant bits of the provided
* mask.
*
* @param x provided array
* @param i starting index
* @param mask indicates most significant bits
*/
public abstract void fillLeastSignificant16bits(int[] x, int i, int mask);
/**
* Add a short to the container if it is not present, otherwise remove it. May generate a new
* container.
*
* @param x short to be added
* @return the new container
*/
public abstract Container flip(short x);
/**
* Size of the underlying array
*
* @return size in bytes
*/
protected abstract int getArraySizeInBytes();
/**
* Computes the distinct number of short values in the container. Can be expected to run in
* constant time.
*
* @return the cardinality
*/
public abstract int getCardinality();
/**
* Get the name of this container.
*
* @return name of the container
*/
public String getContainerName() {
if (this instanceof BitmapContainer) {
return ContainerNames[0];
} else if (this instanceof ArrayContainer) {
return ContainerNames[1];
} else {
return ContainerNames[2];
}
}
/**
* Iterate through the values of this container and pass them
* along to the IntConsumer, using msb as the 16 most significant bits.
*
* @param msb 16 most significant bits
* @param ic consumer
*/
public abstract void forEach(short msb, IntConsumer ic);
/**
* Iterator to visit the short values in the container in descending order.
*
* @return iterator
*/
public abstract ShortIterator getReverseShortIterator();
/**
* Iterator to visit the short values in the container in ascending order.
*
* @return iterator
*/
public abstract PeekableShortIterator getShortIterator();
/**
* Computes an estimate of the memory usage of this container. The estimate is not meant to be
* exact.
*
* @return estimated memory usage in bytes
*/
public abstract int getSizeInBytes();
/**
* Add all shorts in [begin,end) using an unsigned interpretation. May generate a new container.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract Container iadd(int begin, int end);
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iand(ArrayContainer x);
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iand(BitmapContainer x);
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public Container iand(Container x) {
if (x instanceof ArrayContainer) {
return iand((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return iand((BitmapContainer) x);
}
return iand((RunContainer) x);
}
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iand(RunContainer x);
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iandNot(ArrayContainer x);
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iandNot(BitmapContainer x);
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public Container iandNot(Container x) {
if (x instanceof ArrayContainer) {
return iandNot((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return iandNot((BitmapContainer) x);
}
return iandNot((RunContainer) x);
}
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iandNot(RunContainer x);
/**
* Computes the in-place bitwise NOT of this container (complement). Only those bits within the
* range are affected. The current container is generally modified. May generate a new container.
*
* @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
* @param rangeEnd ending of range (exclusive)
* @return (partially) complemented container
*/
public abstract Container inot(int rangeStart, int rangeEnd);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(ArrayContainer x);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(BitmapContainer x);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public boolean intersects(Container x) {
if (x instanceof ArrayContainer) {
return intersects((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return intersects((BitmapContainer) x);
}
return intersects((RunContainer) x);
}
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(RunContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ior(ArrayContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ior(BitmapContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public Container ior(Container x) {
if (x instanceof ArrayContainer) {
return ior((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ior((BitmapContainer) x);
}
return ior((RunContainer) x);
}
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ior(RunContainer x);
/**
* Remove shorts in [begin,end) using an unsigned interpretation. May generate a new container.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract Container iremove(int begin, int end);
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ixor(ArrayContainer x);
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ixor(BitmapContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public Container ixor(Container x) {
if (x instanceof ArrayContainer) {
return ixor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ixor((BitmapContainer) x);
}
return ixor((RunContainer) x);
}
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ixor(RunContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container. The resulting container may not track its cardinality correctly. The resulting
* container may not track its cardinality correctly. This can be fixed as follows:
* if(c.getCardinality()&lt;0) ((BitmapContainer)c).computeCardinality();
*
* @param x other container
* @return aggregated container
*/
public Container lazyIOR(Container x) {
if (this instanceof ArrayContainer) {
if (x instanceof ArrayContainer) {
return ((ArrayContainer) this).lazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ior((BitmapContainer) x);
}
return ((RunContainer) x).lazyor((ArrayContainer) this);
} else if (this instanceof RunContainer) {
if (x instanceof ArrayContainer) {
return ((RunContainer) this).ilazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ior((BitmapContainer) x);
}
return ior((RunContainer) x);
} else {
if (x instanceof ArrayContainer) {
return ((BitmapContainer) this).ilazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ((BitmapContainer) this).ilazyor((BitmapContainer) x);
}
return ((BitmapContainer) this).ilazyor((RunContainer) x);
}
}
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected. The resulting container may not track its cardinality
* correctly. This can be fixed as follows: if(c.getCardinality()&lt;0)
* ((BitmapContainer)c).computeCardinality();
*
* @param x other container
* @return aggregated container
*/
public Container lazyOR(Container x) {
if (this instanceof ArrayContainer) {
if (x instanceof ArrayContainer) {
return ((ArrayContainer) this).lazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ((BitmapContainer) x).lazyor((ArrayContainer) this);
}
return ((RunContainer) x).lazyor((ArrayContainer) this);
} else if (this instanceof RunContainer) {
if (x instanceof ArrayContainer) {
return ((RunContainer) this).lazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ((BitmapContainer) x).lazyor((RunContainer) this);
}
return or((RunContainer) x);
} else {
if (x instanceof ArrayContainer) {
return ((BitmapContainer) this).lazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ((BitmapContainer) this).lazyor((BitmapContainer) x);
}
return ((BitmapContainer) this).lazyor((RunContainer) x);
}
}
/**
* Create a new Container containing at most maxcardinality integers.
*
* @param maxcardinality maximal cardinality
* @return a new bitmap with cardinality no more than maxcardinality
*/
public abstract Container limit(int maxcardinality);
/**
* Computes the bitwise NOT of this container (complement). Only those bits within the range are
* affected. The current container is left unaffected.
*
* @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
* @param rangeEnd ending of range (exclusive)
* @return (partially) complemented container
*/
public abstract Container not(int rangeStart, int rangeEnd);
abstract int numberOfRuns(); // exact
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container or(ArrayContainer x);
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container or(BitmapContainer x);
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public Container or(Container x) {
if (x instanceof ArrayContainer) {
return or((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return or((BitmapContainer) x);
}
return or((RunContainer) x);
}
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container or(RunContainer x);
/**
* Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be
* GetCardinality()).
*
* @param lowbits upper limit
* @return the rank
*/
public abstract int rank(short lowbits);
/**
* Return a new container with all shorts in [begin,end) remove using an unsigned interpretation.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract Container remove(int begin, int end);
/**
* Remove the short from this container. May create a new container.
*
* @param x to be removed
* @return New container
*/
public abstract Container remove(short x);
/**
* The output of a lazyOR or lazyIOR might be an invalid container, this should be called on it.
*
* @return a new valid container
*/
public abstract Container repairAfterLazy();
/**
* Convert to RunContainers, when the result is smaller. Overridden by RunContainer to possibility
* switch from RunContainer to a smaller alternative. Overridden by BitmapContainer with a more
* efficient approach.
*
* @return the new container
*/
public abstract Container runOptimize();
/**
* Return the jth value
*
* @param j index of the value
* @return the value
*/
public abstract short select(int j);
/**
* Serialize the container.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public abstract void serialize(DataOutput out) throws IOException;
/**
* Report the number of bytes required to serialize this container.
*
* @return the size in bytes
*/
public abstract int serializedSizeInBytes();
/**
* Convert to a mappeable container.
*
* @return the mappeable container
*/
public abstract MappeableContainer toMappeableContainer();
/**
* If possible, recover wasted memory.
*/
public abstract void trim();
/**
* Write just the underlying array.
*
* @param out output stream
* @throws IOException in case of failure
*/
protected abstract void writeArray(DataOutput out) throws IOException;
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container xor(ArrayContainer x);
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container xor(BitmapContainer x);
/**
* Computes the bitwise OR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other parameter
* @return aggregated container
*/
public Container xor(Container x) {
if (x instanceof ArrayContainer) {
return xor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return xor((BitmapContainer) x);
}
return xor((RunContainer) x);
}
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container xor(RunContainer x);
/**
* Convert the current container to a BitmapContainer, if a conversion is needed.
* If the container is already a bitmap, the container is returned unchanged.
*
* @return a bitmap container
*/
public abstract BitmapContainer toBitmapContainer();
}

60
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ContainerPointer.java

@ -0,0 +1,60 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* This interface allows you to iterate over the containers in a roaring bitmap.
*/
public interface ContainerPointer extends Comparable<ContainerPointer>, Cloneable {
/**
* Move to the next container
*/
void advance();
/**
* Create a copy
*
* @return return a clone of this pointer
*/
ContainerPointer clone();
/**
* Return the cardinality of the current container
*
* @return the cardinality
*/
int getCardinality();
/**
* This method can be used to check whether there is current a valid container as it returns null
* when there is not.
*
* @return null or the current container
*/
Container getContainer();
/**
* Check whether the current container is a bitmap container.
*
* @return whether it is a bitmap container
*/
boolean isBitmapContainer();
/**
* Check whether the current container is a run container.
*
* @return whether it is a run container
*/
boolean isRunContainer();
/**
* The key is a 16-bit integer that indicates the position of the container in the roaring bitmap.
* To be interpreted as an unsigned integer.
*
* @return the key
*/
short key();
}

532
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/FastAggregation.java

@ -0,0 +1,532 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.PriorityQueue;
/**
* Fast algorithms to aggregate many bitmaps.
*
* @author Daniel Lemire
*/
public final class FastAggregation {
/**
* Private constructor to prevent instantiation of utility class
*/
private FastAggregation() {
}
/**
* Compute the AND aggregate.
* <p>
* In practice, calls {#link naive_and}
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap and(Iterator<RoaringBitmap> bitmaps) {
return naive_and(bitmaps);
}
/**
* Compute the AND aggregate.
* <p>
* In practice, calls {#link naive_and}
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap and(RoaringBitmap... bitmaps) {
return naive_and(bitmaps);
}
/**
* Calls naive_or.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
@Deprecated
public static RoaringBitmap horizontal_or(Iterator<RoaringBitmap> bitmaps) {
return naive_or(bitmaps);
}
/**
* Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #or(RoaringBitmap...)
*/
public static RoaringBitmap horizontal_or(List<RoaringBitmap> bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
if (bitmaps.isEmpty()) {
return answer;
}
PriorityQueue<ContainerPointer> pq = new PriorityQueue<ContainerPointer>(bitmaps.size());
for (int k = 0; k < bitmaps.size(); ++k) {
ContainerPointer x = bitmaps.get(k).highLowContainer.getContainerPointer();
if (x.getContainer() != null) {
pq.add(x);
}
}
while (!pq.isEmpty()) {
ContainerPointer x1 = pq.poll();
if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
answer.highLowContainer.append(x1.key(), x1.getContainer().clone());
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
continue;
}
ContainerPointer x2 = pq.poll();
Container newc = x1.getContainer().lazyOR(x2.getContainer());
while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
ContainerPointer x = pq.poll();
newc = newc.lazyIOR(x.getContainer());
x.advance();
if (x.getContainer() != null) {
pq.add(x);
} else if (pq.isEmpty()) {
break;
}
}
newc = newc.repairAfterLazy();
answer.highLowContainer.append(x1.key(), newc);
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
x2.advance();
if (x2.getContainer() != null) {
pq.add(x2);
}
}
return answer;
}
/**
* Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #or(RoaringBitmap...)
*/
public static RoaringBitmap horizontal_or(RoaringBitmap... bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
if (bitmaps.length == 0) {
return answer;
}
PriorityQueue<ContainerPointer> pq = new PriorityQueue<ContainerPointer>(bitmaps.length);
for (int k = 0; k < bitmaps.length; ++k) {
ContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
if (x.getContainer() != null) {
pq.add(x);
}
}
while (!pq.isEmpty()) {
ContainerPointer x1 = pq.poll();
if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
answer.highLowContainer.append(x1.key(), x1.getContainer().clone());
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
continue;
}
ContainerPointer x2 = pq.poll();
Container newc = x1.getContainer().lazyOR(x2.getContainer());
while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
ContainerPointer x = pq.poll();
newc = newc.lazyIOR(x.getContainer());
x.advance();
if (x.getContainer() != null) {
pq.add(x);
} else if (pq.isEmpty()) {
break;
}
}
newc = newc.repairAfterLazy();
answer.highLowContainer.append(x1.key(), newc);
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
x2.advance();
if (x2.getContainer() != null) {
pq.add(x2);
}
}
return answer;
}
/**
* Minimizes memory usage while computing the xor aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #xor(RoaringBitmap...)
*/
public static RoaringBitmap horizontal_xor(RoaringBitmap... bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
if (bitmaps.length == 0) {
return answer;
}
PriorityQueue<ContainerPointer> pq = new PriorityQueue<ContainerPointer>(bitmaps.length);
for (int k = 0; k < bitmaps.length; ++k) {
ContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
if (x.getContainer() != null) {
pq.add(x);
}
}
while (!pq.isEmpty()) {
ContainerPointer x1 = pq.poll();
if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
answer.highLowContainer.append(x1.key(), x1.getContainer().clone());
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
continue;
}
ContainerPointer x2 = pq.poll();
Container newc = x1.getContainer().xor(x2.getContainer());
while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
ContainerPointer x = pq.poll();
newc = newc.ixor(x.getContainer());
x.advance();
if (x.getContainer() != null) {
pq.add(x);
} else if (pq.isEmpty()) {
break;
}
}
answer.highLowContainer.append(x1.key(), newc);
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
x2.advance();
if (x2.getContainer() != null) {
pq.add(x2);
}
}
return answer;
}
/**
* Compute overall AND between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_and(Iterator<RoaringBitmap> bitmaps) {
if (!bitmaps.hasNext()) {
return new RoaringBitmap();
}
RoaringBitmap answer = bitmaps.next().clone();
while (bitmaps.hasNext()) {
answer.and(bitmaps.next());
}
return answer;
}
/**
* Compute overall AND between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_and(RoaringBitmap... bitmaps) {
if (bitmaps.length == 0) {
return new RoaringBitmap();
}
RoaringBitmap answer = bitmaps[0].clone();
for (int k = 1; k < bitmaps.length; ++k) {
answer.and(bitmaps[k]);
}
return answer;
}
/**
* Compute overall OR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_or(Iterator<RoaringBitmap> bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
while (bitmaps.hasNext()) {
answer.naivelazyor(bitmaps.next());
}
answer.repairAfterLazy();
return answer;
}
/**
* Compute overall OR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_or(RoaringBitmap... bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.naivelazyor(bitmaps[k]);
}
answer.repairAfterLazy();
return answer;
}
/**
* Compute overall XOR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_xor(Iterator<RoaringBitmap> bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
while (bitmaps.hasNext()) {
answer.xor(bitmaps.next());
}
return answer;
}
/**
* Compute overall XOR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_xor(RoaringBitmap... bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.xor(bitmaps[k]);
}
return answer;
}
/**
* Compute overall OR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap or(Iterator<RoaringBitmap> bitmaps) {
return naive_or(bitmaps);
}
/**
* Compute overall OR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap or(RoaringBitmap... bitmaps) {
return naive_or(bitmaps);
}
/**
* Uses a priority queue to compute the or aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_or(RoaringBitmap...)
*/
public static RoaringBitmap priorityqueue_or(Iterator<RoaringBitmap> bitmaps) {
if (!bitmaps.hasNext()) {
return new RoaringBitmap();
}
// we buffer the call to getSizeInBytes(), hence the code complexity
ArrayList<RoaringBitmap> buffer = new ArrayList<RoaringBitmap>();
while (bitmaps.hasNext()) {
buffer.add(bitmaps.next());
}
final long[] sizes = new long[buffer.size()];
final boolean[] istmp = new boolean[buffer.size()];
for (int k = 0; k < sizes.length; ++k) {
sizes[k] = buffer.get(k).getLongSizeInBytes();
}
PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
@Override
public int compare(Integer a, Integer b) {
return (int) (sizes[a] - sizes[b]);
}
});
for (int k = 0; k < sizes.length; ++k) {
pq.add(k);
}
while (pq.size() > 1) {
Integer x1 = pq.poll();
Integer x2 = pq.poll();
if (istmp[x2] && istmp[x1]) {
buffer.set(x1, RoaringBitmap.lazyorfromlazyinputs(buffer.get(x1), buffer.get(x2)));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
istmp[x1] = true;
pq.add(x1);
} else if (istmp[x2]) {
buffer.get(x2).lazyor(buffer.get(x1));
sizes[x2] = buffer.get(x2).getLongSizeInBytes();
pq.add(x2);
} else if (istmp[x1]) {
buffer.get(x1).lazyor(buffer.get(x2));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
pq.add(x1);
} else {
buffer.set(x1, RoaringBitmap.lazyor(buffer.get(x1), buffer.get(x2)));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
istmp[x1] = true;
pq.add(x1);
}
}
RoaringBitmap answer = buffer.get(pq.poll());
answer.repairAfterLazy();
return answer;
}
/**
* Uses a priority queue to compute the or aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_or(RoaringBitmap...)
*/
public static RoaringBitmap priorityqueue_or(RoaringBitmap... bitmaps) {
if (bitmaps.length == 0) {
return new RoaringBitmap();
}
// we buffer the call to getSizeInBytes(), hence the code complexity
final RoaringBitmap[] buffer = Arrays.copyOf(bitmaps, bitmaps.length);
final long[] sizes = new long[buffer.length];
final boolean[] istmp = new boolean[buffer.length];
for (int k = 0; k < sizes.length; ++k) {
sizes[k] = buffer[k].getLongSizeInBytes();
}
PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
@Override
public int compare(Integer a, Integer b) {
return (int) (sizes[a] - sizes[b]);
}
});
for (int k = 0; k < sizes.length; ++k) {
pq.add(k);
}
while (pq.size() > 1) {
Integer x1 = pq.poll();
Integer x2 = pq.poll();
if (istmp[x2] && istmp[x1]) {
buffer[x1] = RoaringBitmap.lazyorfromlazyinputs(buffer[x1], buffer[x2]);
sizes[x1] = buffer[x1].getLongSizeInBytes();
istmp[x1] = true;
pq.add(x1);
} else if (istmp[x2]) {
buffer[x2].lazyor(buffer[x1]);
sizes[x2] = buffer[x2].getLongSizeInBytes();
pq.add(x2);
} else if (istmp[x1]) {
buffer[x1].lazyor(buffer[x2]);
sizes[x1] = buffer[x1].getLongSizeInBytes();
pq.add(x1);
} else {
buffer[x1] = RoaringBitmap.lazyor(buffer[x1], buffer[x2]);
sizes[x1] = buffer[x1].getLongSizeInBytes();
istmp[x1] = true;
pq.add(x1);
}
}
RoaringBitmap answer = buffer[pq.poll()];
answer.repairAfterLazy();
return answer;
}
/**
* Uses a priority queue to compute the xor aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_xor(RoaringBitmap...)
*/
public static RoaringBitmap priorityqueue_xor(RoaringBitmap... bitmaps) {
// TODO: This code could be faster, see priorityqueue_or
if (bitmaps.length == 0) {
return new RoaringBitmap();
}
PriorityQueue<RoaringBitmap> pq =
new PriorityQueue<RoaringBitmap>(bitmaps.length, new Comparator<RoaringBitmap>() {
@Override
public int compare(RoaringBitmap a, RoaringBitmap b) {
return (int) (a.getLongSizeInBytes() - b.getLongSizeInBytes());
}
});
Collections.addAll(pq, bitmaps);
while (pq.size() > 1) {
RoaringBitmap x1 = pq.poll();
RoaringBitmap x2 = pq.poll();
pq.add(RoaringBitmap.xor(x1, x2));
}
return pq.poll();
}
/**
* Compute overall XOR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap xor(Iterator<RoaringBitmap> bitmaps) {
return naive_xor(bitmaps);
}
/**
* Compute overall XOR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap xor(RoaringBitmap... bitmaps) {
return naive_xor(bitmaps);
}
}

157
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ImmutableBitmapDataProvider.java

@ -0,0 +1,157 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
import java.io.DataOutput;
import java.io.IOException;
/**
* Interface representing an immutable bitmap.
*/
public interface ImmutableBitmapDataProvider {
/**
* Checks whether the value in included, which is equivalent to checking if the corresponding bit
* is set (get in BitSet class).
*
* @param x integer value
* @return whether the integer value is included.
*/
public boolean contains(int x);
/**
* Returns the number of distinct integers added to the bitmap (e.g., number of bits set).
* Internally, this is computed as a 64-bit number.
*
* @return the cardinality
*/
public int getCardinality();
/**
* Returns the number of distinct integers added to the bitmap (e.g., number of bits set).
* This returns a full 64-bit result.
*
* @return the cardinality
*/
public long getLongCardinality();
/**
* Visit all values in the bitmap and pass them to the consumer.
* <p>
* * Usage:
* <pre>
* {@code
* bitmap.forEach(new IntConsumer() {
*
* {@literal @}Override
* public void accept(int value) {
* // do something here
*
* }});
* }
* }
* </pre>
*
* @param ic the consumer
*/
public void forEach(IntConsumer ic);
/**
* For better performance, consider the Use the {@link #forEach forEach} method.
*
* @return a custom iterator over set bits, the bits are traversed in ascending sorted order
*/
public PeekableIntIterator getIntIterator();
/**
* @return a custom iterator over set bits, the bits are traversed in descending sorted order
*/
public IntIterator getReverseIntIterator();
/**
* Estimate of the memory usage of this data structure.
* <p>
* Internally, this is computed as a 64-bit counter.
*
* @return estimated memory usage.
*/
public int getSizeInBytes();
/**
* Estimate of the memory usage of this data structure. Provides
* full 64-bit number.
*
* @return estimated memory usage.
*/
public long getLongSizeInBytes();
/**
* Checks whether the bitmap is empty.
*
* @return true if this bitmap contains no set bit
*/
public boolean isEmpty();
/**
* Create a new bitmap of the same class, containing at most maxcardinality integers.
*
* @param x maximal cardinality
* @return a new bitmap with cardinality no more than maxcardinality
*/
public ImmutableBitmapDataProvider limit(int x);
/**
* Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be
* GetCardinality()).
* <p>
* The value is internally computed as a 64-bit number.
*
* @param x upper limit
* @return the rank
*/
public int rank(int x);
/**
* Same as "rank" but produces a full 64-bit value.
*
* @param x upper limit
* @return the rank
*/
public long rankLong(int x);
/**
* Return the jth value stored in this bitmap.
*
* @param j index of the value
* @return the value
*/
public int select(int j);
/**
* Serialize this bitmap.
* <p>
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void serialize(DataOutput out) throws IOException;
/**
* Report the number of bytes required to serialize this bitmap. This is the number of bytes
* written out when using the serialize method. When using the writeExternal method, the count
* will be higher due to the overhead of Java serialization.
*
* @return the size in bytes
*/
public int serializedSizeInBytes();
/**
* Return the set values as an array. The integer values are in sorted order.
*
* @return array representing the set values.
*/
public int[] toArray();
}

29
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/IntConsumer.java

@ -0,0 +1,29 @@
package com.fr.third.bitmap.roaringbitmap;
/**
* An IntConsumer receives the int values contained in a data structure.
* Each value is visited once.
* <p>
* Usage:
* <p>
* <pre>
* {@code
* bitmap.forEach(new IntConsumer() {
*
* &#64;Override
* public void accept(int value) {
* // do something here
*
* }});
* }
* }
* </pre>
*/
public interface IntConsumer {
/**
* Receives the integer
*
* @param value the integer value
*/
void accept(int value);
}

28
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/IntIterator.java

@ -0,0 +1,28 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* A simple iterator over integer values
*/
public interface IntIterator extends Cloneable {
/**
* Creates a copy of the iterator.
*
* @return a clone of the current iterator
*/
IntIterator clone();
/**
* @return whether there is another value
*/
boolean hasNext();
/**
* @return next integer value
*/
int next();
}

124
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/IntIteratorFlyweight.java

@ -0,0 +1,124 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Fast iterator minimizing the stress on the garbage collector. You can create one reusable
* instance of this class and then {@link #wrap(RoaringBitmap)}
* <p>
* For better performance, consider the {@link RoaringBitmap#forEach} method.
*
* @author Borislav Ivanov
**/
public class IntIteratorFlyweight implements PeekableIntIterator {
private int hs;
private PeekableShortIterator iter;
private ArrayContainerShortIterator arrIter = new ArrayContainerShortIterator();
private BitmapContainerShortIterator bitmapIter = new BitmapContainerShortIterator();
private RunContainerShortIterator runIter = new RunContainerShortIterator();
private int pos;
private RoaringBitmap roaringBitmap = null;
/**
* Creates an instance that is not ready for iteration. You must first call
* {@link #wrap(RoaringBitmap)}.
*/
public IntIteratorFlyweight() {
}
/**
* Creates an instance that is ready for iteration.
*
* @param r bitmap to be iterated over
*/
public IntIteratorFlyweight(RoaringBitmap r) {
wrap(r);
}
@Override
public PeekableIntIterator clone() {
try {
IntIteratorFlyweight x = (IntIteratorFlyweight) super.clone();
x.iter = this.iter.clone();
return x;
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public boolean hasNext() {
return pos < this.roaringBitmap.highLowContainer.size();
}
@Override
public int next() {
int x = iter.nextAsInt() | hs;
if (!iter.hasNext()) {
++pos;
nextContainer();
}
return x;
}
private void nextContainer() {
if (pos < this.roaringBitmap.highLowContainer.size()) {
Container container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
if (container instanceof BitmapContainer) {
bitmapIter.wrap(((BitmapContainer) container).bitmap);
iter = bitmapIter;
} else if (container instanceof ArrayContainer) {
arrIter.wrap((ArrayContainer) container);
iter = arrIter;
} else {
runIter.wrap((RunContainer) container);
iter = runIter;
}
hs = Util.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
}
}
/**
* Prepares a bitmap for iteration
*
* @param r bitmap to be iterated over
*/
public void wrap(RoaringBitmap r) {
this.hs = 0;
this.pos = 0;
this.roaringBitmap = r;
this.nextContainer();
}
@Override
public void advanceIfNeeded(final int minval) {
while (hasNext() && ((hs >>> 16) < (minval >>> 16))) {
++pos;
nextContainer();
}
if (hasNext() && ((hs >>> 16) == (minval >>> 16))) {
iter.advanceIfNeeded(Util.lowbits(minval));
if (!iter.hasNext()) {
++pos;
nextContainer();
}
}
}
@Override
public int peekNext() {
return Util.toIntUnsigned(iter.peekNext()) | hs;
}
}

33
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/PeekableIntIterator.java

@ -0,0 +1,33 @@
package com.fr.third.bitmap.roaringbitmap;
/**
* Simple extension to the IntIterator interface.
* It allows you to "skip" values using the advanceIfNeeded
* method, and to look at the value without advancing (peekNext).
*/
public interface PeekableIntIterator extends IntIterator {
/**
* If needed, advance as long as the next value is smaller than minval
*
* @param minval threshold
*/
public void advanceIfNeeded(int minval);
/**
* Look at the next value without advancing
*
* @return next value
*/
public int peekNext();
/**
* Creates a copy of the iterator.
*
* @return a clone of the current iterator
*/
@Override
PeekableIntIterator clone();
}

34
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/PeekableShortIterator.java

@ -0,0 +1,34 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Simple extension to the ShortIterator interface
*/
public interface PeekableShortIterator extends ShortIterator {
/**
* If needed, advance as long as the next value is smaller than minval (as an unsigned
* short)
*
* @param minval threshold
*/
public void advanceIfNeeded(short minval);
/**
* Look at the next value without advancing
*
* @return next value
*/
public short peekNext();
/**
* Creates a copy of the iterator.
*
* @return a clone of the current iterator
*/
@Override
PeekableShortIterator clone();
}

110
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ReverseIntIteratorFlyweight.java

@ -0,0 +1,110 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Fast iterator minimizing the stress on the garbage collector. You can create one reusable
* instance of this class and then {@link #wrap(RoaringBitmap)}
* <p>
* This iterator enumerates the stored values in reverse (starting from the end).
*
* @author Borislav Ivanov
**/
public class ReverseIntIteratorFlyweight implements IntIterator {
private int hs;
private ShortIterator iter;
private ReverseArrayContainerShortIterator arrIter = new ReverseArrayContainerShortIterator();
private ReverseBitmapContainerShortIterator bitmapIter =
new ReverseBitmapContainerShortIterator();
private ReverseRunContainerShortIterator runIter = new ReverseRunContainerShortIterator();
private short pos;
private RoaringBitmap roaringBitmap = null;
/**
* Creates an instance that is not ready for iteration. You must first call
* {@link #wrap(RoaringBitmap)}.
*/
public ReverseIntIteratorFlyweight() {
}
/**
* Creates an instance that is ready for iteration.
*
* @param r bitmap to be iterated over
*/
public ReverseIntIteratorFlyweight(RoaringBitmap r) {
wrap(r);
}
@Override
public IntIterator clone() {
try {
ReverseIntIteratorFlyweight x = (ReverseIntIteratorFlyweight) super.clone();
x.iter = this.iter.clone();
return x;
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public boolean hasNext() {
return pos >= 0;
}
@Override
public int next() {
final int x = iter.nextAsInt() | hs;
if (!iter.hasNext()) {
--pos;
nextContainer();
}
return x;
}
private void nextContainer() {
if (pos >= 0) {
Container container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
if (container instanceof BitmapContainer) {
bitmapIter.wrap(((BitmapContainer) container).bitmap);
iter = bitmapIter;
} else if (container instanceof ArrayContainer) {
arrIter.wrap((ArrayContainer) container);
iter = arrIter;
} else {
runIter.wrap((RunContainer) container);
iter = runIter;
}
hs = Util.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
}
}
/**
* Prepares a bitmap for iteration
*
* @param r bitmap to be iterated over
*/
public void wrap(RoaringBitmap r) {
this.roaringBitmap = r;
this.hs = 0;
this.pos = (short) (this.roaringBitmap.highLowContainer.size() - 1);
this.nextContainer();
}
}

582
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/RoaringArray.java

@ -0,0 +1,582 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.Arrays;
/**
* Specialized array to store the containers used by a RoaringBitmap. This is not meant to be used
* by end users.
*/
public final class RoaringArray implements Cloneable, Externalizable {
protected static final short SERIAL_COOKIE_NO_RUNCONTAINER = 12346;
protected static final short SERIAL_COOKIE = 12347;
protected static final int NO_OFFSET_THRESHOLD = 4;
static final int INITIAL_CAPACITY = 4;
// bumped serialVersionUID with runcontainers, so default serialization
// will not work...
private static final long serialVersionUID = 8L;
short[] keys = null;
Container[] values = null;
int size = 0;
protected RoaringArray() {
this.keys = new short[INITIAL_CAPACITY];
this.values = new Container[INITIAL_CAPACITY];
}
/**
* Find the smallest integer index larger than pos such that array[index].key&gt;=x. If none can
* be found, return size. Based on code by O. Kaser.
*
* @param x minimal value
* @param pos index to exceed
* @return the smallest index greater than pos such that array[index].key is at least as large as
* min, or size if it is not possible.
*/
protected int advanceUntil(short x, int pos) {
int lower = pos + 1;
// special handling for a possibly common sequential case
if (lower >= size || Util.toIntUnsigned(keys[lower]) >= Util.toIntUnsigned(x)) {
return lower;
}
int spansize = 1; // could set larger
// bootstrap an upper limit
while (lower + spansize < size
&& Util.toIntUnsigned(keys[lower + spansize]) < Util.toIntUnsigned(x)) {
spansize *= 2; // hoping for compiler will reduce to shift
}
int upper = (lower + spansize < size) ? lower + spansize : size - 1;
// maybe we are lucky (could be common case when the seek ahead
// expected to be small and sequential will otherwise make us look bad)
if (keys[upper] == x) {
return upper;
}
if (Util.toIntUnsigned(keys[upper]) < Util.toIntUnsigned(x)) {// means array has no item key >=
// x
return size;
}
// we know that the next-smallest span was too small
lower += (spansize / 2);
// else begin binary search
// invariant: array[lower]<x && array[upper]>x
while (lower + 1 != upper) {
int mid = (lower + upper) / 2;
if (keys[mid] == x) {
return mid;
} else if (Util.toIntUnsigned(keys[mid]) < Util.toIntUnsigned(x)) {
lower = mid;
} else {
upper = mid;
}
}
return upper;
}
protected void append(short key, Container value) {
extendArray(1);
this.keys[this.size] = key;
this.values[this.size] = value;
this.size++;
}
/**
* Append copies of the values AFTER a specified key (may or may not be present) to end.
*
* @param sa other array
* @param beforeStart given key is the largest key that we won't copy
*/
protected void appendCopiesAfter(RoaringArray sa, short beforeStart) {
int startLocation = sa.getIndex(beforeStart);
if (startLocation >= 0) {
startLocation++;
} else {
startLocation = -startLocation - 1;
}
extendArray(sa.size - startLocation);
for (int i = startLocation; i < sa.size; ++i) {
this.keys[this.size] = sa.keys[i];
this.values[this.size] = sa.values[i].clone();
this.size++;
}
}
/**
* Append copies of the values from another array, from the start
*
* @param sourceArray The array to copy from
* @param stoppingKey any equal or larger key in other array will terminate copying
*/
protected void appendCopiesUntil(RoaringArray sourceArray, short stoppingKey) {
int stopKey = Util.toIntUnsigned(stoppingKey);
for (int i = 0; i < sourceArray.size; ++i) {
if (Util.toIntUnsigned(sourceArray.keys[i]) >= stopKey) {
break;
}
extendArray(1);
this.keys[this.size] = sourceArray.keys[i];
this.values[this.size] = sourceArray.values[i].clone();
this.size++;
}
}
/**
* Append copy of the one value from another array
*
* @param sa other array
* @param index index in the other array
*/
protected void appendCopy(RoaringArray sa, int index) {
extendArray(1);
this.keys[this.size] = sa.keys[index];
this.values[this.size] = sa.values[index].clone();
this.size++;
}
/**
* Append copies of the values from another array
*
* @param sa other array
* @param startingIndex starting index in the other array
* @param end endingIndex (exclusive) in the other array
*/
protected void appendCopy(RoaringArray sa, int startingIndex, int end) {
extendArray(end - startingIndex);
for (int i = startingIndex; i < end; ++i) {
this.keys[this.size] = sa.keys[i];
this.values[this.size] = sa.values[i].clone();
this.size++;
}
}
/**
* Append the values from another array, no copy is made (use with care)
*
* @param sa other array
* @param startingIndex starting index in the other array
* @param end endingIndex (exclusive) in the other array
*/
protected void append(RoaringArray sa, int startingIndex, int end) {
extendArray(end - startingIndex);
for (int i = startingIndex; i < end; ++i) {
this.keys[this.size] = sa.keys[i];
this.values[this.size] = sa.values[i];
this.size++;
}
}
private int binarySearch(int begin, int end, short key) {
return Util.unsignedBinarySearch(keys, begin, end, key);
}
protected void clear() {
this.keys = null;
this.values = null;
this.size = 0;
}
@Override
public RoaringArray clone() throws CloneNotSupportedException {
RoaringArray sa;
sa = (RoaringArray) super.clone();
sa.keys = Arrays.copyOf(this.keys, this.size);
sa.values = Arrays.copyOf(this.values, this.size);
for (int k = 0; k < this.size; ++k) {
sa.values[k] = sa.values[k].clone();
}
sa.size = this.size;
return sa;
}
protected void copyRange(int begin, int end, int newBegin) {
// assuming begin <= end and newBegin < begin
final int range = end - begin;
System.arraycopy(this.keys, begin, this.keys, newBegin, range);
System.arraycopy(this.values, begin, this.values, newBegin, range);
}
/**
* Deserialize.
*
* @param in the DataInput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void deserialize(DataInput in) throws IOException {
this.clear();
// little endian
final int cookie = Integer.reverseBytes(in.readInt());
if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) {
throw new IOException("I failed to find one of the right cookies.");
}
this.size = ((cookie & 0xFFFF) == SERIAL_COOKIE) ? (cookie >>> 16) + 1
: Integer.reverseBytes(in.readInt());
if ((this.keys == null) || (this.keys.length < this.size)) {
this.keys = new short[this.size];
this.values = new Container[this.size];
}
byte[] bitmapOfRunContainers = null;
boolean hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE;
if (hasrun) {
bitmapOfRunContainers = new byte[(size + 7) / 8];
in.readFully(bitmapOfRunContainers);
}
final short keys[] = new short[this.size];
final int cardinalities[] = new int[this.size];
final boolean isBitmap[] = new boolean[this.size];
for (int k = 0; k < this.size; ++k) {
keys[k] = Short.reverseBytes(in.readShort());
cardinalities[k] = 1 + (0xFFFF & Short.reverseBytes(in.readShort()));
isBitmap[k] = cardinalities[k] > ArrayContainer.DEFAULT_MAX_SIZE;
if (bitmapOfRunContainers != null && (bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) {
isBitmap[k] = false;
}
}
if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
// skipping the offsets
in.skipBytes(this.size * 4);
}
// Reading the containers
for (int k = 0; k < this.size; ++k) {
Container val;
if (isBitmap[k]) {
final long[] bitmapArray = new long[BitmapContainer.MAX_CAPACITY / 64];
// little endian
for (int l = 0; l < bitmapArray.length; ++l) {
bitmapArray[l] = Long.reverseBytes(in.readLong());
}
val = new BitmapContainer(bitmapArray, cardinalities[k]);
} else if (bitmapOfRunContainers != null
&& ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0)) {
// cf RunContainer.writeArray()
int nbrruns = Util.toIntUnsigned(Short.reverseBytes(in.readShort()));
final short lengthsAndValues[] = new short[2 * nbrruns];
for (int j = 0; j < 2 * nbrruns; ++j) {
lengthsAndValues[j] = Short.reverseBytes(in.readShort());
}
val = new RunContainer(lengthsAndValues, nbrruns);
} else {
final short[] shortArray = new short[cardinalities[k]];
for (int l = 0; l < shortArray.length; ++l) {
shortArray[l] = Short.reverseBytes(in.readShort());
}
val = new ArrayContainer(shortArray);
}
this.keys[k] = keys[k];
this.values[k] = val;
}
}
@Override
public boolean equals(Object o) {
if (o instanceof RoaringArray) {
RoaringArray srb = (RoaringArray) o;
if (srb.size != this.size) {
return false;
}
for (int i = 0; i < srb.size; ++i) {
if (this.keys[i] != srb.keys[i] || !this.values[i].equals(srb.values[i])) {
return false;
}
}
return true;
}
return false;
}
// make sure there is capacity for at least k more elements
protected void extendArray(int k) {
// size + 1 could overflow
if (this.size + k >= this.keys.length) {
int newCapacity;
if (this.keys.length < 1024) {
newCapacity = 2 * (this.size + k);
} else {
newCapacity = 5 * (this.size + k) / 4;
}
this.keys = Arrays.copyOf(this.keys, newCapacity);
this.values = Arrays.copyOf(this.values, newCapacity);
}
}
// involves a binary search
protected Container getContainer(short x) {
int i = this.binarySearch(0, size, x);
if (i < 0) {
return null;
}
return this.values[i];
}
protected Container getContainerAtIndex(int i) {
return this.values[i];
}
/**
* Create a ContainerPointer for this RoaringArray
*
* @return a ContainerPointer
*/
public ContainerPointer getContainerPointer() {
return getContainerPointer(0);
}
/**
* Create a ContainerPointer for this RoaringArray
*
* @param startIndex starting index in the container list
* @return a ContainerPointer
*/
public ContainerPointer getContainerPointer(final int startIndex) {
return new ContainerPointer() {
int k = startIndex;
@Override
public void advance() {
++k;
}
@Override
public ContainerPointer clone() {
try {
return (ContainerPointer) super.clone();
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public int compareTo(ContainerPointer o) {
if (key() != o.key()) {
return Util.toIntUnsigned(key()) - Util.toIntUnsigned(o.key());
}
return o.getCardinality() - getCardinality();
}
@Override
public int getCardinality() {
return getContainer().getCardinality();
}
@Override
public Container getContainer() {
if (k >= RoaringArray.this.size) {
return null;
}
return RoaringArray.this.values[k];
}
@Override
public boolean isBitmapContainer() {
return getContainer() instanceof BitmapContainer;
}
@Override
public boolean isRunContainer() {
return getContainer() instanceof RunContainer;
}
@Override
public short key() {
return RoaringArray.this.keys[k];
}
};
}
// involves a binary search
protected int getIndex(short x) {
// before the binary search, we optimize for frequent cases
if ((size == 0) || (keys[size - 1] == x)) {
return size - 1;
}
// no luck we have to go through the list
return this.binarySearch(0, size, x);
}
protected short getKeyAtIndex(int i) {
return this.keys[i];
}
@Override
public int hashCode() {
int hashvalue = 0;
for (int k = 0; k < this.size; ++k) {
hashvalue = 31 * hashvalue + keys[k] * 0xF0F0F0 + values[k].hashCode();
}
return hashvalue;
}
boolean hasRunContainer() {
for (int k = 0; k < size; ++k) {
Container ck = values[k];
if (ck instanceof RunContainer) {
return true;
}
}
return false;
}
protected int headerSize() {
if (hasRunContainer()) {
if (size < NO_OFFSET_THRESHOLD) {// for small bitmaps, we omit the offsets
return 4 + (size + 7) / 8 + 4 * size;
}
return 4 + (size + 7) / 8 + 8 * size;// - 4 because we pack the size with the cookie
} else {
return 4 + 4 + 8 * size;
}
}
// insert a new key, it is assumed that it does not exist
protected void insertNewKeyValueAt(int i, short key, Container value) {
extendArray(1);
System.arraycopy(keys, i, keys, i + 1, size - i);
keys[i] = key;
System.arraycopy(values, i, values, i + 1, size - i);
values[i] = value;
size++;
}
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
deserialize(in);
}
protected void removeAtIndex(int i) {
System.arraycopy(keys, i + 1, keys, i, size - i - 1);
keys[size - 1] = 0;
System.arraycopy(values, i + 1, values, i, size - i - 1);
values[size - 1] = null;
size--;
}
protected void removeIndexRange(int begin, int end) {
if (end <= begin) {
return;
}
final int range = end - begin;
System.arraycopy(keys, end, keys, begin, size - end);
System.arraycopy(values, end, values, begin, size - end);
for (int i = 1; i <= range; ++i) {
keys[size - i] = 0;
values[size - i] = null;
}
size -= range;
}
protected void replaceKeyAndContainerAtIndex(int i, short key, Container c) {
this.keys[i] = key;
this.values[i] = c;
}
protected void resize(int newLength) {
Arrays.fill(this.keys, newLength, this.size, (short) 0);
Arrays.fill(this.values, newLength, this.size, null);
this.size = newLength;
}
/**
* Serialize.
* <p>
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void serialize(DataOutput out) throws IOException {
int startOffset = 0;
boolean hasrun = hasRunContainer();
if (hasrun) {
out.writeInt(Integer.reverseBytes(SERIAL_COOKIE | ((size - 1) << 16)));
byte[] bitmapOfRunContainers = new byte[(size + 7) / 8];
for (int i = 0; i < size; ++i) {
if (this.values[i] instanceof RunContainer) {
bitmapOfRunContainers[i / 8] |= (1 << (i % 8));
}
}
out.write(bitmapOfRunContainers);
if (this.size < NO_OFFSET_THRESHOLD) {
startOffset = 4 + 4 * this.size + bitmapOfRunContainers.length;
} else {
startOffset = 4 + 8 * this.size + bitmapOfRunContainers.length;
}
} else { // backwards compatibility
out.writeInt(Integer.reverseBytes(SERIAL_COOKIE_NO_RUNCONTAINER));
out.writeInt(Integer.reverseBytes(size));
startOffset = 4 + 4 + 4 * this.size + 4 * this.size;
}
for (int k = 0; k < size; ++k) {
out.writeShort(Short.reverseBytes(this.keys[k]));
out.writeShort(Short.reverseBytes((short) (this.values[k].getCardinality() - 1)));
}
if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
// writing the containers offsets
for (int k = 0; k < this.size; k++) {
out.writeInt(Integer.reverseBytes(startOffset));
startOffset = startOffset + this.values[k].getArraySizeInBytes();
}
}
for (int k = 0; k < size; ++k) {
values[k].writeArray(out);
}
}
/**
* Report the number of bytes required for serialization.
*
* @return the size in bytes
*/
public int serializedSizeInBytes() {
int count = headerSize();
for (int k = 0; k < size; ++k) {
count += values[k].getArraySizeInBytes();
}
return count;
}
protected void setContainerAtIndex(int i, Container c) {
this.values[i] = c;
}
protected int size() {
return this.size;
}
@Override
public void writeExternal(ObjectOutput out) throws IOException {
serialize(out);
}
}

2205
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/RoaringBitmap.java

File diff suppressed because it is too large Load Diff

2518
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/RunContainer.java

File diff suppressed because it is too large Load Diff

39
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/ShortIterator.java

@ -0,0 +1,39 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Iterator over short values.
*/
public interface ShortIterator extends Cloneable {
/**
* Creates a copy of the iterator.
*
* @return a clone of the current iterator
*/
ShortIterator clone();
/**
* @return whether there is another value
*/
boolean hasNext();
/**
* @return next short value
*/
short next();
/**
* @return next short value as int value (using the least significant 16 bits)
*/
int nextAsInt();
/**
* If possible, remove the current value
*/
void remove();
}

931
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/Util.java

@ -0,0 +1,931 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Various useful methods for roaring bitmaps.
*/
public final class Util {
/**
* optimization flag: whether to use hybrid binary search: hybrid formats
* combine a binary search with a sequential search
*/
public static boolean USE_HYBRID_BINSEARCH = true;
/**
* Private constructor to prevent instantiation of utility class
*/
private Util() {
}
/**
* Find the smallest integer larger than pos such that array[pos]&gt;= min. If none can be found,
* return length. Based on code by O. Kaser.
*
* @param array array to search within
* @param pos starting position of the search
* @param length length of the array to search
* @param min minimum value
* @return x greater than pos such that array[pos] is at least as large as min, pos is is equal to
* length if it is not possible.
*/
public static int advanceUntil(short[] array, int pos, int length, short min) {
int lower = pos + 1;
// special handling for a possibly common sequential case
if (lower >= length || toIntUnsigned(array[lower]) >= toIntUnsigned(min)) {
return lower;
}
int spansize = 1; // could set larger
// bootstrap an upper limit
while (lower + spansize < length
&& toIntUnsigned(array[lower + spansize]) < toIntUnsigned(min)) {
spansize *= 2; // hoping for compiler will reduce to
}
// shift
int upper = (lower + spansize < length) ? lower + spansize : length - 1;
// maybe we are lucky (could be common case when the seek ahead
// expected
// to be small and sequential will otherwise make us look bad)
if (array[upper] == min) {
return upper;
}
if (toIntUnsigned(array[upper]) < toIntUnsigned(min)) {// means
// array
// has no
// item
// >= min
// pos = array.length;
return length;
}
// we know that the next-smallest span was too small
lower += (spansize / 2);
// else begin binary search
// invariant: array[lower]<min && array[upper]>min
while (lower + 1 != upper) {
int mid = (lower + upper) / 2;
short arraymid = array[mid];
if (arraymid == min) {
return mid;
} else if (toIntUnsigned(arraymid) < toIntUnsigned(min)) {
lower = mid;
} else {
upper = mid;
}
}
return upper;
}
protected static int branchyUnsignedBinarySearch(final short[] array, final int begin,
final int end, final short k) {
int ikey = toIntUnsigned(k);
// next line accelerates the possibly common case where the value would
// be inserted at the end
if ((end > 0) && (toIntUnsigned(array[end - 1]) < ikey)) {
return -end - 1;
}
int low = begin;
int high = end - 1;
while (low <= high) {
final int middleIndex = (low + high) >>> 1;
final int middleValue = toIntUnsigned(array[middleIndex]);
if (middleValue < ikey) {
low = middleIndex + 1;
} else if (middleValue > ikey) {
high = middleIndex - 1;
} else {
return middleIndex;
}
}
return -(low + 1);
}
/**
* Compares the two specified {@code short} values, treating them as unsigned values between
* {@code 0} and {@code 2^16 - 1} inclusive.
*
* @param a the first unsigned {@code short} to compare
* @param b the second unsigned {@code short} to compare
* @return a negative value if {@code a} is less than {@code b}; a positive value if {@code a} is
* greater than {@code b}; or zero if they are equal
*/
public static int compareUnsigned(short a, short b) {
return toIntUnsigned(a) - toIntUnsigned(b);
}
/**
* Compute the bitwise AND between two long arrays and write the set bits in the container.
*
* @param container where we write
* @param bitmap1 first bitmap
* @param bitmap2 second bitmap
*/
public static void fillArrayAND(final short[] container, final long[] bitmap1,
final long[] bitmap2) {
int pos = 0;
if (bitmap1.length != bitmap2.length) {
throw new IllegalArgumentException("not supported");
}
for (int k = 0; k < bitmap1.length; ++k) {
long bitset = bitmap1[k] & bitmap2[k];
while (bitset != 0) {
long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
/**
* Compute the bitwise ANDNOT between two long arrays and write the set bits in the container.
*
* @param container where we write
* @param bitmap1 first bitmap
* @param bitmap2 second bitmap
*/
public static void fillArrayANDNOT(final short[] container, final long[] bitmap1,
final long[] bitmap2) {
int pos = 0;
if (bitmap1.length != bitmap2.length) {
throw new IllegalArgumentException("not supported");
}
for (int k = 0; k < bitmap1.length; ++k) {
long bitset = bitmap1[k] & (~bitmap2[k]);
while (bitset != 0) {
long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
/**
* Compute the bitwise XOR between two long arrays and write the set bits in the container.
*
* @param container where we write
* @param bitmap1 first bitmap
* @param bitmap2 second bitmap
*/
public static void fillArrayXOR(final short[] container, final long[] bitmap1,
final long[] bitmap2) {
int pos = 0;
if (bitmap1.length != bitmap2.length) {
throw new IllegalArgumentException("not supported");
}
for (int k = 0; k < bitmap1.length; ++k) {
long bitset = bitmap1[k] ^ bitmap2[k];
while (bitset != 0) {
long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
/**
* flip bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void flipBitmapRange(long[] bitmap, int start, int end) {
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
bitmap[firstword] ^= ~(~0L << start);
for (int i = firstword; i < endword; i++) {
bitmap[i] = ~bitmap[i];
}
bitmap[endword] ^= ~0L >>> -end;
}
/**
* Hamming weight of the 64-bit words involved in the range
* start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return the hamming weight
*/
public static int cardinalityInBitmapWordRange(long[] bitmap, int start, int end) {
if (start == end) {
return 0;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
int answer = 0;
for (int i = firstword; i <= endword; i++) {
answer += Long.bitCount(bitmap[i]);
}
return answer;
}
protected static short highbits(int x) {
return (short) (x >>> 16);
}
protected static short highbits(long x) {
return (short) (x >>> 16);
}
// starts with binary search and finishes with a sequential search
protected static int hybridUnsignedBinarySearch(final short[] array, final int begin,
final int end, final short k) {
int ikey = toIntUnsigned(k);
// next line accelerates the possibly common case where the value would
// be inserted at the end
if ((end > 0) && (toIntUnsigned(array[end - 1]) < ikey)) {
return -end - 1;
}
int low = begin;
int high = end - 1;
// 32 in the next line matches the size of a cache line
while (low + 32 <= high) {
final int middleIndex = (low + high) >>> 1;
final int middleValue = toIntUnsigned(array[middleIndex]);
if (middleValue < ikey) {
low = middleIndex + 1;
} else if (middleValue > ikey) {
high = middleIndex - 1;
} else {
return middleIndex;
}
}
// we finish the job with a sequential search
int x = low;
for (; x <= high; ++x) {
final int val = toIntUnsigned(array[x]);
if (val >= ikey) {
if (val == ikey) {
return x;
}
break;
}
}
return -(x + 1);
}
protected static short lowbits(int x) {
return (short) (x & 0xFFFF);
}
protected static short lowbits(long x) {
return (short) (x & 0xFFFF);
}
protected static short maxLowBit() {
return (short) 0xFFFF;
}
protected static int maxLowBitAsInteger() {
return 0xFFFF;
}
/**
* clear bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void resetBitmapRange(long[] bitmap, int start, int end) {
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
if (firstword == endword) {
bitmap[firstword] &= ~((~0L << start) & (~0L >>> -end));
return;
}
bitmap[firstword] &= ~(~0L << start);
for (int i = firstword + 1; i < endword; i++) {
bitmap[i] = 0;
}
bitmap[endword] &= ~(~0L >>> -end);
}
/**
* Given a word w, return the position of the jth true bit.
*
* @param w word
* @param j index
* @return position of jth true bit in w
*/
public static int select(long w, int j) {
int seen = 0;
// Divide 64bit
int part = (int) (w & 0xFFFFFFFF);
int n = Integer.bitCount(part);
if (n <= j) {
part = (int) (w >>> 32);
seen += 32;
j -= n;
}
int ww = part;
// Divide 32bit
part = ww & 0xFFFF;
n = Integer.bitCount(part);
if (n <= j) {
part = ww >>> 16;
seen += 16;
j -= n;
}
ww = part;
// Divide 16bit
part = ww & 0xFF;
n = Integer.bitCount(part);
if (n <= j) {
part = ww >>> 8;
seen += 8;
j -= n;
}
ww = part;
// Lookup in final byte
int counter;
for (counter = 0; counter < 8; counter++) {
j -= (ww >>> counter) & 1;
if (j < 0) {
break;
}
}
return seen + counter;
}
/**
* set bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void setBitmapRange(long[] bitmap, int start, int end) {
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
if (firstword == endword) {
bitmap[firstword] |= (~0L << start) & (~0L >>> -end);
return;
}
bitmap[firstword] |= ~0L << start;
for (int i = firstword + 1; i < endword; i++) {
bitmap[i] = ~0L;
}
bitmap[endword] |= ~0L >>> -end;
}
/**
* set bits at start, start+1,..., end-1 and report the
* cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int setBitmapRangeAndCardinalityChange(long[] bitmap, int start, int end) {
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
setBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
/**
* flip bits at start, start+1,..., end-1 and report the
* cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int flipBitmapRangeAndCardinalityChange(long[] bitmap, int start, int end) {
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
flipBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
/**
* reset bits at start, start+1,..., end-1 and report the
* cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int resetBitmapRangeAndCardinalityChange(long[] bitmap, int start, int end) {
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
resetBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
protected static int toIntUnsigned(short x) {
return x & 0xFFFF;
}
/**
* Look for value k in array in the range [begin,end). If the value is found, return its index. If
* not, return -(i+1) where i is the index where the value would be inserted. The array is assumed
* to contain sorted values where shorts are interpreted as unsigned integers.
*
* @param array array where we search
* @param begin first index (inclusive)
* @param end last index (exclusive)
* @param k value we search for
* @return count
*/
public static int unsignedBinarySearch(final short[] array, final int begin, final int end,
final short k) {
if (USE_HYBRID_BINSEARCH) {
return hybridUnsignedBinarySearch(array, begin, end, k);
} else {
return branchyUnsignedBinarySearch(array, begin, end, k);
}
}
/**
* Compute the difference between two sorted lists and write the result to the provided output
* array
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @param buffer output array
* @return cardinality of the difference
*/
public static int unsignedDifference(final short[] set1, final int length1, final short[] set2,
final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
System.arraycopy(set1, 0, buffer, 0, length1);
return length1;
}
if (0 == length1) {
return 0;
}
short s1 = set1[k1];
short s2 = set2[k2];
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
break;
}
s1 = set1[k1];
} else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
++k1;
++k2;
if (k1 >= length1) {
break;
}
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1[k1];
s2 = set2[k2];
} else {// if (val1>val2)
++k2;
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2[k2];
}
}
return pos;
}
/**
* Compute the difference between two sorted lists and write the result to the provided output
* array
*
* @param set1 first array
* @param set2 second array
* @param buffer output array
* @return cardinality of the difference
*/
public static int unsignedDifference(ShortIterator set1, ShortIterator set2,
final short[] buffer) {
int pos = 0;
if (!set2.hasNext()) {
while (set1.hasNext()) {
buffer[pos++] = set1.next();
}
return pos;
}
if (!set1.hasNext()) {
return 0;
}
short v1 = set1.next();
short v2 = set2.next();
while (true) {
if (toIntUnsigned(v1) < toIntUnsigned(v2)) {
buffer[pos++] = v1;
if (!set1.hasNext()) {
return pos;
}
v1 = set1.next();
} else if (v1 == v2) {
if (!set1.hasNext()) {
break;
}
if (!set2.hasNext()) {
while (set1.hasNext()) {
buffer[pos++] = set1.next();
}
return pos;
}
v1 = set1.next();
v2 = set2.next();
} else {// if (val1>val2)
if (!set2.hasNext()) {
buffer[pos++] = v1;
while (set1.hasNext()) {
buffer[pos++] = set1.next();
}
return pos;
}
v2 = set2.next();
}
}
return pos;
}
/**
* Compute the exclusive union of two sorted lists and write the result to the provided output
* array
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @param buffer output array
* @return cardinality of the exclusive union
*/
public static int unsignedExclusiveUnion2by2(final short[] set1, final int length1,
final short[] set2, final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
System.arraycopy(set1, 0, buffer, 0, length1);
return length1;
}
if (0 == length1) {
System.arraycopy(set2, 0, buffer, 0, length2);
return length2;
}
short s1 = set1[k1];
short s2 = set2[k2];
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
System.arraycopy(set2, k2, buffer, pos, length2 - k2);
return pos + length2 - k2;
}
s1 = set1[k1];
} else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
++k1;
++k2;
if (k1 >= length1) {
System.arraycopy(set2, k2, buffer, pos, length2 - k2);
return pos + length2 - k2;
}
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1[k1];
s2 = set2[k2];
} else {// if (val1>val2)
buffer[pos++] = s2;
++k2;
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2[k2];
}
}
// return pos;
}
/**
* Intersect two sorted lists and write the result to the provided output array
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @param buffer output array
* @return cardinality of the intersection
*/
public static int unsignedIntersect2by2(final short[] set1, final int length1, final short[] set2,
final int length2, final short[] buffer) {
if (set1.length * 64 < set2.length) {
return unsignedOneSidedGallopingIntersect2by2(set1, length1, set2, length2, buffer);
} else if (set2.length * 64 < set1.length) {
return unsignedOneSidedGallopingIntersect2by2(set2, length2, set1, length1, buffer);
} else {
return unsignedLocalIntersect2by2(set1, length1, set2, length2, buffer);
}
}
/**
* Checks if two arrays intersect
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @return true if they intersect
*/
public static boolean unsignedIntersects(short[] set1, int length1, short[] set2, int length2) {
// galloping might be faster, but we do not expect this function to be slow
if ((0 == length1) || (0 == length2)) {
return false;
}
int k1 = 0;
int k2 = 0;
short s1 = set1[k1];
short s2 = set2[k2];
mainwhile:
while (true) {
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2[k2];
} while (toIntUnsigned(s2) < toIntUnsigned(s1));
}
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1[k1];
} while (toIntUnsigned(s1) < toIntUnsigned(s2));
} else {
return true;
}
}
return false;
}
protected static int unsignedLocalIntersect2by2(final short[] set1, final int length1,
final short[] set2, final int length2, final short[] buffer) {
if ((0 == length1) || (0 == length2)) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = set1[k1];
short s2 = set2[k2];
mainwhile:
while (true) {
int v1 = toIntUnsigned(s1);
int v2 = toIntUnsigned(s2);
if (v2 < v1) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2[k2];
v2 = toIntUnsigned(s2);
} while (v2 < v1);
}
if (v1 < v2) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1[k1];
v1 = toIntUnsigned(s1);
} while (v1 < v2);
} else {
// (set2[k2] == set1[k1])
buffer[pos++] = s1;
++k1;
if (k1 == length1) {
break;
}
++k2;
if (k2 == length2) {
break;
}
s1 = set1[k1];
s2 = set2[k2];
}
}
return pos;
}
/**
* Compute the cardinality of the intersection
*
* @param set1 first set
* @param length1 how many values to consider in the first set
* @param set2 second set
* @param length2 how many values to consider in the second set
* @return cardinality of the intersection
*/
public static int unsignedLocalIntersect2by2Cardinality(final short[] set1, final int length1,
final short[] set2, final int length2) {
if ((0 == length1) || (0 == length2)) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = set1[k1];
short s2 = set2[k2];
mainwhile:
while (true) {
int v1 = toIntUnsigned(s1);
int v2 = toIntUnsigned(s2);
if (v2 < v1) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2[k2];
v2 = toIntUnsigned(s2);
} while (v2 < v1);
}
if (v1 < v2) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1[k1];
v1 = toIntUnsigned(s1);
} while (v1 < v2);
} else {
// (set2[k2] == set1[k1])
pos++;
++k1;
if (k1 == length1) {
break;
}
++k2;
if (k2 == length2) {
break;
}
s1 = set1[k1];
s2 = set2[k2];
}
}
return pos;
}
protected static int unsignedOneSidedGallopingIntersect2by2(final short[] smallSet,
final int smallLength, final short[] largeSet, final int largeLength, final short[] buffer) {
if (0 == smallLength) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = largeSet[k1];
short s2 = smallSet[k2];
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
k1 = advanceUntil(largeSet, k1, largeLength, s2);
if (k1 == largeLength) {
break;
}
s1 = largeSet[k1];
}
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
++k2;
if (k2 == smallLength) {
break;
}
s2 = smallSet[k2];
} else {
// (set2[k2] == set1[k1])
buffer[pos++] = s2;
++k2;
if (k2 == smallLength) {
break;
}
s2 = smallSet[k2];
k1 = advanceUntil(largeSet, k1, largeLength, s2);
if (k1 == largeLength) {
break;
}
s1 = largeSet[k1];
}
}
return pos;
}
/**
* Unite two sorted lists and write the result to the provided output array
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @param buffer output array
* @return cardinality of the union
*/
public static int unsignedUnion2by2(final short[] set1, final int length1, final short[] set2,
final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
System.arraycopy(set1, 0, buffer, 0, length1);
return length1;
}
if (0 == length1) {
System.arraycopy(set2, 0, buffer, 0, length2);
return length2;
}
short s1 = set1[k1];
short s2 = set2[k2];
while (true) {
int v1 = toIntUnsigned(s1);
int v2 = toIntUnsigned(s2);
if (v1 < v2) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
System.arraycopy(set2, k2, buffer, pos, length2 - k2);
return pos + length2 - k2;
}
s1 = set1[k1];
} else if (v1 == v2) {
buffer[pos++] = s1;
++k1;
++k2;
if (k1 >= length1) {
System.arraycopy(set2, k2, buffer, pos, length2 - k2);
return pos + length2 - k2;
}
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1[k1];
s2 = set2[k2];
} else {// if (set1[k1]>set2[k2])
buffer[pos++] = s2;
++k2;
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2[k2];
}
}
// return pos;
}
}

113
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferBitSetUtil.java

@ -0,0 +1,113 @@
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.IntIterator;
import java.nio.LongBuffer;
import java.nio.ShortBuffer;
import java.util.Arrays;
import java.util.BitSet;
/***
*
* This class provides convenience functions to manipulate BitSet and MutableRoaringBitmap objects.
*
*/
public class BufferBitSetUtil {
// todo: add a method to convert an ImmutableRoaringBitmap to a BitSet using BitSet.valueOf
// a block consists has a maximum of 1024 words, each representing 64 bits,
// thus representing at maximum 65536 bits
static final private int BLOCK_LENGTH = MappeableBitmapContainer.MAX_CAPACITY / Long.SIZE; //
// 64-bit
// word
private static MappeableArrayContainer arrayContainerOf(final int from, final int to,
final int cardinality, final long[] words) {
// precondition: cardinality is max 4096
final short[] content = new short[cardinality];
int index = 0;
for (int i = from, socket = 0; i < to; ++i, socket += Long.SIZE) {
long word = words[i];
while (word != 0) {
long t = word & -word;
content[index++] = (short) (socket + Long.bitCount(t - 1));
word ^= t;
}
}
return new MappeableArrayContainer(ShortBuffer.wrap(content), cardinality);
}
/**
* Generate a MutableRoaringBitmap out of a long[], each long using little-endian representation
* of its bits
*
* @param words array of longs (will not be modified)
* @return roaring bitmap
* @see BitSet#toLongArray() for an equivalent
*/
public static MutableRoaringBitmap bitmapOf(final long[] words) {
// split long[] into blocks.
// each block becomes a single container, if any bit is set
final MutableRoaringBitmap ans = new MutableRoaringBitmap();
int containerIndex = 0;
for (int from = 0; from < words.length; from += BLOCK_LENGTH) {
final int to = Math.min(from + BLOCK_LENGTH, words.length);
final int blockCardinality = cardinality(from, to, words);
if (blockCardinality > 0) {
((MutableRoaringArray) ans.highLowContainer).insertNewKeyValueAt(containerIndex++,
BufferUtil.highbits(from * Long.SIZE),
BufferBitSetUtil.containerOf(from, to, blockCardinality, words));
}
}
return ans;
}
private static int cardinality(final int from, final int to, final long[] words) {
int sum = 0;
for (int i = from; i < to; i++) {
sum += Long.bitCount(words[i]);
}
return sum;
}
private static MappeableContainer containerOf(final int from, final int to,
final int blockCardinality, final long[] words) {
// find the best container available
if (blockCardinality <= MappeableArrayContainer.DEFAULT_MAX_SIZE) {
// containers with DEFAULT_MAX_SIZE or less integers should be
// ArrayContainers
return arrayContainerOf(from, to, blockCardinality, words);
} else {
// otherwise use bitmap container
return new MappeableBitmapContainer(
LongBuffer.wrap(Arrays.copyOfRange(words, from, from + BLOCK_LENGTH)), blockCardinality);
}
}
/**
* Compares a RoaringBitmap and a BitSet. They are equal if and only if they contain the same set
* of integers.
*
* @param bitset first object to be compared
* @param bitmap second object to be compared
* @return whether they are equal
*/
public static boolean equals(final BitSet bitset, final ImmutableRoaringBitmap bitmap) {
if (bitset.cardinality() != bitmap.getCardinality()) {
return false;
}
final IntIterator it = bitmap.getIntIterator();
while (it.hasNext()) {
int val = it.next();
if (!bitset.get(val)) {
return false;
}
}
return true;
}
}

631
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferFastAggregation.java

@ -0,0 +1,631 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.PriorityQueue;
/**
* Fast algorithms to aggregate many bitmaps.
*
* @author Daniel Lemire
*/
public final class BufferFastAggregation {
/**
* Private constructor to prevent instantiation of utility class
*/
private BufferFastAggregation() {
}
/**
* Compute the AND aggregate.
* <p>
* In practice, calls {#link naive_and}
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap and(ImmutableRoaringBitmap... bitmaps) {
return naive_and(bitmaps);
}
/**
* Compute the AND aggregate.
* <p>
* In practice, calls {#link naive_and}
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap and(@SuppressWarnings("rawtypes") Iterator bitmaps) {
return naive_and(bitmaps);
}
/**
* Compute the AND aggregate.
* <p>
* In practice, calls {#link naive_and}
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap and(MutableRoaringBitmap... bitmaps) {
return and(convertToImmutable(bitmaps));
}
/**
* Convenience method converting one type of iterator into another, to avoid unnecessary warnings.
*
* @param i input bitmaps
* @return an iterator over the provided iterator, with a different type
*/
public static Iterator<ImmutableRoaringBitmap> convertToImmutable(
final Iterator<MutableRoaringBitmap> i) {
return new Iterator<ImmutableRoaringBitmap>() {
@Override
public boolean hasNext() {
return i.hasNext();
}
@Override
public ImmutableRoaringBitmap next() {
return i.next();
}
@Override
public void remove() {
}
;
};
}
private static ImmutableRoaringBitmap[] convertToImmutable(MutableRoaringBitmap[] array) {
ImmutableRoaringBitmap[] answer = new ImmutableRoaringBitmap[array.length];
for (int k = 0; k < answer.length; ++k) {
answer[k] = array[k];
}
return answer;
}
/**
* Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #or(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap horizontal_or(ImmutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
if (bitmaps.length == 0) {
return answer;
}
PriorityQueue<MappeableContainerPointer> pq = new PriorityQueue<MappeableContainerPointer>(bitmaps.length);
for (int k = 0; k < bitmaps.length; ++k) {
MappeableContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
if (x.getContainer() != null) {
pq.add(x);
}
}
while (!pq.isEmpty()) {
MappeableContainerPointer x1 = pq.poll();
if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
answer.getMappeableRoaringArray().append(x1.key(), x1.getContainer().clone());
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
continue;
}
MappeableContainerPointer x2 = pq.poll();
MappeableContainer newc = x1.getContainer().lazyOR(x2.getContainer());
while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
MappeableContainerPointer x = pq.poll();
newc = newc.lazyIOR(x.getContainer());
x.advance();
if (x.getContainer() != null) {
pq.add(x);
} else if (pq.isEmpty()) {
break;
}
}
newc = newc.repairAfterLazy();
answer.getMappeableRoaringArray().append(x1.key(), newc);
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
x2.advance();
if (x2.getContainer() != null) {
pq.add(x2);
}
}
return answer;
}
/**
* Calls naive_or.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
@Deprecated
public static MutableRoaringBitmap horizontal_or(@SuppressWarnings("rawtypes") Iterator bitmaps) {
return naive_or(bitmaps);
}
/**
* Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #or(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap horizontal_or(MutableRoaringBitmap... bitmaps) {
return horizontal_or(convertToImmutable(bitmaps));
}
/**
* Minimizes memory usage while computing the xor aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #xor(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap horizontal_xor(ImmutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
if (bitmaps.length == 0) {
return answer;
}
PriorityQueue<MappeableContainerPointer> pq = new PriorityQueue<MappeableContainerPointer>(bitmaps.length);
for (int k = 0; k < bitmaps.length; ++k) {
MappeableContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
if (x.getContainer() != null) {
pq.add(x);
}
}
while (!pq.isEmpty()) {
MappeableContainerPointer x1 = pq.poll();
if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
answer.getMappeableRoaringArray().append(x1.key(), x1.getContainer().clone());
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
continue;
}
MappeableContainerPointer x2 = pq.poll();
MappeableContainer newc = x1.getContainer().xor(x2.getContainer());
while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
MappeableContainerPointer x = pq.poll();
newc = newc.ixor(x.getContainer());
x.advance();
if (x.getContainer() != null) {
pq.add(x);
} else if (pq.isEmpty()) {
break;
}
}
answer.getMappeableRoaringArray().append(x1.key(), newc);
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
x2.advance();
if (x2.getContainer() != null) {
pq.add(x2);
}
}
return answer;
}
/**
* Minimizes memory usage while computing the xor aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #xor(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap horizontal_xor(MutableRoaringBitmap... bitmaps) {
return horizontal_xor(convertToImmutable(bitmaps));
}
/**
* Compute overall AND between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_and(ImmutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer;
if (bitmaps.length > 0) {
answer = (bitmaps[0]).toMutableRoaringBitmap();
for (int k = 1; k < bitmaps.length; ++k) {
answer = ImmutableRoaringBitmap.and(answer, bitmaps[k]);
}
} else {
answer = new MutableRoaringBitmap();
}
return answer;
}
/**
* Compute overall AND between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_and(@SuppressWarnings("rawtypes") Iterator bitmaps) {
if (!bitmaps.hasNext()) {
return new MutableRoaringBitmap();
}
MutableRoaringBitmap answer =
((ImmutableRoaringBitmap) bitmaps.next()).toMutableRoaringBitmap();
while (bitmaps.hasNext()) {
answer.and((ImmutableRoaringBitmap) bitmaps.next());
}
return answer;
}
/**
* Compute overall AND between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_and(MutableRoaringBitmap... bitmaps) {
if (bitmaps.length == 0) {
return new MutableRoaringBitmap();
}
MutableRoaringBitmap answer = bitmaps[0].clone();
for (int k = 1; k < bitmaps.length; ++k) {
answer.and(bitmaps[k]);
}
return answer;
}
/**
* Compute overall OR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_or(ImmutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.naivelazyor(bitmaps[k]);
}
answer.repairAfterLazy();
return answer;
}
/**
* Compute overall OR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_or(@SuppressWarnings("rawtypes") Iterator bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
while (bitmaps.hasNext()) {
answer.naivelazyor((ImmutableRoaringBitmap) bitmaps.next());
}
answer.repairAfterLazy();
return answer;
}
/**
* Compute overall OR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_or(MutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.lazyor(bitmaps[k]);
}
answer.repairAfterLazy();
return answer;
}
/**
* Compute overall XOR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_xor(ImmutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.xor(bitmaps[k]);
}
return answer;
}
/**
* Compute overall XOR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_xor(@SuppressWarnings("rawtypes") Iterator bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
while (bitmaps.hasNext()) {
answer.xor((ImmutableRoaringBitmap) bitmaps.next());
}
return answer;
}
/**
* Compute overall XOR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_xor(MutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.xor(bitmaps[k]);
}
return answer;
}
/**
* Compute overall OR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap or(ImmutableRoaringBitmap... bitmaps) {
return naive_or(bitmaps);
}
/**
* Compute overall OR between bitmaps.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap or(@SuppressWarnings("rawtypes") Iterator bitmaps) {
return naive_or(bitmaps);
}
/**
* Compute overall OR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap or(MutableRoaringBitmap... bitmaps) {
return naive_or(bitmaps);
}
/**
* Uses a priority queue to compute the or aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_or(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap priorityqueue_or(ImmutableRoaringBitmap... bitmaps) {
if (bitmaps.length == 0) {
return new MutableRoaringBitmap();
} else if (bitmaps.length == 1) {
return bitmaps[0].toMutableRoaringBitmap();
}
// we buffer the call to getLongSizeInBytes(), hence the code complexity
final ImmutableRoaringBitmap[] buffer = Arrays.copyOf(bitmaps, bitmaps.length);
final int[] sizes = new int[buffer.length];
final boolean[] istmp = new boolean[buffer.length];
for (int k = 0; k < sizes.length; ++k) {
sizes[k] = buffer[k].serializedSizeInBytes();
}
PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
@Override
public int compare(Integer a, Integer b) {
return sizes[a] - sizes[b];
}
});
for (int k = 0; k < sizes.length; ++k) {
pq.add(k);
}
while (pq.size() > 1) {
Integer x1 = pq.poll();
Integer x2 = pq.poll();
if (istmp[x1] && istmp[x2]) {
buffer[x1] = MutableRoaringBitmap.lazyorfromlazyinputs((MutableRoaringBitmap) buffer[x1],
(MutableRoaringBitmap) buffer[x2]);
sizes[x1] = buffer[x1].serializedSizeInBytes();
pq.add(x1);
} else if (istmp[x2]) {
((MutableRoaringBitmap) buffer[x2]).lazyor(buffer[x1]);
sizes[x2] = buffer[x2].serializedSizeInBytes();
pq.add(x2);
} else if (istmp[x1]) {
((MutableRoaringBitmap) buffer[x1]).lazyor(buffer[x2]);
sizes[x1] = buffer[x1].serializedSizeInBytes();
pq.add(x1);
} else {
buffer[x1] = ImmutableRoaringBitmap.lazyor(buffer[x1], buffer[x2]);
sizes[x1] = buffer[x1].serializedSizeInBytes();
istmp[x1] = true;
pq.add(x1);
}
}
MutableRoaringBitmap answer = (MutableRoaringBitmap) buffer[pq.poll()];
answer.repairAfterLazy();
return answer;
}
/**
* Uses a priority queue to compute the or aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_or(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap priorityqueue_or(
@SuppressWarnings("rawtypes") Iterator bitmaps) {
if (!bitmaps.hasNext()) {
return new MutableRoaringBitmap();
}
// we buffer the call to getLongSizeInBytes(), hence the code complexity
ArrayList<ImmutableRoaringBitmap> buffer = new ArrayList<ImmutableRoaringBitmap>();
while (bitmaps.hasNext()) {
buffer.add((ImmutableRoaringBitmap) bitmaps.next());
}
final long[] sizes = new long[buffer.size()];
final boolean[] istmp = new boolean[buffer.size()];
for (int k = 0; k < sizes.length; ++k) {
sizes[k] = buffer.get(k).getLongSizeInBytes();
}
PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
@Override
public int compare(Integer a, Integer b) {
return (int) (sizes[a] - sizes[b]);
}
});
for (int k = 0; k < sizes.length; ++k) {
pq.add(k);
}
if (pq.size() == 1) {
return buffer.get(pq.poll()).toMutableRoaringBitmap();
}
while (pq.size() > 1) {
Integer x1 = pq.poll();
Integer x2 = pq.poll();
if (istmp[x1] && istmp[x2]) {
buffer.set(x1, MutableRoaringBitmap.lazyorfromlazyinputs(
(MutableRoaringBitmap) buffer.get(x1), (MutableRoaringBitmap) buffer.get(x2)));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
pq.add(x1);
} else if (istmp[x2]) {
((MutableRoaringBitmap) buffer.get(x2)).lazyor(buffer.get(x1));
sizes[x2] = buffer.get(x2).getLongSizeInBytes();
pq.add(x2);
} else if (istmp[x1]) {
((MutableRoaringBitmap) buffer.get(x1)).lazyor(buffer.get(x2));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
pq.add(x1);
} else {
buffer.set(x1, ImmutableRoaringBitmap.lazyor(buffer.get(x1), buffer.get(x2)));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
istmp[x1] = true;
pq.add(x1);
}
}
MutableRoaringBitmap answer = (MutableRoaringBitmap) buffer.get(pq.poll());
answer.repairAfterLazy();
return answer;
}
/**
* Uses a priority queue to compute the xor aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_xor(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap priorityqueue_xor(ImmutableRoaringBitmap... bitmaps) {
// code could be faster, see priorityqueue_or
if (bitmaps.length < 2) {
throw new IllegalArgumentException("Expecting at least 2 bitmaps");
}
final PriorityQueue<ImmutableRoaringBitmap> pq =
new PriorityQueue<ImmutableRoaringBitmap>(bitmaps.length, new Comparator<ImmutableRoaringBitmap>() {
@Override
public int compare(ImmutableRoaringBitmap a, ImmutableRoaringBitmap b) {
return (int) (a.getLongSizeInBytes() - b.getLongSizeInBytes());
}
});
Collections.addAll(pq, bitmaps);
while (pq.size() > 1) {
final ImmutableRoaringBitmap x1 = pq.poll();
final ImmutableRoaringBitmap x2 = pq.poll();
pq.add(ImmutableRoaringBitmap.xor(x1, x2));
}
return (MutableRoaringBitmap) pq.poll();
}
/**
* Compute overall XOR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap xor(ImmutableRoaringBitmap... bitmaps) {
return naive_xor(bitmaps);
}
/**
* Compute overall XOR between bitmaps.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap xor(@SuppressWarnings("rawtypes") Iterator bitmaps) {
return naive_xor(bitmaps);
}
/**
* Compute overall XOR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap xor(MutableRoaringBitmap... bitmaps) {
return naive_xor(bitmaps);
}
}

132
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferIntIteratorFlyweight.java

@ -0,0 +1,132 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.PeekableIntIterator;
import com.fr.third.bitmap.roaringbitmap.PeekableShortIterator;
/**
* Fast iterator minimizing the stress on the garbage collector. You can create one reusable
* instance of this class and then {@link #wrap(ImmutableRoaringBitmap)}
* <p>
* For better performance, consider the {@link ImmutableRoaringBitmap#forEach} method.
*
* @author Borislav Ivanov
**/
public class BufferIntIteratorFlyweight implements PeekableIntIterator {
private int hs;
private PeekableShortIterator iter;
private MappeableArrayContainerShortIterator arrIter = new MappeableArrayContainerShortIterator();
private MappeableBitmapContainerShortIterator bitmapIter =
new MappeableBitmapContainerShortIterator();
private MappeableRunContainerShortIterator runIter = new MappeableRunContainerShortIterator();
private int pos;
private ImmutableRoaringBitmap roaringBitmap = null;
/**
* Creates an instance that is not ready for iteration. You must first call
* {@link #wrap(ImmutableRoaringBitmap)}.
*/
public BufferIntIteratorFlyweight() {
}
/**
* Creates an instance that is ready for iteration.
*
* @param r bitmap to be iterated over
*/
public BufferIntIteratorFlyweight(ImmutableRoaringBitmap r) {
wrap(r);
}
@Override
public PeekableIntIterator clone() {
try {
BufferIntIteratorFlyweight x = (BufferIntIteratorFlyweight) super.clone();
x.iter = this.iter.clone();
return x;
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public boolean hasNext() {
return pos < this.roaringBitmap.highLowContainer.size();
}
@Override
public int next() {
int x = iter.nextAsInt() | hs;
if (!iter.hasNext()) {
++pos;
nextContainer();
}
return x;
}
private void nextContainer() {
if (pos < this.roaringBitmap.highLowContainer.size()) {
MappeableContainer container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
if (container instanceof MappeableBitmapContainer) {
bitmapIter.wrap((MappeableBitmapContainer) container);
iter = bitmapIter;
} else if (container instanceof MappeableRunContainer) {
runIter.wrap((MappeableRunContainer) container);
iter = runIter;
} else {
arrIter.wrap((MappeableArrayContainer) container);
iter = arrIter;
}
hs = BufferUtil.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
}
}
/**
* Prepares a bitmap for iteration
*
* @param r bitmap to be iterated over
*/
public void wrap(ImmutableRoaringBitmap r) {
this.hs = 0;
this.pos = 0;
this.roaringBitmap = r;
this.nextContainer();
}
@Override
public void advanceIfNeeded(int minval) {
while (hasNext() && ((hs >>> 16) < (minval >>> 16))) {
++pos;
nextContainer();
}
if (hasNext() && ((hs >>> 16) == (minval >>> 16))) {
iter.advanceIfNeeded(BufferUtil.lowbits(minval));
if (!iter.hasNext()) {
++pos;
nextContainer();
}
}
}
@Override
public int peekNext() {
return BufferUtil.toIntUnsigned(iter.peekNext()) | hs;
}
}

116
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferReverseIntIteratorFlyweight.java

@ -0,0 +1,116 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.IntIterator;
import com.fr.third.bitmap.roaringbitmap.ShortIterator;
/**
* Fast iterator minimizing the stress on the garbage collector. You can create one reusable
* instance of this class and then {@link #wrap(ImmutableRoaringBitmap)}
* <p>
* This iterator enumerates the stored values in reverse (starting from the end).
*
* @author Borislav Ivanov
**/
public class BufferReverseIntIteratorFlyweight implements IntIterator {
private int hs;
private ShortIterator iter;
private ReverseMappeableArrayContainerShortIterator arrIter =
new ReverseMappeableArrayContainerShortIterator();
private ReverseMappeableBitmapContainerShortIterator bitmapIter =
new ReverseMappeableBitmapContainerShortIterator();
private ReverseMappeableRunContainerShortIterator runIter =
new ReverseMappeableRunContainerShortIterator();
private short pos;
private ImmutableRoaringBitmap roaringBitmap = null;
/**
* Creates an instance that is not ready for iteration. You must first call
* {@link #wrap(ImmutableRoaringBitmap)}.
*/
public BufferReverseIntIteratorFlyweight() {
}
/**
* Creates an instance that is ready for iteration.
*
* @param r bitmap to be iterated over
*/
public BufferReverseIntIteratorFlyweight(ImmutableRoaringBitmap r) {
wrap(r);
}
@Override
public IntIterator clone() {
try {
BufferReverseIntIteratorFlyweight x = (BufferReverseIntIteratorFlyweight) super.clone();
x.iter = this.iter.clone();
return x;
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public boolean hasNext() {
return pos >= 0;
}
@Override
public int next() {
final int x = iter.nextAsInt() | hs;
if (!iter.hasNext()) {
--pos;
nextContainer();
}
return x;
}
private void nextContainer() {
if (pos >= 0) {
MappeableContainer container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
if (container instanceof MappeableBitmapContainer) {
bitmapIter.wrap((MappeableBitmapContainer) container);
iter = bitmapIter;
} else if (container instanceof MappeableRunContainer) {
runIter.wrap((MappeableRunContainer) container);
iter = runIter;
} else {
arrIter.wrap((MappeableArrayContainer) container);
iter = arrIter;
}
hs = BufferUtil.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
}
}
/**
* Prepares a bitmap for iteration
*
* @param r bitmap to be iterated over
*/
public void wrap(ImmutableRoaringBitmap r) {
this.roaringBitmap = r;
this.hs = 0;
this.pos = (short) (this.roaringBitmap.highLowContainer.size() - 1);
this.nextContainer();
}
}

825
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/BufferUtil.java

@ -0,0 +1,825 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.Util;
import java.nio.Buffer;
import java.nio.LongBuffer;
import java.nio.ShortBuffer;
/**
* Various useful methods for roaring bitmaps.
* <p>
* This class is similar to Util but meant to be used with memory mapping.
*/
public final class BufferUtil {
/**
* Private constructor to prevent instantiation of utility class
*/
private BufferUtil() {
}
/**
* Find the smallest integer larger than pos such that array[pos]&gt;= min. If none can be found,
* return length. Based on code by O. Kaser.
*
* @param array container where we search
* @param pos initial position
* @param min minimal threshold
* @param length how big should the array consider to be
* @return x greater than pos such that array[pos] is at least as large as min, pos is is equal to
* length if it is not possible.
*/
protected static int advanceUntil(ShortBuffer array, int pos, int length, short min) {
int lower = pos + 1;
// special handling for a possibly common sequential case
if (lower >= length || toIntUnsigned(array.get(lower)) >= toIntUnsigned(min)) {
return lower;
}
int spansize = 1; // could set larger
// bootstrap an upper limit
while (lower + spansize < length
&& toIntUnsigned(array.get(lower + spansize)) < toIntUnsigned(min)) {
spansize *= 2; // hoping for compiler will reduce to
}
// shift
int upper = (lower + spansize < length) ? lower + spansize : length - 1;
// maybe we are lucky (could be common case when the seek ahead
// expected
// to be small and sequential will otherwise make us look bad)
if (array.get(upper) == min) {
return upper;
}
if (toIntUnsigned(array.get(upper)) < toIntUnsigned(min)) {// means
// array
// has no
// item
// >= min
// pos = array.length;
return length;
}
// we know that the next-smallest span was too small
lower += (spansize / 2);
// else begin binary search
// invariant: array[lower]<min && array[upper]>min
while (lower + 1 != upper) {
int mid = (lower + upper) / 2;
short arraymid = array.get(mid);
if (arraymid == min) {
return mid;
} else if (toIntUnsigned(arraymid) < toIntUnsigned(min)) {
lower = mid;
} else {
upper = mid;
}
}
return upper;
}
protected static void arraycopy(ShortBuffer src, int srcPos, ShortBuffer dest, int destPos,
int length) {
if (BufferUtil.isBackedBySimpleArray(src) && BufferUtil.isBackedBySimpleArray(dest)) {
System.arraycopy(src.array(), srcPos, dest.array(), destPos, length);
} else {
if (srcPos < destPos) {
for (int k = length - 1; k >= 0; --k) {
dest.put(destPos + k, src.get(k + srcPos));
}
} else {
for (int k = 0; k < length; ++k) {
dest.put(destPos + k, src.get(k + srcPos));
}
}
}
}
protected static int branchyUnsignedBinarySearch(final ShortBuffer array, final int begin,
final int end, final short k) {
final int ikey = toIntUnsigned(k);
// next line accelerates the possibly common case where the value would be inserted at the end
if ((end > 0) && (toIntUnsigned(array.get(end - 1)) < ikey)) {
return -end - 1;
}
int low = begin;
int high = end - 1;
while (low <= high) {
final int middleIndex = (low + high) >>> 1;
final int middleValue = toIntUnsigned(array.get(middleIndex));
if (middleValue < ikey) {
low = middleIndex + 1;
} else if (middleValue > ikey) {
high = middleIndex - 1;
} else {
return middleIndex;
}
}
return -(low + 1);
}
/**
* Compares the two specified {@code short} values, treating them as unsigned values between
* {@code 0} and {@code 2^16 - 1} inclusive.
*
* @param a the first unsigned {@code short} to compare
* @param b the second unsigned {@code short} to compare
* @return a negative value if {@code a} is less than {@code b}; a positive value if {@code a} is
* greater than {@code b}; or zero if they are equal
*/
public static int compareUnsigned(short a, short b) {
return toIntUnsigned(a) - toIntUnsigned(b);
}
protected static void fillArrayAND(short[] container, LongBuffer bitmap1, LongBuffer bitmap2) {
int pos = 0;
if (bitmap1.limit() != bitmap2.limit()) {
throw new IllegalArgumentException("not supported");
}
if (BufferUtil.isBackedBySimpleArray(bitmap1) && BufferUtil.isBackedBySimpleArray(bitmap2)) {
int len = bitmap1.limit();
long[] b1 = bitmap1.array();
long[] b2 = bitmap2.array();
for (int k = 0; k < len; ++k) {
long bitset = b1[k] & b2[k];
while (bitset != 0) {
final long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
} else {
int len = bitmap1.limit();
for (int k = 0; k < len; ++k) {
long bitset = bitmap1.get(k) & bitmap2.get(k);
while (bitset != 0) {
final long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
}
protected static void fillArrayANDNOT(short[] container, LongBuffer bitmap1, LongBuffer bitmap2) {
int pos = 0;
if (bitmap1.limit() != bitmap2.limit()) {
throw new IllegalArgumentException("not supported");
}
if (BufferUtil.isBackedBySimpleArray(bitmap1) && BufferUtil.isBackedBySimpleArray(bitmap2)) {
int len = bitmap1.limit();
long[] b1 = bitmap1.array();
long[] b2 = bitmap2.array();
for (int k = 0; k < len; ++k) {
long bitset = b1[k] & (~b2[k]);
while (bitset != 0) {
final long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
} else {
int len = bitmap1.limit();
for (int k = 0; k < len; ++k) {
long bitset = bitmap1.get(k) & (~bitmap2.get(k));
while (bitset != 0) {
final long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
}
protected static void fillArrayXOR(short[] container, LongBuffer bitmap1, LongBuffer bitmap2) {
int pos = 0;
if (bitmap1.limit() != bitmap2.limit()) {
throw new IllegalArgumentException("not supported");
}
if (BufferUtil.isBackedBySimpleArray(bitmap1) && BufferUtil.isBackedBySimpleArray(bitmap2)) {
Util.fillArrayXOR(container, bitmap1.array(), bitmap2.array());
} else {
int len = bitmap1.limit();
for (int k = 0; k < len; ++k) {
long bitset = bitmap1.get(k) ^ bitmap2.get(k);
while (bitset != 0) {
final long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
}
/**
* flip bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void flipBitmapRange(LongBuffer bitmap, int start, int end) {
if (isBackedBySimpleArray(bitmap)) {
Util.flipBitmapRange(bitmap.array(), start, end);
return;
}
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
bitmap.put(firstword, bitmap.get(firstword) ^ ~(~0L << start));
for (int i = firstword; i < endword; i++) {
bitmap.put(i, ~bitmap.get(i));
}
bitmap.put(endword, bitmap.get(endword) ^ (~0L >>> -end));
}
/**
* Hamming weight of the 64-bit words involved in the range start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
private static int cardinalityInBitmapWordRange(LongBuffer bitmap, int start, int end) {
if (isBackedBySimpleArray(bitmap)) {
return Util.cardinalityInBitmapWordRange(bitmap.array(), start, end);
}
if (start == end) {
return 0;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
int answer = 0;
for (int i = firstword; i <= endword; i++) {
answer += Long.bitCount(bitmap.get(i));
}
return answer;
}
/**
* set bits at start, start+1,..., end-1 and report the cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int setBitmapRangeAndCardinalityChange(LongBuffer bitmap, int start, int end) {
if (BufferUtil.isBackedBySimpleArray(bitmap)) {
return Util.setBitmapRangeAndCardinalityChange(bitmap.array(), start, end);
}
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
setBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
/**
* flip bits at start, start+1,..., end-1 and report the cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int flipBitmapRangeAndCardinalityChange(LongBuffer bitmap, int start, int end) {
if (BufferUtil.isBackedBySimpleArray(bitmap)) {
return Util.flipBitmapRangeAndCardinalityChange(bitmap.array(), start, end);
}
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
flipBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
/**
* reset bits at start, start+1,..., end-1 and report the cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int resetBitmapRangeAndCardinalityChange(LongBuffer bitmap, int start, int end) {
if (BufferUtil.isBackedBySimpleArray(bitmap)) {
return Util.resetBitmapRangeAndCardinalityChange(bitmap.array(), start, end);
}
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
resetBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
/**
* From the cardinality of a container, compute the corresponding size in bytes of the container.
* Additional information is required if the container is run encoded.
*
* @param card the cardinality if this is not run encoded, otherwise ignored
* @param numRuns number of runs if run encoded, othewise ignored
* @param isRunEncoded boolean
* @return the size in bytes
*/
protected static int getSizeInBytesFromCardinalityEtc(int card, int numRuns,
boolean isRunEncoded) {
if (isRunEncoded) {
return 2 + numRuns * 2 * 2; // each run uses 2 shorts, plus the initial short giving num runs
}
boolean isBitmap = card > MappeableArrayContainer.DEFAULT_MAX_SIZE;
if (isBitmap) {
return MappeableBitmapContainer.MAX_CAPACITY / 8;
} else {
return card * 2;
}
}
protected static short highbits(int x) {
return (short) (x >>> 16);
}
protected static short highbits(long x) {
return (short) (x >>> 16);
}
/**
* Checks whether the Buffer is backed by a simple array. In java, a Buffer is an abstraction that
* can represent various data, from data on disk all the way to native Java arrays. Like all
* abstractions, a Buffer might carry a performance penalty. Thus, we sometimes check whether the
* Buffer is simply a wrapper around a Java array. In these instances, it might be best, from a
* performance point of view, to access the underlying array (using the array()) method.
*
* @param b the provided Buffer
* @return whether the Buffer is backed by a simple array
*/
protected static boolean isBackedBySimpleArray(Buffer b) {
return b.hasArray() && (b.arrayOffset() == 0);
}
protected static short lowbits(int x) {
return (short) (x & 0xFFFF);
}
protected static short lowbits(long x) {
return (short) (x & 0xFFFF);
}
protected static short maxLowBit() {
return (short) 0xFFFF;
}
protected static int maxLowBitAsInteger() {
return 0xFFFF;
}
/**
* clear bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void resetBitmapRange(LongBuffer bitmap, int start, int end) {
if (isBackedBySimpleArray(bitmap)) {
Util.resetBitmapRange(bitmap.array(), start, end);
return;
}
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
if (firstword == endword) {
bitmap.put(firstword, bitmap.get(firstword) & ~((~0L << start) & (~0L >>> -end)));
return;
}
bitmap.put(firstword, bitmap.get(firstword) & (~(~0L << start)));
for (int i = firstword + 1; i < endword; i++) {
bitmap.put(i, 0L);
}
bitmap.put(endword, bitmap.get(endword) & (~(~0L >>> -end)));
}
/**
* set bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void setBitmapRange(LongBuffer bitmap, int start, int end) {
if (isBackedBySimpleArray(bitmap)) {
Util.setBitmapRange(bitmap.array(), start, end);
return;
}
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
if (firstword == endword) {
bitmap.put(firstword, bitmap.get(firstword) | ((~0L << start) & (~0L >>> -end)));
return;
}
bitmap.put(firstword, bitmap.get(firstword) | (~0L << start));
for (int i = firstword + 1; i < endword; i++) {
bitmap.put(i, ~0L);
}
bitmap.put(endword, bitmap.get(endword) | (~0L >>> -end));
}
protected static int toIntUnsigned(short x) {
return x & 0xFFFF;
}
/**
* Look for value k in buffer in the range [begin,end). If the value is found, return its index.
* If not, return -(i+1) where i is the index where the value would be inserted. The buffer is
* assumed to contain sorted values where shorts are interpreted as unsigned integers.
*
* @param array buffer where we search
* @param begin first index (inclusive)
* @param end last index (exclusive)
* @param k value we search for
* @return count
*/
public static int unsignedBinarySearch(final ShortBuffer array, final int begin, final int end,
final short k) {
return branchyUnsignedBinarySearch(array, begin, end, k);
}
protected static int unsignedDifference(final ShortBuffer set1, final int length1,
final ShortBuffer set2, final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
set1.get(buffer, 0, length1);
return length1;
}
if (0 == length1) {
return 0;
}
short s1 = set1.get(k1);
short s2 = set2.get(k2);
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
break;
}
s1 = set1.get(k1);
} else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
++k1;
++k2;
if (k1 >= length1) {
break;
}
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1.get(k1);
s2 = set2.get(k2);
} else {// if (val1>val2)
++k2;
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2.get(k2);
}
}
return pos;
}
protected static int unsignedExclusiveUnion2by2(final ShortBuffer set1, final int length1,
final ShortBuffer set2, final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
set1.get(buffer, 0, length1);
return length1;
}
if (0 == length1) {
set2.get(buffer, 0, length2);
return length2;
}
short s1 = set1.get(k1);
short s2 = set2.get(k2);
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
set2.position(k2);
set2.get(buffer, pos, length2 - k2);
return pos + length2 - k2;
}
s1 = set1.get(k1);
} else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
++k1;
++k2;
if (k1 >= length1) {
set2.position(k2);
set2.get(buffer, pos, length2 - k2);
return pos + length2 - k2;
}
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1.get(k1);
s2 = set2.get(k2);
} else {// if (val1>val2)
buffer[pos++] = s2;
++k2;
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2.get(k2);
}
}
// return pos;
}
protected static int unsignedIntersect2by2(final ShortBuffer set1, final int length1,
final ShortBuffer set2, final int length2, final short[] buffer) {
if (length1 * 64 < length2) {
return unsignedOneSidedGallopingIntersect2by2(set1, length1, set2, length2, buffer);
} else if (length2 * 64 < length1) {
return unsignedOneSidedGallopingIntersect2by2(set2, length2, set1, length1, buffer);
} else {
return unsignedLocalIntersect2by2(set1, length1, set2, length2, buffer);
}
}
/**
* Checks if two arrays intersect
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @return true if they intersect
*/
public static boolean unsignedIntersects(ShortBuffer set1, int length1, ShortBuffer set2,
int length2) {
if ((0 == length1) || (0 == length2)) {
return false;
}
int k1 = 0;
int k2 = 0;
// could be more efficient with galloping
short s1 = set1.get(k1);
short s2 = set2.get(k2);
mainwhile:
while (true) {
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2.get(k2);
} while (toIntUnsigned(s2) < toIntUnsigned(s1));
}
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1.get(k1);
} while (toIntUnsigned(s1) < toIntUnsigned(s2));
} else {
return true;
}
}
return false;
}
protected static int unsignedLocalIntersect2by2(final ShortBuffer set1, final int length1,
final ShortBuffer set2, final int length2, final short[] buffer) {
if ((0 == length1) || (0 == length2)) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = set1.get(k1);
short s2 = set2.get(k2);
mainwhile:
while (true) {
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2.get(k2);
} while (toIntUnsigned(s2) < toIntUnsigned(s1));
}
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1.get(k1);
} while (toIntUnsigned(s1) < toIntUnsigned(s2));
} else {
// (set2.get(k2) == set1.get(k1))
buffer[pos++] = s1;
++k1;
if (k1 == length1) {
break;
}
s1 = set1.get(k1);
++k2;
if (k2 == length2) {
break;
}
s2 = set2.get(k2);
}
}
return pos;
}
protected static int unsignedLocalIntersect2by2Cardinality(final ShortBuffer set1,
final int length1, final ShortBuffer set2, final int length2) {
if ((0 == length1) || (0 == length2)) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = set1.get(k1);
short s2 = set2.get(k2);
mainwhile:
while (true) {
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2.get(k2);
} while (toIntUnsigned(s2) < toIntUnsigned(s1));
}
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1.get(k1);
} while (toIntUnsigned(s1) < toIntUnsigned(s2));
} else {
++pos;
++k1;
if (k1 == length1) {
break;
}
s1 = set1.get(k1);
++k2;
if (k2 == length2) {
break;
}
s2 = set2.get(k2);
}
}
return pos;
}
protected static int unsignedOneSidedGallopingIntersect2by2(final ShortBuffer smallSet,
final int smallLength, final ShortBuffer largeSet, final int largeLength,
final short[] buffer) {
if (0 == smallLength) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = largeSet.get(k1);
short s2 = smallSet.get(k2);
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
k1 = advanceUntil(largeSet, k1, largeLength, s2);
if (k1 == largeLength) {
break;
}
s1 = largeSet.get(k1);
}
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
++k2;
if (k2 == smallLength) {
break;
}
s2 = smallSet.get(k2);
} else {
// (set2.get(k2) == set1.get(k1))
buffer[pos++] = s2;
++k2;
if (k2 == smallLength) {
break;
}
s2 = smallSet.get(k2);
k1 = advanceUntil(largeSet, k1, largeLength, s2);
if (k1 == largeLength) {
break;
}
s1 = largeSet.get(k1);
}
}
return pos;
}
protected static int unsignedUnion2by2(final ShortBuffer set1, final int length1,
final ShortBuffer set2, final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
set1.get(buffer, 0, length1);
return length1;
}
if (0 == length1) {
set2.get(buffer, 0, length2);
return length2;
}
short s1 = set1.get(k1);
short s2 = set2.get(k2);
while (true) {
int v1 = toIntUnsigned(s1);
int v2 = toIntUnsigned(s2);
if (v1 < v2) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
set2.position(k2);
set2.get(buffer, pos, length2 - k2);
return pos + length2 - k2;
}
s1 = set1.get(k1);
} else if (v1 == v2) {
buffer[pos++] = s1;
++k1;
++k2;
if (k1 >= length1) {
set2.position(k2);
set2.get(buffer, pos, length2 - k2);
return pos + length2 - k2;
}
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1.get(k1);
s2 = set2.get(k2);
} else {// if (set1.get(k1)>set2.get(k2))
buffer[pos++] = s2;
++k2;
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2.get(k2);
}
}
// return pos;
}
}

463
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/ImmutableRoaringArray.java

@ -0,0 +1,463 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import java.io.DataOutput;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.LongBuffer;
import java.nio.ShortBuffer;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
/**
* This is the underlying data structure for an ImmutableRoaringBitmap. This class is not meant for
* end-users.
*/
public final class ImmutableRoaringArray implements PointableRoaringArray {
protected static final short SERIAL_COOKIE = MutableRoaringArray.SERIAL_COOKIE;
protected static final short SERIAL_COOKIE_NO_RUNCONTAINER =
MutableRoaringArray.SERIAL_COOKIE_NO_RUNCONTAINER;
private final static int startofrunbitmap = 4; // if there is a runcontainer bitmap
ByteBuffer buffer;
int size;
/**
* Create an array based on a previously serialized ByteBuffer. The input ByteBuffer is
* effectively copied (with the slice operation) so you should expect the provided ByteBuffer to
* remain unchanged.
*
* @param bbf The source ByteBuffer
*/
protected ImmutableRoaringArray(ByteBuffer bbf) {
buffer = bbf.slice();
buffer.order(ByteOrder.LITTLE_ENDIAN);
final int cookie = buffer.getInt(0);
if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) {
throw new RuntimeException("I failed to find one of the right cookies. " + cookie);
}
boolean hasRunContainers = (cookie & 0xFFFF) == SERIAL_COOKIE;
this.size = hasRunContainers ? (cookie >>> 16) + 1 : buffer.getInt(4);
int theLimit = size > 0 ? computeSerializedSizeInBytes() : headerSize(hasRunContainers);
buffer.limit(theLimit);
}
@Override
public int advanceUntil(short x, int pos) {
int lower = pos + 1;
// special handling for a possibly common sequential case
if (lower >= size || getKey(lower) >= BufferUtil.toIntUnsigned(x)) {
return lower;
}
int spansize = 1; // could set larger
// bootstrap an upper limit
while (lower + spansize < size && getKey(lower + spansize) < BufferUtil.toIntUnsigned(x)) {
spansize *= 2; // hoping for compiler will reduce to shift
}
int upper = (lower + spansize < size) ? lower + spansize : size - 1;
// maybe we are lucky (could be common case when the seek ahead
// expected to be small and sequential will otherwise make us look bad)
if (getKey(upper) == BufferUtil.toIntUnsigned(x)) {
return upper;
}
if (getKey(upper) < BufferUtil.toIntUnsigned(x)) {// means array has no item key >= x
return size;
}
// we know that the next-smallest span was too small
lower += (spansize / 2);
// else begin binary search
// invariant: array[lower]<x && array[upper]>x
while (lower + 1 != upper) {
int mid = (lower + upper) / 2;
if (getKey(mid) == BufferUtil.toIntUnsigned(x)) {
return mid;
} else if (getKey(mid) < BufferUtil.toIntUnsigned(x)) {
lower = mid;
} else {
upper = mid;
}
}
return upper;
}
private int branchyUnsignedBinarySearch(final short k) {
int low = 0;
int high = this.size - 1;
final int ikey = BufferUtil.toIntUnsigned(k);
while (low <= high) {
final int middleIndex = (low + high) >>> 1;
final int middleValue = getKey(middleIndex);
if (middleValue < ikey) {
low = middleIndex + 1;
} else if (middleValue > ikey) {
high = middleIndex - 1;
} else {
return middleIndex;
}
}
return -(low + 1);
}
@Override
public ImmutableRoaringArray clone() {
ImmutableRoaringArray sa;
try {
sa = (ImmutableRoaringArray) super.clone();
} catch (CloneNotSupportedException e) {
return null;// should never happen
}
return sa;
}
private int computeSerializedSizeInBytes() {
if (this.size == 0) {
return headerSize(hasRunCompression());
}
int CardinalityOfLastContainer = getCardinality(this.size - 1);
int PositionOfLastContainer = getOffsetContainer(this.size - 1);
int SizeOfLastContainer;
boolean hasrun = hasRunCompression();
if (isRunContainer(this.size - 1, hasrun)) {
int nbrruns = BufferUtil.toIntUnsigned(buffer.getShort(PositionOfLastContainer));
SizeOfLastContainer = BufferUtil.getSizeInBytesFromCardinalityEtc(0, nbrruns, true);
} else {
SizeOfLastContainer =
BufferUtil.getSizeInBytesFromCardinalityEtc(CardinalityOfLastContainer, 0, false);
}
return SizeOfLastContainer + PositionOfLastContainer;
}
@Override
public int getCardinality(int k) {
if ((k < 0) || (k >= this.size)) {
throw new IllegalArgumentException(
"out of range container index: " + k + " (report as a bug)");
}
return BufferUtil.toIntUnsigned(buffer.getShort(this.getStartOfKeys() + 4 * k + 2)) + 1;
}
// involves a binary search
@Override
public MappeableContainer getContainer(short x) {
final int i = unsignedBinarySearch(x);
if (i < 0) {
return null;
}
return getContainerAtIndex(i);
}
@Override
public MappeableContainer getContainerAtIndex(int i) {
int cardinality = getCardinality(i);
final boolean isBitmap = cardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE; // if not a
// runcontainer
ByteBuffer tmp = buffer.duplicate();// sad but ByteBuffer is not thread-safe so it is either a
// duplicate or a lock
// note that tmp will indeed be garbage-collected some time after the end of this function
tmp.order(buffer.order());
tmp.position(getOffsetContainer(i));
boolean hasrun = hasRunCompression();
if (isRunContainer(i, hasrun)) {
// first, we have a short giving the number of runs
int nbrruns = BufferUtil.toIntUnsigned(tmp.getShort());
final ShortBuffer shortArray = tmp.asShortBuffer();
shortArray.limit(2 * nbrruns);
return new MappeableRunContainer(shortArray, nbrruns);
}
if (isBitmap) {
final LongBuffer bitmapArray = tmp.asLongBuffer();
bitmapArray.limit(MappeableBitmapContainer.MAX_CAPACITY / 64);
return new MappeableBitmapContainer(bitmapArray, cardinality);
} else {
final ShortBuffer shortArray = tmp.asShortBuffer();
shortArray.limit(cardinality);
return new MappeableArrayContainer(shortArray, cardinality);
}
}
@Override
public MappeableContainerPointer getContainerPointer() {
return getContainerPointer(0);
}
@Override
public MappeableContainerPointer getContainerPointer(final int startIndex) {
final boolean hasrun = isEmpty() ? false : hasRunCompression();
return new MappeableContainerPointer() {
int k = startIndex;
@Override
public void advance() {
++k;
}
@Override
public MappeableContainerPointer clone() {
try {
return (MappeableContainerPointer) super.clone();
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public int compareTo(MappeableContainerPointer o) {
if (key() != o.key()) {
return BufferUtil.toIntUnsigned(key()) - BufferUtil.toIntUnsigned(o.key());
}
return o.getCardinality() - this.getCardinality();
}
@Override
public int getCardinality() {
return ImmutableRoaringArray.this.getCardinality(k);
}
@Override
public MappeableContainer getContainer() {
if (k >= ImmutableRoaringArray.this.size) {
return null;
}
return ImmutableRoaringArray.this.getContainerAtIndex(k);
}
@Override
public int getSizeInBytes() {
// might be a tad expensive
if (ImmutableRoaringArray.this.isRunContainer(k, hasrun)) {
int pos = getOffsetContainer(k);
int nbrruns = BufferUtil.toIntUnsigned(buffer.getShort(pos));
return BufferUtil.getSizeInBytesFromCardinalityEtc(0, nbrruns, true);
} else {
int CardinalityOfLastContainer = getCardinality();
return BufferUtil.getSizeInBytesFromCardinalityEtc(CardinalityOfLastContainer, 0, false);
}
}
@Override
public boolean hasContainer() {
return 0 <= k & k < ImmutableRoaringArray.this.size;
}
@Override
public boolean isBitmapContainer() {
if (ImmutableRoaringArray.this.isRunContainer(k, hasrun)) {
return false;
}
return getCardinality() > MappeableArrayContainer.DEFAULT_MAX_SIZE;
}
@Override
public boolean isRunContainer() {
return ImmutableRoaringArray.this.isRunContainer(k, hasrun);
}
@Override
public short key() {
return ImmutableRoaringArray.this.getKeyAtIndex(k);
}
@Override
public void previous() {
--k;
}
};
}
// involves a binary search
@Override
public int getIndex(short x) {
return unsignedBinarySearch(x);
}
private int getKey(int k) {
return BufferUtil.toIntUnsigned(buffer.getShort(getStartOfKeys() + 4 * k));
}
@Override
public short getKeyAtIndex(int i) {
return buffer.getShort(4 * i + getStartOfKeys());
}
private int getOffsetContainer(int k) {
if ((k < 0) || (k >= this.size)) {
throw new IllegalArgumentException(
"out of range container index: " + k + " (report as a bug)");
}
if (hasRunCompression()) { // account for size of runcontainer bitmap
if (this.size < MutableRoaringArray.NO_OFFSET_THRESHOLD) {
// we do it the hard way
return getOffsetContainerSlow(k);
}
return buffer.getInt(4 + 4 * this.size + ((this.size + 7) / 8) + 4 * k);
} else {
return buffer.getInt(4 + 4 + 4 * this.size + 4 * k);
}
}
private int getOffsetContainerSlow(int k) {
boolean hasrun = hasRunCompression();
int pos = this.headerSize(hasrun);
for (int z = 0; z < k; ++z) {
if (isRunContainer(z, hasrun)) {
int nbrruns = BufferUtil.toIntUnsigned(buffer.getShort(pos));
int SizeOfLastContainer = BufferUtil.getSizeInBytesFromCardinalityEtc(0, nbrruns, true);
pos += SizeOfLastContainer;
} else {
int CardinalityOfLastContainer = this.getCardinality(z);
int SizeOfLastContainer =
BufferUtil.getSizeInBytesFromCardinalityEtc(CardinalityOfLastContainer, 0, false);
pos += SizeOfLastContainer;
}
}
return pos;
}
private int getStartOfKeys() {
if (hasRunCompression()) { // info is in the buffer
return 4 + ((this.size + 7) / 8);
} else {
return 8;
}
}
@Override
public int hashCode() {
MappeableContainerPointer cp = this.getContainerPointer();
int hashvalue = 0;
while (cp.hasContainer()) {
int th = cp.key() * 0xF0F0F0 + cp.getContainer().hashCode();
hashvalue = 31 * hashvalue + th;
cp.advance();
}
return hashvalue;
}
@Override
public boolean hasRunCompression() {
return (buffer.getInt(0) & 0xFFFF) == SERIAL_COOKIE;
}
// hasrun should be equal to hasRunCompression()
protected int headerSize(boolean hasrun) {
if (hasrun) {
if (size < MutableRoaringArray.NO_OFFSET_THRESHOLD) {// for small bitmaps, we omit the offsets
return 4 + (size + 7) / 8 + 4 * size;
}
return 4 + (size + 7) / 8 + 8 * size;// - 4 because we pack the size with the cookie
} else {
return 4 + 4 + 8 * size;
}
}
// starts with binary search and finishes with a sequential search
/*private int hybridUnsignedBinarySearch(final short k) {
int low = 0;
int high = this.size - 1;
final int ikey = BufferUtil.toIntUnsigned(k);
// 32 in the next line matches the size of a cache line
while (low + 16 <= high) {
final int middleIndex = (low + high) >>> 1;
final int middleValue = getKey(middleIndex);
if (middleValue < ikey) {
low = middleIndex + 1;
} else if (middleValue > ikey) {
high = middleIndex - 1;
} else {
return middleIndex;
}
}
// we finish the job with a sequential search
int x = low;
for (; x <= high; ++x) {
final int val = getKey(x);
if (val >= ikey) {
if (val == ikey) {
return x;
}
break;
}
}
return -(x + 1);
}*/
/**
* Returns true if this bitmap is empty.
*
* @return true if empty
*/
public boolean isEmpty() {
return this.size == 0;
}
// hasrun should be initialized with hasRunCompression()
private boolean isRunContainer(int i, boolean hasrun) {
if (hasrun) { // info is in the buffer
int j = buffer.get(startofrunbitmap + i / 8);
int mask = 1 << (i % 8);
return (j & mask) != 0;
} else {
return false;
}
}
/**
* Serialize.
* <p>
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
@Override
public void serialize(DataOutput out) throws IOException {
if (buffer.hasArray()) {
out.write(buffer.array(), buffer.arrayOffset(), buffer.limit());
} else {
ByteBuffer tmp = buffer.duplicate();
tmp.position(0);
WritableByteChannel channel = Channels.newChannel((OutputStream) out);
channel.write(tmp);
}
}
/**
* @return the size that the data structure occupies on disk
*/
@Override
public int serializedSizeInBytes() {
return buffer.limit();
}
@Override
public int size() {
return this.size;
}
private int unsignedBinarySearch(short k) {
return branchyUnsignedBinarySearch(k);
}
}

1320
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/ImmutableRoaringBitmap.java

File diff suppressed because it is too large Load Diff

1688
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableArrayContainer.java

File diff suppressed because it is too large Load Diff

2012
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableBitmapContainer.java

File diff suppressed because it is too large Load Diff

759
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableContainer.java

@ -0,0 +1,759 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.Container;
import com.fr.third.bitmap.roaringbitmap.IntConsumer;
import com.fr.third.bitmap.roaringbitmap.PeekableShortIterator;
import com.fr.third.bitmap.roaringbitmap.ShortIterator;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
/**
* Base container class. This class is similar to Container but meant to be used
* with memory mapping.
*/
public abstract class MappeableContainer implements Iterable<Short>, Cloneable, Externalizable {
/**
* Name of the various possible containers
*/
public static String ContainerNames[] = {"mappeablebitmap", "mappeablearray", "mappeablerun"};
/**
* Create a container initialized with a range of consecutive values
*
* @param start first index
* @param last last index (range is exclusive)
* @return a new container initialized with the specified values
*/
public static MappeableContainer rangeOfOnes(final int start, final int last) {
final int sizeAsArrayContainer = MappeableArrayContainer.serializedSizeInBytes(last - start);
final int sizeAsRunContainer = MappeableRunContainer.serializedSizeInBytes(1);
MappeableContainer answer = sizeAsRunContainer < sizeAsArrayContainer
? new MappeableRunContainer() : new MappeableArrayContainer();
answer = answer.iadd(start, last);
return answer;
}
/**
* Return a new container with all shorts in [begin,end) added using an unsigned interpretation.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract MappeableContainer add(int begin, int end);
/**
* Add a short to the container. May generate a new container.
*
* @param x short to be added
* @return the new container
*/
public abstract MappeableContainer add(short x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer and(MappeableArrayContainer x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer and(MappeableBitmapContainer x);
protected MappeableContainer and(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return and((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return and((MappeableRunContainer) x);
}
return and((MappeableBitmapContainer) x);
}
protected abstract int andCardinality(MappeableArrayContainer x);
protected abstract int andCardinality(MappeableBitmapContainer x);
protected abstract int andCardinality(MappeableRunContainer x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public int andCardinality(MappeableContainer x) {
if (this.getCardinality() == 0) {
return 0;
} else if (x.getCardinality() == 0) {
return 0;
} else {
if (x instanceof MappeableArrayContainer) {
return andCardinality((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return andCardinality((MappeableBitmapContainer) x);
}
return andCardinality((MappeableRunContainer) x);
}
}
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer and(MappeableRunContainer x);
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer andNot(MappeableArrayContainer x);
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer andNot(MappeableBitmapContainer x);
protected MappeableContainer andNot(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return andNot((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return andNot((MappeableRunContainer) x);
}
return andNot((MappeableBitmapContainer) x);
}
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer andNot(MappeableRunContainer x);
/**
* Empties the container
*/
public abstract void clear();
@Override
public abstract MappeableContainer clone();
/**
* Checks whether the contain contains the provided value
*
* @param x value to check
* @return whether the value is in the container
*/
public abstract boolean contains(short x);
/**
* Fill the least significant 16 bits of the integer array, starting at index index, with the
* short values from this container. The caller is responsible to allocate enough room. The most
* significant 16 bits of each integer are given by the most significant bits of the provided
* mask.
*
* @param x provided array
* @param i starting index
* @param mask indicates most significant bits
*/
public abstract void fillLeastSignificant16bits(int[] x, int i, int mask);
/**
* Add a short to the container if it is not present, otherwise remove it. May generate a new
* container.
*
* @param x short to be added
* @return the new container
*/
public abstract MappeableContainer flip(short x);
/**
* Size of the underlying array
*
* @return size in bytes
*/
protected abstract int getArraySizeInBytes();
/**
* Computes the distinct number of short values in the container. Can be expected to run in
* constant time.
*
* @return the cardinality
*/
public abstract int getCardinality();
/**
* Get the name of this container.
*
* @return name of the container
*/
public String getContainerName() {
if (this instanceof MappeableBitmapContainer) {
return ContainerNames[0];
} else if (this instanceof MappeableArrayContainer) {
return ContainerNames[1];
} else {
return ContainerNames[2];
}
}
/**
* Iterator to visit the short values in the container in descending order.
*
* @return iterator
*/
public abstract ShortIterator getReverseShortIterator();
/**
* Iterator to visit the short values in the container in ascending order.
*
* @return iterator
*/
public abstract PeekableShortIterator getShortIterator();
/**
* Iterate through the values of this container and pass them
* along to the IntConsumer, using msb as the 16 most significant bits.
*
* @param msb 16 most significant bits
* @param ic consumer
*/
public abstract void forEach(short msb, IntConsumer ic);
/**
* Computes an estimate of the memory usage of this container. The estimate is not meant to be
* exact.
*
* @return estimated memory usage in bytes
*/
public abstract int getSizeInBytes();
/**
* Add all shorts in [begin,end) using an unsigned interpretation. May generate a new container.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract MappeableContainer iadd(int begin, int end);
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iand(MappeableArrayContainer x);
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iand(MappeableBitmapContainer x);
protected MappeableContainer iand(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return iand((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return iand((MappeableRunContainer) x);
}
return iand((MappeableBitmapContainer) x);
}
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iand(MappeableRunContainer x);
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iandNot(MappeableArrayContainer x);
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iandNot(MappeableBitmapContainer x);
protected MappeableContainer iandNot(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return iandNot((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return iandNot((MappeableRunContainer) x);
}
return iandNot((MappeableBitmapContainer) x);
}
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iandNot(MappeableRunContainer x);
/**
* Computes the in-place bitwise NOT of this container (complement). Only those bits within the
* range are affected. The current container is generally modified. May generate a new container.
*
* @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
* @param rangeEnd ending of range (exclusive)
* @return (partially) completmented container
*/
public abstract MappeableContainer inot(int rangeStart, int rangeEnd);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(MappeableArrayContainer x);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(MappeableBitmapContainer x);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public boolean intersects(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return intersects((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return intersects((MappeableBitmapContainer) x);
}
return intersects((MappeableRunContainer) x);
}
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(MappeableRunContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ior(MappeableArrayContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ior(MappeableBitmapContainer x);
protected MappeableContainer ior(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return ior((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return ior((MappeableRunContainer) x);
}
return ior((MappeableBitmapContainer) x);
}
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ior(MappeableRunContainer x);
/**
* Remove shorts in [begin,end) using an unsigned interpretation. May generate a new container.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract MappeableContainer iremove(int begin, int end);
protected abstract boolean isArrayBacked();
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ixor(MappeableArrayContainer x);
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ixor(MappeableBitmapContainer x);
protected MappeableContainer ixor(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return ixor((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return ixor((MappeableRunContainer) x);
}
return ixor((MappeableBitmapContainer) x);
}
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ixor(MappeableRunContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container. The resulting container may not track its cardinality correctly. The resulting
* container may not track its cardinality correctly. This can be fixed as follows:
* if(c.getCardinality()&lt;0) ((MappeableBitmapContainer)c).computeCardinality();
*
* @param x other container
* @return aggregated container
*/
public MappeableContainer lazyIOR(MappeableContainer x) {
if (this instanceof MappeableArrayContainer) {
if (x instanceof MappeableArrayContainer) {
return ((MappeableArrayContainer) this).lazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) x).lazyor((MappeableArrayContainer) this);
}
return ((MappeableRunContainer) x).lazyor((MappeableArrayContainer) this);
} else if (this instanceof MappeableRunContainer) {
if (x instanceof MappeableArrayContainer) {
return ((MappeableRunContainer) this).ilazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) x).lazyor((MappeableRunContainer) this);
}
return ior((MappeableRunContainer) x);
} else {
if (x instanceof MappeableArrayContainer) {
return ((MappeableBitmapContainer) this).ilazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) this).ilazyor((MappeableBitmapContainer) x);
}
return ((MappeableBitmapContainer) this).ilazyor((MappeableRunContainer) x);
}
}
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected. The resulting container may not track its cardinality
* correctly. This can be fixed as follows: if(c.getCardinality()&lt;0)
* ((MappeableBitmapContainer)c).computeCardinality();
*
* @param x other container
* @return aggregated container
*/
public MappeableContainer lazyOR(MappeableContainer x) {
if (this instanceof MappeableArrayContainer) {
if (x instanceof MappeableArrayContainer) {
return ((MappeableArrayContainer) this).lazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) x).lazyor((MappeableArrayContainer) this);
}
return ((MappeableRunContainer) x).lazyor((MappeableArrayContainer) this);
} else if (this instanceof MappeableRunContainer) {
if (x instanceof MappeableArrayContainer) {
return ((MappeableRunContainer) this).lazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) x).lazyor((MappeableRunContainer) this);
}
return or((MappeableRunContainer) x);
} else {
if (x instanceof MappeableArrayContainer) {
return ((MappeableBitmapContainer) this).lazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) this).lazyor((MappeableBitmapContainer) x);
}
return ((MappeableBitmapContainer) this).lazyor((MappeableRunContainer) x);
}
}
/**
* Create a new MappeableContainer containing at most maxcardinality integers.
*
* @param maxcardinality maximal cardinality
* @return a new bitmap with cardinality no more than maxcardinality
*/
public abstract MappeableContainer limit(int maxcardinality);
/**
* Computes the bitwise NOT of this container (complement). Only those bits within the range are
* affected. The current container is left unaffected.
*
* @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
* @param rangeEnd ending of range (exclusive)
* @return (partially) completmented container
*/
public abstract MappeableContainer not(int rangeStart, int rangeEnd);
abstract int numberOfRuns();
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer or(MappeableArrayContainer x);
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer or(MappeableBitmapContainer x);
protected MappeableContainer or(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return or((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return or((MappeableRunContainer) x);
}
return or((MappeableBitmapContainer) x);
}
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer or(MappeableRunContainer x);
/**
* Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be
* GetCardinality()).
*
* @param lowbits upper limit
* @return the rank
*/
public abstract int rank(short lowbits);
/**
* Return a new container with all shorts in [begin,end) remove using an unsigned interpretation.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract MappeableContainer remove(int begin, int end);
/**
* Remove the short from this container. May create a new container.
*
* @param x to be removed
* @return New container
*/
public abstract MappeableContainer remove(short x);
/**
* The output of a lazyOR or lazyIOR might be an invalid container, this should be called on it.
*
* @return a new valid container
*/
public abstract MappeableContainer repairAfterLazy();
/**
* Convert to MappeableRunContainers, when the result is smaller. Overridden by
* MappeableRunContainer to possibly switch from MappeableRunContainer to a smaller alternative.
*
* @return the new container
*/
public abstract MappeableContainer runOptimize();
/**
* Return the jth value
*
* @param j index of the value
* @return the value
*/
public abstract short select(int j);
/**
* Report the number of bytes required to serialize this container.
*
* @return the size in bytes
*/
public abstract int serializedSizeInBytes();
/**
* Convert to a non-mappeable container.
*
* @return the non-mappeable container
*/
public abstract Container toContainer();
/**
* If possible, recover wasted memory.
*/
public abstract void trim();
/**
* Write just the underlying array.
*
* @param out output stream
* @throws IOException in case of failure
*/
protected abstract void writeArray(DataOutput out) throws IOException;
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer xor(MappeableArrayContainer x);
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer xor(MappeableBitmapContainer x);
protected MappeableContainer xor(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return xor((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return xor((MappeableRunContainer) x);
}
return xor((MappeableBitmapContainer) x);
}
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other parameter
* @return aggregated container
*/
public abstract MappeableContainer xor(MappeableRunContainer x);
/**
* Convert the current container to a BitmapContainer, if a conversion is needed.
* If the container is already a bitmap, the container is returned unchanged.
*
* @return a bitmap container
*/
public abstract MappeableBitmapContainer toBitmapContainer();
}

80
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableContainerPointer.java

@ -0,0 +1,80 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
/**
* This interface allows you to iterate over the containers in a roaring bitmap.
*/
public interface MappeableContainerPointer
extends Comparable<MappeableContainerPointer>, Cloneable {
/**
* Move to the next container
*/
void advance();
/**
* Create a copy
*
* @return return a clone of this pointer
*/
MappeableContainerPointer clone();
/**
* Returns the cardinality of the current container. Can be faster than loading the container
* first.
*
* @return cardinality of the current container
*/
int getCardinality();
/**
* This method can be used to check whether there is current a valid container as it returns null
* when there is not.
*
* @return null or the current container
*/
MappeableContainer getContainer();
/**
* Get the size in bytes of the container. Used for sorting.
*
* @return the size in bytes
*/
int getSizeInBytes();
/**
* @return whether there is a container at the current position
*/
boolean hasContainer();
/**
* Returns true if it is a bitmap container (MappeableBitmapContainer).
*
* @return boolean indicated if it is a bitmap container
*/
public boolean isBitmapContainer();
/**
* Returns true if it is a run container (MappeableRunContainer).
*
* @return boolean indicated if it is a run container
*/
boolean isRunContainer();
/**
* The key is a 16-bit integer that indicates the position of the container in the roaring bitmap.
* To be interpreted as an unsigned integer.
*
* @return the key
*/
short key();
/**
* Move to the previous container
*/
void previous();
}

2798
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MappeableRunContainer.java

File diff suppressed because it is too large Load Diff

573
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MutableRoaringArray.java

@ -0,0 +1,573 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.Util;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.nio.LongBuffer;
import java.nio.ShortBuffer;
import java.util.Arrays;
/**
* Specialized array to store the containers used by a RoaringBitmap. This class is similar to
* RoaringArray but meant to be used with memory mapping. This is not meant to be
* used by end users.
* <p>
* Objects of this class reside in RAM.
*/
public final class MutableRoaringArray implements Cloneable, Externalizable, PointableRoaringArray {
protected static final int INITIAL_CAPACITY = 4;
protected static final short SERIAL_COOKIE_NO_RUNCONTAINER = 12346;
protected static final short SERIAL_COOKIE = 12347;
protected static final int NO_OFFSET_THRESHOLD = 4;
private static final long serialVersionUID = 5L; // TODO: OFK was 4L, not sure
protected boolean mayHaveRunContainers = false; // does not necessarily have them, after
// optimization
short[] keys = null;
MappeableContainer[] values = null;
int size = 0;
protected MutableRoaringArray() {
this.keys = new short[INITIAL_CAPACITY];
this.values = new MappeableContainer[INITIAL_CAPACITY];
}
@Override
public int advanceUntil(short x, int pos) {
int lower = pos + 1;
// special handling for a possibly common sequential case
if (lower >= size || BufferUtil.toIntUnsigned(keys[lower]) >= BufferUtil.toIntUnsigned(x)) {
return lower;
}
int spansize = 1; // could set larger
// bootstrap an upper limit
while (lower + spansize < size
&& BufferUtil.toIntUnsigned(keys[lower + spansize]) < BufferUtil.toIntUnsigned(x)) {
spansize *= 2; // hoping for compiler will reduce to shift
}
int upper = (lower + spansize < size) ? lower + spansize : size - 1;
// maybe we are lucky (could be common case when the seek ahead
// expected to be small and sequential will otherwise make us look bad)
if (keys[upper] == x) {
return upper;
}
if (BufferUtil.toIntUnsigned(keys[upper]) < BufferUtil.toIntUnsigned(x)) {// means array has no
// item key >= x
return size;
}
// we know that the next-smallest span was too small
lower += (spansize / 2);
// else begin binary search
// invariant: array[lower]<x && array[upper]>x
while (lower + 1 != upper) {
int mid = (lower + upper) / 2;
if (keys[mid] == x) {
return mid;
} else if (BufferUtil.toIntUnsigned(keys[mid]) < BufferUtil.toIntUnsigned(x)) {
lower = mid;
} else {
upper = mid;
}
}
return upper;
}
protected void append(short key, MappeableContainer value) {
extendArray(1);
this.keys[this.size] = key;
this.values[this.size] = value;
this.size++;
}
/**
* Append copies of the values AFTER a specified key (may or may not be present) to end.
*
* @param highLowContainer the other array
* @param beforeStart given key is the largest key that we won't copy
*/
protected void appendCopiesAfter(PointableRoaringArray highLowContainer, short beforeStart) {
int startLocation = highLowContainer.getIndex(beforeStart);
if (startLocation >= 0) {
startLocation++;
} else {
startLocation = -startLocation - 1;
}
extendArray(highLowContainer.size() - startLocation);
for (int i = startLocation; i < highLowContainer.size(); ++i) {
this.keys[this.size] = highLowContainer.getKeyAtIndex(i);
this.values[this.size] = highLowContainer.getContainerAtIndex(i).clone();
this.size++;
}
}
/**
* Append copies of the values from another array, from the start
*
* @param highLowContainer the other array
* @param stoppingKey any equal or larger key in other array will terminate copying
*/
protected void appendCopiesUntil(PointableRoaringArray highLowContainer, short stoppingKey) {
final int stopKey = BufferUtil.toIntUnsigned(stoppingKey);
MappeableContainerPointer cp = highLowContainer.getContainerPointer();
while (cp.hasContainer()) {
if (BufferUtil.toIntUnsigned(cp.key()) >= stopKey) {
break;
}
extendArray(1);
this.keys[this.size] = cp.key();
this.values[this.size] = cp.getContainer().clone();
this.size++;
cp.advance();
}
}
/**
* Append copies of the values from another array
*
* @param highLowContainer other array
* @param startingIndex starting index in the other array
* @param end last index array in the other array
*/
protected void appendCopy(PointableRoaringArray highLowContainer, int startingIndex, int end) {
extendArray(end - startingIndex);
for (int i = startingIndex; i < end; ++i) {
this.keys[this.size] = highLowContainer.getKeyAtIndex(i);
this.values[this.size] = highLowContainer.getContainerAtIndex(i).clone();
this.size++;
}
}
protected void appendCopy(short key, MappeableContainer value) {
extendArray(1);
this.keys[this.size] = key;
this.values[this.size] = value.clone();
this.size++;
}
private int binarySearch(int begin, int end, short key) {
return Util.unsignedBinarySearch(keys, begin, end, key);
}
protected void clear() {
this.keys = null;
this.values = null;
this.size = 0;
}
@Override
public MutableRoaringArray clone() {
MutableRoaringArray sa;
try {
sa = (MutableRoaringArray) super.clone();
// OFK: do we need runcontainer bitmap? Guess not, this is just a directory
// and each container knows what kind it is.
sa.keys = Arrays.copyOf(this.keys, this.size);
sa.values = Arrays.copyOf(this.values, this.size);
for (int k = 0; k < this.size; ++k) {
sa.values[k] = sa.values[k].clone();
}
sa.size = this.size;
return sa;
} catch (CloneNotSupportedException e) {
return null;
}
}
protected void copyRange(int begin, int end, int newBegin) {
// assuming begin <= end and newBegin < begin
final int range = end - begin;
System.arraycopy(this.keys, begin, this.keys, newBegin, range);
System.arraycopy(this.values, begin, this.values, newBegin, range);
}
/**
* Deserialize.
*
* @param in the DataInput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void deserialize(DataInput in) throws IOException {
this.clear();
// little endian
final int cookie = Integer.reverseBytes(in.readInt());
if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) {
throw new IOException("I failed to find the one of the right cookies.");
}
this.size = ((cookie & 0xFFFF) == SERIAL_COOKIE) ? (cookie >>> 16) + 1
: Integer.reverseBytes(in.readInt());
if ((this.keys == null) || (this.keys.length < this.size)) {
this.keys = new short[this.size];
this.values = new MappeableContainer[this.size];
}
byte[] bitmapOfRunContainers = null;
boolean hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE;
if (hasrun) {
bitmapOfRunContainers = new byte[(size + 7) / 8];
in.readFully(bitmapOfRunContainers);
}
final short keys[] = new short[this.size];
final int cardinalities[] = new int[this.size];
final boolean isBitmap[] = new boolean[this.size];
for (int k = 0; k < this.size; ++k) {
keys[k] = Short.reverseBytes(in.readShort());
cardinalities[k] = 1 + (0xFFFF & Short.reverseBytes(in.readShort()));
isBitmap[k] = cardinalities[k] > MappeableArrayContainer.DEFAULT_MAX_SIZE;
if (bitmapOfRunContainers != null && (bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) {
isBitmap[k] = false;
}
}
if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
// skipping the offsets
in.skipBytes(this.size * 4);
}
// Reading the containers
for (int k = 0; k < this.size; ++k) {
MappeableContainer val;
if (isBitmap[k]) {
final LongBuffer bitmapArray =
LongBuffer.allocate(MappeableBitmapContainer.MAX_CAPACITY / 64);
// little endian
for (int l = 0; l < bitmapArray.limit(); ++l) {
bitmapArray.put(l, Long.reverseBytes(in.readLong()));
}
val = new MappeableBitmapContainer(bitmapArray, cardinalities[k]);
} else if (bitmapOfRunContainers != null
&& ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0)) {
int nbrruns = BufferUtil.toIntUnsigned(Short.reverseBytes(in.readShort()));
final ShortBuffer shortArray = ShortBuffer.allocate(2 * nbrruns);
for (int l = 0; l < shortArray.limit(); ++l) {
shortArray.put(l, Short.reverseBytes(in.readShort()));
}
val = new MappeableRunContainer(shortArray, nbrruns);
} else {
final ShortBuffer shortArray = ShortBuffer.allocate(cardinalities[k]);
for (int l = 0; l < shortArray.limit(); ++l) {
shortArray.put(l, Short.reverseBytes(in.readShort()));
}
val = new MappeableArrayContainer(shortArray, cardinalities[k]);
}
this.keys[k] = keys[k];
this.values[k] = val;
}
}
// make sure there is capacity for at least k more elements
protected void extendArray(int k) {
// size + 1 could overflow
if (this.size + k >= this.keys.length) {
int newCapacity;
if (this.keys.length < 1024) {
newCapacity = 2 * (this.size + k);
} else {
newCapacity = 5 * (this.size + k) / 4;
}
this.keys = Arrays.copyOf(this.keys, newCapacity);
this.values = Arrays.copyOf(this.values, newCapacity);
}
}
@Override
public int getCardinality(int i) {
return getContainerAtIndex(i).getCardinality();
}
// involves a binary search
@Override
public MappeableContainer getContainer(short x) {
final int i = this.binarySearch(0, size, x);
if (i < 0) {
return null;
}
return this.values[i];
}
@Override
public MappeableContainer getContainerAtIndex(int i) {
return this.values[i];
}
@Override
public MappeableContainerPointer getContainerPointer() {
return getContainerPointer(0);
}
@Override
public MappeableContainerPointer getContainerPointer(final int startIndex) {
return new MappeableContainerPointer() {
int k = startIndex;
@Override
public void advance() {
++k;
}
@Override
public MappeableContainerPointer clone() {
try {
return (MappeableContainerPointer) super.clone();
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public int compareTo(MappeableContainerPointer o) {
if (key() != o.key()) {
return BufferUtil.toIntUnsigned(key()) - BufferUtil.toIntUnsigned(o.key());
}
return o.getCardinality() - this.getCardinality();
}
@Override
public int getCardinality() {
return getContainer().getCardinality();
}
@Override
public MappeableContainer getContainer() {
if (k >= MutableRoaringArray.this.size) {
return null;
}
return MutableRoaringArray.this.values[k];
}
@Override
public int getSizeInBytes() {
return getContainer().getArraySizeInBytes();
}
@Override
public boolean hasContainer() {
return 0 <= k & k < MutableRoaringArray.this.size;
}
@Override
public boolean isBitmapContainer() {
return getContainer() instanceof MappeableBitmapContainer;
}
@Override
public boolean isRunContainer() {
return getContainer() instanceof MappeableRunContainer;
}
@Override
public short key() {
return MutableRoaringArray.this.keys[k];
}
@Override
public void previous() {
--k;
}
};
}
// involves a binary search
@Override
public int getIndex(short x) {
// before the binary search, we optimize for frequent cases
if ((size == 0) || (keys[size - 1] == x)) {
return size - 1;
}
// no luck we have to go through the list
return this.binarySearch(0, size, x);
}
@Override
public short getKeyAtIndex(int i) {
return this.keys[i];
}
@Override
public int hashCode() {
int hashvalue = 0;
for (int k = 0; k < this.size; ++k) {
hashvalue = 31 * hashvalue + keys[k] * 0xF0F0F0 + values[k].hashCode();
}
return hashvalue;
}
@Override
public boolean hasRunCompression() {
for (int k = 0; k < size; ++k) {
MappeableContainer ck = values[k];
if (ck instanceof MappeableRunContainer) {
return true;
}
}
return false;
}
protected int headerSize() {
if (hasRunCompression()) {
if (size < NO_OFFSET_THRESHOLD) {// for small bitmaps, we omit the offsets
return 4 + (size + 7) / 8 + 4 * size;
}
return 4 + (size + 7) / 8 + 8 * size;// - 4 because we pack the size with the cookie
} else {
return 4 + 4 + 8 * size;
}
}
// insert a new key, it is assumed that it does not exist
protected void insertNewKeyValueAt(int i, short key, MappeableContainer value) {
extendArray(1);
System.arraycopy(keys, i, keys, i + 1, size - i);
System.arraycopy(values, i, values, i + 1, size - i);
keys[i] = key;
values[i] = value;
size++;
}
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
deserialize(in);
}
protected void removeAtIndex(int i) {
System.arraycopy(keys, i + 1, keys, i, size - i - 1);
keys[size - 1] = 0;
System.arraycopy(values, i + 1, values, i, size - i - 1);
values[size - 1] = null;
size--;
}
protected void removeIndexRange(int begin, int end) {
if (end <= begin) {
return;
}
final int range = end - begin;
System.arraycopy(keys, end, keys, begin, size - end);
System.arraycopy(values, end, values, begin, size - end);
for (int i = 1; i <= range; ++i) {
keys[size - i] = 0;
values[size - i] = null;
}
size -= range;
}
protected void replaceKeyAndContainerAtIndex(int i, short key, MappeableContainer c) {
this.keys[i] = key;
this.values[i] = c;
}
protected void resize(int newLength) {
Arrays.fill(this.keys, newLength, this.size, (short) 0);
Arrays.fill(this.values, newLength, this.size, null);
this.size = newLength;
}
/**
* Serialize.
* <p>
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
@Override
public void serialize(DataOutput out) throws IOException {
int startOffset = 0;
boolean hasrun = hasRunCompression();
if (hasrun) {
out.writeInt(Integer.reverseBytes(SERIAL_COOKIE | ((this.size - 1) << 16)));
byte[] bitmapOfRunContainers = new byte[(size + 7) / 8];
for (int i = 0; i < size; ++i) {
if (this.values[i] instanceof MappeableRunContainer) {
bitmapOfRunContainers[i / 8] |= (1 << (i % 8));
}
}
out.write(bitmapOfRunContainers);
if (this.size < NO_OFFSET_THRESHOLD) {
startOffset = 4 + 4 * this.size + bitmapOfRunContainers.length;
} else {
startOffset = 4 + 8 * this.size + bitmapOfRunContainers.length;
}
} else { // backwards compatibilility
out.writeInt(Integer.reverseBytes(SERIAL_COOKIE_NO_RUNCONTAINER));
out.writeInt(Integer.reverseBytes(size));
startOffset = 4 + 4 + this.size * 4 + this.size * 4;
}
for (int k = 0; k < size; ++k) {
out.writeShort(Short.reverseBytes(this.keys[k]));
out.writeShort(Short.reverseBytes((short) (this.values[k].getCardinality() - 1)));
}
if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
for (int k = 0; k < this.size; k++) {
out.writeInt(Integer.reverseBytes(startOffset));
startOffset = startOffset + values[k].getArraySizeInBytes();
}
}
for (int k = 0; k < size; ++k) {
values[k].writeArray(out);
}
}
/**
* Report the number of bytes required for serialization.
*
* @return the size in bytes
*/
@Override
public int serializedSizeInBytes() {
int count = headerSize();
// for each container, we store cardinality (16 bits), key (16 bits) and location offset (32
// bits).
for (int k = 0; k < this.size; ++k) {
count += values[k].getArraySizeInBytes();
}
return count;
}
protected void setContainerAtIndex(int i, MappeableContainer c) {
this.values[i] = c;
}
@Override
public int size() {
return this.size;
}
@Override
public void writeExternal(ObjectOutput out) throws IOException {
serialize(out);
}
}

1420
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/MutableRoaringBitmap.java

File diff suppressed because it is too large Load Diff

103
fine-roaringbitmap/src/com/fr/third/bitmap/roaringbitmap/buffer/PointableRoaringArray.java

@ -0,0 +1,103 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import java.io.DataOutput;
import java.io.IOException;
/**
* Generic interface for the array underlying roaring bitmap classes.
*/
public interface PointableRoaringArray extends Cloneable {
/**
* Find the smallest integer index larger than pos such that getKeyAtIndex(index)&gt;=x. If none
* can be found, return size.
*
* @param x minimal value
* @param pos index to exceed
* @return the smallest index greater than pos such that getKeyAtIndex(index) is at least as large
* as min, or size if it is not possible.
*/
int advanceUntil(short x, int pos);
/**
* Create an independent copy of the underlying array
*
* @return a copy
*/
PointableRoaringArray clone();
/**
* Returns the cardinality of the container at the given index. This method is expected to be
* fast.
*
* @param i index
* @return the cardinality
*/
public int getCardinality(int i);
/**
* @param x 16-bit key
* @return matching container
*/
MappeableContainer getContainer(short x);
/**
* @param i index
* @return matching container
*/
MappeableContainer getContainerAtIndex(int i);
/**
* @return a ContainerPointer to iterator over the array
*/
MappeableContainerPointer getContainerPointer();
/**
* @param startIndex starting index
* @return a ContainerPointer to iterator over the array initially positioned at startIndex
*/
MappeableContainerPointer getContainerPointer(int startIndex);
/**
* @param x 16-bit key
* @return corresponding index
*/
int getIndex(short x);
/**
* @param i the index
* @return 16-bit key at the index
*/
short getKeyAtIndex(int i);
/**
* Check whether this bitmap has had its runs compressed.
*
* @return whether this bitmap has run compression
*/
public boolean hasRunCompression();
/**
* Serialize.
* <p>
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void serialize(DataOutput out) throws IOException;
/**
* @return the size that the data structure occupies on disk
*/
public int serializedSizeInBytes();
/**
* @return number of keys
*/
int size();
}
Loading…
Cancel
Save