opensource
/
fine-third
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Merge pull request #2792 in CORE/base-third from feature/10.0 to research/10.0 

* commit 'c022dcbbe845c9250660d214345f2b32f7896d86':
  DEC-17950 fix: 删除fine-roaringbitmap
research/10.0
superman 3 years ago
parent
68b319b568 c022dcbbe8
commit
04da534d84
38 changed files with 0 additions and 24026 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      base-third-project/base-third-step1/pom.xml
  2. 3
      build.third_step1-jdk11.gradle
  3. 3
      build.third_step1.gradle
  4. 18
      fine-roaringbitmap/pom.xml
  5. 1240
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/ArrayContainer.java
  6. 108
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/BitSetUtil.java
  7. 1392
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/BitmapContainer.java
  8. 38
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/BitmapDataProvider.java
  9. 808
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/Container.java
  10. 60
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/ContainerPointer.java
  11. 532
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/FastAggregation.java
  12. 157
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/ImmutableBitmapDataProvider.java
  13. 29
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/IntConsumer.java
  14. 28
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/IntIterator.java
  15. 124
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/IntIteratorFlyweight.java
  16. 33
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/PeekableIntIterator.java
  17. 34
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/PeekableShortIterator.java
  18. 110
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/ReverseIntIteratorFlyweight.java
  19. 582
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/RoaringArray.java
  20. 2205
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/RoaringBitmap.java
  21. 2518
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/RunContainer.java
  22. 39
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/ShortIterator.java
  23. 931
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/Util.java
  24. 113
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/BufferBitSetUtil.java
  25. 631
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/BufferFastAggregation.java
  26. 132
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/BufferIntIteratorFlyweight.java
  27. 116
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/BufferReverseIntIteratorFlyweight.java
  28. 825
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/BufferUtil.java
  29. 463
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/ImmutableRoaringArray.java
  30. 1320
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/ImmutableRoaringBitmap.java
  31. 1688
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MappeableArrayContainer.java
  32. 2012
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MappeableBitmapContainer.java
  33. 759
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MappeableContainer.java
  34. 80
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MappeableContainerPointer.java
  35. 2798
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MappeableRunContainer.java
  36. 573
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MutableRoaringArray.java
  37. 1420
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MutableRoaringBitmap.java
  38. 103
      fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/PointableRoaringArray.java

1
base-third-project/base-third-step1/pom.xml
Unescape View File

@ -37,7 +37,6 @@
<module>../../fine-lucene</module>
<module>../../fine-lz4</module>
<module>../../fine-org-dom4j</module>
<module>../../fine-roaringbitmap</module>
<module>../../fine-sense4</module>
<module>../../fine-deprecated-utils</module>
<module>../../fine-third-default</module>

3
build.third_step1-jdk11.gradle
Unescape View File

@ -54,7 +54,6 @@ sourceSets{
"${srcDir}/fine-lucene/src/main/java",
"${srcDir}/fine-lz4/src/main/java",
"${srcDir}/fine-org-dom4j/src/main/java",
"${srcDir}/fine-roaringbitmap/src/main/java",
"${srcDir}/fine-sense4/src/main/java",
"${srcDir}/fine-third-default/fine-mail/src/main/java",
"${srcDir}/fine-third-default/fine-sun-misc/src/main/java",
@ -114,8 +113,6 @@ def resourceDirs = [
"${srcDir}/fine-lz4/src/main/resources",
"${srcDir}/fine-org-dom4j/src/main/java",
"${srcDir}/fine-org-dom4j/src/main/resources",
"${srcDir}/fine-roaringbitmap/src/main/java",
"${srcDir}/fine-roaringbitmap/src/main/resources",
"${srcDir}/fine-sense4/src/main/java",
"${srcDir}/fine-sense4/src/main/resources",
"${srcDir}/fine-third-default/fine-mail/src/main/java",

3
build.third_step1.gradle
Unescape View File

@ -55,7 +55,6 @@ sourceSets{
"${srcDir}/fine-lucene/src/main/java",
"${srcDir}/fine-lz4/src/main/java",
"${srcDir}/fine-org-dom4j/src/main/java",
"${srcDir}/fine-roaringbitmap/src/main/java",
"${srcDir}/fine-sense4/src/main/java",
"${srcDir}/fine-third-default/fine-mail/src/main/java",
"${srcDir}/fine-third-default/fine-sun-misc/src/main/java",
@ -157,8 +156,6 @@ task copyFiles(type:Copy,dependsOn:'compileJava'){
with dataContent.call("${srcDir}/fine-lz4/src/main/resources")
with dataContent.call("${srcDir}/fine-org-dom4j/src/main/java")
with dataContent.call("${srcDir}/fine-org-dom4j/src/main/resources")
with dataContent.call("${srcDir}/fine-roaringbitmap/src/main/java")
with dataContent.call("${srcDir}/fine-roaringbitmap/src/main/resources")
with dataContent.call("${srcDir}/fine-sense4/src/main/java")
with dataContent.call("${srcDir}/fine-sense4/src/main/resources")
with dataContent.call("${srcDir}/fine-third-default/fine-sun-misc/src/main/java")

18
fine-roaringbitmap/pom.xml
Unescape View File

@ -1,18 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.fr.third</groupId>
<artifactId>step1</artifactId>
<version>${revision}</version>
<relativePath>../base-third-project/base-third-step1</relativePath>
</parent>
<artifactId>fine-roaringbitmap</artifactId>
<version>${revision}</version>
</project>

1240
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/ArrayContainer.java
View File

File diff suppressed because it is too large Load Diff

108
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/BitSetUtil.java
Unescape View File

@ -1,108 +0,0 @@
package com.fr.third.bitmap.roaringbitmap;
import java.util.Arrays;
import java.util.BitSet;
/***
*
* This class provides convenience functions to manipulate BitSet and RoaringBitmap objects.
*
*/
public class BitSetUtil {
// todo: add a method to convert a RoaringBitmap to a BitSet using BitSet.valueOf
// a block consists has a maximum of 1024 words, each representing 64 bits,
// thus representing at maximum 65536 bits
static final private int BLOCK_LENGTH = BitmapContainer.MAX_CAPACITY / Long.SIZE; //
// 64-bit
// word
private static ArrayContainer arrayContainerOf(final int from, final int to,
final int cardinality, final long[] words) {
// precondition: cardinality is max 4096
final short[] content = new short[cardinality];
int index = 0;
for (int i = from, socket = 0; i < to; ++i, socket += Long.SIZE) {
long word = words[i];
while (word != 0) {
long t = word & -word;
content[index++] = (short) (socket + Long.bitCount(t - 1));
word ^= t;
}
}
return new ArrayContainer(content);
}
/**
* Generate a RoaringBitmap out of a long[], each long using little-endian representation of its
* bits
*
* @param words array of longs (will not be modified)
* @return roaring bitmap
* @see BitSet#toLongArray() for an equivalent
*/
public static RoaringBitmap bitmapOf(final long[] words) {
// split long[] into blocks.
// each block becomes a single container, if any bit is set
final RoaringBitmap ans = new RoaringBitmap();
int containerIndex = 0;
for (int from = 0; from < words.length; from += BLOCK_LENGTH) {
final int to = Math.min(from + BLOCK_LENGTH, words.length);
final int blockCardinality = cardinality(from, to, words);
if (blockCardinality > 0) {
ans.highLowContainer.insertNewKeyValueAt(containerIndex++, Util.highbits(from * Long.SIZE),
BitSetUtil.containerOf(from, to, blockCardinality, words));
}
}
return ans;
}
private static int cardinality(final int from, final int to, final long[] words) {
int sum = 0;
for (int i = from; i < to; i++) {
sum += Long.bitCount(words[i]);
}
return sum;
}
private static Container containerOf(final int from, final int to, final int blockCardinality,
final long[] words) {
// find the best container available
if (blockCardinality <= ArrayContainer.DEFAULT_MAX_SIZE) {
// containers with DEFAULT_MAX_SIZE or less integers should be
// ArrayContainers
return arrayContainerOf(from, to, blockCardinality, words);
} else {
// otherwise use bitmap container
return new BitmapContainer(Arrays.copyOfRange(words, from, from + BLOCK_LENGTH),
blockCardinality);
}
}
/**
* Compares a RoaringBitmap and a BitSet. They are equal if and only if they contain the same set
* of integers.
*
* @param bitset first object to be compared
* @param bitmap second object to be compared
* @return whether they are equals
*/
public static boolean equals(final BitSet bitset, final RoaringBitmap bitmap) {
if (bitset.cardinality() != bitmap.getCardinality()) {
return false;
}
final IntIterator it = bitmap.getIntIterator();
while (it.hasNext()) {
int val = it.next();
if (!bitset.get(val)) {
return false;
}
}
return true;
}
}

1392
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/BitmapContainer.java
View File

File diff suppressed because it is too large Load Diff

38
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/BitmapDataProvider.java
Unescape View File

@ -1,38 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Representing a general bitmap interface.
*/
public interface BitmapDataProvider extends ImmutableBitmapDataProvider {
/**
* set the value to "true", whether it already appears or not.
*
* @param x integer value
*/
public void add(int x);
/**
* If present remove the specified integers (effectively, sets its bit value to false)
*
* @param x integer value representing the index in a bitmap
*/
public void remove(int x);
/**
* Return the jth value stored in this bitmap.
*
* @param j index of the value
* @return the value
*/
@Override
public int select(int j);
/**
* Recover allocated but unused memory.
*/
public void trim();
}

808
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/Container.java
Unescape View File

@ -1,808 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
import com.fr.third.bitmap.roaringbitmap.buffer.MappeableContainer;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
/**
* Base container class.
*/
public abstract class Container implements Iterable<Short>, Cloneable, Externalizable {
/**
* Name of the various possible containers
*/
public static String ContainerNames[] = {"bitmap", "array", "run"};
/**
* Create a container initialized with a range of consecutive values
*
* @param start first index
* @param last last index (range is exclusive)
* @return a new container initialized with the specified values
*/
public static Container rangeOfOnes(final int start, final int last) {
final int sizeAsArrayContainer = ArrayContainer.serializedSizeInBytes(last - start);
final int sizeAsRunContainer = RunContainer.serializedSizeInBytes(1);
Container answer =
sizeAsRunContainer < sizeAsArrayContainer ? new RunContainer() : new ArrayContainer();
answer = answer.iadd(start, last);
return answer;
}
/**
* Return a new container with all shorts in [begin,end) added using an unsigned interpretation.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract Container add(int begin, int end);
/**
* Add a short to the container. May generate a new container.
*
* @param x short to be added
* @return the new container
*/
public abstract Container add(short x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container and(ArrayContainer x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container and(BitmapContainer x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public Container and(Container x) {
if (x instanceof ArrayContainer) {
return and((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return and((BitmapContainer) x);
}
return and((RunContainer) x);
}
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container and(RunContainer x);
protected abstract int andCardinality(ArrayContainer x);
protected abstract int andCardinality(BitmapContainer x);
protected abstract int andCardinality(RunContainer x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public int andCardinality(Container x) {
if (this.getCardinality() == 0) {
return 0;
} else if (x.getCardinality() == 0) {
return 0;
} else {
if (x instanceof ArrayContainer) {
return andCardinality((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return andCardinality((BitmapContainer) x);
}
return andCardinality((RunContainer) x);
}
}
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container andNot(ArrayContainer x);
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container andNot(BitmapContainer x);
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public Container andNot(Container x) {
if (x instanceof ArrayContainer) {
return andNot((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return andNot((BitmapContainer) x);
}
return andNot((RunContainer) x);
}
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container andNot(RunContainer x);
/**
* Empties the container
*/
public abstract void clear();
@Override
public abstract Container clone();
/**
* Checks whether the contain contains the provided value
*
* @param x value to check
* @return whether the value is in the container
*/
public abstract boolean contains(short x);
/**
* Deserialize (recover) the container.
*
* @param in the DataInput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public abstract void deserialize(DataInput in) throws IOException;
/**
* Fill the least significant 16 bits of the integer array, starting at index i, with the short
* values from this container. The caller is responsible to allocate enough room. The most
* significant 16 bits of each integer are given by the most significant bits of the provided
* mask.
*
* @param x provided array
* @param i starting index
* @param mask indicates most significant bits
*/
public abstract void fillLeastSignificant16bits(int[] x, int i, int mask);
/**
* Add a short to the container if it is not present, otherwise remove it. May generate a new
* container.
*
* @param x short to be added
* @return the new container
*/
public abstract Container flip(short x);
/**
* Size of the underlying array
*
* @return size in bytes
*/
protected abstract int getArraySizeInBytes();
/**
* Computes the distinct number of short values in the container. Can be expected to run in
* constant time.
*
* @return the cardinality
*/
public abstract int getCardinality();
/**
* Get the name of this container.
*
* @return name of the container
*/
public String getContainerName() {
if (this instanceof BitmapContainer) {
return ContainerNames[0];
} else if (this instanceof ArrayContainer) {
return ContainerNames[1];
} else {
return ContainerNames[2];
}
}
/**
* Iterate through the values of this container and pass them
* along to the IntConsumer, using msb as the 16 most significant bits.
*
* @param msb 16 most significant bits
* @param ic consumer
*/
public abstract void forEach(short msb, IntConsumer ic);
/**
* Iterator to visit the short values in the container in descending order.
*
* @return iterator
*/
public abstract ShortIterator getReverseShortIterator();
/**
* Iterator to visit the short values in the container in ascending order.
*
* @return iterator
*/
public abstract PeekableShortIterator getShortIterator();
/**
* Computes an estimate of the memory usage of this container. The estimate is not meant to be
* exact.
*
* @return estimated memory usage in bytes
*/
public abstract int getSizeInBytes();
/**
* Add all shorts in [begin,end) using an unsigned interpretation. May generate a new container.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract Container iadd(int begin, int end);
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iand(ArrayContainer x);
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iand(BitmapContainer x);
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public Container iand(Container x) {
if (x instanceof ArrayContainer) {
return iand((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return iand((BitmapContainer) x);
}
return iand((RunContainer) x);
}
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iand(RunContainer x);
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iandNot(ArrayContainer x);
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iandNot(BitmapContainer x);
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public Container iandNot(Container x) {
if (x instanceof ArrayContainer) {
return iandNot((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return iandNot((BitmapContainer) x);
}
return iandNot((RunContainer) x);
}
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container iandNot(RunContainer x);
/**
* Computes the in-place bitwise NOT of this container (complement). Only those bits within the
* range are affected. The current container is generally modified. May generate a new container.
*
* @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
* @param rangeEnd ending of range (exclusive)
* @return (partially) complemented container
*/
public abstract Container inot(int rangeStart, int rangeEnd);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(ArrayContainer x);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(BitmapContainer x);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public boolean intersects(Container x) {
if (x instanceof ArrayContainer) {
return intersects((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return intersects((BitmapContainer) x);
}
return intersects((RunContainer) x);
}
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(RunContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ior(ArrayContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ior(BitmapContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public Container ior(Container x) {
if (x instanceof ArrayContainer) {
return ior((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ior((BitmapContainer) x);
}
return ior((RunContainer) x);
}
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ior(RunContainer x);
/**
* Remove shorts in [begin,end) using an unsigned interpretation. May generate a new container.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract Container iremove(int begin, int end);
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ixor(ArrayContainer x);
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ixor(BitmapContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public Container ixor(Container x) {
if (x instanceof ArrayContainer) {
return ixor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ixor((BitmapContainer) x);
}
return ixor((RunContainer) x);
}
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract Container ixor(RunContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container. The resulting container may not track its cardinality correctly. The resulting
* container may not track its cardinality correctly. This can be fixed as follows:
* if(c.getCardinality()&lt;0) ((BitmapContainer)c).computeCardinality();
*
* @param x other container
* @return aggregated container
*/
public Container lazyIOR(Container x) {
if (this instanceof ArrayContainer) {
if (x instanceof ArrayContainer) {
return ((ArrayContainer) this).lazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ior((BitmapContainer) x);
}
return ((RunContainer) x).lazyor((ArrayContainer) this);
} else if (this instanceof RunContainer) {
if (x instanceof ArrayContainer) {
return ((RunContainer) this).ilazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ior((BitmapContainer) x);
}
return ior((RunContainer) x);
} else {
if (x instanceof ArrayContainer) {
return ((BitmapContainer) this).ilazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ((BitmapContainer) this).ilazyor((BitmapContainer) x);
}
return ((BitmapContainer) this).ilazyor((RunContainer) x);
}
}
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected. The resulting container may not track its cardinality
* correctly. This can be fixed as follows: if(c.getCardinality()&lt;0)
* ((BitmapContainer)c).computeCardinality();
*
* @param x other container
* @return aggregated container
*/
public Container lazyOR(Container x) {
if (this instanceof ArrayContainer) {
if (x instanceof ArrayContainer) {
return ((ArrayContainer) this).lazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ((BitmapContainer) x).lazyor((ArrayContainer) this);
}
return ((RunContainer) x).lazyor((ArrayContainer) this);
} else if (this instanceof RunContainer) {
if (x instanceof ArrayContainer) {
return ((RunContainer) this).lazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ((BitmapContainer) x).lazyor((RunContainer) this);
}
return or((RunContainer) x);
} else {
if (x instanceof ArrayContainer) {
return ((BitmapContainer) this).lazyor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return ((BitmapContainer) this).lazyor((BitmapContainer) x);
}
return ((BitmapContainer) this).lazyor((RunContainer) x);
}
}
/**
* Create a new Container containing at most maxcardinality integers.
*
* @param maxcardinality maximal cardinality
* @return a new bitmap with cardinality no more than maxcardinality
*/
public abstract Container limit(int maxcardinality);
/**
* Computes the bitwise NOT of this container (complement). Only those bits within the range are
* affected. The current container is left unaffected.
*
* @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
* @param rangeEnd ending of range (exclusive)
* @return (partially) complemented container
*/
public abstract Container not(int rangeStart, int rangeEnd);
abstract int numberOfRuns(); // exact
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container or(ArrayContainer x);
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container or(BitmapContainer x);
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public Container or(Container x) {
if (x instanceof ArrayContainer) {
return or((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return or((BitmapContainer) x);
}
return or((RunContainer) x);
}
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container or(RunContainer x);
/**
* Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be
* GetCardinality()).
*
* @param lowbits upper limit
* @return the rank
*/
public abstract int rank(short lowbits);
/**
* Return a new container with all shorts in [begin,end) remove using an unsigned interpretation.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract Container remove(int begin, int end);
/**
* Remove the short from this container. May create a new container.
*
* @param x to be removed
* @return New container
*/
public abstract Container remove(short x);
/**
* The output of a lazyOR or lazyIOR might be an invalid container, this should be called on it.
*
* @return a new valid container
*/
public abstract Container repairAfterLazy();
/**
* Convert to RunContainers, when the result is smaller. Overridden by RunContainer to possibility
* switch from RunContainer to a smaller alternative. Overridden by BitmapContainer with a more
* efficient approach.
*
* @return the new container
*/
public abstract Container runOptimize();
/**
* Return the jth value
*
* @param j index of the value
* @return the value
*/
public abstract short select(int j);
/**
* Serialize the container.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public abstract void serialize(DataOutput out) throws IOException;
/**
* Report the number of bytes required to serialize this container.
*
* @return the size in bytes
*/
public abstract int serializedSizeInBytes();
/**
* Convert to a mappeable container.
*
* @return the mappeable container
*/
public abstract MappeableContainer toMappeableContainer();
/**
* If possible, recover wasted memory.
*/
public abstract void trim();
/**
* Write just the underlying array.
*
* @param out output stream
* @throws IOException in case of failure
*/
protected abstract void writeArray(DataOutput out) throws IOException;
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container xor(ArrayContainer x);
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container xor(BitmapContainer x);
/**
* Computes the bitwise OR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other parameter
* @return aggregated container
*/
public Container xor(Container x) {
if (x instanceof ArrayContainer) {
return xor((ArrayContainer) x);
} else if (x instanceof BitmapContainer) {
return xor((BitmapContainer) x);
}
return xor((RunContainer) x);
}
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract Container xor(RunContainer x);
/**
* Convert the current container to a BitmapContainer, if a conversion is needed.
* If the container is already a bitmap, the container is returned unchanged.
*
* @return a bitmap container
*/
public abstract BitmapContainer toBitmapContainer();
}

60
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/ContainerPointer.java
Unescape View File

@ -1,60 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* This interface allows you to iterate over the containers in a roaring bitmap.
*/
public interface ContainerPointer extends Comparable<ContainerPointer>, Cloneable {
/**
* Move to the next container
*/
void advance();
/**
* Create a copy
*
* @return return a clone of this pointer
*/
ContainerPointer clone();
/**
* Return the cardinality of the current container
*
* @return the cardinality
*/
int getCardinality();
/**
* This method can be used to check whether there is current a valid container as it returns null
* when there is not.
*
* @return null or the current container
*/
Container getContainer();
/**
* Check whether the current container is a bitmap container.
*
* @return whether it is a bitmap container
*/
boolean isBitmapContainer();
/**
* Check whether the current container is a run container.
*
* @return whether it is a run container
*/
boolean isRunContainer();
/**
* The key is a 16-bit integer that indicates the position of the container in the roaring bitmap.
* To be interpreted as an unsigned integer.
*
* @return the key
*/
short key();
}

532
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/FastAggregation.java
Unescape View File

@ -1,532 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.PriorityQueue;
/**
* Fast algorithms to aggregate many bitmaps.
*
* @author Daniel Lemire
*/
public final class FastAggregation {
/**
* Private constructor to prevent instantiation of utility class
*/
private FastAggregation() {
}
/**
* Compute the AND aggregate.
* <p>
* In practice, calls {#link naive_and}
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap and(Iterator<RoaringBitmap> bitmaps) {
return naive_and(bitmaps);
}
/**
* Compute the AND aggregate.
* <p>
* In practice, calls {#link naive_and}
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap and(RoaringBitmap... bitmaps) {
return naive_and(bitmaps);
}
/**
* Calls naive_or.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
@Deprecated
public static RoaringBitmap horizontal_or(Iterator<RoaringBitmap> bitmaps) {
return naive_or(bitmaps);
}
/**
* Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #or(RoaringBitmap...)
*/
public static RoaringBitmap horizontal_or(List<RoaringBitmap> bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
if (bitmaps.isEmpty()) {
return answer;
}
PriorityQueue<ContainerPointer> pq = new PriorityQueue<ContainerPointer>(bitmaps.size());
for (int k = 0; k < bitmaps.size(); ++k) {
ContainerPointer x = bitmaps.get(k).highLowContainer.getContainerPointer();
if (x.getContainer() != null) {
pq.add(x);
}
}
while (!pq.isEmpty()) {
ContainerPointer x1 = pq.poll();
if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
answer.highLowContainer.append(x1.key(), x1.getContainer().clone());
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
continue;
}
ContainerPointer x2 = pq.poll();
Container newc = x1.getContainer().lazyOR(x2.getContainer());
while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
ContainerPointer x = pq.poll();
newc = newc.lazyIOR(x.getContainer());
x.advance();
if (x.getContainer() != null) {
pq.add(x);
} else if (pq.isEmpty()) {
break;
}
}
newc = newc.repairAfterLazy();
answer.highLowContainer.append(x1.key(), newc);
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
x2.advance();
if (x2.getContainer() != null) {
pq.add(x2);
}
}
return answer;
}
/**
* Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #or(RoaringBitmap...)
*/
public static RoaringBitmap horizontal_or(RoaringBitmap... bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
if (bitmaps.length == 0) {
return answer;
}
PriorityQueue<ContainerPointer> pq = new PriorityQueue<ContainerPointer>(bitmaps.length);
for (int k = 0; k < bitmaps.length; ++k) {
ContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
if (x.getContainer() != null) {
pq.add(x);
}
}
while (!pq.isEmpty()) {
ContainerPointer x1 = pq.poll();
if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
answer.highLowContainer.append(x1.key(), x1.getContainer().clone());
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
continue;
}
ContainerPointer x2 = pq.poll();
Container newc = x1.getContainer().lazyOR(x2.getContainer());
while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
ContainerPointer x = pq.poll();
newc = newc.lazyIOR(x.getContainer());
x.advance();
if (x.getContainer() != null) {
pq.add(x);
} else if (pq.isEmpty()) {
break;
}
}
newc = newc.repairAfterLazy();
answer.highLowContainer.append(x1.key(), newc);
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
x2.advance();
if (x2.getContainer() != null) {
pq.add(x2);
}
}
return answer;
}
/**
* Minimizes memory usage while computing the xor aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #xor(RoaringBitmap...)
*/
public static RoaringBitmap horizontal_xor(RoaringBitmap... bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
if (bitmaps.length == 0) {
return answer;
}
PriorityQueue<ContainerPointer> pq = new PriorityQueue<ContainerPointer>(bitmaps.length);
for (int k = 0; k < bitmaps.length; ++k) {
ContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
if (x.getContainer() != null) {
pq.add(x);
}
}
while (!pq.isEmpty()) {
ContainerPointer x1 = pq.poll();
if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
answer.highLowContainer.append(x1.key(), x1.getContainer().clone());
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
continue;
}
ContainerPointer x2 = pq.poll();
Container newc = x1.getContainer().xor(x2.getContainer());
while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
ContainerPointer x = pq.poll();
newc = newc.ixor(x.getContainer());
x.advance();
if (x.getContainer() != null) {
pq.add(x);
} else if (pq.isEmpty()) {
break;
}
}
answer.highLowContainer.append(x1.key(), newc);
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
x2.advance();
if (x2.getContainer() != null) {
pq.add(x2);
}
}
return answer;
}
/**
* Compute overall AND between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_and(Iterator<RoaringBitmap> bitmaps) {
if (!bitmaps.hasNext()) {
return new RoaringBitmap();
}
RoaringBitmap answer = bitmaps.next().clone();
while (bitmaps.hasNext()) {
answer.and(bitmaps.next());
}
return answer;
}
/**
* Compute overall AND between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_and(RoaringBitmap... bitmaps) {
if (bitmaps.length == 0) {
return new RoaringBitmap();
}
RoaringBitmap answer = bitmaps[0].clone();
for (int k = 1; k < bitmaps.length; ++k) {
answer.and(bitmaps[k]);
}
return answer;
}
/**
* Compute overall OR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_or(Iterator<RoaringBitmap> bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
while (bitmaps.hasNext()) {
answer.naivelazyor(bitmaps.next());
}
answer.repairAfterLazy();
return answer;
}
/**
* Compute overall OR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_or(RoaringBitmap... bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.naivelazyor(bitmaps[k]);
}
answer.repairAfterLazy();
return answer;
}
/**
* Compute overall XOR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_xor(Iterator<RoaringBitmap> bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
while (bitmaps.hasNext()) {
answer.xor(bitmaps.next());
}
return answer;
}
/**
* Compute overall XOR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap naive_xor(RoaringBitmap... bitmaps) {
RoaringBitmap answer = new RoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.xor(bitmaps[k]);
}
return answer;
}
/**
* Compute overall OR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap or(Iterator<RoaringBitmap> bitmaps) {
return naive_or(bitmaps);
}
/**
* Compute overall OR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap or(RoaringBitmap... bitmaps) {
return naive_or(bitmaps);
}
/**
* Uses a priority queue to compute the or aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_or(RoaringBitmap...)
*/
public static RoaringBitmap priorityqueue_or(Iterator<RoaringBitmap> bitmaps) {
if (!bitmaps.hasNext()) {
return new RoaringBitmap();
}
// we buffer the call to getSizeInBytes(), hence the code complexity
ArrayList<RoaringBitmap> buffer = new ArrayList<RoaringBitmap>();
while (bitmaps.hasNext()) {
buffer.add(bitmaps.next());
}
final long[] sizes = new long[buffer.size()];
final boolean[] istmp = new boolean[buffer.size()];
for (int k = 0; k < sizes.length; ++k) {
sizes[k] = buffer.get(k).getLongSizeInBytes();
}
PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
@Override
public int compare(Integer a, Integer b) {
return (int) (sizes[a] - sizes[b]);
}
});
for (int k = 0; k < sizes.length; ++k) {
pq.add(k);
}
while (pq.size() > 1) {
Integer x1 = pq.poll();
Integer x2 = pq.poll();
if (istmp[x2] && istmp[x1]) {
buffer.set(x1, RoaringBitmap.lazyorfromlazyinputs(buffer.get(x1), buffer.get(x2)));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
istmp[x1] = true;
pq.add(x1);
} else if (istmp[x2]) {
buffer.get(x2).lazyor(buffer.get(x1));
sizes[x2] = buffer.get(x2).getLongSizeInBytes();
pq.add(x2);
} else if (istmp[x1]) {
buffer.get(x1).lazyor(buffer.get(x2));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
pq.add(x1);
} else {
buffer.set(x1, RoaringBitmap.lazyor(buffer.get(x1), buffer.get(x2)));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
istmp[x1] = true;
pq.add(x1);
}
}
RoaringBitmap answer = buffer.get(pq.poll());
answer.repairAfterLazy();
return answer;
}
/**
* Uses a priority queue to compute the or aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_or(RoaringBitmap...)
*/
public static RoaringBitmap priorityqueue_or(RoaringBitmap... bitmaps) {
if (bitmaps.length == 0) {
return new RoaringBitmap();
}
// we buffer the call to getSizeInBytes(), hence the code complexity
final RoaringBitmap[] buffer = Arrays.copyOf(bitmaps, bitmaps.length);
final long[] sizes = new long[buffer.length];
final boolean[] istmp = new boolean[buffer.length];
for (int k = 0; k < sizes.length; ++k) {
sizes[k] = buffer[k].getLongSizeInBytes();
}
PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
@Override
public int compare(Integer a, Integer b) {
return (int) (sizes[a] - sizes[b]);
}
});
for (int k = 0; k < sizes.length; ++k) {
pq.add(k);
}
while (pq.size() > 1) {
Integer x1 = pq.poll();
Integer x2 = pq.poll();
if (istmp[x2] && istmp[x1]) {
buffer[x1] = RoaringBitmap.lazyorfromlazyinputs(buffer[x1], buffer[x2]);
sizes[x1] = buffer[x1].getLongSizeInBytes();
istmp[x1] = true;
pq.add(x1);
} else if (istmp[x2]) {
buffer[x2].lazyor(buffer[x1]);
sizes[x2] = buffer[x2].getLongSizeInBytes();
pq.add(x2);
} else if (istmp[x1]) {
buffer[x1].lazyor(buffer[x2]);
sizes[x1] = buffer[x1].getLongSizeInBytes();
pq.add(x1);
} else {
buffer[x1] = RoaringBitmap.lazyor(buffer[x1], buffer[x2]);
sizes[x1] = buffer[x1].getLongSizeInBytes();
istmp[x1] = true;
pq.add(x1);
}
}
RoaringBitmap answer = buffer[pq.poll()];
answer.repairAfterLazy();
return answer;
}
/**
* Uses a priority queue to compute the xor aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_xor(RoaringBitmap...)
*/
public static RoaringBitmap priorityqueue_xor(RoaringBitmap... bitmaps) {
// TODO: This code could be faster, see priorityqueue_or
if (bitmaps.length == 0) {
return new RoaringBitmap();
}
PriorityQueue<RoaringBitmap> pq =
new PriorityQueue<RoaringBitmap>(bitmaps.length, new Comparator<RoaringBitmap>() {
@Override
public int compare(RoaringBitmap a, RoaringBitmap b) {
return (int) (a.getLongSizeInBytes() - b.getLongSizeInBytes());
}
});
Collections.addAll(pq, bitmaps);
while (pq.size() > 1) {
RoaringBitmap x1 = pq.poll();
RoaringBitmap x2 = pq.poll();
pq.add(RoaringBitmap.xor(x1, x2));
}
return pq.poll();
}
/**
* Compute overall XOR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap xor(Iterator<RoaringBitmap> bitmaps) {
return naive_xor(bitmaps);
}
/**
* Compute overall XOR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static RoaringBitmap xor(RoaringBitmap... bitmaps) {
return naive_xor(bitmaps);
}
}

157
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/ImmutableBitmapDataProvider.java
Unescape View File

@ -1,157 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
import java.io.DataOutput;
import java.io.IOException;
/**
* Interface representing an immutable bitmap.
*/
public interface ImmutableBitmapDataProvider {
/**
* Checks whether the value in included, which is equivalent to checking if the corresponding bit
* is set (get in BitSet class).
*
* @param x integer value
* @return whether the integer value is included.
*/
public boolean contains(int x);
/**
* Returns the number of distinct integers added to the bitmap (e.g., number of bits set).
* Internally, this is computed as a 64-bit number.
*
* @return the cardinality
*/
public int getCardinality();
/**
* Returns the number of distinct integers added to the bitmap (e.g., number of bits set).
* This returns a full 64-bit result.
*
* @return the cardinality
*/
public long getLongCardinality();
/**
* Visit all values in the bitmap and pass them to the consumer.
* <p>
* * Usage:
* <pre>
* {@code
* bitmap.forEach(new IntConsumer() {
*
* {@literal @}Override
* public void accept(int value) {
* // do something here
*
* }});
* }
* }
* </pre>
*
* @param ic the consumer
*/
public void forEach(IntConsumer ic);
/**
* For better performance, consider the Use the {@link #forEach forEach} method.
*
* @return a custom iterator over set bits, the bits are traversed in ascending sorted order
*/
public PeekableIntIterator getIntIterator();
/**
* @return a custom iterator over set bits, the bits are traversed in descending sorted order
*/
public IntIterator getReverseIntIterator();
/**
* Estimate of the memory usage of this data structure.
* <p>
* Internally, this is computed as a 64-bit counter.
*
* @return estimated memory usage.
*/
public int getSizeInBytes();
/**
* Estimate of the memory usage of this data structure. Provides
* full 64-bit number.
*
* @return estimated memory usage.
*/
public long getLongSizeInBytes();
/**
* Checks whether the bitmap is empty.
*
* @return true if this bitmap contains no set bit
*/
public boolean isEmpty();
/**
* Create a new bitmap of the same class, containing at most maxcardinality integers.
*
* @param x maximal cardinality
* @return a new bitmap with cardinality no more than maxcardinality
*/
public ImmutableBitmapDataProvider limit(int x);
/**
* Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be
* GetCardinality()).
* <p>
* The value is internally computed as a 64-bit number.
*
* @param x upper limit
* @return the rank
*/
public int rank(int x);
/**
* Same as "rank" but produces a full 64-bit value.
*
* @param x upper limit
* @return the rank
*/
public long rankLong(int x);
/**
* Return the jth value stored in this bitmap.
*
* @param j index of the value
* @return the value
*/
public int select(int j);
/**
* Serialize this bitmap.
* <p>
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void serialize(DataOutput out) throws IOException;
/**
* Report the number of bytes required to serialize this bitmap. This is the number of bytes
* written out when using the serialize method. When using the writeExternal method, the count
* will be higher due to the overhead of Java serialization.
*
* @return the size in bytes
*/
public int serializedSizeInBytes();
/**
* Return the set values as an array. The integer values are in sorted order.
*
* @return array representing the set values.
*/
public int[] toArray();
}

29
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/IntConsumer.java
Unescape View File

@ -1,29 +0,0 @@
package com.fr.third.bitmap.roaringbitmap;
/**
* An IntConsumer receives the int values contained in a data structure.
* Each value is visited once.
* <p>
* Usage:
* <p>
* <pre>
* {@code
* bitmap.forEach(new IntConsumer() {
*
* &#64;Override
* public void accept(int value) {
* // do something here
*
* }});
* }
* }
* </pre>
*/
public interface IntConsumer {
/**
* Receives the integer
*
* @param value the integer value
*/
void accept(int value);
}

28
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/IntIterator.java
Unescape View File

@ -1,28 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* A simple iterator over integer values
*/
public interface IntIterator extends Cloneable {
/**
* Creates a copy of the iterator.
*
* @return a clone of the current iterator
*/
IntIterator clone();
/**
* @return whether there is another value
*/
boolean hasNext();
/**
* @return next integer value
*/
int next();
}

124
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/IntIteratorFlyweight.java
Unescape View File

@ -1,124 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Fast iterator minimizing the stress on the garbage collector. You can create one reusable
* instance of this class and then {@link #wrap(RoaringBitmap)}
* <p>
* For better performance, consider the {@link RoaringBitmap#forEach} method.
*
* @author Borislav Ivanov
**/
public class IntIteratorFlyweight implements PeekableIntIterator {
private int hs;
private PeekableShortIterator iter;
private ArrayContainerShortIterator arrIter = new ArrayContainerShortIterator();
private BitmapContainerShortIterator bitmapIter = new BitmapContainerShortIterator();
private RunContainerShortIterator runIter = new RunContainerShortIterator();
private int pos;
private RoaringBitmap roaringBitmap = null;
/**
* Creates an instance that is not ready for iteration. You must first call
* {@link #wrap(RoaringBitmap)}.
*/
public IntIteratorFlyweight() {
}
/**
* Creates an instance that is ready for iteration.
*
* @param r bitmap to be iterated over
*/
public IntIteratorFlyweight(RoaringBitmap r) {
wrap(r);
}
@Override
public PeekableIntIterator clone() {
try {
IntIteratorFlyweight x = (IntIteratorFlyweight) super.clone();
x.iter = this.iter.clone();
return x;
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public boolean hasNext() {
return pos < this.roaringBitmap.highLowContainer.size();
}
@Override
public int next() {
int x = iter.nextAsInt() | hs;
if (!iter.hasNext()) {
++pos;
nextContainer();
}
return x;
}
private void nextContainer() {
if (pos < this.roaringBitmap.highLowContainer.size()) {
Container container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
if (container instanceof BitmapContainer) {
bitmapIter.wrap(((BitmapContainer) container).bitmap);
iter = bitmapIter;
} else if (container instanceof ArrayContainer) {
arrIter.wrap((ArrayContainer) container);
iter = arrIter;
} else {
runIter.wrap((RunContainer) container);
iter = runIter;
}
hs = Util.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
}
}
/**
* Prepares a bitmap for iteration
*
* @param r bitmap to be iterated over
*/
public void wrap(RoaringBitmap r) {
this.hs = 0;
this.pos = 0;
this.roaringBitmap = r;
this.nextContainer();
}
@Override
public void advanceIfNeeded(final int minval) {
while (hasNext() && ((hs >>> 16) < (minval >>> 16))) {
++pos;
nextContainer();
}
if (hasNext() && ((hs >>> 16) == (minval >>> 16))) {
iter.advanceIfNeeded(Util.lowbits(minval));
if (!iter.hasNext()) {
++pos;
nextContainer();
}
}
}
@Override
public int peekNext() {
return Util.toIntUnsigned(iter.peekNext()) | hs;
}
}

33
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/PeekableIntIterator.java
Unescape View File

@ -1,33 +0,0 @@
package com.fr.third.bitmap.roaringbitmap;
/**
* Simple extension to the IntIterator interface.
* It allows you to "skip" values using the advanceIfNeeded
* method, and to look at the value without advancing (peekNext).
*/
public interface PeekableIntIterator extends IntIterator {
/**
* If needed, advance as long as the next value is smaller than minval
*
* @param minval threshold
*/
public void advanceIfNeeded(int minval);
/**
* Look at the next value without advancing
*
* @return next value
*/
public int peekNext();
/**
* Creates a copy of the iterator.
*
* @return a clone of the current iterator
*/
@Override
PeekableIntIterator clone();
}

34
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/PeekableShortIterator.java
Unescape View File

@ -1,34 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Simple extension to the ShortIterator interface
*/
public interface PeekableShortIterator extends ShortIterator {
/**
* If needed, advance as long as the next value is smaller than minval (as an unsigned
* short)
*
* @param minval threshold
*/
public void advanceIfNeeded(short minval);
/**
* Look at the next value without advancing
*
* @return next value
*/
public short peekNext();
/**
* Creates a copy of the iterator.
*
* @return a clone of the current iterator
*/
@Override
PeekableShortIterator clone();
}

110
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/ReverseIntIteratorFlyweight.java
Unescape View File

@ -1,110 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Fast iterator minimizing the stress on the garbage collector. You can create one reusable
* instance of this class and then {@link #wrap(RoaringBitmap)}
* <p>
* This iterator enumerates the stored values in reverse (starting from the end).
*
* @author Borislav Ivanov
**/
public class ReverseIntIteratorFlyweight implements IntIterator {
private int hs;
private ShortIterator iter;
private ReverseArrayContainerShortIterator arrIter = new ReverseArrayContainerShortIterator();
private ReverseBitmapContainerShortIterator bitmapIter =
new ReverseBitmapContainerShortIterator();
private ReverseRunContainerShortIterator runIter = new ReverseRunContainerShortIterator();
private short pos;
private RoaringBitmap roaringBitmap = null;
/**
* Creates an instance that is not ready for iteration. You must first call
* {@link #wrap(RoaringBitmap)}.
*/
public ReverseIntIteratorFlyweight() {
}
/**
* Creates an instance that is ready for iteration.
*
* @param r bitmap to be iterated over
*/
public ReverseIntIteratorFlyweight(RoaringBitmap r) {
wrap(r);
}
@Override
public IntIterator clone() {
try {
ReverseIntIteratorFlyweight x = (ReverseIntIteratorFlyweight) super.clone();
x.iter = this.iter.clone();
return x;
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public boolean hasNext() {
return pos >= 0;
}
@Override
public int next() {
final int x = iter.nextAsInt() | hs;
if (!iter.hasNext()) {
--pos;
nextContainer();
}
return x;
}
private void nextContainer() {
if (pos >= 0) {
Container container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
if (container instanceof BitmapContainer) {
bitmapIter.wrap(((BitmapContainer) container).bitmap);
iter = bitmapIter;
} else if (container instanceof ArrayContainer) {
arrIter.wrap((ArrayContainer) container);
iter = arrIter;
} else {
runIter.wrap((RunContainer) container);
iter = runIter;
}
hs = Util.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
}
}
/**
* Prepares a bitmap for iteration
*
* @param r bitmap to be iterated over
*/
public void wrap(RoaringBitmap r) {
this.roaringBitmap = r;
this.hs = 0;
this.pos = (short) (this.roaringBitmap.highLowContainer.size() - 1);
this.nextContainer();
}
}

582
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/RoaringArray.java
Unescape View File

@ -1,582 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.Arrays;
/**
* Specialized array to store the containers used by a RoaringBitmap. This is not meant to be used
* by end users.
*/
public final class RoaringArray implements Cloneable, Externalizable {
protected static final short SERIAL_COOKIE_NO_RUNCONTAINER = 12346;
protected static final short SERIAL_COOKIE = 12347;
protected static final int NO_OFFSET_THRESHOLD = 4;
static final int INITIAL_CAPACITY = 4;
// bumped serialVersionUID with runcontainers, so default serialization
// will not work...
private static final long serialVersionUID = 8L;
short[] keys = null;
Container[] values = null;
int size = 0;
protected RoaringArray() {
this.keys = new short[INITIAL_CAPACITY];
this.values = new Container[INITIAL_CAPACITY];
}
/**
* Find the smallest integer index larger than pos such that array[index].key&gt;=x. If none can
* be found, return size. Based on code by O. Kaser.
*
* @param x minimal value
* @param pos index to exceed
* @return the smallest index greater than pos such that array[index].key is at least as large as
* min, or size if it is not possible.
*/
protected int advanceUntil(short x, int pos) {
int lower = pos + 1;
// special handling for a possibly common sequential case
if (lower >= size || Util.toIntUnsigned(keys[lower]) >= Util.toIntUnsigned(x)) {
return lower;
}
int spansize = 1; // could set larger
// bootstrap an upper limit
while (lower + spansize < size
&& Util.toIntUnsigned(keys[lower + spansize]) < Util.toIntUnsigned(x)) {
spansize *= 2; // hoping for compiler will reduce to shift
}
int upper = (lower + spansize < size) ? lower + spansize : size - 1;
// maybe we are lucky (could be common case when the seek ahead
// expected to be small and sequential will otherwise make us look bad)
if (keys[upper] == x) {
return upper;
}
if (Util.toIntUnsigned(keys[upper]) < Util.toIntUnsigned(x)) {// means array has no item key >=
// x
return size;
}
// we know that the next-smallest span was too small
lower += (spansize / 2);
// else begin binary search
// invariant: array[lower]<x && array[upper]>x
while (lower + 1 != upper) {
int mid = (lower + upper) / 2;
if (keys[mid] == x) {
return mid;
} else if (Util.toIntUnsigned(keys[mid]) < Util.toIntUnsigned(x)) {
lower = mid;
} else {
upper = mid;
}
}
return upper;
}
protected void append(short key, Container value) {
extendArray(1);
this.keys[this.size] = key;
this.values[this.size] = value;
this.size++;
}
/**
* Append copies of the values AFTER a specified key (may or may not be present) to end.
*
* @param sa other array
* @param beforeStart given key is the largest key that we won't copy
*/
protected void appendCopiesAfter(RoaringArray sa, short beforeStart) {
int startLocation = sa.getIndex(beforeStart);
if (startLocation >= 0) {
startLocation++;
} else {
startLocation = -startLocation - 1;
}
extendArray(sa.size - startLocation);
for (int i = startLocation; i < sa.size; ++i) {
this.keys[this.size] = sa.keys[i];
this.values[this.size] = sa.values[i].clone();
this.size++;
}
}
/**
* Append copies of the values from another array, from the start
*
* @param sourceArray The array to copy from
* @param stoppingKey any equal or larger key in other array will terminate copying
*/
protected void appendCopiesUntil(RoaringArray sourceArray, short stoppingKey) {
int stopKey = Util.toIntUnsigned(stoppingKey);
for (int i = 0; i < sourceArray.size; ++i) {
if (Util.toIntUnsigned(sourceArray.keys[i]) >= stopKey) {
break;
}
extendArray(1);
this.keys[this.size] = sourceArray.keys[i];
this.values[this.size] = sourceArray.values[i].clone();
this.size++;
}
}
/**
* Append copy of the one value from another array
*
* @param sa other array
* @param index index in the other array
*/
protected void appendCopy(RoaringArray sa, int index) {
extendArray(1);
this.keys[this.size] = sa.keys[index];
this.values[this.size] = sa.values[index].clone();
this.size++;
}
/**
* Append copies of the values from another array
*
* @param sa other array
* @param startingIndex starting index in the other array
* @param end endingIndex (exclusive) in the other array
*/
protected void appendCopy(RoaringArray sa, int startingIndex, int end) {
extendArray(end - startingIndex);
for (int i = startingIndex; i < end; ++i) {
this.keys[this.size] = sa.keys[i];
this.values[this.size] = sa.values[i].clone();
this.size++;
}
}
/**
* Append the values from another array, no copy is made (use with care)
*
* @param sa other array
* @param startingIndex starting index in the other array
* @param end endingIndex (exclusive) in the other array
*/
protected void append(RoaringArray sa, int startingIndex, int end) {
extendArray(end - startingIndex);
for (int i = startingIndex; i < end; ++i) {
this.keys[this.size] = sa.keys[i];
this.values[this.size] = sa.values[i];
this.size++;
}
}
private int binarySearch(int begin, int end, short key) {
return Util.unsignedBinarySearch(keys, begin, end, key);
}
protected void clear() {
this.keys = null;
this.values = null;
this.size = 0;
}
@Override
public RoaringArray clone() throws CloneNotSupportedException {
RoaringArray sa;
sa = (RoaringArray) super.clone();
sa.keys = Arrays.copyOf(this.keys, this.size);
sa.values = Arrays.copyOf(this.values, this.size);
for (int k = 0; k < this.size; ++k) {
sa.values[k] = sa.values[k].clone();
}
sa.size = this.size;
return sa;
}
protected void copyRange(int begin, int end, int newBegin) {
// assuming begin <= end and newBegin < begin
final int range = end - begin;
System.arraycopy(this.keys, begin, this.keys, newBegin, range);
System.arraycopy(this.values, begin, this.values, newBegin, range);
}
/**
* Deserialize.
*
* @param in the DataInput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void deserialize(DataInput in) throws IOException {
this.clear();
// little endian
final int cookie = Integer.reverseBytes(in.readInt());
if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) {
throw new IOException("I failed to find one of the right cookies.");
}
this.size = ((cookie & 0xFFFF) == SERIAL_COOKIE) ? (cookie >>> 16) + 1
: Integer.reverseBytes(in.readInt());
if ((this.keys == null) || (this.keys.length < this.size)) {
this.keys = new short[this.size];
this.values = new Container[this.size];
}
byte[] bitmapOfRunContainers = null;
boolean hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE;
if (hasrun) {
bitmapOfRunContainers = new byte[(size + 7) / 8];
in.readFully(bitmapOfRunContainers);
}
final short keys[] = new short[this.size];
final int cardinalities[] = new int[this.size];
final boolean isBitmap[] = new boolean[this.size];
for (int k = 0; k < this.size; ++k) {
keys[k] = Short.reverseBytes(in.readShort());
cardinalities[k] = 1 + (0xFFFF & Short.reverseBytes(in.readShort()));
isBitmap[k] = cardinalities[k] > ArrayContainer.DEFAULT_MAX_SIZE;
if (bitmapOfRunContainers != null && (bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) {
isBitmap[k] = false;
}
}
if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
// skipping the offsets
in.skipBytes(this.size * 4);
}
// Reading the containers
for (int k = 0; k < this.size; ++k) {
Container val;
if (isBitmap[k]) {
final long[] bitmapArray = new long[BitmapContainer.MAX_CAPACITY / 64];
// little endian
for (int l = 0; l < bitmapArray.length; ++l) {
bitmapArray[l] = Long.reverseBytes(in.readLong());
}
val = new BitmapContainer(bitmapArray, cardinalities[k]);
} else if (bitmapOfRunContainers != null
&& ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0)) {
// cf RunContainer.writeArray()
int nbrruns = Util.toIntUnsigned(Short.reverseBytes(in.readShort()));
final short lengthsAndValues[] = new short[2 * nbrruns];
for (int j = 0; j < 2 * nbrruns; ++j) {
lengthsAndValues[j] = Short.reverseBytes(in.readShort());
}
val = new RunContainer(lengthsAndValues, nbrruns);
} else {
final short[] shortArray = new short[cardinalities[k]];
for (int l = 0; l < shortArray.length; ++l) {
shortArray[l] = Short.reverseBytes(in.readShort());
}
val = new ArrayContainer(shortArray);
}
this.keys[k] = keys[k];
this.values[k] = val;
}
}
@Override
public boolean equals(Object o) {
if (o instanceof RoaringArray) {
RoaringArray srb = (RoaringArray) o;
if (srb.size != this.size) {
return false;
}
for (int i = 0; i < srb.size; ++i) {
if (this.keys[i] != srb.keys[i] || !this.values[i].equals(srb.values[i])) {
return false;
}
}
return true;
}
return false;
}
// make sure there is capacity for at least k more elements
protected void extendArray(int k) {
// size + 1 could overflow
if (this.size + k >= this.keys.length) {
int newCapacity;
if (this.keys.length < 1024) {
newCapacity = 2 * (this.size + k);
} else {
newCapacity = 5 * (this.size + k) / 4;
}
this.keys = Arrays.copyOf(this.keys, newCapacity);
this.values = Arrays.copyOf(this.values, newCapacity);
}
}
// involves a binary search
protected Container getContainer(short x) {
int i = this.binarySearch(0, size, x);
if (i < 0) {
return null;
}
return this.values[i];
}
protected Container getContainerAtIndex(int i) {
return this.values[i];
}
/**
* Create a ContainerPointer for this RoaringArray
*
* @return a ContainerPointer
*/
public ContainerPointer getContainerPointer() {
return getContainerPointer(0);
}
/**
* Create a ContainerPointer for this RoaringArray
*
* @param startIndex starting index in the container list
* @return a ContainerPointer
*/
public ContainerPointer getContainerPointer(final int startIndex) {
return new ContainerPointer() {
int k = startIndex;
@Override
public void advance() {
++k;
}
@Override
public ContainerPointer clone() {
try {
return (ContainerPointer) super.clone();
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public int compareTo(ContainerPointer o) {
if (key() != o.key()) {
return Util.toIntUnsigned(key()) - Util.toIntUnsigned(o.key());
}
return o.getCardinality() - getCardinality();
}
@Override
public int getCardinality() {
return getContainer().getCardinality();
}
@Override
public Container getContainer() {
if (k >= RoaringArray.this.size) {
return null;
}
return RoaringArray.this.values[k];
}
@Override
public boolean isBitmapContainer() {
return getContainer() instanceof BitmapContainer;
}
@Override
public boolean isRunContainer() {
return getContainer() instanceof RunContainer;
}
@Override
public short key() {
return RoaringArray.this.keys[k];
}
};
}
// involves a binary search
protected int getIndex(short x) {
// before the binary search, we optimize for frequent cases
if ((size == 0) || (keys[size - 1] == x)) {
return size - 1;
}
// no luck we have to go through the list
return this.binarySearch(0, size, x);
}
protected short getKeyAtIndex(int i) {
return this.keys[i];
}
@Override
public int hashCode() {
int hashvalue = 0;
for (int k = 0; k < this.size; ++k) {
hashvalue = 31 * hashvalue + keys[k] * 0xF0F0F0 + values[k].hashCode();
}
return hashvalue;
}
boolean hasRunContainer() {
for (int k = 0; k < size; ++k) {
Container ck = values[k];
if (ck instanceof RunContainer) {
return true;
}
}
return false;
}
protected int headerSize() {
if (hasRunContainer()) {
if (size < NO_OFFSET_THRESHOLD) {// for small bitmaps, we omit the offsets
return 4 + (size + 7) / 8 + 4 * size;
}
return 4 + (size + 7) / 8 + 8 * size;// - 4 because we pack the size with the cookie
} else {
return 4 + 4 + 8 * size;
}
}
// insert a new key, it is assumed that it does not exist
protected void insertNewKeyValueAt(int i, short key, Container value) {
extendArray(1);
System.arraycopy(keys, i, keys, i + 1, size - i);
keys[i] = key;
System.arraycopy(values, i, values, i + 1, size - i);
values[i] = value;
size++;
}
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
deserialize(in);
}
protected void removeAtIndex(int i) {
System.arraycopy(keys, i + 1, keys, i, size - i - 1);
keys[size - 1] = 0;
System.arraycopy(values, i + 1, values, i, size - i - 1);
values[size - 1] = null;
size--;
}
protected void removeIndexRange(int begin, int end) {
if (end <= begin) {
return;
}
final int range = end - begin;
System.arraycopy(keys, end, keys, begin, size - end);
System.arraycopy(values, end, values, begin, size - end);
for (int i = 1; i <= range; ++i) {
keys[size - i] = 0;
values[size - i] = null;
}
size -= range;
}
protected void replaceKeyAndContainerAtIndex(int i, short key, Container c) {
this.keys[i] = key;
this.values[i] = c;
}
protected void resize(int newLength) {
Arrays.fill(this.keys, newLength, this.size, (short) 0);
Arrays.fill(this.values, newLength, this.size, null);
this.size = newLength;
}
/**
* Serialize.
* <p>
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void serialize(DataOutput out) throws IOException {
int startOffset = 0;
boolean hasrun = hasRunContainer();
if (hasrun) {
out.writeInt(Integer.reverseBytes(SERIAL_COOKIE | ((size - 1) << 16)));
byte[] bitmapOfRunContainers = new byte[(size + 7) / 8];
for (int i = 0; i < size; ++i) {
if (this.values[i] instanceof RunContainer) {
bitmapOfRunContainers[i / 8] |= (1 << (i % 8));
}
}
out.write(bitmapOfRunContainers);
if (this.size < NO_OFFSET_THRESHOLD) {
startOffset = 4 + 4 * this.size + bitmapOfRunContainers.length;
} else {
startOffset = 4 + 8 * this.size + bitmapOfRunContainers.length;
}
} else { // backwards compatibility
out.writeInt(Integer.reverseBytes(SERIAL_COOKIE_NO_RUNCONTAINER));
out.writeInt(Integer.reverseBytes(size));
startOffset = 4 + 4 + 4 * this.size + 4 * this.size;
}
for (int k = 0; k < size; ++k) {
out.writeShort(Short.reverseBytes(this.keys[k]));
out.writeShort(Short.reverseBytes((short) (this.values[k].getCardinality() - 1)));
}
if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
// writing the containers offsets
for (int k = 0; k < this.size; k++) {
out.writeInt(Integer.reverseBytes(startOffset));
startOffset = startOffset + this.values[k].getArraySizeInBytes();
}
}
for (int k = 0; k < size; ++k) {
values[k].writeArray(out);
}
}
/**
* Report the number of bytes required for serialization.
*
* @return the size in bytes
*/
public int serializedSizeInBytes() {
int count = headerSize();
for (int k = 0; k < size; ++k) {
count += values[k].getArraySizeInBytes();
}
return count;
}
protected void setContainerAtIndex(int i, Container c) {
this.values[i] = c;
}
protected int size() {
return this.size;
}
@Override
public void writeExternal(ObjectOutput out) throws IOException {
serialize(out);
}
}

2205
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/RoaringBitmap.java
View File

File diff suppressed because it is too large Load Diff

2518
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/RunContainer.java
View File

File diff suppressed because it is too large Load Diff

39
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/ShortIterator.java
Unescape View File

@ -1,39 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Iterator over short values.
*/
public interface ShortIterator extends Cloneable {
/**
* Creates a copy of the iterator.
*
* @return a clone of the current iterator
*/
ShortIterator clone();
/**
* @return whether there is another value
*/
boolean hasNext();
/**
* @return next short value
*/
short next();
/**
* @return next short value as int value (using the least significant 16 bits)
*/
int nextAsInt();
/**
* If possible, remove the current value
*/
void remove();
}

931
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/Util.java
Unescape View File

@ -1,931 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap;
/**
* Various useful methods for roaring bitmaps.
*/
public final class Util {
/**
* optimization flag: whether to use hybrid binary search: hybrid formats
* combine a binary search with a sequential search
*/
public static boolean USE_HYBRID_BINSEARCH = true;
/**
* Private constructor to prevent instantiation of utility class
*/
private Util() {
}
/**
* Find the smallest integer larger than pos such that array[pos]&gt;= min. If none can be found,
* return length. Based on code by O. Kaser.
*
* @param array array to search within
* @param pos starting position of the search
* @param length length of the array to search
* @param min minimum value
* @return x greater than pos such that array[pos] is at least as large as min, pos is is equal to
* length if it is not possible.
*/
public static int advanceUntil(short[] array, int pos, int length, short min) {
int lower = pos + 1;
// special handling for a possibly common sequential case
if (lower >= length || toIntUnsigned(array[lower]) >= toIntUnsigned(min)) {
return lower;
}
int spansize = 1; // could set larger
// bootstrap an upper limit
while (lower + spansize < length
&& toIntUnsigned(array[lower + spansize]) < toIntUnsigned(min)) {
spansize *= 2; // hoping for compiler will reduce to
}
// shift
int upper = (lower + spansize < length) ? lower + spansize : length - 1;
// maybe we are lucky (could be common case when the seek ahead
// expected
// to be small and sequential will otherwise make us look bad)
if (array[upper] == min) {
return upper;
}
if (toIntUnsigned(array[upper]) < toIntUnsigned(min)) {// means
// array
// has no
// item
// >= min
// pos = array.length;
return length;
}
// we know that the next-smallest span was too small
lower += (spansize / 2);
// else begin binary search
// invariant: array[lower]<min && array[upper]>min
while (lower + 1 != upper) {
int mid = (lower + upper) / 2;
short arraymid = array[mid];
if (arraymid == min) {
return mid;
} else if (toIntUnsigned(arraymid) < toIntUnsigned(min)) {
lower = mid;
} else {
upper = mid;
}
}
return upper;
}
protected static int branchyUnsignedBinarySearch(final short[] array, final int begin,
final int end, final short k) {
int ikey = toIntUnsigned(k);
// next line accelerates the possibly common case where the value would
// be inserted at the end
if ((end > 0) && (toIntUnsigned(array[end - 1]) < ikey)) {
return -end - 1;
}
int low = begin;
int high = end - 1;
while (low <= high) {
final int middleIndex = (low + high) >>> 1;
final int middleValue = toIntUnsigned(array[middleIndex]);
if (middleValue < ikey) {
low = middleIndex + 1;
} else if (middleValue > ikey) {
high = middleIndex - 1;
} else {
return middleIndex;
}
}
return -(low + 1);
}
/**
* Compares the two specified {@code short} values, treating them as unsigned values between
* {@code 0} and {@code 2^16 - 1} inclusive.
*
* @param a the first unsigned {@code short} to compare
* @param b the second unsigned {@code short} to compare
* @return a negative value if {@code a} is less than {@code b}; a positive value if {@code a} is
* greater than {@code b}; or zero if they are equal
*/
public static int compareUnsigned(short a, short b) {
return toIntUnsigned(a) - toIntUnsigned(b);
}
/**
* Compute the bitwise AND between two long arrays and write the set bits in the container.
*
* @param container where we write
* @param bitmap1 first bitmap
* @param bitmap2 second bitmap
*/
public static void fillArrayAND(final short[] container, final long[] bitmap1,
final long[] bitmap2) {
int pos = 0;
if (bitmap1.length != bitmap2.length) {
throw new IllegalArgumentException("not supported");
}
for (int k = 0; k < bitmap1.length; ++k) {
long bitset = bitmap1[k] & bitmap2[k];
while (bitset != 0) {
long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
/**
* Compute the bitwise ANDNOT between two long arrays and write the set bits in the container.
*
* @param container where we write
* @param bitmap1 first bitmap
* @param bitmap2 second bitmap
*/
public static void fillArrayANDNOT(final short[] container, final long[] bitmap1,
final long[] bitmap2) {
int pos = 0;
if (bitmap1.length != bitmap2.length) {
throw new IllegalArgumentException("not supported");
}
for (int k = 0; k < bitmap1.length; ++k) {
long bitset = bitmap1[k] & (~bitmap2[k]);
while (bitset != 0) {
long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
/**
* Compute the bitwise XOR between two long arrays and write the set bits in the container.
*
* @param container where we write
* @param bitmap1 first bitmap
* @param bitmap2 second bitmap
*/
public static void fillArrayXOR(final short[] container, final long[] bitmap1,
final long[] bitmap2) {
int pos = 0;
if (bitmap1.length != bitmap2.length) {
throw new IllegalArgumentException("not supported");
}
for (int k = 0; k < bitmap1.length; ++k) {
long bitset = bitmap1[k] ^ bitmap2[k];
while (bitset != 0) {
long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
/**
* flip bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void flipBitmapRange(long[] bitmap, int start, int end) {
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
bitmap[firstword] ^= ~(~0L << start);
for (int i = firstword; i < endword; i++) {
bitmap[i] = ~bitmap[i];
}
bitmap[endword] ^= ~0L >>> -end;
}
/**
* Hamming weight of the 64-bit words involved in the range
* start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return the hamming weight
*/
public static int cardinalityInBitmapWordRange(long[] bitmap, int start, int end) {
if (start == end) {
return 0;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
int answer = 0;
for (int i = firstword; i <= endword; i++) {
answer += Long.bitCount(bitmap[i]);
}
return answer;
}
protected static short highbits(int x) {
return (short) (x >>> 16);
}
protected static short highbits(long x) {
return (short) (x >>> 16);
}
// starts with binary search and finishes with a sequential search
protected static int hybridUnsignedBinarySearch(final short[] array, final int begin,
final int end, final short k) {
int ikey = toIntUnsigned(k);
// next line accelerates the possibly common case where the value would
// be inserted at the end
if ((end > 0) && (toIntUnsigned(array[end - 1]) < ikey)) {
return -end - 1;
}
int low = begin;
int high = end - 1;
// 32 in the next line matches the size of a cache line
while (low + 32 <= high) {
final int middleIndex = (low + high) >>> 1;
final int middleValue = toIntUnsigned(array[middleIndex]);
if (middleValue < ikey) {
low = middleIndex + 1;
} else if (middleValue > ikey) {
high = middleIndex - 1;
} else {
return middleIndex;
}
}
// we finish the job with a sequential search
int x = low;
for (; x <= high; ++x) {
final int val = toIntUnsigned(array[x]);
if (val >= ikey) {
if (val == ikey) {
return x;
}
break;
}
}
return -(x + 1);
}
protected static short lowbits(int x) {
return (short) (x & 0xFFFF);
}
protected static short lowbits(long x) {
return (short) (x & 0xFFFF);
}
protected static short maxLowBit() {
return (short) 0xFFFF;
}
protected static int maxLowBitAsInteger() {
return 0xFFFF;
}
/**
* clear bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void resetBitmapRange(long[] bitmap, int start, int end) {
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
if (firstword == endword) {
bitmap[firstword] &= ~((~0L << start) & (~0L >>> -end));
return;
}
bitmap[firstword] &= ~(~0L << start);
for (int i = firstword + 1; i < endword; i++) {
bitmap[i] = 0;
}
bitmap[endword] &= ~(~0L >>> -end);
}
/**
* Given a word w, return the position of the jth true bit.
*
* @param w word
* @param j index
* @return position of jth true bit in w
*/
public static int select(long w, int j) {
int seen = 0;
// Divide 64bit
int part = (int) (w & 0xFFFFFFFF);
int n = Integer.bitCount(part);
if (n <= j) {
part = (int) (w >>> 32);
seen += 32;
j -= n;
}
int ww = part;
// Divide 32bit
part = ww & 0xFFFF;
n = Integer.bitCount(part);
if (n <= j) {
part = ww >>> 16;
seen += 16;
j -= n;
}
ww = part;
// Divide 16bit
part = ww & 0xFF;
n = Integer.bitCount(part);
if (n <= j) {
part = ww >>> 8;
seen += 8;
j -= n;
}
ww = part;
// Lookup in final byte
int counter;
for (counter = 0; counter < 8; counter++) {
j -= (ww >>> counter) & 1;
if (j < 0) {
break;
}
}
return seen + counter;
}
/**
* set bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void setBitmapRange(long[] bitmap, int start, int end) {
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
if (firstword == endword) {
bitmap[firstword] |= (~0L << start) & (~0L >>> -end);
return;
}
bitmap[firstword] |= ~0L << start;
for (int i = firstword + 1; i < endword; i++) {
bitmap[i] = ~0L;
}
bitmap[endword] |= ~0L >>> -end;
}
/**
* set bits at start, start+1,..., end-1 and report the
* cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int setBitmapRangeAndCardinalityChange(long[] bitmap, int start, int end) {
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
setBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
/**
* flip bits at start, start+1,..., end-1 and report the
* cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int flipBitmapRangeAndCardinalityChange(long[] bitmap, int start, int end) {
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
flipBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
/**
* reset bits at start, start+1,..., end-1 and report the
* cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int resetBitmapRangeAndCardinalityChange(long[] bitmap, int start, int end) {
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
resetBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
protected static int toIntUnsigned(short x) {
return x & 0xFFFF;
}
/**
* Look for value k in array in the range [begin,end). If the value is found, return its index. If
* not, return -(i+1) where i is the index where the value would be inserted. The array is assumed
* to contain sorted values where shorts are interpreted as unsigned integers.
*
* @param array array where we search
* @param begin first index (inclusive)
* @param end last index (exclusive)
* @param k value we search for
* @return count
*/
public static int unsignedBinarySearch(final short[] array, final int begin, final int end,
final short k) {
if (USE_HYBRID_BINSEARCH) {
return hybridUnsignedBinarySearch(array, begin, end, k);
} else {
return branchyUnsignedBinarySearch(array, begin, end, k);
}
}
/**
* Compute the difference between two sorted lists and write the result to the provided output
* array
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @param buffer output array
* @return cardinality of the difference
*/
public static int unsignedDifference(final short[] set1, final int length1, final short[] set2,
final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
System.arraycopy(set1, 0, buffer, 0, length1);
return length1;
}
if (0 == length1) {
return 0;
}
short s1 = set1[k1];
short s2 = set2[k2];
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
break;
}
s1 = set1[k1];
} else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
++k1;
++k2;
if (k1 >= length1) {
break;
}
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1[k1];
s2 = set2[k2];
} else {// if (val1>val2)
++k2;
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2[k2];
}
}
return pos;
}
/**
* Compute the difference between two sorted lists and write the result to the provided output
* array
*
* @param set1 first array
* @param set2 second array
* @param buffer output array
* @return cardinality of the difference
*/
public static int unsignedDifference(ShortIterator set1, ShortIterator set2,
final short[] buffer) {
int pos = 0;
if (!set2.hasNext()) {
while (set1.hasNext()) {
buffer[pos++] = set1.next();
}
return pos;
}
if (!set1.hasNext()) {
return 0;
}
short v1 = set1.next();
short v2 = set2.next();
while (true) {
if (toIntUnsigned(v1) < toIntUnsigned(v2)) {
buffer[pos++] = v1;
if (!set1.hasNext()) {
return pos;
}
v1 = set1.next();
} else if (v1 == v2) {
if (!set1.hasNext()) {
break;
}
if (!set2.hasNext()) {
while (set1.hasNext()) {
buffer[pos++] = set1.next();
}
return pos;
}
v1 = set1.next();
v2 = set2.next();
} else {// if (val1>val2)
if (!set2.hasNext()) {
buffer[pos++] = v1;
while (set1.hasNext()) {
buffer[pos++] = set1.next();
}
return pos;
}
v2 = set2.next();
}
}
return pos;
}
/**
* Compute the exclusive union of two sorted lists and write the result to the provided output
* array
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @param buffer output array
* @return cardinality of the exclusive union
*/
public static int unsignedExclusiveUnion2by2(final short[] set1, final int length1,
final short[] set2, final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
System.arraycopy(set1, 0, buffer, 0, length1);
return length1;
}
if (0 == length1) {
System.arraycopy(set2, 0, buffer, 0, length2);
return length2;
}
short s1 = set1[k1];
short s2 = set2[k2];
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
System.arraycopy(set2, k2, buffer, pos, length2 - k2);
return pos + length2 - k2;
}
s1 = set1[k1];
} else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
++k1;
++k2;
if (k1 >= length1) {
System.arraycopy(set2, k2, buffer, pos, length2 - k2);
return pos + length2 - k2;
}
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1[k1];
s2 = set2[k2];
} else {// if (val1>val2)
buffer[pos++] = s2;
++k2;
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2[k2];
}
}
// return pos;
}
/**
* Intersect two sorted lists and write the result to the provided output array
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @param buffer output array
* @return cardinality of the intersection
*/
public static int unsignedIntersect2by2(final short[] set1, final int length1, final short[] set2,
final int length2, final short[] buffer) {
if (set1.length * 64 < set2.length) {
return unsignedOneSidedGallopingIntersect2by2(set1, length1, set2, length2, buffer);
} else if (set2.length * 64 < set1.length) {
return unsignedOneSidedGallopingIntersect2by2(set2, length2, set1, length1, buffer);
} else {
return unsignedLocalIntersect2by2(set1, length1, set2, length2, buffer);
}
}
/**
* Checks if two arrays intersect
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @return true if they intersect
*/
public static boolean unsignedIntersects(short[] set1, int length1, short[] set2, int length2) {
// galloping might be faster, but we do not expect this function to be slow
if ((0 == length1) || (0 == length2)) {
return false;
}
int k1 = 0;
int k2 = 0;
short s1 = set1[k1];
short s2 = set2[k2];
mainwhile:
while (true) {
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2[k2];
} while (toIntUnsigned(s2) < toIntUnsigned(s1));
}
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1[k1];
} while (toIntUnsigned(s1) < toIntUnsigned(s2));
} else {
return true;
}
}
return false;
}
protected static int unsignedLocalIntersect2by2(final short[] set1, final int length1,
final short[] set2, final int length2, final short[] buffer) {
if ((0 == length1) || (0 == length2)) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = set1[k1];
short s2 = set2[k2];
mainwhile:
while (true) {
int v1 = toIntUnsigned(s1);
int v2 = toIntUnsigned(s2);
if (v2 < v1) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2[k2];
v2 = toIntUnsigned(s2);
} while (v2 < v1);
}
if (v1 < v2) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1[k1];
v1 = toIntUnsigned(s1);
} while (v1 < v2);
} else {
// (set2[k2] == set1[k1])
buffer[pos++] = s1;
++k1;
if (k1 == length1) {
break;
}
++k2;
if (k2 == length2) {
break;
}
s1 = set1[k1];
s2 = set2[k2];
}
}
return pos;
}
/**
* Compute the cardinality of the intersection
*
* @param set1 first set
* @param length1 how many values to consider in the first set
* @param set2 second set
* @param length2 how many values to consider in the second set
* @return cardinality of the intersection
*/
public static int unsignedLocalIntersect2by2Cardinality(final short[] set1, final int length1,
final short[] set2, final int length2) {
if ((0 == length1) || (0 == length2)) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = set1[k1];
short s2 = set2[k2];
mainwhile:
while (true) {
int v1 = toIntUnsigned(s1);
int v2 = toIntUnsigned(s2);
if (v2 < v1) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2[k2];
v2 = toIntUnsigned(s2);
} while (v2 < v1);
}
if (v1 < v2) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1[k1];
v1 = toIntUnsigned(s1);
} while (v1 < v2);
} else {
// (set2[k2] == set1[k1])
pos++;
++k1;
if (k1 == length1) {
break;
}
++k2;
if (k2 == length2) {
break;
}
s1 = set1[k1];
s2 = set2[k2];
}
}
return pos;
}
protected static int unsignedOneSidedGallopingIntersect2by2(final short[] smallSet,
final int smallLength, final short[] largeSet, final int largeLength, final short[] buffer) {
if (0 == smallLength) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = largeSet[k1];
short s2 = smallSet[k2];
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
k1 = advanceUntil(largeSet, k1, largeLength, s2);
if (k1 == largeLength) {
break;
}
s1 = largeSet[k1];
}
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
++k2;
if (k2 == smallLength) {
break;
}
s2 = smallSet[k2];
} else {
// (set2[k2] == set1[k1])
buffer[pos++] = s2;
++k2;
if (k2 == smallLength) {
break;
}
s2 = smallSet[k2];
k1 = advanceUntil(largeSet, k1, largeLength, s2);
if (k1 == largeLength) {
break;
}
s1 = largeSet[k1];
}
}
return pos;
}
/**
* Unite two sorted lists and write the result to the provided output array
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @param buffer output array
* @return cardinality of the union
*/
public static int unsignedUnion2by2(final short[] set1, final int length1, final short[] set2,
final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
System.arraycopy(set1, 0, buffer, 0, length1);
return length1;
}
if (0 == length1) {
System.arraycopy(set2, 0, buffer, 0, length2);
return length2;
}
short s1 = set1[k1];
short s2 = set2[k2];
while (true) {
int v1 = toIntUnsigned(s1);
int v2 = toIntUnsigned(s2);
if (v1 < v2) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
System.arraycopy(set2, k2, buffer, pos, length2 - k2);
return pos + length2 - k2;
}
s1 = set1[k1];
} else if (v1 == v2) {
buffer[pos++] = s1;
++k1;
++k2;
if (k1 >= length1) {
System.arraycopy(set2, k2, buffer, pos, length2 - k2);
return pos + length2 - k2;
}
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1[k1];
s2 = set2[k2];
} else {// if (set1[k1]>set2[k2])
buffer[pos++] = s2;
++k2;
if (k2 >= length2) {
System.arraycopy(set1, k1, buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2[k2];
}
}
// return pos;
}
}

113
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/BufferBitSetUtil.java
Unescape View File

@ -1,113 +0,0 @@
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.IntIterator;
import java.nio.LongBuffer;
import java.nio.ShortBuffer;
import java.util.Arrays;
import java.util.BitSet;
/***
*
* This class provides convenience functions to manipulate BitSet and MutableRoaringBitmap objects.
*
*/
public class BufferBitSetUtil {
// todo: add a method to convert an ImmutableRoaringBitmap to a BitSet using BitSet.valueOf
// a block consists has a maximum of 1024 words, each representing 64 bits,
// thus representing at maximum 65536 bits
static final private int BLOCK_LENGTH = MappeableBitmapContainer.MAX_CAPACITY / Long.SIZE; //
// 64-bit
// word
private static MappeableArrayContainer arrayContainerOf(final int from, final int to,
final int cardinality, final long[] words) {
// precondition: cardinality is max 4096
final short[] content = new short[cardinality];
int index = 0;
for (int i = from, socket = 0; i < to; ++i, socket += Long.SIZE) {
long word = words[i];
while (word != 0) {
long t = word & -word;
content[index++] = (short) (socket + Long.bitCount(t - 1));
word ^= t;
}
}
return new MappeableArrayContainer(ShortBuffer.wrap(content), cardinality);
}
/**
* Generate a MutableRoaringBitmap out of a long[], each long using little-endian representation
* of its bits
*
* @param words array of longs (will not be modified)
* @return roaring bitmap
* @see BitSet#toLongArray() for an equivalent
*/
public static MutableRoaringBitmap bitmapOf(final long[] words) {
// split long[] into blocks.
// each block becomes a single container, if any bit is set
final MutableRoaringBitmap ans = new MutableRoaringBitmap();
int containerIndex = 0;
for (int from = 0; from < words.length; from += BLOCK_LENGTH) {
final int to = Math.min(from + BLOCK_LENGTH, words.length);
final int blockCardinality = cardinality(from, to, words);
if (blockCardinality > 0) {
((MutableRoaringArray) ans.highLowContainer).insertNewKeyValueAt(containerIndex++,
BufferUtil.highbits(from * Long.SIZE),
BufferBitSetUtil.containerOf(from, to, blockCardinality, words));
}
}
return ans;
}
private static int cardinality(final int from, final int to, final long[] words) {
int sum = 0;
for (int i = from; i < to; i++) {
sum += Long.bitCount(words[i]);
}
return sum;
}
private static MappeableContainer containerOf(final int from, final int to,
final int blockCardinality, final long[] words) {
// find the best container available
if (blockCardinality <= MappeableArrayContainer.DEFAULT_MAX_SIZE) {
// containers with DEFAULT_MAX_SIZE or less integers should be
// ArrayContainers
return arrayContainerOf(from, to, blockCardinality, words);
} else {
// otherwise use bitmap container
return new MappeableBitmapContainer(
LongBuffer.wrap(Arrays.copyOfRange(words, from, from + BLOCK_LENGTH)), blockCardinality);
}
}
/**
* Compares a RoaringBitmap and a BitSet. They are equal if and only if they contain the same set
* of integers.
*
* @param bitset first object to be compared
* @param bitmap second object to be compared
* @return whether they are equal
*/
public static boolean equals(final BitSet bitset, final ImmutableRoaringBitmap bitmap) {
if (bitset.cardinality() != bitmap.getCardinality()) {
return false;
}
final IntIterator it = bitmap.getIntIterator();
while (it.hasNext()) {
int val = it.next();
if (!bitset.get(val)) {
return false;
}
}
return true;
}
}

631
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/BufferFastAggregation.java
Unescape View File

@ -1,631 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.PriorityQueue;
/**
* Fast algorithms to aggregate many bitmaps.
*
* @author Daniel Lemire
*/
public final class BufferFastAggregation {
/**
* Private constructor to prevent instantiation of utility class
*/
private BufferFastAggregation() {
}
/**
* Compute the AND aggregate.
* <p>
* In practice, calls {#link naive_and}
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap and(ImmutableRoaringBitmap... bitmaps) {
return naive_and(bitmaps);
}
/**
* Compute the AND aggregate.
* <p>
* In practice, calls {#link naive_and}
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap and(@SuppressWarnings("rawtypes") Iterator bitmaps) {
return naive_and(bitmaps);
}
/**
* Compute the AND aggregate.
* <p>
* In practice, calls {#link naive_and}
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap and(MutableRoaringBitmap... bitmaps) {
return and(convertToImmutable(bitmaps));
}
/**
* Convenience method converting one type of iterator into another, to avoid unnecessary warnings.
*
* @param i input bitmaps
* @return an iterator over the provided iterator, with a different type
*/
public static Iterator<ImmutableRoaringBitmap> convertToImmutable(
final Iterator<MutableRoaringBitmap> i) {
return new Iterator<ImmutableRoaringBitmap>() {
@Override
public boolean hasNext() {
return i.hasNext();
}
@Override
public ImmutableRoaringBitmap next() {
return i.next();
}
@Override
public void remove() {
}
;
};
}
private static ImmutableRoaringBitmap[] convertToImmutable(MutableRoaringBitmap[] array) {
ImmutableRoaringBitmap[] answer = new ImmutableRoaringBitmap[array.length];
for (int k = 0; k < answer.length; ++k) {
answer[k] = array[k];
}
return answer;
}
/**
* Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #or(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap horizontal_or(ImmutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
if (bitmaps.length == 0) {
return answer;
}
PriorityQueue<MappeableContainerPointer> pq = new PriorityQueue<MappeableContainerPointer>(bitmaps.length);
for (int k = 0; k < bitmaps.length; ++k) {
MappeableContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
if (x.getContainer() != null) {
pq.add(x);
}
}
while (!pq.isEmpty()) {
MappeableContainerPointer x1 = pq.poll();
if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
answer.getMappeableRoaringArray().append(x1.key(), x1.getContainer().clone());
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
continue;
}
MappeableContainerPointer x2 = pq.poll();
MappeableContainer newc = x1.getContainer().lazyOR(x2.getContainer());
while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
MappeableContainerPointer x = pq.poll();
newc = newc.lazyIOR(x.getContainer());
x.advance();
if (x.getContainer() != null) {
pq.add(x);
} else if (pq.isEmpty()) {
break;
}
}
newc = newc.repairAfterLazy();
answer.getMappeableRoaringArray().append(x1.key(), newc);
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
x2.advance();
if (x2.getContainer() != null) {
pq.add(x2);
}
}
return answer;
}
/**
* Calls naive_or.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
@Deprecated
public static MutableRoaringBitmap horizontal_or(@SuppressWarnings("rawtypes") Iterator bitmaps) {
return naive_or(bitmaps);
}
/**
* Minimizes memory usage while computing the or aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #or(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap horizontal_or(MutableRoaringBitmap... bitmaps) {
return horizontal_or(convertToImmutable(bitmaps));
}
/**
* Minimizes memory usage while computing the xor aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #xor(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap horizontal_xor(ImmutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
if (bitmaps.length == 0) {
return answer;
}
PriorityQueue<MappeableContainerPointer> pq = new PriorityQueue<MappeableContainerPointer>(bitmaps.length);
for (int k = 0; k < bitmaps.length; ++k) {
MappeableContainerPointer x = bitmaps[k].highLowContainer.getContainerPointer();
if (x.getContainer() != null) {
pq.add(x);
}
}
while (!pq.isEmpty()) {
MappeableContainerPointer x1 = pq.poll();
if (pq.isEmpty() || (pq.peek().key() != x1.key())) {
answer.getMappeableRoaringArray().append(x1.key(), x1.getContainer().clone());
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
continue;
}
MappeableContainerPointer x2 = pq.poll();
MappeableContainer newc = x1.getContainer().xor(x2.getContainer());
while (!pq.isEmpty() && (pq.peek().key() == x1.key())) {
MappeableContainerPointer x = pq.poll();
newc = newc.ixor(x.getContainer());
x.advance();
if (x.getContainer() != null) {
pq.add(x);
} else if (pq.isEmpty()) {
break;
}
}
answer.getMappeableRoaringArray().append(x1.key(), newc);
x1.advance();
if (x1.getContainer() != null) {
pq.add(x1);
}
x2.advance();
if (x2.getContainer() != null) {
pq.add(x2);
}
}
return answer;
}
/**
* Minimizes memory usage while computing the xor aggregate on a moderate number of bitmaps.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #xor(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap horizontal_xor(MutableRoaringBitmap... bitmaps) {
return horizontal_xor(convertToImmutable(bitmaps));
}
/**
* Compute overall AND between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_and(ImmutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer;
if (bitmaps.length > 0) {
answer = (bitmaps[0]).toMutableRoaringBitmap();
for (int k = 1; k < bitmaps.length; ++k) {
answer = ImmutableRoaringBitmap.and(answer, bitmaps[k]);
}
} else {
answer = new MutableRoaringBitmap();
}
return answer;
}
/**
* Compute overall AND between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_and(@SuppressWarnings("rawtypes") Iterator bitmaps) {
if (!bitmaps.hasNext()) {
return new MutableRoaringBitmap();
}
MutableRoaringBitmap answer =
((ImmutableRoaringBitmap) bitmaps.next()).toMutableRoaringBitmap();
while (bitmaps.hasNext()) {
answer.and((ImmutableRoaringBitmap) bitmaps.next());
}
return answer;
}
/**
* Compute overall AND between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_and(MutableRoaringBitmap... bitmaps) {
if (bitmaps.length == 0) {
return new MutableRoaringBitmap();
}
MutableRoaringBitmap answer = bitmaps[0].clone();
for (int k = 1; k < bitmaps.length; ++k) {
answer.and(bitmaps[k]);
}
return answer;
}
/**
* Compute overall OR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_or(ImmutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.naivelazyor(bitmaps[k]);
}
answer.repairAfterLazy();
return answer;
}
/**
* Compute overall OR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_or(@SuppressWarnings("rawtypes") Iterator bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
while (bitmaps.hasNext()) {
answer.naivelazyor((ImmutableRoaringBitmap) bitmaps.next());
}
answer.repairAfterLazy();
return answer;
}
/**
* Compute overall OR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_or(MutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.lazyor(bitmaps[k]);
}
answer.repairAfterLazy();
return answer;
}
/**
* Compute overall XOR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_xor(ImmutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.xor(bitmaps[k]);
}
return answer;
}
/**
* Compute overall XOR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_xor(@SuppressWarnings("rawtypes") Iterator bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
while (bitmaps.hasNext()) {
answer.xor((ImmutableRoaringBitmap) bitmaps.next());
}
return answer;
}
/**
* Compute overall XOR between bitmaps two-by-two.
* <p>
* This function runs in linear time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap naive_xor(MutableRoaringBitmap... bitmaps) {
MutableRoaringBitmap answer = new MutableRoaringBitmap();
for (int k = 0; k < bitmaps.length; ++k) {
answer.xor(bitmaps[k]);
}
return answer;
}
/**
* Compute overall OR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap or(ImmutableRoaringBitmap... bitmaps) {
return naive_or(bitmaps);
}
/**
* Compute overall OR between bitmaps.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap or(@SuppressWarnings("rawtypes") Iterator bitmaps) {
return naive_or(bitmaps);
}
/**
* Compute overall OR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap or(MutableRoaringBitmap... bitmaps) {
return naive_or(bitmaps);
}
/**
* Uses a priority queue to compute the or aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_or(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap priorityqueue_or(ImmutableRoaringBitmap... bitmaps) {
if (bitmaps.length == 0) {
return new MutableRoaringBitmap();
} else if (bitmaps.length == 1) {
return bitmaps[0].toMutableRoaringBitmap();
}
// we buffer the call to getLongSizeInBytes(), hence the code complexity
final ImmutableRoaringBitmap[] buffer = Arrays.copyOf(bitmaps, bitmaps.length);
final int[] sizes = new int[buffer.length];
final boolean[] istmp = new boolean[buffer.length];
for (int k = 0; k < sizes.length; ++k) {
sizes[k] = buffer[k].serializedSizeInBytes();
}
PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
@Override
public int compare(Integer a, Integer b) {
return sizes[a] - sizes[b];
}
});
for (int k = 0; k < sizes.length; ++k) {
pq.add(k);
}
while (pq.size() > 1) {
Integer x1 = pq.poll();
Integer x2 = pq.poll();
if (istmp[x1] && istmp[x2]) {
buffer[x1] = MutableRoaringBitmap.lazyorfromlazyinputs((MutableRoaringBitmap) buffer[x1],
(MutableRoaringBitmap) buffer[x2]);
sizes[x1] = buffer[x1].serializedSizeInBytes();
pq.add(x1);
} else if (istmp[x2]) {
((MutableRoaringBitmap) buffer[x2]).lazyor(buffer[x1]);
sizes[x2] = buffer[x2].serializedSizeInBytes();
pq.add(x2);
} else if (istmp[x1]) {
((MutableRoaringBitmap) buffer[x1]).lazyor(buffer[x2]);
sizes[x1] = buffer[x1].serializedSizeInBytes();
pq.add(x1);
} else {
buffer[x1] = ImmutableRoaringBitmap.lazyor(buffer[x1], buffer[x2]);
sizes[x1] = buffer[x1].serializedSizeInBytes();
istmp[x1] = true;
pq.add(x1);
}
}
MutableRoaringBitmap answer = (MutableRoaringBitmap) buffer[pq.poll()];
answer.repairAfterLazy();
return answer;
}
/**
* Uses a priority queue to compute the or aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_or(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap priorityqueue_or(
@SuppressWarnings("rawtypes") Iterator bitmaps) {
if (!bitmaps.hasNext()) {
return new MutableRoaringBitmap();
}
// we buffer the call to getLongSizeInBytes(), hence the code complexity
ArrayList<ImmutableRoaringBitmap> buffer = new ArrayList<ImmutableRoaringBitmap>();
while (bitmaps.hasNext()) {
buffer.add((ImmutableRoaringBitmap) bitmaps.next());
}
final long[] sizes = new long[buffer.size()];
final boolean[] istmp = new boolean[buffer.size()];
for (int k = 0; k < sizes.length; ++k) {
sizes[k] = buffer.get(k).getLongSizeInBytes();
}
PriorityQueue<Integer> pq = new PriorityQueue<Integer>(128, new Comparator<Integer>() {
@Override
public int compare(Integer a, Integer b) {
return (int) (sizes[a] - sizes[b]);
}
});
for (int k = 0; k < sizes.length; ++k) {
pq.add(k);
}
if (pq.size() == 1) {
return buffer.get(pq.poll()).toMutableRoaringBitmap();
}
while (pq.size() > 1) {
Integer x1 = pq.poll();
Integer x2 = pq.poll();
if (istmp[x1] && istmp[x2]) {
buffer.set(x1, MutableRoaringBitmap.lazyorfromlazyinputs(
(MutableRoaringBitmap) buffer.get(x1), (MutableRoaringBitmap) buffer.get(x2)));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
pq.add(x1);
} else if (istmp[x2]) {
((MutableRoaringBitmap) buffer.get(x2)).lazyor(buffer.get(x1));
sizes[x2] = buffer.get(x2).getLongSizeInBytes();
pq.add(x2);
} else if (istmp[x1]) {
((MutableRoaringBitmap) buffer.get(x1)).lazyor(buffer.get(x2));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
pq.add(x1);
} else {
buffer.set(x1, ImmutableRoaringBitmap.lazyor(buffer.get(x1), buffer.get(x2)));
sizes[x1] = buffer.get(x1).getLongSizeInBytes();
istmp[x1] = true;
pq.add(x1);
}
}
MutableRoaringBitmap answer = (MutableRoaringBitmap) buffer.get(pq.poll());
answer.repairAfterLazy();
return answer;
}
/**
* Uses a priority queue to compute the xor aggregate.
* <p>
* This function runs in linearithmic (O(n log n)) time with respect to the number of bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
* @see #horizontal_xor(ImmutableRoaringBitmap...)
*/
public static MutableRoaringBitmap priorityqueue_xor(ImmutableRoaringBitmap... bitmaps) {
// code could be faster, see priorityqueue_or
if (bitmaps.length < 2) {
throw new IllegalArgumentException("Expecting at least 2 bitmaps");
}
final PriorityQueue<ImmutableRoaringBitmap> pq =
new PriorityQueue<ImmutableRoaringBitmap>(bitmaps.length, new Comparator<ImmutableRoaringBitmap>() {
@Override
public int compare(ImmutableRoaringBitmap a, ImmutableRoaringBitmap b) {
return (int) (a.getLongSizeInBytes() - b.getLongSizeInBytes());
}
});
Collections.addAll(pq, bitmaps);
while (pq.size() > 1) {
final ImmutableRoaringBitmap x1 = pq.poll();
final ImmutableRoaringBitmap x2 = pq.poll();
pq.add(ImmutableRoaringBitmap.xor(x1, x2));
}
return (MutableRoaringBitmap) pq.poll();
}
/**
* Compute overall XOR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap xor(ImmutableRoaringBitmap... bitmaps) {
return naive_xor(bitmaps);
}
/**
* Compute overall XOR between bitmaps.
*
* @param bitmaps input bitmaps (ImmutableRoaringBitmap or MutableRoaringBitmap)
* @return aggregated bitmap
*/
public static MutableRoaringBitmap xor(@SuppressWarnings("rawtypes") Iterator bitmaps) {
return naive_xor(bitmaps);
}
/**
* Compute overall XOR between bitmaps.
*
* @param bitmaps input bitmaps
* @return aggregated bitmap
*/
public static MutableRoaringBitmap xor(MutableRoaringBitmap... bitmaps) {
return naive_xor(bitmaps);
}
}

132
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/BufferIntIteratorFlyweight.java
Unescape View File

@ -1,132 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.PeekableIntIterator;
import com.fr.third.bitmap.roaringbitmap.PeekableShortIterator;
/**
* Fast iterator minimizing the stress on the garbage collector. You can create one reusable
* instance of this class and then {@link #wrap(ImmutableRoaringBitmap)}
* <p>
* For better performance, consider the {@link ImmutableRoaringBitmap#forEach} method.
*
* @author Borislav Ivanov
**/
public class BufferIntIteratorFlyweight implements PeekableIntIterator {
private int hs;
private PeekableShortIterator iter;
private MappeableArrayContainerShortIterator arrIter = new MappeableArrayContainerShortIterator();
private MappeableBitmapContainerShortIterator bitmapIter =
new MappeableBitmapContainerShortIterator();
private MappeableRunContainerShortIterator runIter = new MappeableRunContainerShortIterator();
private int pos;
private ImmutableRoaringBitmap roaringBitmap = null;
/**
* Creates an instance that is not ready for iteration. You must first call
* {@link #wrap(ImmutableRoaringBitmap)}.
*/
public BufferIntIteratorFlyweight() {
}
/**
* Creates an instance that is ready for iteration.
*
* @param r bitmap to be iterated over
*/
public BufferIntIteratorFlyweight(ImmutableRoaringBitmap r) {
wrap(r);
}
@Override
public PeekableIntIterator clone() {
try {
BufferIntIteratorFlyweight x = (BufferIntIteratorFlyweight) super.clone();
x.iter = this.iter.clone();
return x;
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public boolean hasNext() {
return pos < this.roaringBitmap.highLowContainer.size();
}
@Override
public int next() {
int x = iter.nextAsInt() | hs;
if (!iter.hasNext()) {
++pos;
nextContainer();
}
return x;
}
private void nextContainer() {
if (pos < this.roaringBitmap.highLowContainer.size()) {
MappeableContainer container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
if (container instanceof MappeableBitmapContainer) {
bitmapIter.wrap((MappeableBitmapContainer) container);
iter = bitmapIter;
} else if (container instanceof MappeableRunContainer) {
runIter.wrap((MappeableRunContainer) container);
iter = runIter;
} else {
arrIter.wrap((MappeableArrayContainer) container);
iter = arrIter;
}
hs = BufferUtil.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
}
}
/**
* Prepares a bitmap for iteration
*
* @param r bitmap to be iterated over
*/
public void wrap(ImmutableRoaringBitmap r) {
this.hs = 0;
this.pos = 0;
this.roaringBitmap = r;
this.nextContainer();
}
@Override
public void advanceIfNeeded(int minval) {
while (hasNext() && ((hs >>> 16) < (minval >>> 16))) {
++pos;
nextContainer();
}
if (hasNext() && ((hs >>> 16) == (minval >>> 16))) {
iter.advanceIfNeeded(BufferUtil.lowbits(minval));
if (!iter.hasNext()) {
++pos;
nextContainer();
}
}
}
@Override
public int peekNext() {
return BufferUtil.toIntUnsigned(iter.peekNext()) | hs;
}
}

116
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/BufferReverseIntIteratorFlyweight.java
Unescape View File

@ -1,116 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.IntIterator;
import com.fr.third.bitmap.roaringbitmap.ShortIterator;
/**
* Fast iterator minimizing the stress on the garbage collector. You can create one reusable
* instance of this class and then {@link #wrap(ImmutableRoaringBitmap)}
* <p>
* This iterator enumerates the stored values in reverse (starting from the end).
*
* @author Borislav Ivanov
**/
public class BufferReverseIntIteratorFlyweight implements IntIterator {
private int hs;
private ShortIterator iter;
private ReverseMappeableArrayContainerShortIterator arrIter =
new ReverseMappeableArrayContainerShortIterator();
private ReverseMappeableBitmapContainerShortIterator bitmapIter =
new ReverseMappeableBitmapContainerShortIterator();
private ReverseMappeableRunContainerShortIterator runIter =
new ReverseMappeableRunContainerShortIterator();
private short pos;
private ImmutableRoaringBitmap roaringBitmap = null;
/**
* Creates an instance that is not ready for iteration. You must first call
* {@link #wrap(ImmutableRoaringBitmap)}.
*/
public BufferReverseIntIteratorFlyweight() {
}
/**
* Creates an instance that is ready for iteration.
*
* @param r bitmap to be iterated over
*/
public BufferReverseIntIteratorFlyweight(ImmutableRoaringBitmap r) {
wrap(r);
}
@Override
public IntIterator clone() {
try {
BufferReverseIntIteratorFlyweight x = (BufferReverseIntIteratorFlyweight) super.clone();
x.iter = this.iter.clone();
return x;
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public boolean hasNext() {
return pos >= 0;
}
@Override
public int next() {
final int x = iter.nextAsInt() | hs;
if (!iter.hasNext()) {
--pos;
nextContainer();
}
return x;
}
private void nextContainer() {
if (pos >= 0) {
MappeableContainer container = this.roaringBitmap.highLowContainer.getContainerAtIndex(pos);
if (container instanceof MappeableBitmapContainer) {
bitmapIter.wrap((MappeableBitmapContainer) container);
iter = bitmapIter;
} else if (container instanceof MappeableRunContainer) {
runIter.wrap((MappeableRunContainer) container);
iter = runIter;
} else {
arrIter.wrap((MappeableArrayContainer) container);
iter = arrIter;
}
hs = BufferUtil.toIntUnsigned(this.roaringBitmap.highLowContainer.getKeyAtIndex(pos)) << 16;
}
}
/**
* Prepares a bitmap for iteration
*
* @param r bitmap to be iterated over
*/
public void wrap(ImmutableRoaringBitmap r) {
this.roaringBitmap = r;
this.hs = 0;
this.pos = (short) (this.roaringBitmap.highLowContainer.size() - 1);
this.nextContainer();
}
}

825
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/BufferUtil.java
Unescape View File

@ -1,825 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.Util;
import java.nio.Buffer;
import java.nio.LongBuffer;
import java.nio.ShortBuffer;
/**
* Various useful methods for roaring bitmaps.
* <p>
* This class is similar to Util but meant to be used with memory mapping.
*/
public final class BufferUtil {
/**
* Private constructor to prevent instantiation of utility class
*/
private BufferUtil() {
}
/**
* Find the smallest integer larger than pos such that array[pos]&gt;= min. If none can be found,
* return length. Based on code by O. Kaser.
*
* @param array container where we search
* @param pos initial position
* @param min minimal threshold
* @param length how big should the array consider to be
* @return x greater than pos such that array[pos] is at least as large as min, pos is is equal to
* length if it is not possible.
*/
protected static int advanceUntil(ShortBuffer array, int pos, int length, short min) {
int lower = pos + 1;
// special handling for a possibly common sequential case
if (lower >= length || toIntUnsigned(array.get(lower)) >= toIntUnsigned(min)) {
return lower;
}
int spansize = 1; // could set larger
// bootstrap an upper limit
while (lower + spansize < length
&& toIntUnsigned(array.get(lower + spansize)) < toIntUnsigned(min)) {
spansize *= 2; // hoping for compiler will reduce to
}
// shift
int upper = (lower + spansize < length) ? lower + spansize : length - 1;
// maybe we are lucky (could be common case when the seek ahead
// expected
// to be small and sequential will otherwise make us look bad)
if (array.get(upper) == min) {
return upper;
}
if (toIntUnsigned(array.get(upper)) < toIntUnsigned(min)) {// means
// array
// has no
// item
// >= min
// pos = array.length;
return length;
}
// we know that the next-smallest span was too small
lower += (spansize / 2);
// else begin binary search
// invariant: array[lower]<min && array[upper]>min
while (lower + 1 != upper) {
int mid = (lower + upper) / 2;
short arraymid = array.get(mid);
if (arraymid == min) {
return mid;
} else if (toIntUnsigned(arraymid) < toIntUnsigned(min)) {
lower = mid;
} else {
upper = mid;
}
}
return upper;
}
protected static void arraycopy(ShortBuffer src, int srcPos, ShortBuffer dest, int destPos,
int length) {
if (BufferUtil.isBackedBySimpleArray(src) && BufferUtil.isBackedBySimpleArray(dest)) {
System.arraycopy(src.array(), srcPos, dest.array(), destPos, length);
} else {
if (srcPos < destPos) {
for (int k = length - 1; k >= 0; --k) {
dest.put(destPos + k, src.get(k + srcPos));
}
} else {
for (int k = 0; k < length; ++k) {
dest.put(destPos + k, src.get(k + srcPos));
}
}
}
}
protected static int branchyUnsignedBinarySearch(final ShortBuffer array, final int begin,
final int end, final short k) {
final int ikey = toIntUnsigned(k);
// next line accelerates the possibly common case where the value would be inserted at the end
if ((end > 0) && (toIntUnsigned(array.get(end - 1)) < ikey)) {
return -end - 1;
}
int low = begin;
int high = end - 1;
while (low <= high) {
final int middleIndex = (low + high) >>> 1;
final int middleValue = toIntUnsigned(array.get(middleIndex));
if (middleValue < ikey) {
low = middleIndex + 1;
} else if (middleValue > ikey) {
high = middleIndex - 1;
} else {
return middleIndex;
}
}
return -(low + 1);
}
/**
* Compares the two specified {@code short} values, treating them as unsigned values between
* {@code 0} and {@code 2^16 - 1} inclusive.
*
* @param a the first unsigned {@code short} to compare
* @param b the second unsigned {@code short} to compare
* @return a negative value if {@code a} is less than {@code b}; a positive value if {@code a} is
* greater than {@code b}; or zero if they are equal
*/
public static int compareUnsigned(short a, short b) {
return toIntUnsigned(a) - toIntUnsigned(b);
}
protected static void fillArrayAND(short[] container, LongBuffer bitmap1, LongBuffer bitmap2) {
int pos = 0;
if (bitmap1.limit() != bitmap2.limit()) {
throw new IllegalArgumentException("not supported");
}
if (BufferUtil.isBackedBySimpleArray(bitmap1) && BufferUtil.isBackedBySimpleArray(bitmap2)) {
int len = bitmap1.limit();
long[] b1 = bitmap1.array();
long[] b2 = bitmap2.array();
for (int k = 0; k < len; ++k) {
long bitset = b1[k] & b2[k];
while (bitset != 0) {
final long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
} else {
int len = bitmap1.limit();
for (int k = 0; k < len; ++k) {
long bitset = bitmap1.get(k) & bitmap2.get(k);
while (bitset != 0) {
final long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
}
protected static void fillArrayANDNOT(short[] container, LongBuffer bitmap1, LongBuffer bitmap2) {
int pos = 0;
if (bitmap1.limit() != bitmap2.limit()) {
throw new IllegalArgumentException("not supported");
}
if (BufferUtil.isBackedBySimpleArray(bitmap1) && BufferUtil.isBackedBySimpleArray(bitmap2)) {
int len = bitmap1.limit();
long[] b1 = bitmap1.array();
long[] b2 = bitmap2.array();
for (int k = 0; k < len; ++k) {
long bitset = b1[k] & (~b2[k]);
while (bitset != 0) {
final long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
} else {
int len = bitmap1.limit();
for (int k = 0; k < len; ++k) {
long bitset = bitmap1.get(k) & (~bitmap2.get(k));
while (bitset != 0) {
final long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
}
protected static void fillArrayXOR(short[] container, LongBuffer bitmap1, LongBuffer bitmap2) {
int pos = 0;
if (bitmap1.limit() != bitmap2.limit()) {
throw new IllegalArgumentException("not supported");
}
if (BufferUtil.isBackedBySimpleArray(bitmap1) && BufferUtil.isBackedBySimpleArray(bitmap2)) {
Util.fillArrayXOR(container, bitmap1.array(), bitmap2.array());
} else {
int len = bitmap1.limit();
for (int k = 0; k < len; ++k) {
long bitset = bitmap1.get(k) ^ bitmap2.get(k);
while (bitset != 0) {
final long t = bitset & -bitset;
container[pos++] = (short) (k * 64 + Long.bitCount(t - 1));
bitset ^= t;
}
}
}
}
/**
* flip bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void flipBitmapRange(LongBuffer bitmap, int start, int end) {
if (isBackedBySimpleArray(bitmap)) {
Util.flipBitmapRange(bitmap.array(), start, end);
return;
}
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
bitmap.put(firstword, bitmap.get(firstword) ^ ~(~0L << start));
for (int i = firstword; i < endword; i++) {
bitmap.put(i, ~bitmap.get(i));
}
bitmap.put(endword, bitmap.get(endword) ^ (~0L >>> -end));
}
/**
* Hamming weight of the 64-bit words involved in the range start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
private static int cardinalityInBitmapWordRange(LongBuffer bitmap, int start, int end) {
if (isBackedBySimpleArray(bitmap)) {
return Util.cardinalityInBitmapWordRange(bitmap.array(), start, end);
}
if (start == end) {
return 0;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
int answer = 0;
for (int i = firstword; i <= endword; i++) {
answer += Long.bitCount(bitmap.get(i));
}
return answer;
}
/**
* set bits at start, start+1,..., end-1 and report the cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int setBitmapRangeAndCardinalityChange(LongBuffer bitmap, int start, int end) {
if (BufferUtil.isBackedBySimpleArray(bitmap)) {
return Util.setBitmapRangeAndCardinalityChange(bitmap.array(), start, end);
}
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
setBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
/**
* flip bits at start, start+1,..., end-1 and report the cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int flipBitmapRangeAndCardinalityChange(LongBuffer bitmap, int start, int end) {
if (BufferUtil.isBackedBySimpleArray(bitmap)) {
return Util.flipBitmapRangeAndCardinalityChange(bitmap.array(), start, end);
}
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
flipBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
/**
* reset bits at start, start+1,..., end-1 and report the cardinality change
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
* @return cardinality change
*/
public static int resetBitmapRangeAndCardinalityChange(LongBuffer bitmap, int start, int end) {
if (BufferUtil.isBackedBySimpleArray(bitmap)) {
return Util.resetBitmapRangeAndCardinalityChange(bitmap.array(), start, end);
}
int cardbefore = cardinalityInBitmapWordRange(bitmap, start, end);
resetBitmapRange(bitmap, start, end);
int cardafter = cardinalityInBitmapWordRange(bitmap, start, end);
return cardafter - cardbefore;
}
/**
* From the cardinality of a container, compute the corresponding size in bytes of the container.
* Additional information is required if the container is run encoded.
*
* @param card the cardinality if this is not run encoded, otherwise ignored
* @param numRuns number of runs if run encoded, othewise ignored
* @param isRunEncoded boolean
* @return the size in bytes
*/
protected static int getSizeInBytesFromCardinalityEtc(int card, int numRuns,
boolean isRunEncoded) {
if (isRunEncoded) {
return 2 + numRuns * 2 * 2; // each run uses 2 shorts, plus the initial short giving num runs
}
boolean isBitmap = card > MappeableArrayContainer.DEFAULT_MAX_SIZE;
if (isBitmap) {
return MappeableBitmapContainer.MAX_CAPACITY / 8;
} else {
return card * 2;
}
}
protected static short highbits(int x) {
return (short) (x >>> 16);
}
protected static short highbits(long x) {
return (short) (x >>> 16);
}
/**
* Checks whether the Buffer is backed by a simple array. In java, a Buffer is an abstraction that
* can represent various data, from data on disk all the way to native Java arrays. Like all
* abstractions, a Buffer might carry a performance penalty. Thus, we sometimes check whether the
* Buffer is simply a wrapper around a Java array. In these instances, it might be best, from a
* performance point of view, to access the underlying array (using the array()) method.
*
* @param b the provided Buffer
* @return whether the Buffer is backed by a simple array
*/
protected static boolean isBackedBySimpleArray(Buffer b) {
return b.hasArray() && (b.arrayOffset() == 0);
}
protected static short lowbits(int x) {
return (short) (x & 0xFFFF);
}
protected static short lowbits(long x) {
return (short) (x & 0xFFFF);
}
protected static short maxLowBit() {
return (short) 0xFFFF;
}
protected static int maxLowBitAsInteger() {
return 0xFFFF;
}
/**
* clear bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void resetBitmapRange(LongBuffer bitmap, int start, int end) {
if (isBackedBySimpleArray(bitmap)) {
Util.resetBitmapRange(bitmap.array(), start, end);
return;
}
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
if (firstword == endword) {
bitmap.put(firstword, bitmap.get(firstword) & ~((~0L << start) & (~0L >>> -end)));
return;
}
bitmap.put(firstword, bitmap.get(firstword) & (~(~0L << start)));
for (int i = firstword + 1; i < endword; i++) {
bitmap.put(i, 0L);
}
bitmap.put(endword, bitmap.get(endword) & (~(~0L >>> -end)));
}
/**
* set bits at start, start+1,..., end-1
*
* @param bitmap array of words to be modified
* @param start first index to be modified (inclusive)
* @param end last index to be modified (exclusive)
*/
public static void setBitmapRange(LongBuffer bitmap, int start, int end) {
if (isBackedBySimpleArray(bitmap)) {
Util.setBitmapRange(bitmap.array(), start, end);
return;
}
if (start == end) {
return;
}
int firstword = start / 64;
int endword = (end - 1) / 64;
if (firstword == endword) {
bitmap.put(firstword, bitmap.get(firstword) | ((~0L << start) & (~0L >>> -end)));
return;
}
bitmap.put(firstword, bitmap.get(firstword) | (~0L << start));
for (int i = firstword + 1; i < endword; i++) {
bitmap.put(i, ~0L);
}
bitmap.put(endword, bitmap.get(endword) | (~0L >>> -end));
}
protected static int toIntUnsigned(short x) {
return x & 0xFFFF;
}
/**
* Look for value k in buffer in the range [begin,end). If the value is found, return its index.
* If not, return -(i+1) where i is the index where the value would be inserted. The buffer is
* assumed to contain sorted values where shorts are interpreted as unsigned integers.
*
* @param array buffer where we search
* @param begin first index (inclusive)
* @param end last index (exclusive)
* @param k value we search for
* @return count
*/
public static int unsignedBinarySearch(final ShortBuffer array, final int begin, final int end,
final short k) {
return branchyUnsignedBinarySearch(array, begin, end, k);
}
protected static int unsignedDifference(final ShortBuffer set1, final int length1,
final ShortBuffer set2, final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
set1.get(buffer, 0, length1);
return length1;
}
if (0 == length1) {
return 0;
}
short s1 = set1.get(k1);
short s2 = set2.get(k2);
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
break;
}
s1 = set1.get(k1);
} else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
++k1;
++k2;
if (k1 >= length1) {
break;
}
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1.get(k1);
s2 = set2.get(k2);
} else {// if (val1>val2)
++k2;
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2.get(k2);
}
}
return pos;
}
protected static int unsignedExclusiveUnion2by2(final ShortBuffer set1, final int length1,
final ShortBuffer set2, final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
set1.get(buffer, 0, length1);
return length1;
}
if (0 == length1) {
set2.get(buffer, 0, length2);
return length2;
}
short s1 = set1.get(k1);
short s2 = set2.get(k2);
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
set2.position(k2);
set2.get(buffer, pos, length2 - k2);
return pos + length2 - k2;
}
s1 = set1.get(k1);
} else if (toIntUnsigned(s1) == toIntUnsigned(s2)) {
++k1;
++k2;
if (k1 >= length1) {
set2.position(k2);
set2.get(buffer, pos, length2 - k2);
return pos + length2 - k2;
}
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1.get(k1);
s2 = set2.get(k2);
} else {// if (val1>val2)
buffer[pos++] = s2;
++k2;
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2.get(k2);
}
}
// return pos;
}
protected static int unsignedIntersect2by2(final ShortBuffer set1, final int length1,
final ShortBuffer set2, final int length2, final short[] buffer) {
if (length1 * 64 < length2) {
return unsignedOneSidedGallopingIntersect2by2(set1, length1, set2, length2, buffer);
} else if (length2 * 64 < length1) {
return unsignedOneSidedGallopingIntersect2by2(set2, length2, set1, length1, buffer);
} else {
return unsignedLocalIntersect2by2(set1, length1, set2, length2, buffer);
}
}
/**
* Checks if two arrays intersect
*
* @param set1 first array
* @param length1 length of first array
* @param set2 second array
* @param length2 length of second array
* @return true if they intersect
*/
public static boolean unsignedIntersects(ShortBuffer set1, int length1, ShortBuffer set2,
int length2) {
if ((0 == length1) || (0 == length2)) {
return false;
}
int k1 = 0;
int k2 = 0;
// could be more efficient with galloping
short s1 = set1.get(k1);
short s2 = set2.get(k2);
mainwhile:
while (true) {
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2.get(k2);
} while (toIntUnsigned(s2) < toIntUnsigned(s1));
}
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1.get(k1);
} while (toIntUnsigned(s1) < toIntUnsigned(s2));
} else {
return true;
}
}
return false;
}
protected static int unsignedLocalIntersect2by2(final ShortBuffer set1, final int length1,
final ShortBuffer set2, final int length2, final short[] buffer) {
if ((0 == length1) || (0 == length2)) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = set1.get(k1);
short s2 = set2.get(k2);
mainwhile:
while (true) {
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2.get(k2);
} while (toIntUnsigned(s2) < toIntUnsigned(s1));
}
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1.get(k1);
} while (toIntUnsigned(s1) < toIntUnsigned(s2));
} else {
// (set2.get(k2) == set1.get(k1))
buffer[pos++] = s1;
++k1;
if (k1 == length1) {
break;
}
s1 = set1.get(k1);
++k2;
if (k2 == length2) {
break;
}
s2 = set2.get(k2);
}
}
return pos;
}
protected static int unsignedLocalIntersect2by2Cardinality(final ShortBuffer set1,
final int length1, final ShortBuffer set2, final int length2) {
if ((0 == length1) || (0 == length2)) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = set1.get(k1);
short s2 = set2.get(k2);
mainwhile:
while (true) {
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
do {
++k2;
if (k2 == length2) {
break mainwhile;
}
s2 = set2.get(k2);
} while (toIntUnsigned(s2) < toIntUnsigned(s1));
}
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
do {
++k1;
if (k1 == length1) {
break mainwhile;
}
s1 = set1.get(k1);
} while (toIntUnsigned(s1) < toIntUnsigned(s2));
} else {
++pos;
++k1;
if (k1 == length1) {
break;
}
s1 = set1.get(k1);
++k2;
if (k2 == length2) {
break;
}
s2 = set2.get(k2);
}
}
return pos;
}
protected static int unsignedOneSidedGallopingIntersect2by2(final ShortBuffer smallSet,
final int smallLength, final ShortBuffer largeSet, final int largeLength,
final short[] buffer) {
if (0 == smallLength) {
return 0;
}
int k1 = 0;
int k2 = 0;
int pos = 0;
short s1 = largeSet.get(k1);
short s2 = smallSet.get(k2);
while (true) {
if (toIntUnsigned(s1) < toIntUnsigned(s2)) {
k1 = advanceUntil(largeSet, k1, largeLength, s2);
if (k1 == largeLength) {
break;
}
s1 = largeSet.get(k1);
}
if (toIntUnsigned(s2) < toIntUnsigned(s1)) {
++k2;
if (k2 == smallLength) {
break;
}
s2 = smallSet.get(k2);
} else {
// (set2.get(k2) == set1.get(k1))
buffer[pos++] = s2;
++k2;
if (k2 == smallLength) {
break;
}
s2 = smallSet.get(k2);
k1 = advanceUntil(largeSet, k1, largeLength, s2);
if (k1 == largeLength) {
break;
}
s1 = largeSet.get(k1);
}
}
return pos;
}
protected static int unsignedUnion2by2(final ShortBuffer set1, final int length1,
final ShortBuffer set2, final int length2, final short[] buffer) {
int pos = 0;
int k1 = 0, k2 = 0;
if (0 == length2) {
set1.get(buffer, 0, length1);
return length1;
}
if (0 == length1) {
set2.get(buffer, 0, length2);
return length2;
}
short s1 = set1.get(k1);
short s2 = set2.get(k2);
while (true) {
int v1 = toIntUnsigned(s1);
int v2 = toIntUnsigned(s2);
if (v1 < v2) {
buffer[pos++] = s1;
++k1;
if (k1 >= length1) {
set2.position(k2);
set2.get(buffer, pos, length2 - k2);
return pos + length2 - k2;
}
s1 = set1.get(k1);
} else if (v1 == v2) {
buffer[pos++] = s1;
++k1;
++k2;
if (k1 >= length1) {
set2.position(k2);
set2.get(buffer, pos, length2 - k2);
return pos + length2 - k2;
}
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s1 = set1.get(k1);
s2 = set2.get(k2);
} else {// if (set1.get(k1)>set2.get(k2))
buffer[pos++] = s2;
++k2;
if (k2 >= length2) {
set1.position(k1);
set1.get(buffer, pos, length1 - k1);
return pos + length1 - k1;
}
s2 = set2.get(k2);
}
}
// return pos;
}
}

463
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/ImmutableRoaringArray.java
Unescape View File

@ -1,463 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import java.io.DataOutput;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.LongBuffer;
import java.nio.ShortBuffer;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
/**
* This is the underlying data structure for an ImmutableRoaringBitmap. This class is not meant for
* end-users.
*/
public final class ImmutableRoaringArray implements PointableRoaringArray {
protected static final short SERIAL_COOKIE = MutableRoaringArray.SERIAL_COOKIE;
protected static final short SERIAL_COOKIE_NO_RUNCONTAINER =
MutableRoaringArray.SERIAL_COOKIE_NO_RUNCONTAINER;
private final static int startofrunbitmap = 4; // if there is a runcontainer bitmap
ByteBuffer buffer;
int size;
/**
* Create an array based on a previously serialized ByteBuffer. The input ByteBuffer is
* effectively copied (with the slice operation) so you should expect the provided ByteBuffer to
* remain unchanged.
*
* @param bbf The source ByteBuffer
*/
protected ImmutableRoaringArray(ByteBuffer bbf) {
buffer = bbf.slice();
buffer.order(ByteOrder.LITTLE_ENDIAN);
final int cookie = buffer.getInt(0);
if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) {
throw new RuntimeException("I failed to find one of the right cookies. " + cookie);
}
boolean hasRunContainers = (cookie & 0xFFFF) == SERIAL_COOKIE;
this.size = hasRunContainers ? (cookie >>> 16) + 1 : buffer.getInt(4);
int theLimit = size > 0 ? computeSerializedSizeInBytes() : headerSize(hasRunContainers);
buffer.limit(theLimit);
}
@Override
public int advanceUntil(short x, int pos) {
int lower = pos + 1;
// special handling for a possibly common sequential case
if (lower >= size || getKey(lower) >= BufferUtil.toIntUnsigned(x)) {
return lower;
}
int spansize = 1; // could set larger
// bootstrap an upper limit
while (lower + spansize < size && getKey(lower + spansize) < BufferUtil.toIntUnsigned(x)) {
spansize *= 2; // hoping for compiler will reduce to shift
}
int upper = (lower + spansize < size) ? lower + spansize : size - 1;
// maybe we are lucky (could be common case when the seek ahead
// expected to be small and sequential will otherwise make us look bad)
if (getKey(upper) == BufferUtil.toIntUnsigned(x)) {
return upper;
}
if (getKey(upper) < BufferUtil.toIntUnsigned(x)) {// means array has no item key >= x
return size;
}
// we know that the next-smallest span was too small
lower += (spansize / 2);
// else begin binary search
// invariant: array[lower]<x && array[upper]>x
while (lower + 1 != upper) {
int mid = (lower + upper) / 2;
if (getKey(mid) == BufferUtil.toIntUnsigned(x)) {
return mid;
} else if (getKey(mid) < BufferUtil.toIntUnsigned(x)) {
lower = mid;
} else {
upper = mid;
}
}
return upper;
}
private int branchyUnsignedBinarySearch(final short k) {
int low = 0;
int high = this.size - 1;
final int ikey = BufferUtil.toIntUnsigned(k);
while (low <= high) {
final int middleIndex = (low + high) >>> 1;
final int middleValue = getKey(middleIndex);
if (middleValue < ikey) {
low = middleIndex + 1;
} else if (middleValue > ikey) {
high = middleIndex - 1;
} else {
return middleIndex;
}
}
return -(low + 1);
}
@Override
public ImmutableRoaringArray clone() {
ImmutableRoaringArray sa;
try {
sa = (ImmutableRoaringArray) super.clone();
} catch (CloneNotSupportedException e) {
return null;// should never happen
}
return sa;
}
private int computeSerializedSizeInBytes() {
if (this.size == 0) {
return headerSize(hasRunCompression());
}
int CardinalityOfLastContainer = getCardinality(this.size - 1);
int PositionOfLastContainer = getOffsetContainer(this.size - 1);
int SizeOfLastContainer;
boolean hasrun = hasRunCompression();
if (isRunContainer(this.size - 1, hasrun)) {
int nbrruns = BufferUtil.toIntUnsigned(buffer.getShort(PositionOfLastContainer));
SizeOfLastContainer = BufferUtil.getSizeInBytesFromCardinalityEtc(0, nbrruns, true);
} else {
SizeOfLastContainer =
BufferUtil.getSizeInBytesFromCardinalityEtc(CardinalityOfLastContainer, 0, false);
}
return SizeOfLastContainer + PositionOfLastContainer;
}
@Override
public int getCardinality(int k) {
if ((k < 0) || (k >= this.size)) {
throw new IllegalArgumentException(
"out of range container index: " + k + " (report as a bug)");
}
return BufferUtil.toIntUnsigned(buffer.getShort(this.getStartOfKeys() + 4 * k + 2)) + 1;
}
// involves a binary search
@Override
public MappeableContainer getContainer(short x) {
final int i = unsignedBinarySearch(x);
if (i < 0) {
return null;
}
return getContainerAtIndex(i);
}
@Override
public MappeableContainer getContainerAtIndex(int i) {
int cardinality = getCardinality(i);
final boolean isBitmap = cardinality > MappeableArrayContainer.DEFAULT_MAX_SIZE; // if not a
// runcontainer
ByteBuffer tmp = buffer.duplicate();// sad but ByteBuffer is not thread-safe so it is either a
// duplicate or a lock
// note that tmp will indeed be garbage-collected some time after the end of this function
tmp.order(buffer.order());
tmp.position(getOffsetContainer(i));
boolean hasrun = hasRunCompression();
if (isRunContainer(i, hasrun)) {
// first, we have a short giving the number of runs
int nbrruns = BufferUtil.toIntUnsigned(tmp.getShort());
final ShortBuffer shortArray = tmp.asShortBuffer();
shortArray.limit(2 * nbrruns);
return new MappeableRunContainer(shortArray, nbrruns);
}
if (isBitmap) {
final LongBuffer bitmapArray = tmp.asLongBuffer();
bitmapArray.limit(MappeableBitmapContainer.MAX_CAPACITY / 64);
return new MappeableBitmapContainer(bitmapArray, cardinality);
} else {
final ShortBuffer shortArray = tmp.asShortBuffer();
shortArray.limit(cardinality);
return new MappeableArrayContainer(shortArray, cardinality);
}
}
@Override
public MappeableContainerPointer getContainerPointer() {
return getContainerPointer(0);
}
@Override
public MappeableContainerPointer getContainerPointer(final int startIndex) {
final boolean hasrun = isEmpty() ? false : hasRunCompression();
return new MappeableContainerPointer() {
int k = startIndex;
@Override
public void advance() {
++k;
}
@Override
public MappeableContainerPointer clone() {
try {
return (MappeableContainerPointer) super.clone();
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public int compareTo(MappeableContainerPointer o) {
if (key() != o.key()) {
return BufferUtil.toIntUnsigned(key()) - BufferUtil.toIntUnsigned(o.key());
}
return o.getCardinality() - this.getCardinality();
}
@Override
public int getCardinality() {
return ImmutableRoaringArray.this.getCardinality(k);
}
@Override
public MappeableContainer getContainer() {
if (k >= ImmutableRoaringArray.this.size) {
return null;
}
return ImmutableRoaringArray.this.getContainerAtIndex(k);
}
@Override
public int getSizeInBytes() {
// might be a tad expensive
if (ImmutableRoaringArray.this.isRunContainer(k, hasrun)) {
int pos = getOffsetContainer(k);
int nbrruns = BufferUtil.toIntUnsigned(buffer.getShort(pos));
return BufferUtil.getSizeInBytesFromCardinalityEtc(0, nbrruns, true);
} else {
int CardinalityOfLastContainer = getCardinality();
return BufferUtil.getSizeInBytesFromCardinalityEtc(CardinalityOfLastContainer, 0, false);
}
}
@Override
public boolean hasContainer() {
return 0 <= k & k < ImmutableRoaringArray.this.size;
}
@Override
public boolean isBitmapContainer() {
if (ImmutableRoaringArray.this.isRunContainer(k, hasrun)) {
return false;
}
return getCardinality() > MappeableArrayContainer.DEFAULT_MAX_SIZE;
}
@Override
public boolean isRunContainer() {
return ImmutableRoaringArray.this.isRunContainer(k, hasrun);
}
@Override
public short key() {
return ImmutableRoaringArray.this.getKeyAtIndex(k);
}
@Override
public void previous() {
--k;
}
};
}
// involves a binary search
@Override
public int getIndex(short x) {
return unsignedBinarySearch(x);
}
private int getKey(int k) {
return BufferUtil.toIntUnsigned(buffer.getShort(getStartOfKeys() + 4 * k));
}
@Override
public short getKeyAtIndex(int i) {
return buffer.getShort(4 * i + getStartOfKeys());
}
private int getOffsetContainer(int k) {
if ((k < 0) || (k >= this.size)) {
throw new IllegalArgumentException(
"out of range container index: " + k + " (report as a bug)");
}
if (hasRunCompression()) { // account for size of runcontainer bitmap
if (this.size < MutableRoaringArray.NO_OFFSET_THRESHOLD) {
// we do it the hard way
return getOffsetContainerSlow(k);
}
return buffer.getInt(4 + 4 * this.size + ((this.size + 7) / 8) + 4 * k);
} else {
return buffer.getInt(4 + 4 + 4 * this.size + 4 * k);
}
}
private int getOffsetContainerSlow(int k) {
boolean hasrun = hasRunCompression();
int pos = this.headerSize(hasrun);
for (int z = 0; z < k; ++z) {
if (isRunContainer(z, hasrun)) {
int nbrruns = BufferUtil.toIntUnsigned(buffer.getShort(pos));
int SizeOfLastContainer = BufferUtil.getSizeInBytesFromCardinalityEtc(0, nbrruns, true);
pos += SizeOfLastContainer;
} else {
int CardinalityOfLastContainer = this.getCardinality(z);
int SizeOfLastContainer =
BufferUtil.getSizeInBytesFromCardinalityEtc(CardinalityOfLastContainer, 0, false);
pos += SizeOfLastContainer;
}
}
return pos;
}
private int getStartOfKeys() {
if (hasRunCompression()) { // info is in the buffer
return 4 + ((this.size + 7) / 8);
} else {
return 8;
}
}
@Override
public int hashCode() {
MappeableContainerPointer cp = this.getContainerPointer();
int hashvalue = 0;
while (cp.hasContainer()) {
int th = cp.key() * 0xF0F0F0 + cp.getContainer().hashCode();
hashvalue = 31 * hashvalue + th;
cp.advance();
}
return hashvalue;
}
@Override
public boolean hasRunCompression() {
return (buffer.getInt(0) & 0xFFFF) == SERIAL_COOKIE;
}
// hasrun should be equal to hasRunCompression()
protected int headerSize(boolean hasrun) {
if (hasrun) {
if (size < MutableRoaringArray.NO_OFFSET_THRESHOLD) {// for small bitmaps, we omit the offsets
return 4 + (size + 7) / 8 + 4 * size;
}
return 4 + (size + 7) / 8 + 8 * size;// - 4 because we pack the size with the cookie
} else {
return 4 + 4 + 8 * size;
}
}
// starts with binary search and finishes with a sequential search
/*private int hybridUnsignedBinarySearch(final short k) {
int low = 0;
int high = this.size - 1;
final int ikey = BufferUtil.toIntUnsigned(k);
// 32 in the next line matches the size of a cache line
while (low + 16 <= high) {
final int middleIndex = (low + high) >>> 1;
final int middleValue = getKey(middleIndex);
if (middleValue < ikey) {
low = middleIndex + 1;
} else if (middleValue > ikey) {
high = middleIndex - 1;
} else {
return middleIndex;
}
}
// we finish the job with a sequential search
int x = low;
for (; x <= high; ++x) {
final int val = getKey(x);
if (val >= ikey) {
if (val == ikey) {
return x;
}
break;
}
}
return -(x + 1);
}*/
/**
* Returns true if this bitmap is empty.
*
* @return true if empty
*/
public boolean isEmpty() {
return this.size == 0;
}
// hasrun should be initialized with hasRunCompression()
private boolean isRunContainer(int i, boolean hasrun) {
if (hasrun) { // info is in the buffer
int j = buffer.get(startofrunbitmap + i / 8);
int mask = 1 << (i % 8);
return (j & mask) != 0;
} else {
return false;
}
}
/**
* Serialize.
* <p>
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
@Override
public void serialize(DataOutput out) throws IOException {
if (buffer.hasArray()) {
out.write(buffer.array(), buffer.arrayOffset(), buffer.limit());
} else {
ByteBuffer tmp = buffer.duplicate();
tmp.position(0);
WritableByteChannel channel = Channels.newChannel((OutputStream) out);
channel.write(tmp);
}
}
/**
* @return the size that the data structure occupies on disk
*/
@Override
public int serializedSizeInBytes() {
return buffer.limit();
}
@Override
public int size() {
return this.size;
}
private int unsignedBinarySearch(short k) {
return branchyUnsignedBinarySearch(k);
}
}

1320
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/ImmutableRoaringBitmap.java
View File

File diff suppressed because it is too large Load Diff

1688
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MappeableArrayContainer.java
View File

File diff suppressed because it is too large Load Diff

2012
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MappeableBitmapContainer.java
View File

File diff suppressed because it is too large Load Diff

759
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MappeableContainer.java
Unescape View File

@ -1,759 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.Container;
import com.fr.third.bitmap.roaringbitmap.IntConsumer;
import com.fr.third.bitmap.roaringbitmap.PeekableShortIterator;
import com.fr.third.bitmap.roaringbitmap.ShortIterator;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
/**
* Base container class. This class is similar to Container but meant to be used
* with memory mapping.
*/
public abstract class MappeableContainer implements Iterable<Short>, Cloneable, Externalizable {
/**
* Name of the various possible containers
*/
public static String ContainerNames[] = {"mappeablebitmap", "mappeablearray", "mappeablerun"};
/**
* Create a container initialized with a range of consecutive values
*
* @param start first index
* @param last last index (range is exclusive)
* @return a new container initialized with the specified values
*/
public static MappeableContainer rangeOfOnes(final int start, final int last) {
final int sizeAsArrayContainer = MappeableArrayContainer.serializedSizeInBytes(last - start);
final int sizeAsRunContainer = MappeableRunContainer.serializedSizeInBytes(1);
MappeableContainer answer = sizeAsRunContainer < sizeAsArrayContainer
? new MappeableRunContainer() : new MappeableArrayContainer();
answer = answer.iadd(start, last);
return answer;
}
/**
* Return a new container with all shorts in [begin,end) added using an unsigned interpretation.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract MappeableContainer add(int begin, int end);
/**
* Add a short to the container. May generate a new container.
*
* @param x short to be added
* @return the new container
*/
public abstract MappeableContainer add(short x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer and(MappeableArrayContainer x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer and(MappeableBitmapContainer x);
protected MappeableContainer and(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return and((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return and((MappeableRunContainer) x);
}
return and((MappeableBitmapContainer) x);
}
protected abstract int andCardinality(MappeableArrayContainer x);
protected abstract int andCardinality(MappeableBitmapContainer x);
protected abstract int andCardinality(MappeableRunContainer x);
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public int andCardinality(MappeableContainer x) {
if (this.getCardinality() == 0) {
return 0;
} else if (x.getCardinality() == 0) {
return 0;
} else {
if (x instanceof MappeableArrayContainer) {
return andCardinality((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return andCardinality((MappeableBitmapContainer) x);
}
return andCardinality((MappeableRunContainer) x);
}
}
/**
* Computes the bitwise AND of this container with another (intersection). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer and(MappeableRunContainer x);
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer andNot(MappeableArrayContainer x);
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer andNot(MappeableBitmapContainer x);
protected MappeableContainer andNot(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return andNot((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return andNot((MappeableRunContainer) x);
}
return andNot((MappeableBitmapContainer) x);
}
/**
* Computes the bitwise ANDNOT of this container with another (difference). This container as well
* as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer andNot(MappeableRunContainer x);
/**
* Empties the container
*/
public abstract void clear();
@Override
public abstract MappeableContainer clone();
/**
* Checks whether the contain contains the provided value
*
* @param x value to check
* @return whether the value is in the container
*/
public abstract boolean contains(short x);
/**
* Fill the least significant 16 bits of the integer array, starting at index index, with the
* short values from this container. The caller is responsible to allocate enough room. The most
* significant 16 bits of each integer are given by the most significant bits of the provided
* mask.
*
* @param x provided array
* @param i starting index
* @param mask indicates most significant bits
*/
public abstract void fillLeastSignificant16bits(int[] x, int i, int mask);
/**
* Add a short to the container if it is not present, otherwise remove it. May generate a new
* container.
*
* @param x short to be added
* @return the new container
*/
public abstract MappeableContainer flip(short x);
/**
* Size of the underlying array
*
* @return size in bytes
*/
protected abstract int getArraySizeInBytes();
/**
* Computes the distinct number of short values in the container. Can be expected to run in
* constant time.
*
* @return the cardinality
*/
public abstract int getCardinality();
/**
* Get the name of this container.
*
* @return name of the container
*/
public String getContainerName() {
if (this instanceof MappeableBitmapContainer) {
return ContainerNames[0];
} else if (this instanceof MappeableArrayContainer) {
return ContainerNames[1];
} else {
return ContainerNames[2];
}
}
/**
* Iterator to visit the short values in the container in descending order.
*
* @return iterator
*/
public abstract ShortIterator getReverseShortIterator();
/**
* Iterator to visit the short values in the container in ascending order.
*
* @return iterator
*/
public abstract PeekableShortIterator getShortIterator();
/**
* Iterate through the values of this container and pass them
* along to the IntConsumer, using msb as the 16 most significant bits.
*
* @param msb 16 most significant bits
* @param ic consumer
*/
public abstract void forEach(short msb, IntConsumer ic);
/**
* Computes an estimate of the memory usage of this container. The estimate is not meant to be
* exact.
*
* @return estimated memory usage in bytes
*/
public abstract int getSizeInBytes();
/**
* Add all shorts in [begin,end) using an unsigned interpretation. May generate a new container.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract MappeableContainer iadd(int begin, int end);
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iand(MappeableArrayContainer x);
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iand(MappeableBitmapContainer x);
protected MappeableContainer iand(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return iand((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return iand((MappeableRunContainer) x);
}
return iand((MappeableBitmapContainer) x);
}
/**
* Computes the in-place bitwise AND of this container with another (intersection). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iand(MappeableRunContainer x);
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iandNot(MappeableArrayContainer x);
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iandNot(MappeableBitmapContainer x);
protected MappeableContainer iandNot(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return iandNot((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return iandNot((MappeableRunContainer) x);
}
return iandNot((MappeableBitmapContainer) x);
}
/**
* Computes the in-place bitwise ANDNOT of this container with another (difference). The current
* container is generally modified, whereas the provided container (x) is unaffected. May generate
* a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer iandNot(MappeableRunContainer x);
/**
* Computes the in-place bitwise NOT of this container (complement). Only those bits within the
* range are affected. The current container is generally modified. May generate a new container.
*
* @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
* @param rangeEnd ending of range (exclusive)
* @return (partially) completmented container
*/
public abstract MappeableContainer inot(int rangeStart, int rangeEnd);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(MappeableArrayContainer x);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(MappeableBitmapContainer x);
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public boolean intersects(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return intersects((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return intersects((MappeableBitmapContainer) x);
}
return intersects((MappeableRunContainer) x);
}
/**
* Returns true if the current container intersects the other container.
*
* @param x other container
* @return whether they intersect
*/
public abstract boolean intersects(MappeableRunContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ior(MappeableArrayContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ior(MappeableBitmapContainer x);
protected MappeableContainer ior(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return ior((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return ior((MappeableRunContainer) x);
}
return ior((MappeableBitmapContainer) x);
}
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ior(MappeableRunContainer x);
/**
* Remove shorts in [begin,end) using an unsigned interpretation. May generate a new container.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract MappeableContainer iremove(int begin, int end);
protected abstract boolean isArrayBacked();
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ixor(MappeableArrayContainer x);
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ixor(MappeableBitmapContainer x);
protected MappeableContainer ixor(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return ixor((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return ixor((MappeableRunContainer) x);
}
return ixor((MappeableBitmapContainer) x);
}
/**
* Computes the in-place bitwise XOR of this container with another (symmetric difference). The
* current container is generally modified, whereas the provided container (x) is unaffected. May
* generate a new container.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer ixor(MappeableRunContainer x);
/**
* Computes the in-place bitwise OR of this container with another (union). The current container
* is generally modified, whereas the provided container (x) is unaffected. May generate a new
* container. The resulting container may not track its cardinality correctly. The resulting
* container may not track its cardinality correctly. This can be fixed as follows:
* if(c.getCardinality()&lt;0) ((MappeableBitmapContainer)c).computeCardinality();
*
* @param x other container
* @return aggregated container
*/
public MappeableContainer lazyIOR(MappeableContainer x) {
if (this instanceof MappeableArrayContainer) {
if (x instanceof MappeableArrayContainer) {
return ((MappeableArrayContainer) this).lazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) x).lazyor((MappeableArrayContainer) this);
}
return ((MappeableRunContainer) x).lazyor((MappeableArrayContainer) this);
} else if (this instanceof MappeableRunContainer) {
if (x instanceof MappeableArrayContainer) {
return ((MappeableRunContainer) this).ilazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) x).lazyor((MappeableRunContainer) this);
}
return ior((MappeableRunContainer) x);
} else {
if (x instanceof MappeableArrayContainer) {
return ((MappeableBitmapContainer) this).ilazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) this).ilazyor((MappeableBitmapContainer) x);
}
return ((MappeableBitmapContainer) this).ilazyor((MappeableRunContainer) x);
}
}
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected. The resulting container may not track its cardinality
* correctly. This can be fixed as follows: if(c.getCardinality()&lt;0)
* ((MappeableBitmapContainer)c).computeCardinality();
*
* @param x other container
* @return aggregated container
*/
public MappeableContainer lazyOR(MappeableContainer x) {
if (this instanceof MappeableArrayContainer) {
if (x instanceof MappeableArrayContainer) {
return ((MappeableArrayContainer) this).lazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) x).lazyor((MappeableArrayContainer) this);
}
return ((MappeableRunContainer) x).lazyor((MappeableArrayContainer) this);
} else if (this instanceof MappeableRunContainer) {
if (x instanceof MappeableArrayContainer) {
return ((MappeableRunContainer) this).lazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) x).lazyor((MappeableRunContainer) this);
}
return or((MappeableRunContainer) x);
} else {
if (x instanceof MappeableArrayContainer) {
return ((MappeableBitmapContainer) this).lazyor((MappeableArrayContainer) x);
} else if (x instanceof MappeableBitmapContainer) {
return ((MappeableBitmapContainer) this).lazyor((MappeableBitmapContainer) x);
}
return ((MappeableBitmapContainer) this).lazyor((MappeableRunContainer) x);
}
}
/**
* Create a new MappeableContainer containing at most maxcardinality integers.
*
* @param maxcardinality maximal cardinality
* @return a new bitmap with cardinality no more than maxcardinality
*/
public abstract MappeableContainer limit(int maxcardinality);
/**
* Computes the bitwise NOT of this container (complement). Only those bits within the range are
* affected. The current container is left unaffected.
*
* @param rangeStart beginning of range (inclusive); 0 is beginning of this container.
* @param rangeEnd ending of range (exclusive)
* @return (partially) completmented container
*/
public abstract MappeableContainer not(int rangeStart, int rangeEnd);
abstract int numberOfRuns();
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer or(MappeableArrayContainer x);
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer or(MappeableBitmapContainer x);
protected MappeableContainer or(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return or((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return or((MappeableRunContainer) x);
}
return or((MappeableBitmapContainer) x);
}
/**
* Computes the bitwise OR of this container with another (union). This container as well as the
* provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer or(MappeableRunContainer x);
/**
* Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be
* GetCardinality()).
*
* @param lowbits upper limit
* @return the rank
*/
public abstract int rank(short lowbits);
/**
* Return a new container with all shorts in [begin,end) remove using an unsigned interpretation.
*
* @param begin start of range (inclusive)
* @param end end of range (exclusive)
* @return the new container
*/
public abstract MappeableContainer remove(int begin, int end);
/**
* Remove the short from this container. May create a new container.
*
* @param x to be removed
* @return New container
*/
public abstract MappeableContainer remove(short x);
/**
* The output of a lazyOR or lazyIOR might be an invalid container, this should be called on it.
*
* @return a new valid container
*/
public abstract MappeableContainer repairAfterLazy();
/**
* Convert to MappeableRunContainers, when the result is smaller. Overridden by
* MappeableRunContainer to possibly switch from MappeableRunContainer to a smaller alternative.
*
* @return the new container
*/
public abstract MappeableContainer runOptimize();
/**
* Return the jth value
*
* @param j index of the value
* @return the value
*/
public abstract short select(int j);
/**
* Report the number of bytes required to serialize this container.
*
* @return the size in bytes
*/
public abstract int serializedSizeInBytes();
/**
* Convert to a non-mappeable container.
*
* @return the non-mappeable container
*/
public abstract Container toContainer();
/**
* If possible, recover wasted memory.
*/
public abstract void trim();
/**
* Write just the underlying array.
*
* @param out output stream
* @throws IOException in case of failure
*/
protected abstract void writeArray(DataOutput out) throws IOException;
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer xor(MappeableArrayContainer x);
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other container
* @return aggregated container
*/
public abstract MappeableContainer xor(MappeableBitmapContainer x);
protected MappeableContainer xor(MappeableContainer x) {
if (x instanceof MappeableArrayContainer) {
return xor((MappeableArrayContainer) x);
} else if (x instanceof MappeableRunContainer) {
return xor((MappeableRunContainer) x);
}
return xor((MappeableBitmapContainer) x);
}
/**
* Computes the bitwise XOR of this container with another (symmetric difference). This container
* as well as the provided container are left unaffected.
*
* @param x other parameter
* @return aggregated container
*/
public abstract MappeableContainer xor(MappeableRunContainer x);
/**
* Convert the current container to a BitmapContainer, if a conversion is needed.
* If the container is already a bitmap, the container is returned unchanged.
*
* @return a bitmap container
*/
public abstract MappeableBitmapContainer toBitmapContainer();
}

80
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MappeableContainerPointer.java
Unescape View File

@ -1,80 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
/**
* This interface allows you to iterate over the containers in a roaring bitmap.
*/
public interface MappeableContainerPointer
extends Comparable<MappeableContainerPointer>, Cloneable {
/**
* Move to the next container
*/
void advance();
/**
* Create a copy
*
* @return return a clone of this pointer
*/
MappeableContainerPointer clone();
/**
* Returns the cardinality of the current container. Can be faster than loading the container
* first.
*
* @return cardinality of the current container
*/
int getCardinality();
/**
* This method can be used to check whether there is current a valid container as it returns null
* when there is not.
*
* @return null or the current container
*/
MappeableContainer getContainer();
/**
* Get the size in bytes of the container. Used for sorting.
*
* @return the size in bytes
*/
int getSizeInBytes();
/**
* @return whether there is a container at the current position
*/
boolean hasContainer();
/**
* Returns true if it is a bitmap container (MappeableBitmapContainer).
*
* @return boolean indicated if it is a bitmap container
*/
public boolean isBitmapContainer();
/**
* Returns true if it is a run container (MappeableRunContainer).
*
* @return boolean indicated if it is a run container
*/
boolean isRunContainer();
/**
* The key is a 16-bit integer that indicates the position of the container in the roaring bitmap.
* To be interpreted as an unsigned integer.
*
* @return the key
*/
short key();
/**
* Move to the previous container
*/
void previous();
}

2798
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MappeableRunContainer.java
View File

File diff suppressed because it is too large Load Diff

573
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MutableRoaringArray.java
Unescape View File

@ -1,573 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import com.fr.third.bitmap.roaringbitmap.Util;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.nio.LongBuffer;
import java.nio.ShortBuffer;
import java.util.Arrays;
/**
* Specialized array to store the containers used by a RoaringBitmap. This class is similar to
* RoaringArray but meant to be used with memory mapping. This is not meant to be
* used by end users.
* <p>
* Objects of this class reside in RAM.
*/
public final class MutableRoaringArray implements Cloneable, Externalizable, PointableRoaringArray {
protected static final int INITIAL_CAPACITY = 4;
protected static final short SERIAL_COOKIE_NO_RUNCONTAINER = 12346;
protected static final short SERIAL_COOKIE = 12347;
protected static final int NO_OFFSET_THRESHOLD = 4;
private static final long serialVersionUID = 5L; // TODO: OFK was 4L, not sure
protected boolean mayHaveRunContainers = false; // does not necessarily have them, after
// optimization
short[] keys = null;
MappeableContainer[] values = null;
int size = 0;
protected MutableRoaringArray() {
this.keys = new short[INITIAL_CAPACITY];
this.values = new MappeableContainer[INITIAL_CAPACITY];
}
@Override
public int advanceUntil(short x, int pos) {
int lower = pos + 1;
// special handling for a possibly common sequential case
if (lower >= size || BufferUtil.toIntUnsigned(keys[lower]) >= BufferUtil.toIntUnsigned(x)) {
return lower;
}
int spansize = 1; // could set larger
// bootstrap an upper limit
while (lower + spansize < size
&& BufferUtil.toIntUnsigned(keys[lower + spansize]) < BufferUtil.toIntUnsigned(x)) {
spansize *= 2; // hoping for compiler will reduce to shift
}
int upper = (lower + spansize < size) ? lower + spansize : size - 1;
// maybe we are lucky (could be common case when the seek ahead
// expected to be small and sequential will otherwise make us look bad)
if (keys[upper] == x) {
return upper;
}
if (BufferUtil.toIntUnsigned(keys[upper]) < BufferUtil.toIntUnsigned(x)) {// means array has no
// item key >= x
return size;
}
// we know that the next-smallest span was too small
lower += (spansize / 2);
// else begin binary search
// invariant: array[lower]<x && array[upper]>x
while (lower + 1 != upper) {
int mid = (lower + upper) / 2;
if (keys[mid] == x) {
return mid;
} else if (BufferUtil.toIntUnsigned(keys[mid]) < BufferUtil.toIntUnsigned(x)) {
lower = mid;
} else {
upper = mid;
}
}
return upper;
}
protected void append(short key, MappeableContainer value) {
extendArray(1);
this.keys[this.size] = key;
this.values[this.size] = value;
this.size++;
}
/**
* Append copies of the values AFTER a specified key (may or may not be present) to end.
*
* @param highLowContainer the other array
* @param beforeStart given key is the largest key that we won't copy
*/
protected void appendCopiesAfter(PointableRoaringArray highLowContainer, short beforeStart) {
int startLocation = highLowContainer.getIndex(beforeStart);
if (startLocation >= 0) {
startLocation++;
} else {
startLocation = -startLocation - 1;
}
extendArray(highLowContainer.size() - startLocation);
for (int i = startLocation; i < highLowContainer.size(); ++i) {
this.keys[this.size] = highLowContainer.getKeyAtIndex(i);
this.values[this.size] = highLowContainer.getContainerAtIndex(i).clone();
this.size++;
}
}
/**
* Append copies of the values from another array, from the start
*
* @param highLowContainer the other array
* @param stoppingKey any equal or larger key in other array will terminate copying
*/
protected void appendCopiesUntil(PointableRoaringArray highLowContainer, short stoppingKey) {
final int stopKey = BufferUtil.toIntUnsigned(stoppingKey);
MappeableContainerPointer cp = highLowContainer.getContainerPointer();
while (cp.hasContainer()) {
if (BufferUtil.toIntUnsigned(cp.key()) >= stopKey) {
break;
}
extendArray(1);
this.keys[this.size] = cp.key();
this.values[this.size] = cp.getContainer().clone();
this.size++;
cp.advance();
}
}
/**
* Append copies of the values from another array
*
* @param highLowContainer other array
* @param startingIndex starting index in the other array
* @param end last index array in the other array
*/
protected void appendCopy(PointableRoaringArray highLowContainer, int startingIndex, int end) {
extendArray(end - startingIndex);
for (int i = startingIndex; i < end; ++i) {
this.keys[this.size] = highLowContainer.getKeyAtIndex(i);
this.values[this.size] = highLowContainer.getContainerAtIndex(i).clone();
this.size++;
}
}
protected void appendCopy(short key, MappeableContainer value) {
extendArray(1);
this.keys[this.size] = key;
this.values[this.size] = value.clone();
this.size++;
}
private int binarySearch(int begin, int end, short key) {
return Util.unsignedBinarySearch(keys, begin, end, key);
}
protected void clear() {
this.keys = null;
this.values = null;
this.size = 0;
}
@Override
public MutableRoaringArray clone() {
MutableRoaringArray sa;
try {
sa = (MutableRoaringArray) super.clone();
// OFK: do we need runcontainer bitmap? Guess not, this is just a directory
// and each container knows what kind it is.
sa.keys = Arrays.copyOf(this.keys, this.size);
sa.values = Arrays.copyOf(this.values, this.size);
for (int k = 0; k < this.size; ++k) {
sa.values[k] = sa.values[k].clone();
}
sa.size = this.size;
return sa;
} catch (CloneNotSupportedException e) {
return null;
}
}
protected void copyRange(int begin, int end, int newBegin) {
// assuming begin <= end and newBegin < begin
final int range = end - begin;
System.arraycopy(this.keys, begin, this.keys, newBegin, range);
System.arraycopy(this.values, begin, this.values, newBegin, range);
}
/**
* Deserialize.
*
* @param in the DataInput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void deserialize(DataInput in) throws IOException {
this.clear();
// little endian
final int cookie = Integer.reverseBytes(in.readInt());
if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) {
throw new IOException("I failed to find the one of the right cookies.");
}
this.size = ((cookie & 0xFFFF) == SERIAL_COOKIE) ? (cookie >>> 16) + 1
: Integer.reverseBytes(in.readInt());
if ((this.keys == null) || (this.keys.length < this.size)) {
this.keys = new short[this.size];
this.values = new MappeableContainer[this.size];
}
byte[] bitmapOfRunContainers = null;
boolean hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE;
if (hasrun) {
bitmapOfRunContainers = new byte[(size + 7) / 8];
in.readFully(bitmapOfRunContainers);
}
final short keys[] = new short[this.size];
final int cardinalities[] = new int[this.size];
final boolean isBitmap[] = new boolean[this.size];
for (int k = 0; k < this.size; ++k) {
keys[k] = Short.reverseBytes(in.readShort());
cardinalities[k] = 1 + (0xFFFF & Short.reverseBytes(in.readShort()));
isBitmap[k] = cardinalities[k] > MappeableArrayContainer.DEFAULT_MAX_SIZE;
if (bitmapOfRunContainers != null && (bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) {
isBitmap[k] = false;
}
}
if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
// skipping the offsets
in.skipBytes(this.size * 4);
}
// Reading the containers
for (int k = 0; k < this.size; ++k) {
MappeableContainer val;
if (isBitmap[k]) {
final LongBuffer bitmapArray =
LongBuffer.allocate(MappeableBitmapContainer.MAX_CAPACITY / 64);
// little endian
for (int l = 0; l < bitmapArray.limit(); ++l) {
bitmapArray.put(l, Long.reverseBytes(in.readLong()));
}
val = new MappeableBitmapContainer(bitmapArray, cardinalities[k]);
} else if (bitmapOfRunContainers != null
&& ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0)) {
int nbrruns = BufferUtil.toIntUnsigned(Short.reverseBytes(in.readShort()));
final ShortBuffer shortArray = ShortBuffer.allocate(2 * nbrruns);
for (int l = 0; l < shortArray.limit(); ++l) {
shortArray.put(l, Short.reverseBytes(in.readShort()));
}
val = new MappeableRunContainer(shortArray, nbrruns);
} else {
final ShortBuffer shortArray = ShortBuffer.allocate(cardinalities[k]);
for (int l = 0; l < shortArray.limit(); ++l) {
shortArray.put(l, Short.reverseBytes(in.readShort()));
}
val = new MappeableArrayContainer(shortArray, cardinalities[k]);
}
this.keys[k] = keys[k];
this.values[k] = val;
}
}
// make sure there is capacity for at least k more elements
protected void extendArray(int k) {
// size + 1 could overflow
if (this.size + k >= this.keys.length) {
int newCapacity;
if (this.keys.length < 1024) {
newCapacity = 2 * (this.size + k);
} else {
newCapacity = 5 * (this.size + k) / 4;
}
this.keys = Arrays.copyOf(this.keys, newCapacity);
this.values = Arrays.copyOf(this.values, newCapacity);
}
}
@Override
public int getCardinality(int i) {
return getContainerAtIndex(i).getCardinality();
}
// involves a binary search
@Override
public MappeableContainer getContainer(short x) {
final int i = this.binarySearch(0, size, x);
if (i < 0) {
return null;
}
return this.values[i];
}
@Override
public MappeableContainer getContainerAtIndex(int i) {
return this.values[i];
}
@Override
public MappeableContainerPointer getContainerPointer() {
return getContainerPointer(0);
}
@Override
public MappeableContainerPointer getContainerPointer(final int startIndex) {
return new MappeableContainerPointer() {
int k = startIndex;
@Override
public void advance() {
++k;
}
@Override
public MappeableContainerPointer clone() {
try {
return (MappeableContainerPointer) super.clone();
} catch (CloneNotSupportedException e) {
return null;// will not happen
}
}
@Override
public int compareTo(MappeableContainerPointer o) {
if (key() != o.key()) {
return BufferUtil.toIntUnsigned(key()) - BufferUtil.toIntUnsigned(o.key());
}
return o.getCardinality() - this.getCardinality();
}
@Override
public int getCardinality() {
return getContainer().getCardinality();
}
@Override
public MappeableContainer getContainer() {
if (k >= MutableRoaringArray.this.size) {
return null;
}
return MutableRoaringArray.this.values[k];
}
@Override
public int getSizeInBytes() {
return getContainer().getArraySizeInBytes();
}
@Override
public boolean hasContainer() {
return 0 <= k & k < MutableRoaringArray.this.size;
}
@Override
public boolean isBitmapContainer() {
return getContainer() instanceof MappeableBitmapContainer;
}
@Override
public boolean isRunContainer() {
return getContainer() instanceof MappeableRunContainer;
}
@Override
public short key() {
return MutableRoaringArray.this.keys[k];
}
@Override
public void previous() {
--k;
}
};
}
// involves a binary search
@Override
public int getIndex(short x) {
// before the binary search, we optimize for frequent cases
if ((size == 0) || (keys[size - 1] == x)) {
return size - 1;
}
// no luck we have to go through the list
return this.binarySearch(0, size, x);
}
@Override
public short getKeyAtIndex(int i) {
return this.keys[i];
}
@Override
public int hashCode() {
int hashvalue = 0;
for (int k = 0; k < this.size; ++k) {
hashvalue = 31 * hashvalue + keys[k] * 0xF0F0F0 + values[k].hashCode();
}
return hashvalue;
}
@Override
public boolean hasRunCompression() {
for (int k = 0; k < size; ++k) {
MappeableContainer ck = values[k];
if (ck instanceof MappeableRunContainer) {
return true;
}
}
return false;
}
protected int headerSize() {
if (hasRunCompression()) {
if (size < NO_OFFSET_THRESHOLD) {// for small bitmaps, we omit the offsets
return 4 + (size + 7) / 8 + 4 * size;
}
return 4 + (size + 7) / 8 + 8 * size;// - 4 because we pack the size with the cookie
} else {
return 4 + 4 + 8 * size;
}
}
// insert a new key, it is assumed that it does not exist
protected void insertNewKeyValueAt(int i, short key, MappeableContainer value) {
extendArray(1);
System.arraycopy(keys, i, keys, i + 1, size - i);
System.arraycopy(values, i, values, i + 1, size - i);
keys[i] = key;
values[i] = value;
size++;
}
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
deserialize(in);
}
protected void removeAtIndex(int i) {
System.arraycopy(keys, i + 1, keys, i, size - i - 1);
keys[size - 1] = 0;
System.arraycopy(values, i + 1, values, i, size - i - 1);
values[size - 1] = null;
size--;
}
protected void removeIndexRange(int begin, int end) {
if (end <= begin) {
return;
}
final int range = end - begin;
System.arraycopy(keys, end, keys, begin, size - end);
System.arraycopy(values, end, values, begin, size - end);
for (int i = 1; i <= range; ++i) {
keys[size - i] = 0;
values[size - i] = null;
}
size -= range;
}
protected void replaceKeyAndContainerAtIndex(int i, short key, MappeableContainer c) {
this.keys[i] = key;
this.values[i] = c;
}
protected void resize(int newLength) {
Arrays.fill(this.keys, newLength, this.size, (short) 0);
Arrays.fill(this.values, newLength, this.size, null);
this.size = newLength;
}
/**
* Serialize.
* <p>
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
@Override
public void serialize(DataOutput out) throws IOException {
int startOffset = 0;
boolean hasrun = hasRunCompression();
if (hasrun) {
out.writeInt(Integer.reverseBytes(SERIAL_COOKIE | ((this.size - 1) << 16)));
byte[] bitmapOfRunContainers = new byte[(size + 7) / 8];
for (int i = 0; i < size; ++i) {
if (this.values[i] instanceof MappeableRunContainer) {
bitmapOfRunContainers[i / 8] |= (1 << (i % 8));
}
}
out.write(bitmapOfRunContainers);
if (this.size < NO_OFFSET_THRESHOLD) {
startOffset = 4 + 4 * this.size + bitmapOfRunContainers.length;
} else {
startOffset = 4 + 8 * this.size + bitmapOfRunContainers.length;
}
} else { // backwards compatibilility
out.writeInt(Integer.reverseBytes(SERIAL_COOKIE_NO_RUNCONTAINER));
out.writeInt(Integer.reverseBytes(size));
startOffset = 4 + 4 + this.size * 4 + this.size * 4;
}
for (int k = 0; k < size; ++k) {
out.writeShort(Short.reverseBytes(this.keys[k]));
out.writeShort(Short.reverseBytes((short) (this.values[k].getCardinality() - 1)));
}
if ((!hasrun) || (this.size >= NO_OFFSET_THRESHOLD)) {
for (int k = 0; k < this.size; k++) {
out.writeInt(Integer.reverseBytes(startOffset));
startOffset = startOffset + values[k].getArraySizeInBytes();
}
}
for (int k = 0; k < size; ++k) {
values[k].writeArray(out);
}
}
/**
* Report the number of bytes required for serialization.
*
* @return the size in bytes
*/
@Override
public int serializedSizeInBytes() {
int count = headerSize();
// for each container, we store cardinality (16 bits), key (16 bits) and location offset (32
// bits).
for (int k = 0; k < this.size; ++k) {
count += values[k].getArraySizeInBytes();
}
return count;
}
protected void setContainerAtIndex(int i, MappeableContainer c) {
this.values[i] = c;
}
@Override
public int size() {
return this.size;
}
@Override
public void writeExternal(ObjectOutput out) throws IOException {
serialize(out);
}
}

1420
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/MutableRoaringBitmap.java
View File

File diff suppressed because it is too large Load Diff

103
fine-roaringbitmap/src/main/java/com/fr/third/bitmap/roaringbitmap/buffer/PointableRoaringArray.java
Unescape View File

@ -1,103 +0,0 @@
/*
* (c) the authors Licensed under the Apache License, Version 2.0.
*/
package com.fr.third.bitmap.roaringbitmap.buffer;
import java.io.DataOutput;
import java.io.IOException;
/**
* Generic interface for the array underlying roaring bitmap classes.
*/
public interface PointableRoaringArray extends Cloneable {
/**
* Find the smallest integer index larger than pos such that getKeyAtIndex(index)&gt;=x. If none
* can be found, return size.
*
* @param x minimal value
* @param pos index to exceed
* @return the smallest index greater than pos such that getKeyAtIndex(index) is at least as large
* as min, or size if it is not possible.
*/
int advanceUntil(short x, int pos);
/**
* Create an independent copy of the underlying array
*
* @return a copy
*/
PointableRoaringArray clone();
/**
* Returns the cardinality of the container at the given index. This method is expected to be
* fast.
*
* @param i index
* @return the cardinality
*/
public int getCardinality(int i);
/**
* @param x 16-bit key
* @return matching container
*/
MappeableContainer getContainer(short x);
/**
* @param i index
* @return matching container
*/
MappeableContainer getContainerAtIndex(int i);
/**
* @return a ContainerPointer to iterator over the array
*/
MappeableContainerPointer getContainerPointer();
/**
* @param startIndex starting index
* @return a ContainerPointer to iterator over the array initially positioned at startIndex
*/
MappeableContainerPointer getContainerPointer(int startIndex);
/**
* @param x 16-bit key
* @return corresponding index
*/
int getIndex(short x);
/**
* @param i the index
* @return 16-bit key at the index
*/
short getKeyAtIndex(int i);
/**
* Check whether this bitmap has had its runs compressed.
*
* @return whether this bitmap has run compression
*/
public boolean hasRunCompression();
/**
* Serialize.
* <p>
* The current bitmap is not modified.
*
* @param out the DataOutput stream
* @throws IOException Signals that an I/O exception has occurred.
*/
public void serialize(DataOutput out) throws IOException;
/**
* @return the size that the data structure occupies on disk
*/
public int serializedSizeInBytes();
/**
* @return number of keys
*/
int size();
}
Write Preview
Loading…
Cancel
Save