Browse Source

REPORT-78067 三方组件批量升级

feature/x
Yuan.Wang 2 years ago
parent
commit
ca10d09020
  1. 2
      fine-lz4/README.md
  2. 25
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4BlockInputStream.java
  3. 6
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4BlockOutputStream.java
  4. 37
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4ByteBufferUtils.java
  5. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Compressor.java
  6. 201
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4CompressorWithLength.java
  7. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Constants.java
  8. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Decompressor.java
  9. 277
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4DecompressorWithLength.java
  10. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Exception.java
  11. 43
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Factory.java
  12. 9
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4FastDecompressor.java
  13. 112
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4FrameInputStream.java
  14. 40
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4FrameOutputStream.java
  15. 6
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4HCJNICompressor.java
  16. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4JNI.java
  17. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4JNICompressor.java
  18. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4JNIFastDecompressor.java
  19. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4JNISafeDecompressor.java
  20. 3
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4SafeDecompressor.java
  21. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4SafeUtils.java
  22. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4UnknownSizeDecompressor.java
  23. 72
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4UnsafeUtils.java
  24. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Utils.java
  25. 8
      fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/package.html
  26. 16
      fine-lz4/src/main/java/com/fr/third/net/jpountz/util/ByteBufferUtils.java
  27. 106
      fine-lz4/src/main/java/com/fr/third/net/jpountz/util/Native.java
  28. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/util/SafeUtils.java
  29. 4
      fine-lz4/src/main/java/com/fr/third/net/jpountz/util/UnsafeUtils.java
  30. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/util/Utils.java
  31. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/util/package.html
  32. 11
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/AbstractStreamingXXHash32Java.java
  33. 11
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/AbstractStreamingXXHash64Java.java
  34. 15
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/StreamingXXHash32.java
  35. 36
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/StreamingXXHash32JNI.java
  36. 15
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/StreamingXXHash64.java
  37. 36
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/StreamingXXHash64JNI.java
  38. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHash32.java
  39. 9
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHash32JNI.java
  40. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHash64.java
  41. 9
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHash64JNI.java
  42. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHashConstants.java
  43. 10
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHashFactory.java
  44. 2
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHashJNI.java
  45. 8
      fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/package.html
  46. BIN
      fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/darwin/aarch64/liblz4-java.dylib
  47. BIN
      fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/darwin/x86_64/liblz4-java.dylib
  48. BIN
      fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/linux/aarch64/liblz4-java.so
  49. BIN
      fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/linux/amd64/liblz4-java.so
  50. BIN
      fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/linux/i386/liblz4-java.so
  51. BIN
      fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/linux/ppc64le/liblz4-java.so
  52. BIN
      fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/linux/s390x/liblz4-java.so
  53. BIN
      fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/win32/amd64/liblz4-java.so

2
fine-lz4/README.md

@ -0,0 +1,2 @@
lz4的源码地址:https://github.com/lz4/lz4 <br>
版本:1.8.0<br>

25
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4BlockInputStream.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -28,8 +30,8 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.zip.Checksum; import java.util.zip.Checksum;
import com.fr.third.net.jpountz.util.SafeUtils;
import com.fr.third.net.jpountz.xxhash.StreamingXXHash32; import com.fr.third.net.jpountz.xxhash.StreamingXXHash32;
import com.fr.third.net.jpountz.util.SafeUtils;
import com.fr.third.net.jpountz.xxhash.XXHash32; import com.fr.third.net.jpountz.xxhash.XXHash32;
import com.fr.third.net.jpountz.xxhash.XXHashFactory; import com.fr.third.net.jpountz.xxhash.XXHashFactory;
@ -39,7 +41,7 @@ import com.fr.third.net.jpountz.xxhash.XXHashFactory;
* support {@link #mark(int)}/{@link #reset()}. * support {@link #mark(int)}/{@link #reset()}.
* @see LZ4BlockOutputStream * @see LZ4BlockOutputStream
*/ */
public final class LZ4BlockInputStream extends FilterInputStream { public class LZ4BlockInputStream extends FilterInputStream {
private final LZ4FastDecompressor decompressor; private final LZ4FastDecompressor decompressor;
private final Checksum checksum; private final Checksum checksum;
@ -187,13 +189,11 @@ public final class LZ4BlockInputStream extends FilterInputStream {
} }
private void refill() throws IOException { private void refill() throws IOException {
try { if (!tryReadFully(compressedBuffer, HEADER_LENGTH)) {
readFully(compressedBuffer, HEADER_LENGTH);
} catch (EOFException e) {
if (!stopOnEmptyBlock) { if (!stopOnEmptyBlock) {
finished = true; finished = true;
} else { } else {
throw e; throw new EOFException("Stream ended prematurely");
} }
return; return;
} }
@ -263,16 +263,25 @@ public final class LZ4BlockInputStream extends FilterInputStream {
o = 0; o = 0;
} }
private void readFully(byte[] b, int len) throws IOException { // Like readFully(), except it signals incomplete reads by returning
// false instead of throwing EOFException.
private boolean tryReadFully(byte[] b, int len) throws IOException {
int read = 0; int read = 0;
while (read < len) { while (read < len) {
final int r = in.read(b, read, len - read); final int r = in.read(b, read, len - read);
if (r < 0) { if (r < 0) {
throw new EOFException("Stream ended prematurely"); return false;
} }
read += r; read += r;
} }
assert len == read; assert len == read;
return true;
}
private void readFully(byte[] b, int len) throws IOException {
if (!tryReadFully(b, len)) {
throw new EOFException("Stream ended prematurely");
}
} }
@Override @Override

6
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4BlockOutputStream.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -19,9 +21,9 @@ import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.util.zip.Checksum; import java.util.zip.Checksum;
import com.fr.third.net.jpountz.util.SafeUtils;
import com.fr.third.net.jpountz.xxhash.StreamingXXHash32; import com.fr.third.net.jpountz.xxhash.StreamingXXHash32;
import com.fr.third.net.jpountz.xxhash.XXHashFactory; import com.fr.third.net.jpountz.xxhash.XXHashFactory;
import com.fr.third.net.jpountz.util.SafeUtils;
/** /**
* Streaming LZ4 (not compatible with the LZ4 Frame format). * Streaming LZ4 (not compatible with the LZ4 Frame format).
@ -32,7 +34,7 @@ import com.fr.third.net.jpountz.xxhash.XXHashFactory;
* @see LZ4BlockInputStream * @see LZ4BlockInputStream
* @see LZ4FrameOutputStream * @see LZ4FrameOutputStream
*/ */
public final class LZ4BlockOutputStream extends FilterOutputStream { public class LZ4BlockOutputStream extends FilterOutputStream {
static final byte[] MAGIC = new byte[] { 'L', 'Z', '4', 'B', 'l', 'o', 'c', 'k' }; static final byte[] MAGIC = new byte[] { 'L', 'Z', '4', 'B', 'l', 'o', 'c', 'k' };
static final int MAGIC_LENGTH = MAGIC.length; static final int MAGIC_LENGTH = MAGIC.length;

37
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4ByteBufferUtils.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -14,11 +16,6 @@ package com.fr.third.net.jpountz.lz4;
* limitations under the License. * limitations under the License.
*/ */
import static com.fr.third.net.jpountz.lz4.LZ4Constants.COPY_LENGTH;
import static com.fr.third.net.jpountz.lz4.LZ4Constants.LAST_LITERALS;
import static com.fr.third.net.jpountz.lz4.LZ4Constants.ML_BITS;
import static com.fr.third.net.jpountz.lz4.LZ4Constants.ML_MASK;
import static com.fr.third.net.jpountz.lz4.LZ4Constants.RUN_MASK;
import static com.fr.third.net.jpountz.util.ByteBufferUtils.readByte; import static com.fr.third.net.jpountz.util.ByteBufferUtils.readByte;
import static com.fr.third.net.jpountz.util.ByteBufferUtils.readInt; import static com.fr.third.net.jpountz.util.ByteBufferUtils.readInt;
import static com.fr.third.net.jpountz.util.ByteBufferUtils.readLong; import static com.fr.third.net.jpountz.util.ByteBufferUtils.readLong;
@ -84,7 +81,7 @@ enum LZ4ByteBufferUtils {
writeInt(dest, dOff, readInt(dest, matchOff)); writeInt(dest, dOff, readInt(dest, matchOff));
dOff += 4; dOff += 4;
matchOff -= dec; matchOff -= dec;
} else if (dOff - matchOff < COPY_LENGTH) { } else if (dOff - matchOff < LZ4Constants.COPY_LENGTH) {
writeLong(dest, dOff, readLong(dest, matchOff)); writeLong(dest, dOff, readLong(dest, matchOff));
dOff += dOff - matchOff; dOff += dOff - matchOff;
} }
@ -147,16 +144,16 @@ enum LZ4ByteBufferUtils {
final int runLen = matchOff - anchor; final int runLen = matchOff - anchor;
final int tokenOff = dOff++; final int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) { if (dOff + runLen + (2 + 1 + LZ4Constants.LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small"); throw new LZ4Exception("maxDestLen is too small");
} }
int token; int token;
if (runLen >= RUN_MASK) { if (runLen >= LZ4Constants.RUN_MASK) {
token = (byte) (RUN_MASK << ML_BITS); token = (byte) (LZ4Constants.RUN_MASK << LZ4Constants.ML_BITS);
dOff = writeLen(runLen - RUN_MASK, dest, dOff); dOff = writeLen(runLen - LZ4Constants.RUN_MASK, dest, dOff);
} else { } else {
token = runLen << ML_BITS; token = runLen << LZ4Constants.ML_BITS;
} }
// copy literals // copy literals
@ -170,12 +167,12 @@ enum LZ4ByteBufferUtils {
// encode match len // encode match len
matchLen -= 4; matchLen -= 4;
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) { if (dOff + (1 + LZ4Constants.LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small"); throw new LZ4Exception("maxDestLen is too small");
} }
if (matchLen >= ML_MASK) { if (matchLen >= LZ4Constants.ML_MASK) {
token |= ML_MASK; token |= LZ4Constants.ML_MASK;
dOff = writeLen(matchLen - RUN_MASK, dest, dOff); dOff = writeLen(matchLen - LZ4Constants.RUN_MASK, dest, dOff);
} else { } else {
token |= matchLen; token |= matchLen;
} }
@ -188,15 +185,15 @@ enum LZ4ByteBufferUtils {
static int lastLiterals(ByteBuffer src, int sOff, int srcLen, ByteBuffer dest, int dOff, int destEnd) { static int lastLiterals(ByteBuffer src, int sOff, int srcLen, ByteBuffer dest, int dOff, int destEnd) {
final int runLen = srcLen; final int runLen = srcLen;
if (dOff + runLen + 1 + (runLen + 255 - RUN_MASK) / 255 > destEnd) { if (dOff + runLen + 1 + (runLen + 255 - LZ4Constants.RUN_MASK) / 255 > destEnd) {
throw new LZ4Exception(); throw new LZ4Exception();
} }
if (runLen >= RUN_MASK) { if (runLen >= LZ4Constants.RUN_MASK) {
dest.put(dOff++, (byte) (RUN_MASK << ML_BITS)); dest.put(dOff++, (byte) (LZ4Constants.RUN_MASK << LZ4Constants.ML_BITS));
dOff = writeLen(runLen - RUN_MASK, dest, dOff); dOff = writeLen(runLen - LZ4Constants.RUN_MASK, dest, dOff);
} else { } else {
dest.put(dOff++, (byte) (runLen << ML_BITS)); dest.put(dOff++, (byte) (runLen << LZ4Constants.ML_BITS));
} }
// copy literals // copy literals
safeArraycopy(src, sOff, dest, dOff, runLen); safeArraycopy(src, sOff, dest, dOff, runLen);

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Compressor.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

201
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4CompressorWithLength.java

@ -0,0 +1,201 @@
package com.fr.third.net.jpountz.lz4;
/*
* Copyright 2020 Rei Odaira and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteBuffer;
import java.util.Arrays;
/**
* Covenience class to include the length of the original decompressed data
* in the output compressed data, so that the user does not need to save
* the length at anywhere else. The compressed data must be decompressed by
* {@link LZ4DecompressorWithLength} and is NOT compatible with any other
* decompressors in lz4-java or any other lz4 tools. This class deliberately
* does not extend {@link LZ4Compressor} because they are not interchangable.
*/
public class LZ4CompressorWithLength {
private final LZ4Compressor compressor;
/**
* Creates a new compressor that includes the length of the original
* decompressed data in the output compressed data.
*
* @param compressor compressor to use
*/
public LZ4CompressorWithLength(LZ4Compressor compressor) {
this.compressor = compressor;
}
private void putOriginalLength(byte[] dest, int destOff, int originalLength) {
dest[destOff] = (byte)originalLength;
dest[destOff + 1] = (byte)(originalLength >> 8);
dest[destOff + 2] = (byte)(originalLength >> 16);
dest[destOff + 3] = (byte)(originalLength >> 24);
}
private void putOriginalLength(ByteBuffer dest, int destOff, int originalLength) {
dest.put(destOff, (byte)originalLength);
dest.put(destOff + 1, (byte)(originalLength >> 8));
dest.put(destOff + 2, (byte)(originalLength >> 16));
dest.put(destOff + 3, (byte)(originalLength >> 24));
}
/**
* Returns the maximum compressed length for an input of size <code>length</code>.
*
* @param length the input size in bytes
* @return the maximum compressed length in bytes
*/
public int maxCompressedLength(int length) {
return compressor.maxCompressedLength(length) + 4;
}
/**
* Convenience method, equivalent to calling
* {@link #compress(byte[], int, int) compress(src, 0, src.length)}.
*
* @param src the source data
* @return the compressed data
*/
public byte[] compress(byte[] src) {
return compress(src, 0, src.length);
}
/**
* Convenience method which returns <code>src[srcOff:srcOff+srcLen]</code>
* compressed.
* <p><b><span style="color:red">Warning</span></b>: this method has an
* important overhead due to the fact that it needs to allocate a buffer to
* compress into, and then needs to resize this buffer to the actual
* compressed length.</p>
* <p>Here is how this method is implemented:</p>
* <pre>
* final int maxCompressedLength = maxCompressedLength(srcLen);
* final byte[] compressed = new byte[maxCompressedLength];
* final int compressedLength = compress(src, srcOff, srcLen, compressed, 0);
* return Arrays.copyOf(compressed, compressedLength);
* </pre>
*
* @param src the source data
* @param srcOff the start offset in src
* @param srcLen the number of bytes to compress
* @return the compressed data
*/
public byte[] compress(byte[] src, int srcOff, int srcLen) {
final int maxCompressedLength = maxCompressedLength(srcLen);
final byte[] compressed = new byte[maxCompressedLength];
final int compressedLength = compress(src, srcOff, srcLen, compressed, 0);
return Arrays.copyOf(compressed, compressedLength);
}
/**
* Convenience method, equivalent to calling
* {@link #compress(byte[], int, int, byte[], int) compress(src, 0, src.length, dest, 0)}.
*
* @param src the source data
* @param dest the destination buffer
* @throws LZ4Exception if dest is too small
* @return the compressed size
*/
public int compress(byte[] src, byte[] dest) {
return compress(src, 0, src.length, dest, 0);
}
/**
* Convenience method, equivalent to calling
* {@link #compress(byte[], int, int, byte[], int, int) compress(src, srcOff, srcLen, dest, destOff, dest.length - destOff)}.
*
* @param src the source data
* @param srcOff the start offset in src
* @param srcLen the number of bytes to compress
* @param dest the destination buffer
* @param destOff the start offset in dest
* @throws LZ4Exception if dest is too small
* @return the compressed size
*/
public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff) {
return compress(src, srcOff, srcLen, dest, destOff, dest.length - destOff);
}
/**
* Compresses <code>src[srcOff:srcOff+srcLen]</code> into
* <code>dest[destOff:destOff+maxDestLen]</code> and returns the compressed
* length.
*
* This method will throw a {@link LZ4Exception} if this compressor is unable
* to compress the input into less than <code>maxDestLen</code> bytes. To
* prevent this exception to be thrown, you should make sure that
* <code>maxDestLen &gt;= maxCompressedLength(srcLen)</code>.
*
* @param src the source data
* @param srcOff the start offset in src
* @param srcLen the number of bytes to compress
* @param dest the destination buffer
* @param destOff the start offset in dest
* @param maxDestLen the maximum number of bytes to write in dest
* @throws LZ4Exception if maxDestLen is too small
* @return the compressed size
*/
public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
final int compressedLength = compressor.compress(src, srcOff, srcLen, dest, destOff + 4, maxDestLen - 4);
putOriginalLength(dest, destOff, srcLen);
return compressedLength + 4;
}
/**
* Compresses <code>src</code> into <code>dest</code>. Calling this method
* will update the positions of both {@link ByteBuffer}s.
*
* @param src the source data
* @param dest the destination buffer
* @throws LZ4Exception if dest is too small
*/
public void compress(ByteBuffer src, ByteBuffer dest) {
final int compressedLength = compress(src, src.position(), src.remaining(), dest, dest.position(), dest.remaining());
src.position(src.limit());
dest.position(dest.position() + compressedLength);
}
/**
* Compresses <code>src[srcOff:srcOff+srcLen]</code> into
* <code>dest[destOff:destOff+maxDestLen]</code> and returns the compressed
* length.
*
* This method will throw a {@link LZ4Exception} if this compressor is unable
* to compress the input into less than <code>maxDestLen</code> bytes. To
* prevent this exception to be thrown, you should make sure that
* <code>maxDestLen &gt;= maxCompressedLength(srcLen)</code>.
*
* {@link ByteBuffer} positions remain unchanged.
*
* @param src the source data
* @param srcOff the start offset in src
* @param srcLen the number of bytes to compress
* @param dest the destination buffer
* @param destOff the start offset in dest
* @param maxDestLen the maximum number of bytes to write in dest
* @throws LZ4Exception if maxDestLen is too small
* @return the compressed size
*/
public int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
final int compressedLength = compressor.compress(src, srcOff, srcLen, dest, destOff + 4, maxDestLen - 4);
putOriginalLength(dest, destOff, srcLen);
return compressedLength + 4;
}
}

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Constants.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Decompressor.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

277
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4DecompressorWithLength.java

@ -0,0 +1,277 @@
package com.fr.third.net.jpountz.lz4;
/*
* Copyright 2020 Rei Odaira and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteBuffer;
/**
* Convenience class to decompress data compressed by {@link LZ4CompressorWithLength}.
* This decompressor is NOT compatible with any other compressors in lz4-java
* or any other lz4 tools.
* The user does not need to specify the length of the compressed data or
* original data because the length of the original decompressed data is
* included in the compressed data.
*/
public class LZ4DecompressorWithLength {
private final LZ4FastDecompressor fastDecompressor;
private final LZ4SafeDecompressor safeDecompressor;
/**
* Returns the decompressed length of compressed data in <code>src</code>.
*
* @param src the compressed data
* @return the decompressed length
*/
public static int getDecompressedLength(byte[] src) {
return getDecompressedLength(src, 0);
}
/**
* Returns the decompressed length of compressed data in <code>src[srcOff:]</code>.
*
* @param src the compressed data
* @param srcOff the start offset in src
* @return the decompressed length
*/
public static int getDecompressedLength(byte[] src, int srcOff) {
return (src[srcOff] & 0xFF) | (src[srcOff + 1] & 0xFF) << 8 | (src[srcOff + 2] & 0xFF) << 16 | src[srcOff + 3] << 24;
}
/**
* Returns the decompressed length of compressed data in <code>src</code>.
*
* @param src the compressed data
* @return the decompressed length
*/
public static int getDecompressedLength(ByteBuffer src) {
return getDecompressedLength(src, src.position());
}
/**
* Returns the decompressed length of compressed data in <code>src[srcOff:]</code>.
*
* @param src the compressed data
* @param srcOff the start offset in src
* @return the decompressed length
*/
public static int getDecompressedLength(ByteBuffer src, int srcOff) {
return (src.get(srcOff) & 0xFF) | (src.get(srcOff + 1) & 0xFF) << 8 | (src.get(srcOff + 2) & 0xFF) << 16 | src.get(srcOff + 3) << 24;
}
/**
* Creates a new decompressor to decompress data compressed by {@link LZ4CompressorWithLength}.
* Note that it is deprecated to use a JNI-binding instance of {@link LZ4FastDecompressor}.
* Please see {@link LZ4Factory#nativeInstance()} for details.
*
* @param fastDecompressor fast decompressor to use
*/
public LZ4DecompressorWithLength(LZ4FastDecompressor fastDecompressor) {
this.fastDecompressor = fastDecompressor;
this.safeDecompressor = null;
}
/**
* Creates a new decompressor to decompress data compressed by {@link LZ4CompressorWithLength}.
*
* @param safeDecompressor safe decompressor to use
*/
public LZ4DecompressorWithLength(LZ4SafeDecompressor safeDecompressor) {
this.fastDecompressor = null;
this.safeDecompressor = safeDecompressor;
}
/**
* Convenience method, equivalent to calling
* {@link #decompress(byte[], int, byte[], int) decompress(src, 0, dest, 0)}.
*
* @param src the compressed data
* @param dest the destination buffer to store the decompressed data
* @return the number of bytes read to restore the original input
*/
public int decompress(byte[] src, byte[] dest) {
return decompress(src, 0, dest, 0);
}
/**
* When {@link LZ4FastDecompressor} was specified to the constructor,
* decompresses <code>src[srcOff:]</code> into <code>dest[destOff:]</code>
* and returns the number of bytes read from <code>src</code>, and
* when {@link LZ4SafeDecompressor} was specified to the constructor,
* decompresses <code>src[srcOff:src.length]</code> into <code>dest[destOff:]</code>
* and returns the number of decompressed bytes written into <code>dest</code>.
*
* @param src the compressed data
* @param srcOff the start offset in src
* @param dest the destination buffer to store the decompressed data
* @param destOff the start offset in dest
* @return the number of bytes read to restore the original input (when {@link LZ4FastDecompressor} is used), or the number of decompressed bytes (when {@link LZ4SafeDecompressor} is used)
*/
public int decompress(byte[] src, int srcOff, byte[] dest, int destOff) {
if (safeDecompressor != null) {
return decompress(src, srcOff, src.length - srcOff, dest, destOff);
}
final int destLen = getDecompressedLength(src, srcOff);
return fastDecompressor.decompress(src, srcOff + 4, dest, destOff, destLen) + 4;
}
/**
* When {@link LZ4FastDecompressor} was specified to the constructor,
* decompresses <code>src[srcOff:]</code> into <code>dest[destOff:]</code>
* and returns the number of bytes read from <code>src</code>, and
* when {@link LZ4SafeDecompressor} was specified to the constructor,
* decompresses <code>src[srcOff:srcOff+srcLen]</code> into <code>dest[destOff:]</code>
* and returns the number of decompressed bytes written into <code>dest</code>.
*
* @param src the compressed data
* @param srcOff the start offset in src
* @param srcLen the exact size of the compressed data (ignored when {@link LZ4FastDecompressor} is used)
* @param dest the destination buffer to store the decompressed data
* @param destOff the start offset in dest
* @return the number of bytes read to restore the original input (when {@link LZ4FastDecompressor} is used), or the number of decompressed bytes (when {@link LZ4SafeDecompressor} is used)
*/
public int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff) {
if (safeDecompressor == null) {
return decompress(src, srcOff, dest, destOff);
}
final int destLen = getDecompressedLength(src, srcOff);
return safeDecompressor.decompress(src, srcOff + 4, srcLen - 4, dest, destOff, destLen);
}
/**
* Convenience method, equivalent to calling
* {@link #decompress(byte[], int) decompress(src, 0)}.
*
* @param src the compressed data
* @return the decompressed data
*/
public byte[] decompress(byte[] src) {
return decompress(src, 0);
}
/**
* Convenience method which returns <code>src[srcOff:]</code>
* decompressed when {@link LZ4FastDecompressor} was specified to the constructor,
* or <code>src[srcOff:src.length]</code> decompressed when
* {@link LZ4SafeDecompressor} was specified to the constructor.
* <p><b><span style="color:red">Warning</span></b>: this method has an
* important overhead due to the fact that it needs to allocate a buffer to
* decompress into.
*
* @param src the compressed data
* @param srcOff the start offset in src
* @return the decompressed data
*/
public byte[] decompress(byte[] src, int srcOff) {
if (safeDecompressor != null) {
return decompress(src, srcOff, src.length - srcOff);
}
final int destLen = getDecompressedLength(src, srcOff);
return fastDecompressor.decompress(src, srcOff + 4, destLen);
}
/**
* Convenience method which returns <code>src[srcOff:]</code>
* decompressed when {@link LZ4FastDecompressor} was specified to the constructor,
* or <code>src[srcOff:srcOff+srcLen]</code> decompressed when
* {@link LZ4SafeDecompressor} was specified to the constructor.
* <p><b><span style="color:red">Warning</span></b>: this method has an
* important overhead due to the fact that it needs to allocate a buffer to
* decompress into.
*
* @param src the compressed data
* @param srcOff the start offset in src
* @param srcLen the exact size of the compressed data (ignored when {@link LZ4FastDecompressor} is used)
* @return the decompressed data
*/
public byte[] decompress(byte[] src, int srcOff, int srcLen) {
if (safeDecompressor == null) {
return decompress(src, srcOff);
}
final int destLen = getDecompressedLength(src, srcOff);
return safeDecompressor.decompress(src, srcOff + 4, srcLen - 4, destLen);
}
/**
* Decompresses <code>src</code> into <code>dest</code>.
* When {@link LZ4SafeDecompressor} was specified to the constructor,
* <code>src</code>'s {@link ByteBuffer#remaining()} must be exactly the size
* of the compressed data. This method moves the positions of the buffers.
*
* @param src the compressed data
* @param dest the destination buffer to store the decompressed data
*/
public void decompress(ByteBuffer src, ByteBuffer dest) {
final int destLen = getDecompressedLength(src, src.position());
if (safeDecompressor == null) {
final int read = fastDecompressor.decompress(src, src.position() + 4, dest, dest.position(), destLen);
src.position(src.position() + 4 + read);
dest.position(dest.position() + destLen);
} else {
final int written = safeDecompressor.decompress(src, src.position() + 4, src.remaining() - 4, dest, dest.position(), destLen);
src.position(src.limit());
dest.position(dest.position() + written);
}
}
/** When {@link LZ4FastDecompressor} was specified to the constructor,
* decompresses <code>src[srcOff:]</code> into <code>dest[destOff:]</code>
* and returns the number of bytes read from <code>src</code>, and
* when {@link LZ4SafeDecompressor} was specified to the constructor,
* decompresses <code>src[srcOff:src.remaining()]</code> into <code>dest[destOff:]</code>
* and returns the number of decompressed bytes written into <code>dest</code>.
* The positions and limits of the {@link ByteBuffer}s remain unchanged.
*
* @param src the compressed data
* @param srcOff the start offset in src
* @param dest the destination buffer to store the decompressed data
* @param destOff the start offset in dest
* @return the number of bytes read to restore the original input (when {@link LZ4FastDecompressor} is used), or the number of decompressed bytes (when {@link LZ4SafeDecompressor} is used)
*/
public int decompress(ByteBuffer src, int srcOff, ByteBuffer dest, int destOff) {
if (safeDecompressor != null) {
return decompress(src, srcOff, src.remaining() - srcOff, dest, destOff);
}
final int destLen = getDecompressedLength(src, srcOff);
return fastDecompressor.decompress(src, srcOff + 4, dest, destOff, destLen) + 4;
}
/**
* When {@link LZ4FastDecompressor} was specified to the constructor,
* decompresses <code>src[srcOff:]</code> into <code>dest[destOff:]</code>
* and returns the number of bytes read from <code>src</code>, and
* when {@link LZ4SafeDecompressor} was specified to the constructor,
* decompresses <code>src[srcOff:srcOff+srcLen]</code> into <code>dest[destOff:]</code>
* and returns the number of decompressed bytes written into <code>dest</code>.
* The positions and limits of the {@link ByteBuffer}s remain unchanged.
*
* @param src the compressed data
* @param srcOff the start offset in src
* @param srcLen the exact size of the compressed data (ignored when {@link LZ4FastDecompressor} is used)
* @param dest the destination buffer to store the decompressed data
* @param destOff the start offset in dest
* @return the number of bytes read to restore the original input (when {@link LZ4FastDecompressor} is used), or the number of decompressed bytes (when {@link LZ4SafeDecompressor} is used)
*/
public int decompress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff) {
if (safeDecompressor == null) {
return decompress(src, srcOff, dest, destOff);
}
final int destLen = getDecompressedLength(src, srcOff);
return safeDecompressor.decompress(src, srcOff + 4, srcLen - 4, dest, destOff, destLen);
}
}

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Exception.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

43
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Factory.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -21,13 +23,15 @@ import java.util.Arrays;
import com.fr.third.net.jpountz.util.Native; import com.fr.third.net.jpountz.util.Native;
import com.fr.third.net.jpountz.util.Utils; import com.fr.third.net.jpountz.util.Utils;
import static com.fr.third.net.jpountz.lz4.LZ4Constants.DEFAULT_COMPRESSION_LEVEL;
import static com.fr.third.net.jpountz.lz4.LZ4Constants.MAX_COMPRESSION_LEVEL;
/** /**
* Entry point for the LZ4 API. * Entry point for the LZ4 API.
* <p> * <p>
* This class has 3 instances<ul> * This class has 3 instances<ul>
* <li>a {@link #nativeInstance() native} instance which is a JNI binding to * <li>a {@link #nativeInstance() native} instance which is a JNI binding to
* <a href="http://code.google.com/p/lz4/">the original LZ4 C implementation</a>. * <a href="https://github.com/lz4/lz4">the original LZ4 C implementation</a>.
* <li>a {@link #safeInstance() safe Java} instance which is a pure Java port * <li>a {@link #safeInstance() safe Java} instance which is a pure Java port
* of the original C library,</li> * of the original C library,</li>
* <li>an {@link #unsafeInstance() unsafe Java} instance which is a Java port * <li>an {@link #unsafeInstance() unsafe Java} instance which is a Java port
@ -72,6 +76,13 @@ public final class LZ4Factory {
* either not use this instance in webapps or to put this library in the lib * either not use this instance in webapps or to put this library in the lib
* directory of your servlet container so that it is loaded by the system * directory of your servlet container so that it is loaded by the system
* class loader. * class loader.
* <li>From lz4-java version 1.6.0, a {@link LZ4FastDecompressor} instance
* returned by {@link #fastDecompressor()} of this instance is SLOWER
* than a {@link LZ4SafeDecompressor} instance returned by
* {@link #safeDecompressor()}, due to a change in the original LZ4
* C implementation. The corresponding C API function is deprecated.
* Hence use of {@link #fastDecompressor()} is deprecated
* for this instance.
* </ol> * </ol>
* *
* @return a {@link LZ4Factory} instance that returns compressors and * @return a {@link LZ4Factory} instance that returns compressors and
@ -175,18 +186,18 @@ public final class LZ4Factory {
private final LZ4Compressor highCompressor; private final LZ4Compressor highCompressor;
private final LZ4FastDecompressor fastDecompressor; private final LZ4FastDecompressor fastDecompressor;
private final LZ4SafeDecompressor safeDecompressor; private final LZ4SafeDecompressor safeDecompressor;
private final LZ4Compressor[] highCompressors = new LZ4Compressor[LZ4Constants.MAX_COMPRESSION_LEVEL+1]; private final LZ4Compressor[] highCompressors = new LZ4Compressor[MAX_COMPRESSION_LEVEL+1];
private LZ4Factory(String impl) throws ClassNotFoundException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, NoSuchMethodException, InstantiationException, InvocationTargetException { private LZ4Factory(String impl) throws ClassNotFoundException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, NoSuchMethodException, InstantiationException, InvocationTargetException {
this.impl = impl; this.impl = impl;
fastCompressor = classInstance("com.fr.third.net.jpountz.lz4.LZ4" + impl + "Compressor"); fastCompressor = classInstance("net.jpountz.lz4.LZ4" + impl + "Compressor");
highCompressor = classInstance("com.fr.third.net.jpountz.lz4.LZ4HC" + impl + "Compressor"); highCompressor = classInstance("net.jpountz.lz4.LZ4HC" + impl + "Compressor");
fastDecompressor = classInstance("com.fr.third.net.jpountz.lz4.LZ4" + impl + "FastDecompressor"); fastDecompressor = classInstance("net.jpountz.lz4.LZ4" + impl + "FastDecompressor");
safeDecompressor = classInstance("com.fr.third.net.jpountz.lz4.LZ4" + impl + "SafeDecompressor"); safeDecompressor = classInstance("net.jpountz.lz4.LZ4" + impl + "SafeDecompressor");
Constructor<? extends LZ4Compressor> highConstructor = highCompressor.getClass().getDeclaredConstructor(int.class); Constructor<? extends LZ4Compressor> highConstructor = highCompressor.getClass().getDeclaredConstructor(int.class);
highCompressors[LZ4Constants.DEFAULT_COMPRESSION_LEVEL] = highCompressor; highCompressors[DEFAULT_COMPRESSION_LEVEL] = highCompressor;
for(int level = 1; level <= LZ4Constants.MAX_COMPRESSION_LEVEL; level++) { for(int level = 1; level <= MAX_COMPRESSION_LEVEL; level++) {
if(level == LZ4Constants.DEFAULT_COMPRESSION_LEVEL) continue; if(level == DEFAULT_COMPRESSION_LEVEL) continue;
highCompressors[level] = highConstructor.newInstance(level); highCompressors[level] = highConstructor.newInstance(level);
} }
@ -239,24 +250,32 @@ public final class LZ4Factory {
* <li>A compression level higher than 17 would be treated as 17.</li> * <li>A compression level higher than 17 would be treated as 17.</li>
* <li>A compression level lower than 1 would be treated as 9.</li> * <li>A compression level lower than 1 would be treated as 9.</li>
* </ol> * </ol>
* Note that compression levels from different implementations
* (native, unsafe Java, and safe Java) cannot be compared with one another.
* Specifically, the native implementation of a high compression level
* is not necessarily faster than the safe/unsafe Java implementation
* of the same compression level.
* *
* @param compressionLevel the compression level between [1, 17]; the higher the level, the higher the compression ratio * @param compressionLevel the compression level between [1, 17]; the higher the level, the higher the compression ratio
* @return a {@link LZ4Compressor} which requires more memory than * @return a {@link LZ4Compressor} which requires more memory than
* {@link #fastCompressor()} and is slower but compresses more efficiently. * {@link #fastCompressor()} and is slower but compresses more efficiently.
*/ */
public LZ4Compressor highCompressor(int compressionLevel) { public LZ4Compressor highCompressor(int compressionLevel) {
if(compressionLevel > LZ4Constants.MAX_COMPRESSION_LEVEL) { if(compressionLevel > MAX_COMPRESSION_LEVEL) {
compressionLevel = LZ4Constants.MAX_COMPRESSION_LEVEL; compressionLevel = MAX_COMPRESSION_LEVEL;
} else if (compressionLevel < 1) { } else if (compressionLevel < 1) {
compressionLevel = LZ4Constants.DEFAULT_COMPRESSION_LEVEL; compressionLevel = DEFAULT_COMPRESSION_LEVEL;
} }
return highCompressors[compressionLevel]; return highCompressors[compressionLevel];
} }
/** /**
* Returns a {@link LZ4FastDecompressor} instance. * Returns a {@link LZ4FastDecompressor} instance.
* Use of this method is deprecated for the {@link #nativeInstance() native instance}.
* *
* @return a {@link LZ4FastDecompressor} instance * @return a {@link LZ4FastDecompressor} instance
*
* @see #nativeInstance()
*/ */
public LZ4FastDecompressor fastDecompressor() { public LZ4FastDecompressor fastDecompressor() {
return fastDecompressor; return fastDecompressor;

9
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4FastDecompressor.java

@ -3,6 +3,8 @@ package com.fr.third.net.jpountz.lz4;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -21,7 +23,14 @@ import java.nio.ByteBuffer;
* Use {@link LZ4SafeDecompressor} if you only know the size of the * Use {@link LZ4SafeDecompressor} if you only know the size of the
* compressed stream. * compressed stream.
* <p> * <p>
* From lz4-java 1.6.0, it is deprecated to use a JNI-binding instance
* of this class; i.e., an instasnce returned by
* {@link LZ4Factory#fastDecompressor()} of {@link LZ4Factory#nativeInstance()}.
* Please see {@link LZ4Factory#nativeInstance()} for details.
* <p>
* Instances of this class are thread-safe. * Instances of this class are thread-safe.
*
* @see LZ4Factory#nativeInstance()
*/ */
public abstract class LZ4FastDecompressor implements LZ4Decompressor { public abstract class LZ4FastDecompressor implements LZ4Decompressor {

112
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4FrameInputStream.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 The Apache Software Foundation and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -48,17 +50,20 @@ public class LZ4FrameInputStream extends FilterInputStream {
private final XXHash32 checksum; private final XXHash32 checksum;
private final byte[] headerArray = new byte[LZ4FrameOutputStream.LZ4_MAX_HEADER_LENGTH]; private final byte[] headerArray = new byte[LZ4FrameOutputStream.LZ4_MAX_HEADER_LENGTH];
private final ByteBuffer headerBuffer = ByteBuffer.wrap(headerArray).order(ByteOrder.LITTLE_ENDIAN); private final ByteBuffer headerBuffer = ByteBuffer.wrap(headerArray).order(ByteOrder.LITTLE_ENDIAN);
private final boolean readSingleFrame;
private byte[] compressedBuffer; private byte[] compressedBuffer;
private ByteBuffer buffer = null; private ByteBuffer buffer = null;
private byte[] rawBuffer = null; private byte[] rawBuffer = null;
private int maxBlockSize = -1; private int maxBlockSize = -1;
private long expectedContentSize = -1L; private long expectedContentSize = -1L;
private long totalContentSize = 0L; private long totalContentSize = 0L;
private boolean firstFrameHeaderRead = false;
private LZ4FrameOutputStream.FrameInfo frameInfo = null; private LZ4FrameOutputStream.FrameInfo frameInfo = null;
/** /**
* Creates a new {@link InputStream} that will decompress data using fastest instances of {@link LZ4SafeDecompressor} and {@link XXHash32}. * Creates a new {@link InputStream} that will decompress data using fastest instances of {@link LZ4SafeDecompressor} and {@link XXHash32}.
* This instance will decompress all concatenated frames in their sequential order.
* *
* @param in the stream to decompress * @param in the stream to decompress
* @throws IOException if an I/O error occurs * @throws IOException if an I/O error occurs
@ -71,19 +76,50 @@ public class LZ4FrameInputStream extends FilterInputStream {
this(in, LZ4Factory.fastestInstance().safeDecompressor(), XXHashFactory.fastestInstance().hash32()); this(in, LZ4Factory.fastestInstance().safeDecompressor(), XXHashFactory.fastestInstance().hash32());
} }
/**
* Creates a new {@link InputStream} that will decompress data using fastest instances of {@link LZ4SafeDecompressor} and {@link XXHash32}.
*
* @param in the stream to decompress
* @param readSingleFrame whether read is stopped after the first non-skippable frame
* @throws IOException if an I/O error occurs
*
* @see #LZ4FrameInputStream(InputStream, LZ4SafeDecompressor, XXHash32)
* @see LZ4Factory#fastestInstance()
* @see XXHashFactory#fastestInstance()
*/
public LZ4FrameInputStream(InputStream in, boolean readSingleFrame) throws IOException {
this(in, LZ4Factory.fastestInstance().safeDecompressor(), XXHashFactory.fastestInstance().hash32(), readSingleFrame);
}
/** /**
* Creates a new {@link InputStream} that will decompress data using the LZ4 algorithm. * Creates a new {@link InputStream} that will decompress data using the LZ4 algorithm.
* This instance will decompress all concatenated frames in their sequential order.
* *
* @param in the stream to decompress * @param in the stream to decompress
* @param decompressor the decompressor to use * @param decompressor the decompressor to use
* @param checksum the hash function to use * @param checksum the hash function to use
* @throws IOException if an I/O error occurs * @throws IOException if an I/O error occurs
*
* @see #LZ4FrameInputStream(InputStream, LZ4SafeDecompressor, XXHash32, boolean)
*/ */
public LZ4FrameInputStream(InputStream in, LZ4SafeDecompressor decompressor, XXHash32 checksum) throws IOException { public LZ4FrameInputStream(InputStream in, LZ4SafeDecompressor decompressor, XXHash32 checksum) throws IOException {
this(in, decompressor, checksum, false);
}
/**
* Creates a new {@link InputStream} that will decompress data using the LZ4 algorithm.
*
* @param in the stream to decompress
* @param decompressor the decompressor to use
* @param checksum the hash function to use
* @param readSingleFrame whether read is stopped after the first non-skippable frame
* @throws IOException if an I/O error occurs
*/
public LZ4FrameInputStream(InputStream in, LZ4SafeDecompressor decompressor, XXHash32 checksum, boolean readSingleFrame) throws IOException {
super(in); super(in);
this.decompressor = decompressor; this.decompressor = decompressor;
this.checksum = checksum; this.checksum = checksum;
nextFrameInfo(); this.readSingleFrame = readSingleFrame;
} }
@ -99,7 +135,15 @@ public class LZ4FrameInputStream extends FilterInputStream {
do { do {
final int mySize = in.read(readNumberBuff.array(), size, LZ4FrameOutputStream.INTEGER_BYTES - size); final int mySize = in.read(readNumberBuff.array(), size, LZ4FrameOutputStream.INTEGER_BYTES - size);
if (mySize < 0) { if (mySize < 0) {
return false; if (firstFrameHeaderRead) {
if (size > 0) {
throw new IOException(PREMATURE_EOS);
} else {
return false;
}
} else {
throw new IOException(PREMATURE_EOS);
}
} }
size += mySize; size += mySize;
} while (size < LZ4FrameOutputStream.INTEGER_BYTES); } while (size < LZ4FrameOutputStream.INTEGER_BYTES);
@ -125,6 +169,7 @@ public class LZ4FrameInputStream extends FilterInputStream {
} }
skipSize -= mySize; skipSize -= mySize;
} }
firstFrameHeaderRead = true;
} }
/** /**
@ -175,6 +220,7 @@ public class LZ4FrameInputStream extends FilterInputStream {
rawBuffer = new byte[maxBlockSize]; rawBuffer = new byte[maxBlockSize];
buffer = ByteBuffer.wrap(rawBuffer); buffer = ByteBuffer.wrap(rawBuffer);
buffer.limit(0); buffer.limit(0);
firstFrameHeaderRead = true;
} }
private final ByteBuffer readNumberBuff = ByteBuffer.allocate(LZ4FrameOutputStream.LONG_BYTES).order(ByteOrder.LITTLE_ENDIAN); private final ByteBuffer readNumberBuff = ByteBuffer.allocate(LZ4FrameOutputStream.LONG_BYTES).order(ByteOrder.LITTLE_ENDIAN);
@ -276,8 +322,11 @@ public class LZ4FrameInputStream extends FilterInputStream {
@Override @Override
public int read() throws IOException { public int read() throws IOException {
while (buffer.remaining() == 0) { while (!firstFrameHeaderRead || buffer.remaining() == 0) {
if (frameInfo.isFinished()) { if (!firstFrameHeaderRead || frameInfo.isFinished()) {
if (firstFrameHeaderRead && readSingleFrame) {
return -1;
}
if (!nextFrameInfo()) { if (!nextFrameInfo()) {
return -1; return -1;
} }
@ -292,8 +341,11 @@ public class LZ4FrameInputStream extends FilterInputStream {
if ((off < 0) || (len < 0) || (off + len > b.length)) { if ((off < 0) || (len < 0) || (off + len > b.length)) {
throw new IndexOutOfBoundsException(); throw new IndexOutOfBoundsException();
} }
while (buffer.remaining() == 0) { while (!firstFrameHeaderRead || buffer.remaining() == 0) {
if (frameInfo.isFinished()) { if (!firstFrameHeaderRead || frameInfo.isFinished()) {
if (firstFrameHeaderRead && readSingleFrame) {
return -1;
}
if (!nextFrameInfo()) { if (!nextFrameInfo()) {
return -1; return -1;
} }
@ -310,8 +362,11 @@ public class LZ4FrameInputStream extends FilterInputStream {
if (n <= 0) { if (n <= 0) {
return 0; return 0;
} }
while (buffer.remaining() == 0) { while (!firstFrameHeaderRead || buffer.remaining() == 0) {
if (frameInfo.isFinished()) { if (!firstFrameHeaderRead || frameInfo.isFinished()) {
if (firstFrameHeaderRead && readSingleFrame) {
return 0;
}
if (!nextFrameInfo()) { if (!nextFrameInfo()) {
return 0; return 0;
} }
@ -348,4 +403,45 @@ public class LZ4FrameInputStream extends FilterInputStream {
return false; return false;
} }
/**
* Returns the optional Content Size value set in Frame Descriptor.
* If the Content Size is not set (FLG.Bits.CONTENT_SIZE not enabled) in compressed stream, -1L is returned.
* A call to this method is valid only when this instance is supposed to read only one frame (readSingleFrame == true).
*
* @return the expected content size, or -1L if no expected content size is set in the frame.
* @throws IOException On input stream read exception
*
* @see #LZ4FrameInputStream(InputStream, LZ4SafeDecompressor, XXHash32, boolean)
*/
public long getExpectedContentSize() throws IOException {
if (!readSingleFrame) {
throw new UnsupportedOperationException("Operation not permitted when multiple frames can be read");
}
if (!firstFrameHeaderRead) {
if (!nextFrameInfo()) {
return -1L;
}
}
return expectedContentSize;
}
/**
* Checks if the optionnal Content Size is set (FLG.Bits.CONTENT_SIZE is enabled).
*
* @return true if this instance is supposed to read only one frame and if the optional content size is set in the frame.
* @throws IOException On input stream read exception
*/
public boolean isExpectedContentSizeDefined() throws IOException {
if (readSingleFrame) {
if (!firstFrameHeaderRead) {
if (!nextFrameInfo()) {
return false;
}
}
return expectedContentSize >= 0;
} else {
return false;
}
}
} }

40
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4FrameOutputStream.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 The Apache Software Foundation and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -112,13 +114,30 @@ public class LZ4FrameOutputStream extends FilterOutputStream {
* @throws IOException if an I/O error occurs * @throws IOException if an I/O error occurs
*/ */
public LZ4FrameOutputStream(OutputStream out, BLOCKSIZE blockSize, long knownSize, FLG.Bits... bits) throws IOException { public LZ4FrameOutputStream(OutputStream out, BLOCKSIZE blockSize, long knownSize, FLG.Bits... bits) throws IOException {
this(out, blockSize, knownSize, LZ4Factory.fastestInstance().fastCompressor(),
XXHashFactory.fastestInstance().hash32(), bits);
}
/**
* Creates a new {@link OutputStream} that will compress data using the specified instances of {@link LZ4Compressor} and {@link XXHash32}.
*
* @param out the output stream to compress
* @param blockSize the BLOCKSIZE to use
* @param knownSize the size of the uncompressed data. A value less than zero means unknown.
* @param compressor the {@link LZ4Compressor} instance to use to compress data
* @param checksum the {@link XXHash32} instance to use to check data for integrity
* @param bits a set of features to use
* @throws IOException if an I/O error occurs
*/
public LZ4FrameOutputStream(OutputStream out, BLOCKSIZE blockSize, long knownSize,
LZ4Compressor compressor, XXHash32 checksum, FLG.Bits... bits) throws IOException {
super(out); super(out);
compressor = LZ4Factory.fastestInstance().fastCompressor(); this.compressor = compressor;
checksum = XXHashFactory.fastestInstance().hash32(); this.checksum = checksum;
frameInfo = new FrameInfo(new FLG(FLG.DEFAULT_VERSION, bits), new BD(blockSize)); frameInfo = new FrameInfo(new FLG(FLG.DEFAULT_VERSION, bits), new BD(blockSize));
maxBlockSize = frameInfo.getBD().getBlockMaximumSize(); maxBlockSize = frameInfo.getBD().getBlockMaximumSize();
buffer = ByteBuffer.allocate(maxBlockSize).order(ByteOrder.LITTLE_ENDIAN); buffer = ByteBuffer.allocate(maxBlockSize).order(ByteOrder.LITTLE_ENDIAN);
compressedBuffer = new byte[compressor.maxCompressedLength(maxBlockSize)]; compressedBuffer = new byte[this.compressor.maxCompressedLength(maxBlockSize)];
if (frameInfo.getFLG().isEnabled(FLG.Bits.CONTENT_SIZE) && knownSize < 0) { if (frameInfo.getFLG().isEnabled(FLG.Bits.CONTENT_SIZE) && knownSize < 0) {
throw new IllegalArgumentException("Known size must be greater than zero in order to use the known size feature"); throw new IllegalArgumentException("Known size must be greater than zero in order to use the known size feature");
} }
@ -184,6 +203,10 @@ public class LZ4FrameOutputStream extends FilterOutputStream {
// Make sure there's no stale data // Make sure there's no stale data
Arrays.fill(compressedBuffer, (byte) 0); Arrays.fill(compressedBuffer, (byte) 0);
if (frameInfo.isEnabled(FLG.Bits.CONTENT_CHECKSUM)) {
frameInfo.updateStreamHash(buffer.array(), 0, buffer.position());
}
int compressedLength = compressor.compress(buffer.array(), 0, buffer.position(), compressedBuffer, 0); int compressedLength = compressor.compress(buffer.array(), 0, buffer.position(), compressedBuffer, 0);
final byte[] bufferToWrite; final byte[] bufferToWrite;
final int compressMethod; final int compressMethod;
@ -234,10 +257,6 @@ public class LZ4FrameOutputStream extends FilterOutputStream {
writeBlock(); writeBlock();
} }
buffer.put((byte) b); buffer.put((byte) b);
if (frameInfo.isEnabled(FLG.Bits.CONTENT_CHECKSUM)) {
frameInfo.updateStreamHash(new byte[]{(byte) b}, 0, 1);
}
} }
@Override @Override
@ -252,19 +271,12 @@ public class LZ4FrameOutputStream extends FilterOutputStream {
int sizeWritten = buffer.remaining(); int sizeWritten = buffer.remaining();
// fill remaining space in buffer // fill remaining space in buffer
buffer.put(b, off, sizeWritten); buffer.put(b, off, sizeWritten);
if (frameInfo.isEnabled(FLG.Bits.CONTENT_CHECKSUM)) {
frameInfo.updateStreamHash(b, off, sizeWritten);
}
writeBlock(); writeBlock();
// compute new offset and length // compute new offset and length
off += sizeWritten; off += sizeWritten;
len -= sizeWritten; len -= sizeWritten;
} }
buffer.put(b, off, len); buffer.put(b, off, len);
if (frameInfo.isEnabled(FLG.Bits.CONTENT_CHECKSUM)) {
frameInfo.updateStreamHash(b, off, len);
}
} }
@Override @Override

6
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4HCJNICompressor.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -14,6 +16,8 @@ package com.fr.third.net.jpountz.lz4;
* limitations under the License. * limitations under the License.
*/ */
import static com.fr.third.net.jpountz.lz4.LZ4Constants.DEFAULT_COMPRESSION_LEVEL;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import com.fr.third.net.jpountz.util.ByteBufferUtils; import com.fr.third.net.jpountz.util.ByteBufferUtils;
@ -30,7 +34,7 @@ final class LZ4HCJNICompressor extends LZ4Compressor {
private final int compressionLevel; private final int compressionLevel;
LZ4HCJNICompressor() { this(LZ4Constants.DEFAULT_COMPRESSION_LEVEL); } LZ4HCJNICompressor() { this(DEFAULT_COMPRESSION_LEVEL); }
LZ4HCJNICompressor(int compressionLevel) { LZ4HCJNICompressor(int compressionLevel) {
this.compressionLevel = compressionLevel; this.compressionLevel = compressionLevel;
} }

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4JNI.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4JNICompressor.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4JNIFastDecompressor.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4JNISafeDecompressor.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

3
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4SafeDecompressor.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -46,6 +48,7 @@ public abstract class LZ4SafeDecompressor implements LZ4UnknownSizeDecompressor
* Decompresses <code>src[srcOff:srcOff+srcLen]</code> into * Decompresses <code>src[srcOff:srcOff+srcLen]</code> into
* <code>dest[destOff:destOff+maxDestLen]</code> and returns the number of * <code>dest[destOff:destOff+maxDestLen]</code> and returns the number of
* decompressed bytes written into <code>dest</code>. * decompressed bytes written into <code>dest</code>.
* The positions and limits of the {@link ByteBuffer}s remain unchanged.
* *
* @param src the compressed data * @param src the compressed data
* @param srcOff the start offset in src * @param srcOff the start offset in src

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4SafeUtils.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4UnknownSizeDecompressor.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

72
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4UnsafeUtils.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -14,14 +16,20 @@ package com.fr.third.net.jpountz.lz4;
* limitations under the License. * limitations under the License.
*/ */
import com.fr.third.net.jpountz.util.UnsafeUtils; import static net.jpountz.lz4.LZ4Constants.COPY_LENGTH;
import com.fr.third.net.jpountz.util.Utils; import static net.jpountz.lz4.LZ4Constants.LAST_LITERALS;
import static net.jpountz.lz4.LZ4Constants.ML_BITS;
import static net.jpountz.lz4.LZ4Constants.ML_MASK;
import static net.jpountz.lz4.LZ4Constants.RUN_MASK;
import static com.fr.third.net.jpountz.util.UnsafeUtils.readByte;
import static com.fr.third.net.jpountz.util.UnsafeUtils.readInt; import static com.fr.third.net.jpountz.util.UnsafeUtils.readInt;
import static com.fr.third.net.jpountz.util.UnsafeUtils.readLong;
import static com.fr.third.net.jpountz.util.UnsafeUtils.readShort; import static com.fr.third.net.jpountz.util.UnsafeUtils.readShort;
import static com.fr.third.net.jpountz.util.UnsafeUtils.writeByte; import static com.fr.third.net.jpountz.util.UnsafeUtils.writeByte;
import static com.fr.third.net.jpountz.util.UnsafeUtils.writeInt; import static com.fr.third.net.jpountz.util.UnsafeUtils.writeInt;
import static com.fr.third.net.jpountz.util.UnsafeUtils.writeLong;
import static com.fr.third.net.jpountz.util.UnsafeUtils.writeShort; import static com.fr.third.net.jpountz.util.UnsafeUtils.writeShort;
import static net.jpountz.util.Utils.NATIVE_BYTE_ORDER;
import java.nio.ByteOrder; import java.nio.ByteOrder;
@ -32,20 +40,20 @@ enum LZ4UnsafeUtils {
final int fastLen = len & 0xFFFFFFF8; final int fastLen = len & 0xFFFFFFF8;
wildArraycopy(src, srcOff, dest, destOff, fastLen); wildArraycopy(src, srcOff, dest, destOff, fastLen);
for (int i = 0, slowLen = len & 0x7; i < slowLen; i += 1) { for (int i = 0, slowLen = len & 0x7; i < slowLen; i += 1) {
UnsafeUtils.writeByte(dest, destOff + fastLen + i, UnsafeUtils.readByte(src, srcOff + fastLen + i)); writeByte(dest, destOff + fastLen + i, readByte(src, srcOff + fastLen + i));
} }
} }
static void wildArraycopy(byte[] src, int srcOff, byte[] dest, int destOff, int len) { static void wildArraycopy(byte[] src, int srcOff, byte[] dest, int destOff, int len) {
for (int i = 0; i < len; i += 8) { for (int i = 0; i < len; i += 8) {
UnsafeUtils.writeLong(dest, destOff + i, UnsafeUtils.readLong(src, srcOff + i)); writeLong(dest, destOff + i, readLong(src, srcOff + i));
} }
} }
static void wildIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchCopyEnd) { static void wildIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchCopyEnd) {
if (dOff - matchOff < 4) { if (dOff - matchOff < 4) {
for (int i = 0; i < 4; ++i) { for (int i = 0; i < 4; ++i) {
UnsafeUtils.writeByte(dest, dOff+i, UnsafeUtils.readByte(dest, matchOff+i)); writeByte(dest, dOff+i, readByte(dest, matchOff+i));
} }
dOff += 4; dOff += 4;
matchOff += 4; matchOff += 4;
@ -74,15 +82,15 @@ enum LZ4UnsafeUtils {
default: default:
break; break;
} }
UnsafeUtils.writeInt(dest, dOff, UnsafeUtils.readInt(dest, matchOff)); writeInt(dest, dOff, readInt(dest, matchOff));
dOff += 4; dOff += 4;
matchOff -= dec; matchOff -= dec;
} else if (dOff - matchOff < LZ4Constants.COPY_LENGTH) { } else if (dOff - matchOff < COPY_LENGTH) {
UnsafeUtils.writeLong(dest, dOff, UnsafeUtils.readLong(dest, matchOff)); writeLong(dest, dOff, readLong(dest, matchOff));
dOff += dOff - matchOff; dOff += dOff - matchOff;
} }
while (dOff < matchCopyEnd) { while (dOff < matchCopyEnd) {
UnsafeUtils.writeLong(dest, dOff, UnsafeUtils.readLong(dest, matchOff)); writeLong(dest, dOff, readLong(dest, matchOff));
dOff += 8; dOff += 8;
matchOff += 8; matchOff += 8;
} }
@ -91,13 +99,13 @@ enum LZ4UnsafeUtils {
static void safeIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchLen) { static void safeIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchLen) {
for (int i = 0; i < matchLen; ++i) { for (int i = 0; i < matchLen; ++i) {
dest[dOff + i] = dest[matchOff + i]; dest[dOff + i] = dest[matchOff + i];
UnsafeUtils.writeByte(dest, dOff + i, UnsafeUtils.readByte(dest, matchOff + i)); writeByte(dest, dOff + i, readByte(dest, matchOff + i));
} }
} }
static int readShortLittleEndian(byte[] src, int srcOff) { static int readShortLittleEndian(byte[] src, int srcOff) {
short s = UnsafeUtils.readShort(src, srcOff); short s = readShort(src, srcOff);
if (Utils.NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
s = Short.reverseBytes(s); s = Short.reverseBytes(s);
} }
return s & 0xFFFF; return s & 0xFFFF;
@ -105,34 +113,34 @@ enum LZ4UnsafeUtils {
static void writeShortLittleEndian(byte[] dest, int destOff, int value) { static void writeShortLittleEndian(byte[] dest, int destOff, int value) {
short s = (short) value; short s = (short) value;
if (Utils.NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
s = Short.reverseBytes(s); s = Short.reverseBytes(s);
} }
UnsafeUtils.writeShort(dest, destOff, s); writeShort(dest, destOff, s);
} }
static boolean readIntEquals(byte[] src, int ref, int sOff) { static boolean readIntEquals(byte[] src, int ref, int sOff) {
return UnsafeUtils.readInt(src, ref) == UnsafeUtils.readInt(src, sOff); return readInt(src, ref) == readInt(src, sOff);
} }
static int commonBytes(byte[] src, int ref, int sOff, int srcLimit) { static int commonBytes(byte[] src, int ref, int sOff, int srcLimit) {
int matchLen = 0; int matchLen = 0;
while (sOff <= srcLimit - 8) { while (sOff <= srcLimit - 8) {
if (UnsafeUtils.readLong(src, sOff) == UnsafeUtils.readLong(src, ref)) { if (readLong(src, sOff) == readLong(src, ref)) {
matchLen += 8; matchLen += 8;
ref += 8; ref += 8;
sOff += 8; sOff += 8;
} else { } else {
final int zeroBits; final int zeroBits;
if (Utils.NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
zeroBits = Long.numberOfLeadingZeros(UnsafeUtils.readLong(src, sOff) ^ UnsafeUtils.readLong(src, ref)); zeroBits = Long.numberOfLeadingZeros(readLong(src, sOff) ^ readLong(src, ref));
} else { } else {
zeroBits = Long.numberOfTrailingZeros(UnsafeUtils.readLong(src, sOff) ^ UnsafeUtils.readLong(src, ref)); zeroBits = Long.numberOfTrailingZeros(readLong(src, sOff) ^ readLong(src, ref));
} }
return matchLen + (zeroBits >>> 3); return matchLen + (zeroBits >>> 3);
} }
} }
while (sOff < srcLimit && UnsafeUtils.readByte(src, ref++) == UnsafeUtils.readByte(src, sOff++)) { while (sOff < srcLimit && readByte(src, ref++) == readByte(src, sOff++)) {
++matchLen; ++matchLen;
} }
return matchLen; return matchLen;
@ -140,10 +148,10 @@ enum LZ4UnsafeUtils {
static int writeLen(int len, byte[] dest, int dOff) { static int writeLen(int len, byte[] dest, int dOff) {
while (len >= 0xFF) { while (len >= 0xFF) {
UnsafeUtils.writeByte(dest, dOff++, 0xFF); writeByte(dest, dOff++, 0xFF);
len -= 0xFF; len -= 0xFF;
} }
UnsafeUtils.writeByte(dest, dOff++, len); writeByte(dest, dOff++, len);
return dOff; return dOff;
} }
@ -152,11 +160,11 @@ enum LZ4UnsafeUtils {
final int tokenOff = dOff++; final int tokenOff = dOff++;
int token; int token;
if (runLen >= LZ4Constants.RUN_MASK) { if (runLen >= RUN_MASK) {
token = (byte) (LZ4Constants.RUN_MASK << LZ4Constants.ML_BITS); token = (byte) (RUN_MASK << ML_BITS);
dOff = writeLen(runLen - LZ4Constants.RUN_MASK, dest, dOff); dOff = writeLen(runLen - RUN_MASK, dest, dOff);
} else { } else {
token = runLen << LZ4Constants.ML_BITS; token = runLen << ML_BITS;
} }
// copy literals // copy literals
@ -170,12 +178,12 @@ enum LZ4UnsafeUtils {
// encode match len // encode match len
matchLen -= 4; matchLen -= 4;
if (dOff + (1 + LZ4Constants.LAST_LITERALS) + (matchLen >>> 8) > destEnd) { if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small"); throw new LZ4Exception("maxDestLen is too small");
} }
if (matchLen >= LZ4Constants.ML_MASK) { if (matchLen >= ML_MASK) {
token |= LZ4Constants.ML_MASK; token |= ML_MASK;
dOff = writeLen(matchLen - LZ4Constants.RUN_MASK, dest, dOff); dOff = writeLen(matchLen - RUN_MASK, dest, dOff);
} else { } else {
token |= matchLen; token |= matchLen;
} }
@ -187,7 +195,7 @@ enum LZ4UnsafeUtils {
static int commonBytesBackward(byte[] b, int o1, int o2, int l1, int l2) { static int commonBytesBackward(byte[] b, int o1, int o2, int l1, int l2) {
int count = 0; int count = 0;
while (o1 > l1 && o2 > l2 && UnsafeUtils.readByte(b, --o1) == UnsafeUtils.readByte(b, --o2)) { while (o1 > l1 && o2 > l2 && readByte(b, --o1) == readByte(b, --o2)) {
++count; ++count;
} }
return count; return count;

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/LZ4Utils.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.lz4; package com.fr.third.net.jpountz.lz4;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

8
fine-lz4/src/main/java/com/fr/third/net/jpountz/lz4/package.html

@ -1,5 +1,7 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en"> <!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!-- <!--
Copyright 2020 Adrien Grand and the lz4-java contributors.
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
You may obtain a copy of the License at You may obtain a copy of the License at
@ -18,9 +20,9 @@
</head> </head>
<body> <body>
<p>LZ4 compression. The entry point of the API is the <p>LZ4 compression. The entry point of the API is the
{@link net.jpountz.lz4.LZ4Factory} class, which gives access to {@link com.fr.third.net.jpountz.lz4.LZ4Factory} class, which gives access to
{@link com.fr.third.net.jpountz.lz4.LZ4Compressor compressors} and {@link com.fr.third.com.fr.third.net.jpountz.lz4.LZ4Compressor compressors} and
{@link com.fr.third.net.jpountz.lz4.LZ4SafeDecompressor decompressors}.</p> {@link com.fr.third.com.fr.third.net.jpountz.lz4.LZ4SafeDecompressor decompressors}.</p>
<p>Sample usage:</p> <p>Sample usage:</p>

16
fine-lz4/src/main/java/com/fr/third/net/jpountz/util/ByteBufferUtils.java

@ -1,5 +1,21 @@
package com.fr.third.net.jpountz.util; package com.fr.third.net.jpountz.util;
/*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.ByteOrder; import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException; import java.nio.ReadOnlyBufferException;

106
fine-lz4/src/main/java/com/fr/third/net/jpountz/util/Native.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.util; package com.fr.third.net.jpountz.util;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -18,6 +20,7 @@ import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.FilenameFilter;
/** FOR INTERNAL USE ONLY */ /** FOR INTERNAL USE ONLY */
public enum Native { public enum Native {
@ -67,11 +70,38 @@ public enum Native {
return loaded; return loaded;
} }
private static void cleanupOldTempLibs() {
String tempFolder = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath();
File dir = new File(tempFolder);
File[] tempLibFiles = dir.listFiles(new FilenameFilter() {
private final String searchPattern = "liblz4-java-";
public boolean accept(File dir, String name) {
return name.startsWith(searchPattern) && !name.endsWith(".lck");
}
});
if(tempLibFiles != null) {
for(File tempLibFile : tempLibFiles) {
File lckFile = new File(tempLibFile.getAbsolutePath() + ".lck");
if(!lckFile.exists()) {
try {
tempLibFile.delete();
}
catch(SecurityException e) {
System.err.println("Failed to delete old temp lib" + e.getMessage());
}
}
}
}
}
public static synchronized void load() { public static synchronized void load() {
if (loaded) { if (loaded) {
return; return;
} }
cleanupOldTempLibs();
// Try to load lz4-java (liblz4-java.so on Linux) from the java.library.path. // Try to load lz4-java (liblz4-java.so on Linux) from the java.library.path.
try { try {
System.loadLibrary("lz4-java"); System.loadLibrary("lz4-java");
@ -86,48 +116,48 @@ public enum Native {
if (is == null) { if (is == null) {
throw new UnsupportedOperationException("Unsupported OS/arch, cannot find " + resourceName + ". Please try building from source."); throw new UnsupportedOperationException("Unsupported OS/arch, cannot find " + resourceName + ". Please try building from source.");
} }
File tempLib; File tempLib = null;
File tempLibLock = null;
try { try {
tempLib = File.createTempFile("liblz4-java", "." + os().libExtension); // Create the .lck file first to avoid a race condition
// with other concurrently running Java processes using lz4-java.
tempLibLock = File.createTempFile("liblz4-java-", "." + os().libExtension + ".lck");
tempLib = new File(tempLibLock.getAbsolutePath().replaceFirst(".lck$", ""));
// copy to tempLib // copy to tempLib
FileOutputStream out = new FileOutputStream(tempLib); try (FileOutputStream out = new FileOutputStream(tempLib)) {
try { byte[] buf = new byte[4096];
byte[] buf = new byte[4096]; while (true) {
while (true) { int read = is.read(buf);
int read = is.read(buf); if (read == -1) {
if (read == -1) { break;
break; }
} out.write(buf, 0, read);
out.write(buf, 0, read); }
}
try {
out.close();
out = null;
} catch (IOException e) {
// ignore
}
System.load(tempLib.getAbsolutePath());
loaded = true;
} finally {
try {
if (out != null) {
out.close();
}
} catch (IOException e) {
// ignore
}
if (tempLib != null && tempLib.exists()) {
if (!loaded) {
tempLib.delete();
} else {
// try to delete on exit, does it work on Windows?
tempLib.deleteOnExit();
}
}
} }
System.load(tempLib.getAbsolutePath());
loaded = true;
} catch (IOException e) { } catch (IOException e) {
throw new ExceptionInInitializerError("Cannot unpack liblz4-java"); throw new ExceptionInInitializerError("Cannot unpack liblz4-java: " + e);
} finally {
if (!loaded) {
if (tempLib != null && tempLib.exists()) {
if (!tempLib.delete()) {
throw new ExceptionInInitializerError("Cannot unpack liblz4-java / cannot delete a temporary native library " + tempLib);
}
}
if (tempLibLock != null && tempLibLock.exists()) {
if (!tempLibLock.delete()) {
throw new ExceptionInInitializerError("Cannot unpack liblz4-java / cannot delete a temporary lock file " + tempLibLock);
}
}
} else {
final String keepEnv = System.getenv("LZ4JAVA_KEEP_TEMP_JNI_LIB");
final String keepProp = System.getProperty("lz4java.jnilib.temp.keep");
if ((keepEnv == null || !keepEnv.equals("true")) &&
(keepProp == null || !keepProp.equals("true")))
tempLib.deleteOnExit();
tempLibLock.deleteOnExit();
}
} }
} }
} }

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/util/SafeUtils.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.util; package com.fr.third.net.jpountz.util;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

4
fine-lz4/src/main/java/com/fr/third/net/jpountz/util/UnsafeUtils.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.util; package com.fr.third.net.jpountz.util;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -14,7 +16,7 @@ package com.fr.third.net.jpountz.util;
* limitations under the License. * limitations under the License.
*/ */
import static com.fr.third.net.jpountz.util.Utils.NATIVE_BYTE_ORDER; import static net.jpountz.util.Utils.NATIVE_BYTE_ORDER;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.nio.ByteOrder; import java.nio.ByteOrder;

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/util/Utils.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.util; package com.fr.third.net.jpountz.util;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/util/package.html

@ -1,5 +1,7 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en"> <!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!-- <!--
Copyright 2020 Adrien Grand and the lz4-java contributors.
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
You may obtain a copy of the License at You may obtain a copy of the License at

11
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/AbstractStreamingXXHash32Java.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -14,6 +16,9 @@ package com.fr.third.net.jpountz.xxhash;
* limitations under the License. * limitations under the License.
*/ */
import static com.fr.third.net.jpountz.xxhash.XXHashConstants.PRIME1;
import static com.fr.third.net.jpountz.xxhash.XXHashConstants.PRIME2;
abstract class AbstractStreamingXXHash32Java extends StreamingXXHash32 { abstract class AbstractStreamingXXHash32Java extends StreamingXXHash32 {
int v1, v2, v3, v4, memSize; int v1, v2, v3, v4, memSize;
@ -28,10 +33,10 @@ abstract class AbstractStreamingXXHash32Java extends StreamingXXHash32 {
@Override @Override
public void reset() { public void reset() {
v1 = seed + XXHashConstants.PRIME1 + XXHashConstants.PRIME2; v1 = seed + PRIME1 + PRIME2;
v2 = seed + XXHashConstants.PRIME2; v2 = seed + PRIME2;
v3 = seed + 0; v3 = seed + 0;
v4 = seed - XXHashConstants.PRIME1; v4 = seed - PRIME1;
totalLen = 0; totalLen = 0;
memSize = 0; memSize = 0;
} }

11
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/AbstractStreamingXXHash64Java.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
/* /*
* Copyright 2020 Linnaea Von Lavia and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -14,6 +16,9 @@ package com.fr.third.net.jpountz.xxhash;
* limitations under the License. * limitations under the License.
*/ */
import static com.fr.third.net.jpountz.xxhash.XXHashConstants.PRIME64_1;
import static com.fr.third.net.jpountz.xxhash.XXHashConstants.PRIME64_2;
abstract class AbstractStreamingXXHash64Java extends StreamingXXHash64 { abstract class AbstractStreamingXXHash64Java extends StreamingXXHash64 {
int memSize; int memSize;
@ -29,10 +34,10 @@ abstract class AbstractStreamingXXHash64Java extends StreamingXXHash64 {
@Override @Override
public void reset() { public void reset() {
v1 = seed + XXHashConstants.PRIME64_1 + XXHashConstants.PRIME64_2; v1 = seed + PRIME64_1 + PRIME64_2;
v2 = seed + XXHashConstants.PRIME64_2; v2 = seed + PRIME64_2;
v3 = seed + 0; v3 = seed + 0;
v4 = seed - XXHashConstants.PRIME64_1; v4 = seed - PRIME64_1;
totalLen = 0; totalLen = 0;
memSize = 0; memSize = 0;
} }

15
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/StreamingXXHash32.java

@ -1,8 +1,11 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
import java.util.zip.Checksum; import java.util.zip.Checksum;
import java.io.Closeable;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -38,7 +41,7 @@ import java.util.zip.Checksum;
* <p> * <p>
* Instances of this class are <b>not</b> thread-safe. * Instances of this class are <b>not</b> thread-safe.
*/ */
public abstract class StreamingXXHash32 { public abstract class StreamingXXHash32 implements Closeable {
interface Factory { interface Factory {
@ -74,6 +77,16 @@ public abstract class StreamingXXHash32 {
*/ */
public abstract void reset(); public abstract void reset();
/**
* Releases any system resources associated with this instance.
* It is not mandatory to call this method after using this instance
* because the system resources are released anyway when this instance
* is reclaimed by GC.
*/
@Override
public void close() {
}
@Override @Override
public String toString() { public String toString() {
return getClass().getSimpleName() + "(seed=" + seed + ")"; return getClass().getSimpleName() + "(seed=" + seed + ")";

36
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/StreamingXXHash32JNI.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -14,7 +16,14 @@ package com.fr.third.net.jpountz.xxhash;
* limitations under the License. * limitations under the License.
*/ */
/**
* Fast {@link StreamingXXHash32} implemented with JNI bindings.
* The methods are synchronized to avoid a race condition
* between freeing the native memory in finalize() and using it in
* reset(), getValue(), and update(). Note that GC can call finalize()
* after calling checkState() and before using XXHashJNI if the caller
* does not retain a reference to this object.
*/
final class StreamingXXHash32JNI extends StreamingXXHash32 { final class StreamingXXHash32JNI extends StreamingXXHash32 {
static class Factory implements StreamingXXHash32.Factory { static class Factory implements StreamingXXHash32.Factory {
@ -42,30 +51,41 @@ final class StreamingXXHash32JNI extends StreamingXXHash32 {
} }
@Override @Override
public void reset() { public synchronized void reset() {
checkState(); checkState();
XXHashJNI.XXH32_free(state); XXHashJNI.XXH32_free(state);
state = XXHashJNI.XXH32_init(seed); state = XXHashJNI.XXH32_init(seed);
} }
@Override @Override
public int getValue() { public synchronized int getValue() {
checkState(); checkState();
return XXHashJNI.XXH32_digest(state); return XXHashJNI.XXH32_digest(state);
} }
@Override @Override
public void update(byte[] bytes, int off, int len) { public synchronized void update(byte[] bytes, int off, int len) {
checkState(); checkState();
XXHashJNI.XXH32_update(state, bytes, off, len); XXHashJNI.XXH32_update(state, bytes, off, len);
} }
@Override @Override
protected void finalize() throws Throwable { public synchronized void close() {
if (state != 0) {
super.close();
XXHashJNI.XXH32_free(state);
state = 0;
}
}
@Override
protected synchronized void finalize() throws Throwable {
super.finalize(); super.finalize();
// free memory if (state != 0) {
XXHashJNI.XXH32_free(state); // free memory
state = 0; XXHashJNI.XXH32_free(state);
state = 0;
}
} }
} }

15
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/StreamingXXHash64.java

@ -1,8 +1,11 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
import java.util.zip.Checksum; import java.util.zip.Checksum;
import java.io.Closeable;
/* /*
* Copyright 2020 Linnaea Von Lavia and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -38,7 +41,7 @@ import java.util.zip.Checksum;
* <p> * <p>
* Instances of this class are <b>not</b> thread-safe. * Instances of this class are <b>not</b> thread-safe.
*/ */
public abstract class StreamingXXHash64 { public abstract class StreamingXXHash64 implements Closeable {
interface Factory { interface Factory {
@ -74,6 +77,16 @@ public abstract class StreamingXXHash64 {
*/ */
public abstract void reset(); public abstract void reset();
/**
* Releases any system resources associated with this instance.
* It is not mandatory to call this method after using this instance
* because the system resources are released anyway when this instance
* is reclaimed by GC.
*/
@Override
public void close() {
}
@Override @Override
public String toString() { public String toString() {
return getClass().getSimpleName() + "(seed=" + seed + ")"; return getClass().getSimpleName() + "(seed=" + seed + ")";

36
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/StreamingXXHash64JNI.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
/* /*
* Copyright 2020 Linnaea Von Lavia and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -15,6 +17,14 @@ package com.fr.third.net.jpountz.xxhash;
*/ */
/**
* Fast {@link StreamingXXHash64} implemented with JNI bindings.
* The methods are synchronized to avoid a race condition
* between freeing the native memory in finalize() and using it in
* reset(), getValue(), and update(). Note that GC can call finalize()
* after calling checkState() and before using XXHashJNI if the caller
* does not retain a reference to this object.
*/
final class StreamingXXHash64JNI extends StreamingXXHash64 { final class StreamingXXHash64JNI extends StreamingXXHash64 {
static class Factory implements StreamingXXHash64.Factory { static class Factory implements StreamingXXHash64.Factory {
@ -42,30 +52,42 @@ final class StreamingXXHash64JNI extends StreamingXXHash64 {
} }
@Override @Override
public void reset() { public synchronized void reset() {
checkState(); checkState();
XXHashJNI.XXH64_free(state); XXHashJNI.XXH64_free(state);
state = XXHashJNI.XXH64_init(seed); state = XXHashJNI.XXH64_init(seed);
} }
@Override @Override
public long getValue() { public synchronized long getValue() {
checkState(); checkState();
return XXHashJNI.XXH64_digest(state); return XXHashJNI.XXH64_digest(state);
} }
@Override @Override
public void update(byte[] bytes, int off, int len) { public synchronized void update(byte[] bytes, int off, int len) {
checkState(); checkState();
XXHashJNI.XXH64_update(state, bytes, off, len); XXHashJNI.XXH64_update(state, bytes, off, len);
} }
@Override @Override
protected void finalize() throws Throwable { public synchronized void close() {
if (state != 0) {
super.close();
XXHashJNI.XXH64_free(state);
state = 0;
}
}
@Override
protected synchronized void finalize() throws Throwable {
super.finalize(); super.finalize();
// free memory if (state != 0) {
XXHashJNI.XXH64_free(state); // free memory
state = 0; XXHashJNI.XXH64_free(state);
state = 0;
}
} }
} }

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHash32.java

@ -3,6 +3,8 @@ package com.fr.third.net.jpountz.xxhash;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

9
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHash32JNI.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -14,9 +16,6 @@ package com.fr.third.net.jpountz.xxhash;
* limitations under the License. * limitations under the License.
*/ */
import com.fr.third.net.jpountz.util.ByteBufferUtils;
import com.fr.third.net.jpountz.util.SafeUtils;
import static com.fr.third.net.jpountz.util.ByteBufferUtils.checkRange; import static com.fr.third.net.jpountz.util.ByteBufferUtils.checkRange;
import static com.fr.third.net.jpountz.util.SafeUtils.checkRange; import static com.fr.third.net.jpountz.util.SafeUtils.checkRange;
@ -29,14 +28,14 @@ final class XXHash32JNI extends XXHash32 {
@Override @Override
public int hash(byte[] buf, int off, int len, int seed) { public int hash(byte[] buf, int off, int len, int seed) {
SafeUtils.checkRange(buf, off, len); checkRange(buf, off, len);
return XXHashJNI.XXH32(buf, off, len, seed); return XXHashJNI.XXH32(buf, off, len, seed);
} }
@Override @Override
public int hash(ByteBuffer buf, int off, int len, int seed) { public int hash(ByteBuffer buf, int off, int len, int seed) {
if (buf.isDirect()) { if (buf.isDirect()) {
ByteBufferUtils.checkRange(buf, off, len); checkRange(buf, off, len);
return XXHashJNI.XXH32BB(buf, off, len, seed); return XXHashJNI.XXH32BB(buf, off, len, seed);
} else if (buf.hasArray()) { } else if (buf.hasArray()) {
return hash(buf.array(), off + buf.arrayOffset(), len, seed); return hash(buf.array(), off + buf.arrayOffset(), len, seed);

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHash64.java

@ -3,6 +3,8 @@ package com.fr.third.net.jpountz.xxhash;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
/* /*
* Copyright 2020 Linnaea Von Lavia and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

9
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHash64JNI.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
/* /*
* Copyright 2020 Linnaea Von Lavia and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -14,9 +16,6 @@ package com.fr.third.net.jpountz.xxhash;
* limitations under the License. * limitations under the License.
*/ */
import com.fr.third.net.jpountz.util.ByteBufferUtils;
import com.fr.third.net.jpountz.util.SafeUtils;
import static com.fr.third.net.jpountz.util.ByteBufferUtils.checkRange; import static com.fr.third.net.jpountz.util.ByteBufferUtils.checkRange;
import static com.fr.third.net.jpountz.util.SafeUtils.checkRange; import static com.fr.third.net.jpountz.util.SafeUtils.checkRange;
@ -29,14 +28,14 @@ final class XXHash64JNI extends XXHash64 {
@Override @Override
public long hash(byte[] buf, int off, int len, long seed) { public long hash(byte[] buf, int off, int len, long seed) {
SafeUtils.checkRange(buf, off, len); checkRange(buf, off, len);
return XXHashJNI.XXH64(buf, off, len, seed); return XXHashJNI.XXH64(buf, off, len, seed);
} }
@Override @Override
public long hash(ByteBuffer buf, int off, int len, long seed) { public long hash(ByteBuffer buf, int off, int len, long seed) {
if (buf.isDirect()) { if (buf.isDirect()) {
ByteBufferUtils.checkRange(buf, off, len); checkRange(buf, off, len);
return XXHashJNI.XXH64BB(buf, off, len, seed); return XXHashJNI.XXH64BB(buf, off, len, seed);
} else if (buf.hasArray()) { } else if (buf.hasArray()) {
return hash(buf.array(), off + buf.arrayOffset(), len, seed); return hash(buf.array(), off + buf.arrayOffset(), len, seed);

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHashConstants.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

10
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHashFactory.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
@ -174,10 +176,10 @@ public final class XXHashFactory {
private XXHashFactory(String impl) throws ClassNotFoundException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { private XXHashFactory(String impl) throws ClassNotFoundException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException {
this.impl = impl; this.impl = impl;
hash32 = classInstance("com.fr.third.net.jpountz.xxhash.XXHash32" + impl); hash32 = classInstance("net.jpountz.xxhash.XXHash32" + impl);
streamingHash32Factory = classInstance("com.fr.third.net.jpountz.xxhash.StreamingXXHash32" + impl + "$Factory"); streamingHash32Factory = classInstance("net.jpountz.xxhash.StreamingXXHash32" + impl + "$Factory");
hash64 = classInstance("com.fr.third.net.jpountz.xxhash.XXHash64" + impl); hash64 = classInstance("net.jpountz.xxhash.XXHash64" + impl);
streamingHash64Factory = classInstance("com.fr.third.net.jpountz.xxhash.StreamingXXHash64" + impl + "$Factory"); streamingHash64Factory = classInstance("net.jpountz.xxhash.StreamingXXHash64" + impl + "$Factory");
// make sure it can run // make sure it can run
final byte[] bytes = new byte[100]; final byte[] bytes = new byte[100];

2
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/XXHashJNI.java

@ -1,6 +1,8 @@
package com.fr.third.net.jpountz.xxhash; package com.fr.third.net.jpountz.xxhash;
/* /*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at

8
fine-lz4/src/main/java/com/fr/third/net/jpountz/xxhash/package.html

@ -1,5 +1,7 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en"> <!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!-- <!--
Copyright 2020 Adrien Grand and the lz4-java contributors.
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
You may obtain a copy of the License at You may obtain a copy of the License at
@ -18,9 +20,9 @@
</head> </head>
<body> <body>
<p>xxhash hashing. This package supports both block hashing via <p>xxhash hashing. This package supports both block hashing via
{@link net.jpountz.xxhash.XXHash32} and streaming hashing via {@link com.fr.third.net.jpountz.xxhash.XXHash32} and streaming hashing via
{@link net.jpountz.xxhash.StreamingXXHash32}. Have a look at {@link com.fr.third.net.jpountz.xxhash.StreamingXXHash32}. Have a look at
{@link net.jpountz.xxhash.XXHashFactory} to know how to get instances of these {@link com.fr.third.net.jpountz.xxhash.XXHashFactory} to know how to get instances of these
interfaces.</p> interfaces.</p>
<p>Streaming hashing is a little slower but doesn't require to load the whole <p>Streaming hashing is a little slower but doesn't require to load the whole

BIN
fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/darwin/aarch64/liblz4-java.dylib

Binary file not shown.

BIN
fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/darwin/x86_64/liblz4-java.dylib

Binary file not shown.

BIN
fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/linux/aarch64/liblz4-java.so

Binary file not shown.

BIN
fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/linux/amd64/liblz4-java.so

Binary file not shown.

BIN
fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/linux/i386/liblz4-java.so

Binary file not shown.

BIN
fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/linux/ppc64le/liblz4-java.so

Binary file not shown.

BIN
fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/linux/s390x/liblz4-java.so

Binary file not shown.

BIN
fine-lz4/src/main/resources/com/fr/third/net/jpountz/util/win32/amd64/liblz4-java.so

Binary file not shown.
Loading…
Cancel
Save