Browse Source
Very large delta instruction streams, or deltas which use very large base objects, are now streamed through as large objects rather than being inflated into a byte array. This isn't the most efficient way to access delta encoded content, as we may need to rewind and reprocess the base object when there was a block moved within the file, but it will at least prevent the JVM from having its heap explode. When streaming a delta we have an inflater open for each level in the delta chain, to inflate the instruction set of the delta, as well as an inflater for the base level object. The base object is buffered, as is the top level delta requested by the application, but we do not buffer the intermediate delta streams. This keeps memory usage lower, so its closer to 1024 bytes per level in the chain, without having an adverse impact on raw throughput as the top-level buffer gets pushed down to the lowest stream that has the next region. Delta instructions transparently collapse here, if the top level does not copy a region from its base, the base won't materialize that part from its own base, etc. This allows us to avoid copying around a lot of segments which have been deleted from the final version. Change-Id: I724d45245cebb4bad2deeae7b896fc55b2dd49b3 Signed-off-by: Shawn O. Pearce <spearce@spearce.org>stable-0.9
Shawn O. Pearce
15 years ago
11 changed files with 1387 additions and 93 deletions
@ -0,0 +1,273 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2010, Google Inc. |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.eclipse.jgit.storage.pack; |
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream; |
||||||
|
import java.io.ByteArrayOutputStream; |
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStream; |
||||||
|
import java.util.Arrays; |
||||||
|
|
||||||
|
import junit.framework.TestCase; |
||||||
|
|
||||||
|
import org.eclipse.jgit.JGitText; |
||||||
|
import org.eclipse.jgit.errors.CorruptObjectException; |
||||||
|
import org.eclipse.jgit.junit.TestRng; |
||||||
|
import org.eclipse.jgit.lib.Constants; |
||||||
|
import org.eclipse.jgit.util.IO; |
||||||
|
|
||||||
|
public class DeltaStreamTest extends TestCase { |
||||||
|
private TestRng rng; |
||||||
|
|
||||||
|
private ByteArrayOutputStream deltaBuf; |
||||||
|
|
||||||
|
private DeltaEncoder deltaEnc; |
||||||
|
|
||||||
|
private byte[] base; |
||||||
|
|
||||||
|
private byte[] data; |
||||||
|
|
||||||
|
private int dataPtr; |
||||||
|
|
||||||
|
private byte[] delta; |
||||||
|
|
||||||
|
protected void setUp() throws Exception { |
||||||
|
super.setUp(); |
||||||
|
rng = new TestRng(getName()); |
||||||
|
deltaBuf = new ByteArrayOutputStream(); |
||||||
|
} |
||||||
|
|
||||||
|
public void testCopy_SingleOp() throws IOException { |
||||||
|
init((1 << 16) + 1, (1 << 8) + 1); |
||||||
|
copy(0, data.length); |
||||||
|
assertValidState(); |
||||||
|
} |
||||||
|
|
||||||
|
public void testCopy_MaxSize() throws IOException { |
||||||
|
int max = (0xff << 16) + (0xff << 8) + 0xff; |
||||||
|
init(1 + max, max); |
||||||
|
copy(1, max); |
||||||
|
assertValidState(); |
||||||
|
} |
||||||
|
|
||||||
|
public void testCopy_64k() throws IOException { |
||||||
|
init(0x10000 + 2, 0x10000 + 1); |
||||||
|
copy(1, 0x10000); |
||||||
|
copy(0x10001, 1); |
||||||
|
assertValidState(); |
||||||
|
} |
||||||
|
|
||||||
|
public void testCopy_Gap() throws IOException { |
||||||
|
init(256, 8); |
||||||
|
copy(4, 4); |
||||||
|
copy(128, 4); |
||||||
|
assertValidState(); |
||||||
|
} |
||||||
|
|
||||||
|
public void testCopy_OutOfOrder() throws IOException { |
||||||
|
init((1 << 16) + 1, (1 << 16) + 1); |
||||||
|
copy(1 << 8, 1 << 8); |
||||||
|
copy(0, data.length - dataPtr); |
||||||
|
assertValidState(); |
||||||
|
} |
||||||
|
|
||||||
|
public void testInsert_SingleOp() throws IOException { |
||||||
|
init((1 << 16) + 1, 2); |
||||||
|
insert("hi"); |
||||||
|
assertValidState(); |
||||||
|
} |
||||||
|
|
||||||
|
public void testInsertAndCopy() throws IOException { |
||||||
|
init(8, 512); |
||||||
|
insert(new byte[127]); |
||||||
|
insert(new byte[127]); |
||||||
|
insert(new byte[127]); |
||||||
|
insert(new byte[125]); |
||||||
|
copy(2, 6); |
||||||
|
assertValidState(); |
||||||
|
} |
||||||
|
|
||||||
|
public void testSkip() throws IOException { |
||||||
|
init(32, 15); |
||||||
|
copy(2, 2); |
||||||
|
insert("ab"); |
||||||
|
insert("cd"); |
||||||
|
copy(4, 4); |
||||||
|
copy(0, 2); |
||||||
|
insert("efg"); |
||||||
|
assertValidState(); |
||||||
|
|
||||||
|
for (int p = 0; p < data.length; p++) { |
||||||
|
byte[] act = new byte[data.length]; |
||||||
|
System.arraycopy(data, 0, act, 0, p); |
||||||
|
DeltaStream in = open(); |
||||||
|
IO.skipFully(in, p); |
||||||
|
assertEquals(data.length - p, in.read(act, p, data.length - p)); |
||||||
|
assertEquals(-1, in.read()); |
||||||
|
assertTrue("skipping " + p, Arrays.equals(data, act)); |
||||||
|
} |
||||||
|
|
||||||
|
// Skip all the way to the end should still recognize EOF.
|
||||||
|
DeltaStream in = open(); |
||||||
|
IO.skipFully(in, data.length); |
||||||
|
assertEquals(-1, in.read()); |
||||||
|
assertEquals(0, in.skip(1)); |
||||||
|
|
||||||
|
// Skip should not open the base as we move past it, but it
|
||||||
|
// will open when we need to start copying data from it.
|
||||||
|
final boolean[] opened = new boolean[1]; |
||||||
|
in = new DeltaStream(new ByteArrayInputStream(delta)) { |
||||||
|
@Override |
||||||
|
protected long getBaseSize() throws IOException { |
||||||
|
return base.length; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected InputStream openBase() throws IOException { |
||||||
|
opened[0] = true; |
||||||
|
return new ByteArrayInputStream(base); |
||||||
|
} |
||||||
|
}; |
||||||
|
IO.skipFully(in, 7); |
||||||
|
assertFalse("not yet open", opened[0]); |
||||||
|
assertEquals(data[7], in.read()); |
||||||
|
assertTrue("now open", opened[0]); |
||||||
|
} |
||||||
|
|
||||||
|
public void testIncorrectBaseSize() throws IOException { |
||||||
|
init(4, 4); |
||||||
|
copy(0, 4); |
||||||
|
assertValidState(); |
||||||
|
|
||||||
|
DeltaStream in = new DeltaStream(new ByteArrayInputStream(delta)) { |
||||||
|
@Override |
||||||
|
protected long getBaseSize() throws IOException { |
||||||
|
return 128; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected InputStream openBase() throws IOException { |
||||||
|
return new ByteArrayInputStream(base); |
||||||
|
} |
||||||
|
}; |
||||||
|
try { |
||||||
|
in.read(new byte[4]); |
||||||
|
fail("did not throw an exception"); |
||||||
|
} catch (CorruptObjectException e) { |
||||||
|
assertEquals(JGitText.get().baseLengthIncorrect, e.getMessage()); |
||||||
|
} |
||||||
|
|
||||||
|
in = new DeltaStream(new ByteArrayInputStream(delta)) { |
||||||
|
@Override |
||||||
|
protected long getBaseSize() throws IOException { |
||||||
|
return 4; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected InputStream openBase() throws IOException { |
||||||
|
return new ByteArrayInputStream(new byte[0]); |
||||||
|
} |
||||||
|
}; |
||||||
|
try { |
||||||
|
in.read(new byte[4]); |
||||||
|
fail("did not throw an exception"); |
||||||
|
} catch (CorruptObjectException e) { |
||||||
|
assertEquals(JGitText.get().baseLengthIncorrect, e.getMessage()); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private void init(int baseSize, int dataSize) throws IOException { |
||||||
|
base = rng.nextBytes(baseSize); |
||||||
|
data = new byte[dataSize]; |
||||||
|
deltaEnc = new DeltaEncoder(deltaBuf, baseSize, dataSize); |
||||||
|
} |
||||||
|
|
||||||
|
private void copy(int offset, int len) throws IOException { |
||||||
|
System.arraycopy(base, offset, data, dataPtr, len); |
||||||
|
deltaEnc.copy(offset, len); |
||||||
|
assertEquals(deltaBuf.size(), deltaEnc.getSize()); |
||||||
|
dataPtr += len; |
||||||
|
} |
||||||
|
|
||||||
|
private void insert(String text) throws IOException { |
||||||
|
insert(Constants.encode(text)); |
||||||
|
} |
||||||
|
|
||||||
|
private void insert(byte[] text) throws IOException { |
||||||
|
System.arraycopy(text, 0, data, dataPtr, text.length); |
||||||
|
deltaEnc.insert(text); |
||||||
|
assertEquals(deltaBuf.size(), deltaEnc.getSize()); |
||||||
|
dataPtr += text.length; |
||||||
|
} |
||||||
|
|
||||||
|
private void assertValidState() throws IOException { |
||||||
|
assertEquals("test filled example result", data.length, dataPtr); |
||||||
|
|
||||||
|
delta = deltaBuf.toByteArray(); |
||||||
|
assertEquals(base.length, BinaryDelta.getBaseSize(delta)); |
||||||
|
assertEquals(data.length, BinaryDelta.getResultSize(delta)); |
||||||
|
assertTrue(Arrays.equals(data, BinaryDelta.apply(base, delta))); |
||||||
|
|
||||||
|
byte[] act = new byte[data.length]; |
||||||
|
DeltaStream in = open(); |
||||||
|
assertEquals(data.length, in.getSize()); |
||||||
|
assertEquals(data.length, in.read(act)); |
||||||
|
assertEquals(-1, in.read()); |
||||||
|
assertTrue(Arrays.equals(data, act)); |
||||||
|
} |
||||||
|
|
||||||
|
private DeltaStream open() throws IOException { |
||||||
|
return new DeltaStream(new ByteArrayInputStream(delta)) { |
||||||
|
@Override |
||||||
|
protected long getBaseSize() throws IOException { |
||||||
|
return base.length; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected InputStream openBase() throws IOException { |
||||||
|
return new ByteArrayInputStream(base); |
||||||
|
} |
||||||
|
}; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,153 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2010, Google Inc. |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.eclipse.jgit.storage.file; |
||||||
|
|
||||||
|
import java.io.BufferedInputStream; |
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStream; |
||||||
|
import java.util.zip.InflaterInputStream; |
||||||
|
|
||||||
|
import org.eclipse.jgit.errors.IncorrectObjectTypeException; |
||||||
|
import org.eclipse.jgit.errors.LargeObjectException; |
||||||
|
import org.eclipse.jgit.errors.MissingObjectException; |
||||||
|
import org.eclipse.jgit.lib.ObjectId; |
||||||
|
import org.eclipse.jgit.lib.ObjectLoader; |
||||||
|
import org.eclipse.jgit.lib.ObjectStream; |
||||||
|
import org.eclipse.jgit.storage.pack.DeltaStream; |
||||||
|
|
||||||
|
class LargePackedDeltaObject extends ObjectLoader { |
||||||
|
private final int type; |
||||||
|
|
||||||
|
private final long size; |
||||||
|
|
||||||
|
private final long objectOffset; |
||||||
|
|
||||||
|
private final long baseOffset; |
||||||
|
|
||||||
|
private final int headerLength; |
||||||
|
|
||||||
|
private final PackFile pack; |
||||||
|
|
||||||
|
private final FileObjectDatabase db; |
||||||
|
|
||||||
|
LargePackedDeltaObject(int type, long size, long objectOffset, |
||||||
|
long baseOffset, int headerLength, PackFile pack, |
||||||
|
FileObjectDatabase db) { |
||||||
|
this.type = type; |
||||||
|
this.size = size; |
||||||
|
this.objectOffset = objectOffset; |
||||||
|
this.baseOffset = baseOffset; |
||||||
|
this.headerLength = headerLength; |
||||||
|
this.pack = pack; |
||||||
|
this.db = db; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public int getType() { |
||||||
|
return type; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public long getSize() { |
||||||
|
return size; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public byte[] getCachedBytes() throws LargeObjectException { |
||||||
|
try { |
||||||
|
throw new LargeObjectException(getObjectId()); |
||||||
|
} catch (IOException cannotObtainId) { |
||||||
|
throw new LargeObjectException(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public ObjectStream openStream() throws MissingObjectException, IOException { |
||||||
|
final WindowCursor wc = new WindowCursor(db); |
||||||
|
InputStream in = open(wc); |
||||||
|
in = new BufferedInputStream(in, 8192); |
||||||
|
return new ObjectStream.Filter(type, size, in) { |
||||||
|
@Override |
||||||
|
public void close() throws IOException { |
||||||
|
wc.release(); |
||||||
|
super.close(); |
||||||
|
} |
||||||
|
}; |
||||||
|
} |
||||||
|
|
||||||
|
private InputStream open(final WindowCursor wc) |
||||||
|
throws MissingObjectException, IOException, |
||||||
|
IncorrectObjectTypeException { |
||||||
|
InputStream delta; |
||||||
|
try { |
||||||
|
delta = new PackInputStream(pack, objectOffset + headerLength, wc); |
||||||
|
} catch (IOException packGone) { |
||||||
|
// If the pack file cannot be pinned into the cursor, it
|
||||||
|
// probably was repacked recently. Go find the object
|
||||||
|
// again and open the stream from that location instead.
|
||||||
|
//
|
||||||
|
return wc.open(getObjectId(), type).openStream(); |
||||||
|
} |
||||||
|
delta = new InflaterInputStream(delta); |
||||||
|
|
||||||
|
final ObjectLoader base = pack.load(wc, baseOffset); |
||||||
|
return new DeltaStream(delta) { |
||||||
|
@Override |
||||||
|
protected InputStream openBase() throws IOException { |
||||||
|
if (base instanceof LargePackedDeltaObject) |
||||||
|
return ((LargePackedDeltaObject) base).open(wc); |
||||||
|
return base.openStream(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected long getBaseSize() throws IOException { |
||||||
|
return base.getSize(); |
||||||
|
} |
||||||
|
}; |
||||||
|
} |
||||||
|
|
||||||
|
private ObjectId getObjectId() throws IOException { |
||||||
|
return pack.findObjectForOffset(objectOffset); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,202 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2010, Google Inc. |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.eclipse.jgit.storage.pack; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.io.OutputStream; |
||||||
|
|
||||||
|
import org.eclipse.jgit.lib.Constants; |
||||||
|
|
||||||
|
/** Encodes an instruction stream for {@link BinaryDelta}. */ |
||||||
|
public class DeltaEncoder { |
||||||
|
private static final int MAX_COPY = (0xff << 16) + (0xff << 8) + 0xff; |
||||||
|
|
||||||
|
private final OutputStream out; |
||||||
|
|
||||||
|
private final byte[] buf = new byte[16]; |
||||||
|
|
||||||
|
private int size; |
||||||
|
|
||||||
|
/** |
||||||
|
* Create an encoder. |
||||||
|
* |
||||||
|
* @param out |
||||||
|
* buffer to store the instructions written. |
||||||
|
* @param baseSize |
||||||
|
* size of the base object, in bytes. |
||||||
|
* @param resultSize |
||||||
|
* size of the resulting object, after applying this instruction |
||||||
|
* stream to the base object, in bytes. |
||||||
|
* @throws IOException |
||||||
|
* the output buffer cannot store the instruction stream's |
||||||
|
* header with the size fields. |
||||||
|
*/ |
||||||
|
public DeltaEncoder(OutputStream out, long baseSize, long resultSize) |
||||||
|
throws IOException { |
||||||
|
this.out = out; |
||||||
|
writeVarint(baseSize); |
||||||
|
writeVarint(resultSize); |
||||||
|
} |
||||||
|
|
||||||
|
private void writeVarint(long sz) throws IOException { |
||||||
|
int p = 0; |
||||||
|
while (sz > 0x80) { |
||||||
|
buf[p++] = (byte) (0x80 | (((int) sz) & 0x7f)); |
||||||
|
sz >>>= 7; |
||||||
|
} |
||||||
|
buf[p++] = (byte) (((int) sz) & 0x7f); |
||||||
|
out.write(buf, 0, p); |
||||||
|
size += p; |
||||||
|
} |
||||||
|
|
||||||
|
/** @return current size of the delta stream, in bytes. */ |
||||||
|
public int getSize() { |
||||||
|
return size; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Insert a literal string of text, in UTF-8 encoding. |
||||||
|
* |
||||||
|
* @param text |
||||||
|
* the string to insert. |
||||||
|
* @throws IOException |
||||||
|
* the instruction buffer can't store the instructions. |
||||||
|
*/ |
||||||
|
public void insert(String text) throws IOException { |
||||||
|
insert(Constants.encode(text)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Insert a literal binary sequence. |
||||||
|
* |
||||||
|
* @param text |
||||||
|
* the binary to insert. |
||||||
|
* @throws IOException |
||||||
|
* the instruction buffer can't store the instructions. |
||||||
|
*/ |
||||||
|
public void insert(byte[] text) throws IOException { |
||||||
|
insert(text, 0, text.length); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Insert a literal binary sequence. |
||||||
|
* |
||||||
|
* @param text |
||||||
|
* the binary to insert. |
||||||
|
* @param off |
||||||
|
* offset within {@code text} to start copying from. |
||||||
|
* @param cnt |
||||||
|
* number of bytes to insert. |
||||||
|
* @throws IOException |
||||||
|
* the instruction buffer can't store the instructions. |
||||||
|
*/ |
||||||
|
public void insert(byte[] text, int off, int cnt) throws IOException { |
||||||
|
while (0 < cnt) { |
||||||
|
int n = Math.min(127, cnt); |
||||||
|
out.write((byte) n); |
||||||
|
out.write(text, off, n); |
||||||
|
off += n; |
||||||
|
cnt -= n; |
||||||
|
size += 1 + n; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Create a copy instruction to copy from the base object. |
||||||
|
* |
||||||
|
* @param offset |
||||||
|
* position in the base object to copy from. This is absolute, |
||||||
|
* from the beginning of the base. |
||||||
|
* @param cnt |
||||||
|
* number of bytes to copy. |
||||||
|
* @throws IOException |
||||||
|
* the instruction buffer cannot store the instructions. |
||||||
|
*/ |
||||||
|
public void copy(long offset, int cnt) throws IOException { |
||||||
|
if (cnt > MAX_COPY) { |
||||||
|
copy(offset, MAX_COPY); |
||||||
|
offset += MAX_COPY; |
||||||
|
cnt -= MAX_COPY; |
||||||
|
} |
||||||
|
|
||||||
|
int cmd = 0x80; |
||||||
|
int p = 1; |
||||||
|
|
||||||
|
if ((offset & 0xff) != 0) { |
||||||
|
cmd |= 0x01; |
||||||
|
buf[p++] = (byte) (offset & 0xff); |
||||||
|
} |
||||||
|
if ((offset & (0xff << 8)) != 0) { |
||||||
|
cmd |= 0x02; |
||||||
|
buf[p++] = (byte) ((offset >>> 8) & 0xff); |
||||||
|
} |
||||||
|
if ((offset & (0xff << 16)) != 0) { |
||||||
|
cmd |= 0x04; |
||||||
|
buf[p++] = (byte) ((offset >>> 16) & 0xff); |
||||||
|
} |
||||||
|
if ((offset & (0xff << 24)) != 0) { |
||||||
|
cmd |= 0x08; |
||||||
|
buf[p++] = (byte) ((offset >>> 24) & 0xff); |
||||||
|
} |
||||||
|
|
||||||
|
if (cnt != 0x10000) { |
||||||
|
if ((cnt & 0xff) != 0) { |
||||||
|
cmd |= 0x10; |
||||||
|
buf[p++] = (byte) (cnt & 0xff); |
||||||
|
} |
||||||
|
if ((cnt & (0xff << 8)) != 0) { |
||||||
|
cmd |= 0x20; |
||||||
|
buf[p++] = (byte) ((cnt >>> 8) & 0xff); |
||||||
|
} |
||||||
|
if ((cnt & (0xff << 16)) != 0) { |
||||||
|
cmd |= 0x40; |
||||||
|
buf[p++] = (byte) ((cnt >>> 16) & 0xff); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
buf[0] = (byte) cmd; |
||||||
|
out.write(buf, 0, p); |
||||||
|
size += p; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,341 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com> |
||||||
|
* Copyright (C) 2006-2007, Shawn O. Pearce <spearce@spearce.org> |
||||||
|
* Copyright (C) 2010, Google Inc. |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.eclipse.jgit.storage.pack; |
||||||
|
|
||||||
|
import java.io.EOFException; |
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStream; |
||||||
|
|
||||||
|
import org.eclipse.jgit.JGitText; |
||||||
|
import org.eclipse.jgit.errors.CorruptObjectException; |
||||||
|
import org.eclipse.jgit.util.IO; |
||||||
|
|
||||||
|
/** |
||||||
|
* Inflates a delta in an incremental way. |
||||||
|
* <p> |
||||||
|
* Implementations must provide a means to access a stream for the base object. |
||||||
|
* This stream may be accessed multiple times, in order to randomly position it |
||||||
|
* to match the copy instructions. A {@code DeltaStream} performs an efficient |
||||||
|
* skip by only moving through the delta stream, making restarts of stacked |
||||||
|
* deltas reasonably efficient. |
||||||
|
*/ |
||||||
|
public abstract class DeltaStream extends InputStream { |
||||||
|
private static final int CMD_COPY = 0; |
||||||
|
|
||||||
|
private static final int CMD_INSERT = 1; |
||||||
|
|
||||||
|
private static final int CMD_EOF = 2; |
||||||
|
|
||||||
|
private final InputStream deltaStream; |
||||||
|
|
||||||
|
private long baseSize; |
||||||
|
|
||||||
|
private long resultSize; |
||||||
|
|
||||||
|
private final byte[] cmdbuf = new byte[512]; |
||||||
|
|
||||||
|
private int cmdptr; |
||||||
|
|
||||||
|
private int cmdcnt; |
||||||
|
|
||||||
|
/** Stream to read from the base object. */ |
||||||
|
private InputStream baseStream; |
||||||
|
|
||||||
|
/** Current position within {@link #baseStream}. */ |
||||||
|
private long baseOffset; |
||||||
|
|
||||||
|
private int curcmd; |
||||||
|
|
||||||
|
/** If {@code curcmd == CMD_COPY}, position the base has to be at. */ |
||||||
|
private long copyOffset; |
||||||
|
|
||||||
|
/** Total number of bytes in this current command. */ |
||||||
|
private int copySize; |
||||||
|
|
||||||
|
/** |
||||||
|
* Construct a delta application stream, reading instructions. |
||||||
|
* |
||||||
|
* @param deltaStream |
||||||
|
* the stream to read delta instructions from. |
||||||
|
* @throws IOException |
||||||
|
* the delta instruction stream cannot be read, or is |
||||||
|
* inconsistent with the the base object information. |
||||||
|
*/ |
||||||
|
public DeltaStream(final InputStream deltaStream) throws IOException { |
||||||
|
this.deltaStream = deltaStream; |
||||||
|
if (!fill(cmdbuf.length)) |
||||||
|
throw new EOFException(); |
||||||
|
|
||||||
|
// Length of the base object.
|
||||||
|
//
|
||||||
|
int c, shift = 0; |
||||||
|
do { |
||||||
|
c = cmdbuf[cmdptr++] & 0xff; |
||||||
|
baseSize |= (c & 0x7f) << shift; |
||||||
|
shift += 7; |
||||||
|
} while ((c & 0x80) != 0); |
||||||
|
|
||||||
|
// Length of the resulting object.
|
||||||
|
//
|
||||||
|
shift = 0; |
||||||
|
do { |
||||||
|
c = cmdbuf[cmdptr++] & 0xff; |
||||||
|
resultSize |= (c & 0x7f) << shift; |
||||||
|
shift += 7; |
||||||
|
} while ((c & 0x80) != 0); |
||||||
|
|
||||||
|
curcmd = next(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Open the base stream. |
||||||
|
* <p> |
||||||
|
* The {@code DeltaStream} may close and reopen the base stream multiple |
||||||
|
* times if copy instructions use offsets out of order. This can occur if a |
||||||
|
* large block in the file was moved from near the top, to near the bottom. |
||||||
|
* In such cases the reopened stream is skipped to the target offset, so |
||||||
|
* {@code skip(long)} should be as efficient as possible. |
||||||
|
* |
||||||
|
* @return stream to read from the base object. This stream should not be |
||||||
|
* buffered (or should be only minimally buffered), and does not |
||||||
|
* need to support mark/reset. |
||||||
|
* @throws IOException |
||||||
|
* the base object cannot be opened for reading. |
||||||
|
*/ |
||||||
|
protected abstract InputStream openBase() throws IOException; |
||||||
|
|
||||||
|
/** |
||||||
|
* @return length of the base object, in bytes. |
||||||
|
* @throws IOException |
||||||
|
* the length of the base cannot be determined. |
||||||
|
*/ |
||||||
|
protected abstract long getBaseSize() throws IOException; |
||||||
|
|
||||||
|
/** @return total size of this stream, in bytes. */ |
||||||
|
public long getSize() { |
||||||
|
return resultSize; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public int read() throws IOException { |
||||||
|
byte[] buf = new byte[1]; |
||||||
|
int n = read(buf, 0, 1); |
||||||
|
return n == 1 ? buf[0] & 0xff : -1; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void close() throws IOException { |
||||||
|
deltaStream.close(); |
||||||
|
if (baseStream != null) |
||||||
|
baseStream.close(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public long skip(long len) throws IOException { |
||||||
|
long act = 0; |
||||||
|
while (0 < len) { |
||||||
|
long n = Math.min(len, copySize); |
||||||
|
switch (curcmd) { |
||||||
|
case CMD_COPY: |
||||||
|
copyOffset += n; |
||||||
|
break; |
||||||
|
|
||||||
|
case CMD_INSERT: |
||||||
|
cmdptr += n; |
||||||
|
break; |
||||||
|
|
||||||
|
case CMD_EOF: |
||||||
|
return act; |
||||||
|
default: |
||||||
|
throw new CorruptObjectException( |
||||||
|
JGitText.get().unsupportedCommand0); |
||||||
|
} |
||||||
|
|
||||||
|
act += n; |
||||||
|
len -= n; |
||||||
|
copySize -= n; |
||||||
|
if (copySize == 0) |
||||||
|
curcmd = next(); |
||||||
|
} |
||||||
|
return act; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public int read(byte[] buf, int off, int len) throws IOException { |
||||||
|
int act = 0; |
||||||
|
while (0 < len) { |
||||||
|
int n = Math.min(len, copySize); |
||||||
|
switch (curcmd) { |
||||||
|
case CMD_COPY: |
||||||
|
seekBase(); |
||||||
|
n = baseStream.read(buf, off, n); |
||||||
|
if (n < 0) |
||||||
|
throw new CorruptObjectException( |
||||||
|
JGitText.get().baseLengthIncorrect); |
||||||
|
baseOffset += n; |
||||||
|
break; |
||||||
|
|
||||||
|
case CMD_INSERT: |
||||||
|
System.arraycopy(cmdbuf, cmdptr, buf, off, n); |
||||||
|
cmdptr += n; |
||||||
|
break; |
||||||
|
|
||||||
|
case CMD_EOF: |
||||||
|
return 0 < act ? act : -1; |
||||||
|
default: |
||||||
|
throw new CorruptObjectException( |
||||||
|
JGitText.get().unsupportedCommand0); |
||||||
|
} |
||||||
|
|
||||||
|
act += n; |
||||||
|
off += n; |
||||||
|
len -= n; |
||||||
|
copySize -= n; |
||||||
|
if (copySize == 0) |
||||||
|
curcmd = next(); |
||||||
|
} |
||||||
|
return act; |
||||||
|
} |
||||||
|
|
||||||
|
private boolean fill(final int need) throws IOException { |
||||||
|
int n = have(); |
||||||
|
if (need < n) |
||||||
|
return true; |
||||||
|
if (n == 0) { |
||||||
|
cmdptr = 0; |
||||||
|
cmdcnt = 0; |
||||||
|
} else if (cmdbuf.length - cmdptr < need) { |
||||||
|
// There isn't room for the entire worst-case copy command,
|
||||||
|
// so shift the array down to make sure we can use the entire
|
||||||
|
// command without having it span across the end of the array.
|
||||||
|
//
|
||||||
|
System.arraycopy(cmdbuf, cmdptr, cmdbuf, 0, n); |
||||||
|
cmdptr = 0; |
||||||
|
cmdcnt = n; |
||||||
|
} |
||||||
|
|
||||||
|
do { |
||||||
|
n = deltaStream.read(cmdbuf, cmdcnt, cmdbuf.length - cmdcnt); |
||||||
|
if (n < 0) |
||||||
|
return 0 < have(); |
||||||
|
cmdcnt += n; |
||||||
|
} while (cmdcnt < cmdbuf.length); |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
private int next() throws IOException { |
||||||
|
if (!fill(8)) |
||||||
|
return CMD_EOF; |
||||||
|
|
||||||
|
final int cmd = cmdbuf[cmdptr++] & 0xff; |
||||||
|
if ((cmd & 0x80) != 0) { |
||||||
|
// Determine the segment of the base which should
|
||||||
|
// be copied into the output. The segment is given
|
||||||
|
// as an offset and a length.
|
||||||
|
//
|
||||||
|
copyOffset = 0; |
||||||
|
if ((cmd & 0x01) != 0) |
||||||
|
copyOffset = cmdbuf[cmdptr++] & 0xff; |
||||||
|
if ((cmd & 0x02) != 0) |
||||||
|
copyOffset |= (cmdbuf[cmdptr++] & 0xff) << 8; |
||||||
|
if ((cmd & 0x04) != 0) |
||||||
|
copyOffset |= (cmdbuf[cmdptr++] & 0xff) << 16; |
||||||
|
if ((cmd & 0x08) != 0) |
||||||
|
copyOffset |= (cmdbuf[cmdptr++] & 0xff) << 24; |
||||||
|
|
||||||
|
copySize = 0; |
||||||
|
if ((cmd & 0x10) != 0) |
||||||
|
copySize = cmdbuf[cmdptr++] & 0xff; |
||||||
|
if ((cmd & 0x20) != 0) |
||||||
|
copySize |= (cmdbuf[cmdptr++] & 0xff) << 8; |
||||||
|
if ((cmd & 0x40) != 0) |
||||||
|
copySize |= (cmdbuf[cmdptr++] & 0xff) << 16; |
||||||
|
if (copySize == 0) |
||||||
|
copySize = 0x10000; |
||||||
|
return CMD_COPY; |
||||||
|
|
||||||
|
} else if (cmd != 0) { |
||||||
|
// Anything else the data is literal within the delta
|
||||||
|
// itself. Page the entire thing into the cmdbuf, if
|
||||||
|
// its not already there.
|
||||||
|
//
|
||||||
|
fill(cmd); |
||||||
|
copySize = cmd; |
||||||
|
return CMD_INSERT; |
||||||
|
|
||||||
|
} else { |
||||||
|
// cmd == 0 has been reserved for future encoding but
|
||||||
|
// for now its not acceptable.
|
||||||
|
//
|
||||||
|
throw new CorruptObjectException(JGitText.get().unsupportedCommand0); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private int have() { |
||||||
|
return cmdcnt - cmdptr; |
||||||
|
} |
||||||
|
|
||||||
|
private void seekBase() throws IOException { |
||||||
|
if (baseStream == null) { |
||||||
|
baseStream = openBase(); |
||||||
|
if (getBaseSize() != baseSize) |
||||||
|
throw new CorruptObjectException( |
||||||
|
JGitText.get().baseLengthIncorrect); |
||||||
|
IO.skipFully(baseStream, copyOffset); |
||||||
|
baseOffset = copyOffset; |
||||||
|
|
||||||
|
} else if (baseOffset < copyOffset) { |
||||||
|
IO.skipFully(baseStream, copyOffset - baseOffset); |
||||||
|
baseOffset = copyOffset; |
||||||
|
|
||||||
|
} else if (baseOffset > copyOffset) { |
||||||
|
baseStream.close(); |
||||||
|
baseStream = openBase(); |
||||||
|
IO.skipFully(baseStream, copyOffset); |
||||||
|
baseOffset = copyOffset; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue