Browse Source
* delta: (103 commits) Discard the uncompressed delta as soon as its compressed Honor pack.windowlimit to cap memory usage during packing Honor pack.threads and perform delta search in parallel Cache small deltas during packing Implement delta generation during packing debug-show-packdelta: Dump a pack delta to the console Initial pack format delta generator Add debugging toString() method to ObjectToPack Make ObjectToPack clearReuseAsIs signal available to subclasses Correctly classify the compressing objects phase Refactor ObjectToPack's delta depth setting Configure core.bigFileThreshold into PackWriter Add doNotDelta flag to ObjectToPack Add more configuration options to PackWriter Save object path hash codes during packing Add path hash code to ObjectWalk Add getObjectSize to ObjectReader Allow TemporaryBuffer.Heap to allocate smaller than 8 KiB Define a constant for 127 in DeltaEncoder Cap delta copy instructions at 64k ... Conflicts: org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Diff.java org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RewriteTreeFilter.java Change-Id: I7c7a05e443a48d32c836173a409ee7d340c70796stable-0.9
Shawn O. Pearce
15 years ago
243 changed files with 14011 additions and 5615 deletions
@ -0,0 +1,127 @@
|
||||
/* |
||||
* Copyright (C) 2010, Google Inc. |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
|
||||
package org.eclipse.jgit.pgm.debug; |
||||
|
||||
import java.io.ByteArrayInputStream; |
||||
import java.io.ByteArrayOutputStream; |
||||
import java.io.IOException; |
||||
import java.util.zip.InflaterInputStream; |
||||
|
||||
import org.eclipse.jgit.errors.MissingObjectException; |
||||
import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; |
||||
import org.eclipse.jgit.lib.NullProgressMonitor; |
||||
import org.eclipse.jgit.lib.ObjectId; |
||||
import org.eclipse.jgit.lib.ObjectReader; |
||||
import org.eclipse.jgit.pgm.TextBuiltin; |
||||
import org.eclipse.jgit.revwalk.RevObject; |
||||
import org.eclipse.jgit.revwalk.RevWalk; |
||||
import org.eclipse.jgit.storage.pack.BinaryDelta; |
||||
import org.eclipse.jgit.storage.pack.ObjectReuseAsIs; |
||||
import org.eclipse.jgit.storage.pack.ObjectToPack; |
||||
import org.eclipse.jgit.storage.pack.PackOutputStream; |
||||
import org.eclipse.jgit.storage.pack.PackWriter; |
||||
import org.eclipse.jgit.storage.pack.StoredObjectRepresentation; |
||||
import org.eclipse.jgit.util.TemporaryBuffer; |
||||
import org.kohsuke.args4j.Argument; |
||||
|
||||
class ShowPackDelta extends TextBuiltin { |
||||
@Argument(index = 0) |
||||
private ObjectId objectId; |
||||
|
||||
@Override |
||||
protected void run() throws Exception { |
||||
ObjectReader reader = db.newObjectReader(); |
||||
RevObject obj = new RevWalk(reader).parseAny(objectId); |
||||
byte[] delta = getDelta(reader, obj); |
||||
|
||||
// We're crossing our fingers that this will be a delta. Double
|
||||
// check the size field in the header, it should match.
|
||||
//
|
||||
long size = reader.getObjectSize(obj, obj.getType()); |
||||
try { |
||||
if (BinaryDelta.getResultSize(delta) != size) |
||||
throw die("Object " + obj.name() + " is not a delta"); |
||||
} catch (ArrayIndexOutOfBoundsException bad) { |
||||
throw die("Object " + obj.name() + " is not a delta"); |
||||
} |
||||
|
||||
out.println(BinaryDelta.format(delta)); |
||||
} |
||||
|
||||
private byte[] getDelta(ObjectReader reader, RevObject obj) |
||||
throws IOException, MissingObjectException, |
||||
StoredObjectRepresentationNotAvailableException { |
||||
ObjectReuseAsIs asis = (ObjectReuseAsIs) reader; |
||||
ObjectToPack target = asis.newObjectToPack(obj); |
||||
|
||||
PackWriter pw = new PackWriter(reader) { |
||||
@Override |
||||
public void select(ObjectToPack otp, StoredObjectRepresentation next) { |
||||
otp.select(next); |
||||
} |
||||
}; |
||||
|
||||
ByteArrayOutputStream buf = new ByteArrayOutputStream(); |
||||
asis.selectObjectRepresentation(pw, target); |
||||
asis.copyObjectAsIs(new PackOutputStream(NullProgressMonitor.INSTANCE, |
||||
buf, pw), target); |
||||
|
||||
// At this point the object header has no delta information,
|
||||
// because it was output as though it were a whole object.
|
||||
// Skip over the header and inflate.
|
||||
//
|
||||
byte[] bufArray = buf.toByteArray(); |
||||
int ptr = 0; |
||||
while ((bufArray[ptr] & 0x80) != 0) |
||||
ptr++; |
||||
ptr++; |
||||
|
||||
TemporaryBuffer.Heap raw = new TemporaryBuffer.Heap(bufArray.length); |
||||
InflaterInputStream inf = new InflaterInputStream( |
||||
new ByteArrayInputStream(bufArray, ptr, bufArray.length)); |
||||
raw.copy(inf); |
||||
inf.close(); |
||||
return raw.toByteArray(); |
||||
} |
||||
} |
||||
@ -0,0 +1,387 @@
|
||||
/* |
||||
* Copyright (C) 2010, Google Inc. |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
|
||||
package org.eclipse.jgit.storage.file; |
||||
|
||||
import java.io.ByteArrayInputStream; |
||||
import java.io.ByteArrayOutputStream; |
||||
import java.io.IOException; |
||||
import java.security.MessageDigest; |
||||
import java.util.Arrays; |
||||
import java.util.zip.Deflater; |
||||
|
||||
import org.eclipse.jgit.errors.LargeObjectException; |
||||
import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; |
||||
import org.eclipse.jgit.junit.TestRepository; |
||||
import org.eclipse.jgit.junit.TestRng; |
||||
import org.eclipse.jgit.lib.Constants; |
||||
import org.eclipse.jgit.lib.NullProgressMonitor; |
||||
import org.eclipse.jgit.lib.ObjectId; |
||||
import org.eclipse.jgit.lib.ObjectInserter; |
||||
import org.eclipse.jgit.lib.ObjectLoader; |
||||
import org.eclipse.jgit.lib.ObjectStream; |
||||
import org.eclipse.jgit.revwalk.RevBlob; |
||||
import org.eclipse.jgit.storage.pack.DeltaEncoder; |
||||
import org.eclipse.jgit.transport.IndexPack; |
||||
import org.eclipse.jgit.util.IO; |
||||
import org.eclipse.jgit.util.NB; |
||||
import org.eclipse.jgit.util.TemporaryBuffer; |
||||
|
||||
public class PackFileTest extends LocalDiskRepositoryTestCase { |
||||
private TestRng rng; |
||||
|
||||
private FileRepository repo; |
||||
|
||||
private TestRepository<FileRepository> tr; |
||||
|
||||
private WindowCursor wc; |
||||
|
||||
protected void setUp() throws Exception { |
||||
super.setUp(); |
||||
rng = new TestRng(getName()); |
||||
repo = createBareRepository(); |
||||
tr = new TestRepository<FileRepository>(repo); |
||||
wc = (WindowCursor) repo.newObjectReader(); |
||||
} |
||||
|
||||
protected void tearDown() throws Exception { |
||||
if (wc != null) |
||||
wc.release(); |
||||
super.tearDown(); |
||||
} |
||||
|
||||
public void testWhole_SmallObject() throws Exception { |
||||
final int type = Constants.OBJ_BLOB; |
||||
byte[] data = rng.nextBytes(300); |
||||
RevBlob id = tr.blob(data); |
||||
tr.branch("master").commit().add("A", id).create(); |
||||
tr.packAndPrune(); |
||||
assertTrue("has blob", wc.has(id)); |
||||
|
||||
ObjectLoader ol = wc.open(id); |
||||
assertNotNull("created loader", ol); |
||||
assertEquals(type, ol.getType()); |
||||
assertEquals(data.length, ol.getSize()); |
||||
assertFalse("is not large", ol.isLarge()); |
||||
assertTrue("same content", Arrays.equals(data, ol.getCachedBytes())); |
||||
|
||||
ObjectStream in = ol.openStream(); |
||||
assertNotNull("have stream", in); |
||||
assertEquals(type, in.getType()); |
||||
assertEquals(data.length, in.getSize()); |
||||
byte[] data2 = new byte[data.length]; |
||||
IO.readFully(in, data2, 0, data.length); |
||||
assertTrue("same content", Arrays.equals(data2, data)); |
||||
assertEquals("stream at EOF", -1, in.read()); |
||||
in.close(); |
||||
} |
||||
|
||||
public void testWhole_LargeObject() throws Exception { |
||||
final int type = Constants.OBJ_BLOB; |
||||
byte[] data = rng.nextBytes(ObjectLoader.STREAM_THRESHOLD + 5); |
||||
RevBlob id = tr.blob(data); |
||||
tr.branch("master").commit().add("A", id).create(); |
||||
tr.packAndPrune(); |
||||
assertTrue("has blob", wc.has(id)); |
||||
|
||||
ObjectLoader ol = wc.open(id); |
||||
assertNotNull("created loader", ol); |
||||
assertEquals(type, ol.getType()); |
||||
assertEquals(data.length, ol.getSize()); |
||||
assertTrue("is large", ol.isLarge()); |
||||
try { |
||||
ol.getCachedBytes(); |
||||
fail("Should have thrown LargeObjectException"); |
||||
} catch (LargeObjectException tooBig) { |
||||
assertEquals(id.name(), tooBig.getMessage()); |
||||
} |
||||
|
||||
ObjectStream in = ol.openStream(); |
||||
assertNotNull("have stream", in); |
||||
assertEquals(type, in.getType()); |
||||
assertEquals(data.length, in.getSize()); |
||||
byte[] data2 = new byte[data.length]; |
||||
IO.readFully(in, data2, 0, data.length); |
||||
assertTrue("same content", Arrays.equals(data2, data)); |
||||
assertEquals("stream at EOF", -1, in.read()); |
||||
in.close(); |
||||
} |
||||
|
||||
public void testDelta_SmallObjectChain() throws Exception { |
||||
ObjectInserter.Formatter fmt = new ObjectInserter.Formatter(); |
||||
byte[] data0 = new byte[512]; |
||||
Arrays.fill(data0, (byte) 0xf3); |
||||
ObjectId id0 = fmt.idFor(Constants.OBJ_BLOB, data0); |
||||
|
||||
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); |
||||
packHeader(pack, 4); |
||||
objectHeader(pack, Constants.OBJ_BLOB, data0.length); |
||||
deflate(pack, data0); |
||||
|
||||
byte[] data1 = clone(0x01, data0); |
||||
byte[] delta1 = delta(data0, data1); |
||||
ObjectId id1 = fmt.idFor(Constants.OBJ_BLOB, data1); |
||||
objectHeader(pack, Constants.OBJ_REF_DELTA, delta1.length); |
||||
id0.copyRawTo(pack); |
||||
deflate(pack, delta1); |
||||
|
||||
byte[] data2 = clone(0x02, data1); |
||||
byte[] delta2 = delta(data1, data2); |
||||
ObjectId id2 = fmt.idFor(Constants.OBJ_BLOB, data2); |
||||
objectHeader(pack, Constants.OBJ_REF_DELTA, delta2.length); |
||||
id1.copyRawTo(pack); |
||||
deflate(pack, delta2); |
||||
|
||||
byte[] data3 = clone(0x03, data2); |
||||
byte[] delta3 = delta(data2, data3); |
||||
ObjectId id3 = fmt.idFor(Constants.OBJ_BLOB, data3); |
||||
objectHeader(pack, Constants.OBJ_REF_DELTA, delta3.length); |
||||
id2.copyRawTo(pack); |
||||
deflate(pack, delta3); |
||||
|
||||
digest(pack); |
||||
final byte[] raw = pack.toByteArray(); |
||||
IndexPack ip = IndexPack.create(repo, new ByteArrayInputStream(raw)); |
||||
ip.setFixThin(true); |
||||
ip.index(NullProgressMonitor.INSTANCE); |
||||
ip.renameAndOpenPack(); |
||||
|
||||
assertTrue("has blob", wc.has(id3)); |
||||
|
||||
ObjectLoader ol = wc.open(id3); |
||||
assertNotNull("created loader", ol); |
||||
assertEquals(Constants.OBJ_BLOB, ol.getType()); |
||||
assertEquals(data3.length, ol.getSize()); |
||||
assertFalse("is large", ol.isLarge()); |
||||
assertNotNull(ol.getCachedBytes()); |
||||
assertTrue(Arrays.equals(data3, ol.getCachedBytes())); |
||||
|
||||
ObjectStream in = ol.openStream(); |
||||
assertNotNull("have stream", in); |
||||
assertEquals(Constants.OBJ_BLOB, in.getType()); |
||||
assertEquals(data3.length, in.getSize()); |
||||
byte[] act = new byte[data3.length]; |
||||
IO.readFully(in, act, 0, data3.length); |
||||
assertTrue("same content", Arrays.equals(act, data3)); |
||||
assertEquals("stream at EOF", -1, in.read()); |
||||
in.close(); |
||||
} |
||||
|
||||
public void testDelta_LargeObjectChain() throws Exception { |
||||
ObjectInserter.Formatter fmt = new ObjectInserter.Formatter(); |
||||
byte[] data0 = new byte[ObjectLoader.STREAM_THRESHOLD + 5]; |
||||
Arrays.fill(data0, (byte) 0xf3); |
||||
ObjectId id0 = fmt.idFor(Constants.OBJ_BLOB, data0); |
||||
|
||||
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); |
||||
packHeader(pack, 4); |
||||
objectHeader(pack, Constants.OBJ_BLOB, data0.length); |
||||
deflate(pack, data0); |
||||
|
||||
byte[] data1 = clone(0x01, data0); |
||||
byte[] delta1 = delta(data0, data1); |
||||
ObjectId id1 = fmt.idFor(Constants.OBJ_BLOB, data1); |
||||
objectHeader(pack, Constants.OBJ_REF_DELTA, delta1.length); |
||||
id0.copyRawTo(pack); |
||||
deflate(pack, delta1); |
||||
|
||||
byte[] data2 = clone(0x02, data1); |
||||
byte[] delta2 = delta(data1, data2); |
||||
ObjectId id2 = fmt.idFor(Constants.OBJ_BLOB, data2); |
||||
objectHeader(pack, Constants.OBJ_REF_DELTA, delta2.length); |
||||
id1.copyRawTo(pack); |
||||
deflate(pack, delta2); |
||||
|
||||
byte[] data3 = clone(0x03, data2); |
||||
byte[] delta3 = delta(data2, data3); |
||||
ObjectId id3 = fmt.idFor(Constants.OBJ_BLOB, data3); |
||||
objectHeader(pack, Constants.OBJ_REF_DELTA, delta3.length); |
||||
id2.copyRawTo(pack); |
||||
deflate(pack, delta3); |
||||
|
||||
digest(pack); |
||||
final byte[] raw = pack.toByteArray(); |
||||
IndexPack ip = IndexPack.create(repo, new ByteArrayInputStream(raw)); |
||||
ip.setFixThin(true); |
||||
ip.index(NullProgressMonitor.INSTANCE); |
||||
ip.renameAndOpenPack(); |
||||
|
||||
assertTrue("has blob", wc.has(id3)); |
||||
|
||||
ObjectLoader ol = wc.open(id3); |
||||
assertNotNull("created loader", ol); |
||||
assertEquals(Constants.OBJ_BLOB, ol.getType()); |
||||
assertEquals(data3.length, ol.getSize()); |
||||
assertTrue("is large", ol.isLarge()); |
||||
try { |
||||
ol.getCachedBytes(); |
||||
fail("Should have thrown LargeObjectException"); |
||||
} catch (LargeObjectException tooBig) { |
||||
assertEquals(id3.name(), tooBig.getMessage()); |
||||
} |
||||
|
||||
ObjectStream in = ol.openStream(); |
||||
assertNotNull("have stream", in); |
||||
assertEquals(Constants.OBJ_BLOB, in.getType()); |
||||
assertEquals(data3.length, in.getSize()); |
||||
byte[] act = new byte[data3.length]; |
||||
IO.readFully(in, act, 0, data3.length); |
||||
assertTrue("same content", Arrays.equals(act, data3)); |
||||
assertEquals("stream at EOF", -1, in.read()); |
||||
in.close(); |
||||
} |
||||
|
||||
public void testDelta_LargeInstructionStream() throws Exception { |
||||
ObjectInserter.Formatter fmt = new ObjectInserter.Formatter(); |
||||
byte[] data0 = new byte[32]; |
||||
Arrays.fill(data0, (byte) 0xf3); |
||||
ObjectId id0 = fmt.idFor(Constants.OBJ_BLOB, data0); |
||||
|
||||
byte[] data3 = rng.nextBytes(ObjectLoader.STREAM_THRESHOLD + 5); |
||||
ByteArrayOutputStream tmp = new ByteArrayOutputStream(); |
||||
DeltaEncoder de = new DeltaEncoder(tmp, data0.length, data3.length); |
||||
de.insert(data3, 0, data3.length); |
||||
byte[] delta3 = tmp.toByteArray(); |
||||
assertTrue(delta3.length > ObjectLoader.STREAM_THRESHOLD); |
||||
|
||||
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); |
||||
packHeader(pack, 2); |
||||
objectHeader(pack, Constants.OBJ_BLOB, data0.length); |
||||
deflate(pack, data0); |
||||
|
||||
ObjectId id3 = fmt.idFor(Constants.OBJ_BLOB, data3); |
||||
objectHeader(pack, Constants.OBJ_REF_DELTA, delta3.length); |
||||
id0.copyRawTo(pack); |
||||
deflate(pack, delta3); |
||||
|
||||
digest(pack); |
||||
final byte[] raw = pack.toByteArray(); |
||||
IndexPack ip = IndexPack.create(repo, new ByteArrayInputStream(raw)); |
||||
ip.setFixThin(true); |
||||
ip.index(NullProgressMonitor.INSTANCE); |
||||
ip.renameAndOpenPack(); |
||||
|
||||
assertTrue("has blob", wc.has(id3)); |
||||
|
||||
ObjectLoader ol = wc.open(id3); |
||||
assertNotNull("created loader", ol); |
||||
assertEquals(Constants.OBJ_BLOB, ol.getType()); |
||||
assertEquals(data3.length, ol.getSize()); |
||||
assertTrue("is large", ol.isLarge()); |
||||
try { |
||||
ol.getCachedBytes(); |
||||
fail("Should have thrown LargeObjectException"); |
||||
} catch (LargeObjectException tooBig) { |
||||
assertEquals(id3.name(), tooBig.getMessage()); |
||||
} |
||||
|
||||
ObjectStream in = ol.openStream(); |
||||
assertNotNull("have stream", in); |
||||
assertEquals(Constants.OBJ_BLOB, in.getType()); |
||||
assertEquals(data3.length, in.getSize()); |
||||
byte[] act = new byte[data3.length]; |
||||
IO.readFully(in, act, 0, data3.length); |
||||
assertTrue("same content", Arrays.equals(act, data3)); |
||||
assertEquals("stream at EOF", -1, in.read()); |
||||
in.close(); |
||||
} |
||||
|
||||
private byte[] clone(int first, byte[] base) { |
||||
byte[] r = new byte[base.length]; |
||||
System.arraycopy(base, 1, r, 1, r.length - 1); |
||||
r[0] = (byte) first; |
||||
return r; |
||||
} |
||||
|
||||
private byte[] delta(byte[] base, byte[] dest) throws IOException { |
||||
ByteArrayOutputStream tmp = new ByteArrayOutputStream(); |
||||
DeltaEncoder de = new DeltaEncoder(tmp, base.length, dest.length); |
||||
de.insert(dest, 0, 1); |
||||
de.copy(1, base.length - 1); |
||||
return tmp.toByteArray(); |
||||
} |
||||
|
||||
private void packHeader(TemporaryBuffer.Heap pack, int cnt) |
||||
throws IOException { |
||||
final byte[] hdr = new byte[8]; |
||||
NB.encodeInt32(hdr, 0, 2); |
||||
NB.encodeInt32(hdr, 4, cnt); |
||||
pack.write(Constants.PACK_SIGNATURE); |
||||
pack.write(hdr, 0, 8); |
||||
} |
||||
|
||||
private void objectHeader(TemporaryBuffer.Heap pack, int type, int sz) |
||||
throws IOException { |
||||
byte[] buf = new byte[8]; |
||||
int nextLength = sz >>> 4; |
||||
buf[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (sz & 0x0F)); |
||||
sz = nextLength; |
||||
int n = 1; |
||||
while (sz > 0) { |
||||
nextLength >>>= 7; |
||||
buf[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (sz & 0x7F)); |
||||
sz = nextLength; |
||||
} |
||||
pack.write(buf, 0, n); |
||||
} |
||||
|
||||
private void deflate(TemporaryBuffer.Heap pack, final byte[] content) |
||||
throws IOException { |
||||
final Deflater deflater = new Deflater(); |
||||
final byte[] buf = new byte[128]; |
||||
deflater.setInput(content, 0, content.length); |
||||
deflater.finish(); |
||||
do { |
||||
final int n = deflater.deflate(buf, 0, buf.length); |
||||
if (n > 0) |
||||
pack.write(buf, 0, n); |
||||
} while (!deflater.finished()); |
||||
deflater.end(); |
||||
} |
||||
|
||||
private void digest(TemporaryBuffer.Heap buf) throws IOException { |
||||
MessageDigest md = Constants.newMessageDigest(); |
||||
md.update(buf.toByteArray()); |
||||
buf.write(md.digest()); |
||||
} |
||||
} |
||||
@ -0,0 +1,534 @@
|
||||
/* |
||||
* Copyright (C) 2010, Google Inc. |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
|
||||
package org.eclipse.jgit.storage.file; |
||||
|
||||
import java.io.ByteArrayInputStream; |
||||
import java.io.ByteArrayOutputStream; |
||||
import java.io.File; |
||||
import java.io.FileInputStream; |
||||
import java.io.FileOutputStream; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.text.MessageFormat; |
||||
import java.util.Arrays; |
||||
import java.util.zip.DeflaterOutputStream; |
||||
|
||||
import org.eclipse.jgit.JGitText; |
||||
import org.eclipse.jgit.errors.CorruptObjectException; |
||||
import org.eclipse.jgit.errors.LargeObjectException; |
||||
import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; |
||||
import org.eclipse.jgit.junit.TestRng; |
||||
import org.eclipse.jgit.lib.Constants; |
||||
import org.eclipse.jgit.lib.ObjectId; |
||||
import org.eclipse.jgit.lib.ObjectInserter; |
||||
import org.eclipse.jgit.lib.ObjectLoader; |
||||
import org.eclipse.jgit.lib.ObjectStream; |
||||
import org.eclipse.jgit.util.IO; |
||||
|
||||
public class UnpackedObjectTest extends LocalDiskRepositoryTestCase { |
||||
private TestRng rng; |
||||
|
||||
private FileRepository repo; |
||||
|
||||
private WindowCursor wc; |
||||
|
||||
protected void setUp() throws Exception { |
||||
super.setUp(); |
||||
rng = new TestRng(getName()); |
||||
repo = createBareRepository(); |
||||
wc = (WindowCursor) repo.newObjectReader(); |
||||
} |
||||
|
||||
protected void tearDown() throws Exception { |
||||
if (wc != null) |
||||
wc.release(); |
||||
super.tearDown(); |
||||
} |
||||
|
||||
public void testStandardFormat_SmallObject() throws Exception { |
||||
final int type = Constants.OBJ_BLOB; |
||||
byte[] data = rng.nextBytes(300); |
||||
byte[] gz = compressStandardFormat(type, data); |
||||
ObjectId id = ObjectId.zeroId(); |
||||
|
||||
ObjectLoader ol = UnpackedObject.open(new ByteArrayInputStream(gz), |
||||
path(id), id, wc); |
||||
assertNotNull("created loader", ol); |
||||
assertEquals(type, ol.getType()); |
||||
assertEquals(data.length, ol.getSize()); |
||||
assertFalse("is not large", ol.isLarge()); |
||||
assertTrue("same content", Arrays.equals(data, ol.getCachedBytes())); |
||||
|
||||
ObjectStream in = ol.openStream(); |
||||
assertNotNull("have stream", in); |
||||
assertEquals(type, in.getType()); |
||||
assertEquals(data.length, in.getSize()); |
||||
byte[] data2 = new byte[data.length]; |
||||
IO.readFully(in, data2, 0, data.length); |
||||
assertTrue("same content", Arrays.equals(data2, data)); |
||||
assertEquals("stream at EOF", -1, in.read()); |
||||
in.close(); |
||||
} |
||||
|
||||
public void testStandardFormat_LargeObject() throws Exception { |
||||
final int type = Constants.OBJ_BLOB; |
||||
byte[] data = rng.nextBytes(ObjectLoader.STREAM_THRESHOLD + 5); |
||||
ObjectId id = new ObjectInserter.Formatter().idFor(type, data); |
||||
write(id, compressStandardFormat(type, data)); |
||||
|
||||
ObjectLoader ol; |
||||
{ |
||||
FileInputStream fs = new FileInputStream(path(id)); |
||||
try { |
||||
ol = UnpackedObject.open(fs, path(id), id, wc); |
||||
} finally { |
||||
fs.close(); |
||||
} |
||||
} |
||||
|
||||
assertNotNull("created loader", ol); |
||||
assertEquals(type, ol.getType()); |
||||
assertEquals(data.length, ol.getSize()); |
||||
assertTrue("is large", ol.isLarge()); |
||||
try { |
||||
ol.getCachedBytes(); |
||||
fail("Should have thrown LargeObjectException"); |
||||
} catch (LargeObjectException tooBig) { |
||||
assertEquals(id.name(), tooBig.getMessage()); |
||||
} |
||||
|
||||
ObjectStream in = ol.openStream(); |
||||
assertNotNull("have stream", in); |
||||
assertEquals(type, in.getType()); |
||||
assertEquals(data.length, in.getSize()); |
||||
byte[] data2 = new byte[data.length]; |
||||
IO.readFully(in, data2, 0, data.length); |
||||
assertTrue("same content", Arrays.equals(data2, data)); |
||||
assertEquals("stream at EOF", -1, in.read()); |
||||
in.close(); |
||||
} |
||||
|
||||
public void testStandardFormat_NegativeSize() throws Exception { |
||||
ObjectId id = ObjectId.zeroId(); |
||||
byte[] data = rng.nextBytes(300); |
||||
|
||||
try { |
||||
byte[] gz = compressStandardFormat("blob", "-1", data); |
||||
UnpackedObject.open(new ByteArrayInputStream(gz), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectNegativeSize), coe |
||||
.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public void testStandardFormat_InvalidType() throws Exception { |
||||
ObjectId id = ObjectId.zeroId(); |
||||
byte[] data = rng.nextBytes(300); |
||||
|
||||
try { |
||||
byte[] gz = compressStandardFormat("not.a.type", "1", data); |
||||
UnpackedObject.open(new ByteArrayInputStream(gz), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectInvalidType), coe |
||||
.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public void testStandardFormat_NoHeader() throws Exception { |
||||
ObjectId id = ObjectId.zeroId(); |
||||
byte[] data = {}; |
||||
|
||||
try { |
||||
byte[] gz = compressStandardFormat("", "", data); |
||||
UnpackedObject.open(new ByteArrayInputStream(gz), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectNoHeader), coe |
||||
.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public void testStandardFormat_GarbageAfterSize() throws Exception { |
||||
ObjectId id = ObjectId.zeroId(); |
||||
byte[] data = rng.nextBytes(300); |
||||
|
||||
try { |
||||
byte[] gz = compressStandardFormat("blob", "1foo", data); |
||||
UnpackedObject.open(new ByteArrayInputStream(gz), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectGarbageAfterSize), |
||||
coe.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public void testStandardFormat_SmallObject_CorruptZLibStream() |
||||
throws Exception { |
||||
ObjectId id = ObjectId.zeroId(); |
||||
byte[] data = rng.nextBytes(300); |
||||
|
||||
try { |
||||
byte[] gz = compressStandardFormat(Constants.OBJ_BLOB, data); |
||||
for (int i = 5; i < gz.length; i++) |
||||
gz[i] = 0; |
||||
UnpackedObject.open(new ByteArrayInputStream(gz), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectBadStream), coe |
||||
.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public void testStandardFormat_SmallObject_TruncatedZLibStream() |
||||
throws Exception { |
||||
ObjectId id = ObjectId.zeroId(); |
||||
byte[] data = rng.nextBytes(300); |
||||
|
||||
try { |
||||
byte[] gz = compressStandardFormat(Constants.OBJ_BLOB, data); |
||||
byte[] tr = new byte[gz.length - 1]; |
||||
System.arraycopy(gz, 0, tr, 0, tr.length); |
||||
UnpackedObject.open(new ByteArrayInputStream(tr), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectBadStream), coe |
||||
.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public void testStandardFormat_SmallObject_TrailingGarbage() |
||||
throws Exception { |
||||
ObjectId id = ObjectId.zeroId(); |
||||
byte[] data = rng.nextBytes(300); |
||||
|
||||
try { |
||||
byte[] gz = compressStandardFormat(Constants.OBJ_BLOB, data); |
||||
byte[] tr = new byte[gz.length + 1]; |
||||
System.arraycopy(gz, 0, tr, 0, gz.length); |
||||
UnpackedObject.open(new ByteArrayInputStream(tr), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectBadStream), coe |
||||
.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public void testStandardFormat_LargeObject_CorruptZLibStream() |
||||
throws Exception { |
||||
final int type = Constants.OBJ_BLOB; |
||||
byte[] data = rng.nextBytes(ObjectLoader.STREAM_THRESHOLD + 5); |
||||
ObjectId id = new ObjectInserter.Formatter().idFor(type, data); |
||||
byte[] gz = compressStandardFormat(type, data); |
||||
gz[gz.length - 1] = 0; |
||||
gz[gz.length - 2] = 0; |
||||
|
||||
write(id, gz); |
||||
|
||||
ObjectLoader ol; |
||||
{ |
||||
FileInputStream fs = new FileInputStream(path(id)); |
||||
try { |
||||
ol = UnpackedObject.open(fs, path(id), id, wc); |
||||
} finally { |
||||
fs.close(); |
||||
} |
||||
} |
||||
|
||||
try { |
||||
byte[] tmp = new byte[data.length]; |
||||
InputStream in = ol.openStream(); |
||||
try { |
||||
IO.readFully(in, tmp, 0, tmp.length); |
||||
} finally { |
||||
in.close(); |
||||
} |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectBadStream), coe |
||||
.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public void testStandardFormat_LargeObject_TruncatedZLibStream() |
||||
throws Exception { |
||||
final int type = Constants.OBJ_BLOB; |
||||
byte[] data = rng.nextBytes(ObjectLoader.STREAM_THRESHOLD + 5); |
||||
ObjectId id = new ObjectInserter.Formatter().idFor(type, data); |
||||
byte[] gz = compressStandardFormat(type, data); |
||||
byte[] tr = new byte[gz.length - 1]; |
||||
System.arraycopy(gz, 0, tr, 0, tr.length); |
||||
|
||||
write(id, tr); |
||||
|
||||
ObjectLoader ol; |
||||
{ |
||||
FileInputStream fs = new FileInputStream(path(id)); |
||||
try { |
||||
ol = UnpackedObject.open(fs, path(id), id, wc); |
||||
} finally { |
||||
fs.close(); |
||||
} |
||||
} |
||||
|
||||
byte[] tmp = new byte[data.length]; |
||||
InputStream in = ol.openStream(); |
||||
IO.readFully(in, tmp, 0, tmp.length); |
||||
try { |
||||
in.close(); |
||||
fail("close did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectBadStream), coe |
||||
.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public void testStandardFormat_LargeObject_TrailingGarbage() |
||||
throws Exception { |
||||
final int type = Constants.OBJ_BLOB; |
||||
byte[] data = rng.nextBytes(ObjectLoader.STREAM_THRESHOLD + 5); |
||||
ObjectId id = new ObjectInserter.Formatter().idFor(type, data); |
||||
byte[] gz = compressStandardFormat(type, data); |
||||
byte[] tr = new byte[gz.length + 1]; |
||||
System.arraycopy(gz, 0, tr, 0, gz.length); |
||||
|
||||
write(id, tr); |
||||
|
||||
ObjectLoader ol; |
||||
{ |
||||
FileInputStream fs = new FileInputStream(path(id)); |
||||
try { |
||||
ol = UnpackedObject.open(fs, path(id), id, wc); |
||||
} finally { |
||||
fs.close(); |
||||
} |
||||
} |
||||
|
||||
byte[] tmp = new byte[data.length]; |
||||
InputStream in = ol.openStream(); |
||||
IO.readFully(in, tmp, 0, tmp.length); |
||||
try { |
||||
in.close(); |
||||
fail("close did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectBadStream), coe |
||||
.getMessage()); |
||||
} |
||||
} |
||||
|
||||
public void testPackFormat_SmallObject() throws Exception { |
||||
final int type = Constants.OBJ_BLOB; |
||||
byte[] data = rng.nextBytes(300); |
||||
byte[] gz = compressPackFormat(type, data); |
||||
ObjectId id = ObjectId.zeroId(); |
||||
|
||||
ObjectLoader ol = UnpackedObject.open(new ByteArrayInputStream(gz), |
||||
path(id), id, wc); |
||||
assertNotNull("created loader", ol); |
||||
assertEquals(type, ol.getType()); |
||||
assertEquals(data.length, ol.getSize()); |
||||
assertFalse("is not large", ol.isLarge()); |
||||
assertTrue("same content", Arrays.equals(data, ol.getCachedBytes())); |
||||
|
||||
ObjectStream in = ol.openStream(); |
||||
assertNotNull("have stream", in); |
||||
assertEquals(type, in.getType()); |
||||
assertEquals(data.length, in.getSize()); |
||||
byte[] data2 = new byte[data.length]; |
||||
IO.readFully(in, data2, 0, data.length); |
||||
assertTrue("same content", Arrays.equals(data, ol.getCachedBytes())); |
||||
in.close(); |
||||
} |
||||
|
||||
public void testPackFormat_LargeObject() throws Exception { |
||||
final int type = Constants.OBJ_BLOB; |
||||
byte[] data = rng.nextBytes(ObjectLoader.STREAM_THRESHOLD + 5); |
||||
ObjectId id = new ObjectInserter.Formatter().idFor(type, data); |
||||
write(id, compressPackFormat(type, data)); |
||||
|
||||
ObjectLoader ol; |
||||
{ |
||||
FileInputStream fs = new FileInputStream(path(id)); |
||||
try { |
||||
ol = UnpackedObject.open(fs, path(id), id, wc); |
||||
} finally { |
||||
fs.close(); |
||||
} |
||||
} |
||||
|
||||
assertNotNull("created loader", ol); |
||||
assertEquals(type, ol.getType()); |
||||
assertEquals(data.length, ol.getSize()); |
||||
assertTrue("is large", ol.isLarge()); |
||||
try { |
||||
ol.getCachedBytes(); |
||||
fail("Should have thrown LargeObjectException"); |
||||
} catch (LargeObjectException tooBig) { |
||||
assertEquals(id.name(), tooBig.getMessage()); |
||||
} |
||||
|
||||
ObjectStream in = ol.openStream(); |
||||
assertNotNull("have stream", in); |
||||
assertEquals(type, in.getType()); |
||||
assertEquals(data.length, in.getSize()); |
||||
byte[] data2 = new byte[data.length]; |
||||
IO.readFully(in, data2, 0, data.length); |
||||
assertTrue("same content", Arrays.equals(data2, data)); |
||||
assertEquals("stream at EOF", -1, in.read()); |
||||
in.close(); |
||||
} |
||||
|
||||
public void testPackFormat_DeltaNotAllowed() throws Exception { |
||||
ObjectId id = ObjectId.zeroId(); |
||||
byte[] data = rng.nextBytes(300); |
||||
|
||||
try { |
||||
byte[] gz = compressPackFormat(Constants.OBJ_OFS_DELTA, data); |
||||
UnpackedObject.open(new ByteArrayInputStream(gz), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectInvalidType), coe |
||||
.getMessage()); |
||||
} |
||||
|
||||
try { |
||||
byte[] gz = compressPackFormat(Constants.OBJ_REF_DELTA, data); |
||||
UnpackedObject.open(new ByteArrayInputStream(gz), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectInvalidType), coe |
||||
.getMessage()); |
||||
} |
||||
|
||||
try { |
||||
byte[] gz = compressPackFormat(Constants.OBJ_TYPE_5, data); |
||||
UnpackedObject.open(new ByteArrayInputStream(gz), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectInvalidType), coe |
||||
.getMessage()); |
||||
} |
||||
|
||||
try { |
||||
byte[] gz = compressPackFormat(Constants.OBJ_EXT, data); |
||||
UnpackedObject.open(new ByteArrayInputStream(gz), path(id), id, wc); |
||||
fail("Did not throw CorruptObjectException"); |
||||
} catch (CorruptObjectException coe) { |
||||
assertEquals(MessageFormat.format(JGitText.get().objectIsCorrupt, |
||||
id.name(), JGitText.get().corruptObjectInvalidType), coe |
||||
.getMessage()); |
||||
} |
||||
} |
||||
|
||||
private byte[] compressStandardFormat(int type, byte[] data) |
||||
throws IOException { |
||||
String typeString = Constants.typeString(type); |
||||
String length = String.valueOf(data.length); |
||||
return compressStandardFormat(typeString, length, data); |
||||
} |
||||
|
||||
private byte[] compressStandardFormat(String type, String length, |
||||
byte[] data) throws IOException { |
||||
ByteArrayOutputStream out = new ByteArrayOutputStream(); |
||||
DeflaterOutputStream d = new DeflaterOutputStream(out); |
||||
d.write(Constants.encodeASCII(type)); |
||||
d.write(' '); |
||||
d.write(Constants.encodeASCII(length)); |
||||
d.write(0); |
||||
d.write(data); |
||||
d.finish(); |
||||
return out.toByteArray(); |
||||
} |
||||
|
||||
private byte[] compressPackFormat(int type, byte[] data) throws IOException { |
||||
byte[] hdr = new byte[64]; |
||||
int rawLength = data.length; |
||||
int nextLength = rawLength >>> 4; |
||||
hdr[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (rawLength & 0x0F)); |
||||
rawLength = nextLength; |
||||
int n = 1; |
||||
while (rawLength > 0) { |
||||
nextLength >>>= 7; |
||||
hdr[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (rawLength & 0x7F)); |
||||
rawLength = nextLength; |
||||
} |
||||
|
||||
final ByteArrayOutputStream out = new ByteArrayOutputStream(); |
||||
out.write(hdr, 0, n); |
||||
|
||||
DeflaterOutputStream d = new DeflaterOutputStream(out); |
||||
d.write(data); |
||||
d.finish(); |
||||
return out.toByteArray(); |
||||
} |
||||
|
||||
private File path(ObjectId id) { |
||||
return repo.getObjectDatabase().fileFor(id); |
||||
} |
||||
|
||||
private void write(ObjectId id, byte[] data) throws IOException { |
||||
File path = path(id); |
||||
path.getParentFile().mkdirs(); |
||||
FileOutputStream out = new FileOutputStream(path); |
||||
try { |
||||
out.write(data); |
||||
} finally { |
||||
out.close(); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,228 @@
|
||||
/* |
||||
* Copyright (C) 2010, Google Inc. |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
|
||||
package org.eclipse.jgit.storage.pack; |
||||
|
||||
import java.io.ByteArrayOutputStream; |
||||
import java.io.IOException; |
||||
import java.util.Arrays; |
||||
|
||||
import junit.framework.TestCase; |
||||
|
||||
import org.eclipse.jgit.junit.TestRng; |
||||
import org.eclipse.jgit.lib.Constants; |
||||
|
||||
public class DeltaIndexTest extends TestCase { |
||||
private TestRng rng; |
||||
|
||||
private ByteArrayOutputStream actDeltaBuf; |
||||
|
||||
private ByteArrayOutputStream expDeltaBuf; |
||||
|
||||
private DeltaEncoder expDeltaEnc; |
||||
|
||||
private byte[] src; |
||||
|
||||
private byte[] dst; |
||||
|
||||
private ByteArrayOutputStream dstBuf; |
||||
|
||||
protected void setUp() throws Exception { |
||||
super.setUp(); |
||||
rng = new TestRng(getName()); |
||||
actDeltaBuf = new ByteArrayOutputStream(); |
||||
expDeltaBuf = new ByteArrayOutputStream(); |
||||
expDeltaEnc = new DeltaEncoder(expDeltaBuf, 0, 0); |
||||
dstBuf = new ByteArrayOutputStream(); |
||||
} |
||||
|
||||
public void testInsertWholeObject_Length12() throws IOException { |
||||
src = rng.nextBytes(12); |
||||
insert(src); |
||||
doTest(); |
||||
} |
||||
|
||||
public void testCopyWholeObject_Length128() throws IOException { |
||||
src = rng.nextBytes(128); |
||||
copy(0, 128); |
||||
doTest(); |
||||
} |
||||
|
||||
public void testCopyWholeObject_Length123() throws IOException { |
||||
src = rng.nextBytes(123); |
||||
copy(0, 123); |
||||
doTest(); |
||||
} |
||||
|
||||
public void testCopyZeros_Length128() throws IOException { |
||||
src = new byte[2048]; |
||||
copy(0, src.length); |
||||
doTest(); |
||||
|
||||
// The index should be smaller than expected due to the chain
|
||||
// being truncated. Without truncation we would expect to have
|
||||
// more than 3584 bytes used.
|
||||
//
|
||||
assertEquals(2636, new DeltaIndex(src).getIndexSize()); |
||||
} |
||||
|
||||
public void testShuffleSegments() throws IOException { |
||||
src = rng.nextBytes(128); |
||||
copy(64, 64); |
||||
copy(0, 64); |
||||
doTest(); |
||||
} |
||||
|
||||
public void testInsertHeadMiddle() throws IOException { |
||||
src = rng.nextBytes(1024); |
||||
insert("foo"); |
||||
copy(0, 512); |
||||
insert("yet more fooery"); |
||||
copy(0, 512); |
||||
doTest(); |
||||
} |
||||
|
||||
public void testInsertTail() throws IOException { |
||||
src = rng.nextBytes(1024); |
||||
copy(0, 512); |
||||
insert("bar"); |
||||
doTest(); |
||||
} |
||||
|
||||
public void testIndexSize() { |
||||
src = rng.nextBytes(1024); |
||||
DeltaIndex di = new DeltaIndex(src); |
||||
assertEquals(1860, di.getIndexSize()); |
||||
assertEquals("DeltaIndex[2 KiB]", di.toString()); |
||||
} |
||||
|
||||
public void testLimitObjectSize_Length12InsertFails() throws IOException { |
||||
src = rng.nextBytes(12); |
||||
dst = src; |
||||
|
||||
DeltaIndex di = new DeltaIndex(src); |
||||
assertFalse(di.encode(actDeltaBuf, dst, src.length)); |
||||
} |
||||
|
||||
public void testLimitObjectSize_Length130InsertFails() throws IOException { |
||||
src = rng.nextBytes(130); |
||||
dst = rng.nextBytes(130); |
||||
|
||||
DeltaIndex di = new DeltaIndex(src); |
||||
assertFalse(di.encode(actDeltaBuf, dst, src.length)); |
||||
} |
||||
|
||||
public void testLimitObjectSize_Length130CopyOk() throws IOException { |
||||
src = rng.nextBytes(130); |
||||
copy(0, 130); |
||||
dst = dstBuf.toByteArray(); |
||||
|
||||
DeltaIndex di = new DeltaIndex(src); |
||||
assertTrue(di.encode(actDeltaBuf, dst, dst.length)); |
||||
|
||||
byte[] actDelta = actDeltaBuf.toByteArray(); |
||||
byte[] expDelta = expDeltaBuf.toByteArray(); |
||||
|
||||
assertEquals(BinaryDelta.format(expDelta, false), //
|
||||
BinaryDelta.format(actDelta, false)); |
||||
} |
||||
|
||||
public void testLimitObjectSize_Length130CopyFails() throws IOException { |
||||
src = rng.nextBytes(130); |
||||
copy(0, 130); |
||||
dst = dstBuf.toByteArray(); |
||||
|
||||
// The header requires 4 bytes for these objects, so a target length
|
||||
// of 5 is bigger than the copy instruction and should cause an abort.
|
||||
//
|
||||
DeltaIndex di = new DeltaIndex(src); |
||||
assertFalse(di.encode(actDeltaBuf, dst, 5)); |
||||
assertEquals(4, actDeltaBuf.size()); |
||||
} |
||||
|
||||
public void testLimitObjectSize_InsertFrontFails() throws IOException { |
||||
src = rng.nextBytes(130); |
||||
insert("eight"); |
||||
copy(0, 130); |
||||
dst = dstBuf.toByteArray(); |
||||
|
||||
// The header requires 4 bytes for these objects, so a target length
|
||||
// of 5 is bigger than the copy instruction and should cause an abort.
|
||||
//
|
||||
DeltaIndex di = new DeltaIndex(src); |
||||
assertFalse(di.encode(actDeltaBuf, dst, 5)); |
||||
assertEquals(4, actDeltaBuf.size()); |
||||
} |
||||
|
||||
private void copy(int offset, int len) throws IOException { |
||||
dstBuf.write(src, offset, len); |
||||
expDeltaEnc.copy(offset, len); |
||||
} |
||||
|
||||
private void insert(String text) throws IOException { |
||||
insert(Constants.encode(text)); |
||||
} |
||||
|
||||
private void insert(byte[] text) throws IOException { |
||||
dstBuf.write(text); |
||||
expDeltaEnc.insert(text); |
||||
} |
||||
|
||||
private void doTest() throws IOException { |
||||
dst = dstBuf.toByteArray(); |
||||
|
||||
DeltaIndex di = new DeltaIndex(src); |
||||
di.encode(actDeltaBuf, dst); |
||||
|
||||
byte[] actDelta = actDeltaBuf.toByteArray(); |
||||
byte[] expDelta = expDeltaBuf.toByteArray(); |
||||
|
||||
assertEquals(BinaryDelta.format(expDelta, false), //
|
||||
BinaryDelta.format(actDelta, false)); |
||||
|
||||
assertTrue("delta is not empty", actDelta.length > 0); |
||||
assertEquals(src.length, BinaryDelta.getBaseSize(actDelta)); |
||||
assertEquals(dst.length, BinaryDelta.getResultSize(actDelta)); |
||||
assertTrue(Arrays.equals(dst, BinaryDelta.apply(src, actDelta))); |
||||
} |
||||
} |
||||
@ -0,0 +1,273 @@
|
||||
/* |
||||
* Copyright (C) 2010, Google Inc. |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
|
||||
package org.eclipse.jgit.storage.pack; |
||||
|
||||
import java.io.ByteArrayInputStream; |
||||
import java.io.ByteArrayOutputStream; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.util.Arrays; |
||||
|
||||
import junit.framework.TestCase; |
||||
|
||||
import org.eclipse.jgit.JGitText; |
||||
import org.eclipse.jgit.errors.CorruptObjectException; |
||||
import org.eclipse.jgit.junit.TestRng; |
||||
import org.eclipse.jgit.lib.Constants; |
||||
import org.eclipse.jgit.util.IO; |
||||
|
||||
public class DeltaStreamTest extends TestCase { |
||||
private TestRng rng; |
||||
|
||||
private ByteArrayOutputStream deltaBuf; |
||||
|
||||
private DeltaEncoder deltaEnc; |
||||
|
||||
private byte[] base; |
||||
|
||||
private byte[] data; |
||||
|
||||
private int dataPtr; |
||||
|
||||
private byte[] delta; |
||||
|
||||
protected void setUp() throws Exception { |
||||
super.setUp(); |
||||
rng = new TestRng(getName()); |
||||
deltaBuf = new ByteArrayOutputStream(); |
||||
} |
||||
|
||||
public void testCopy_SingleOp() throws IOException { |
||||
init((1 << 16) + 1, (1 << 8) + 1); |
||||
copy(0, data.length); |
||||
assertValidState(); |
||||
} |
||||
|
||||
public void testCopy_MaxSize() throws IOException { |
||||
int max = (0xff << 16) + (0xff << 8) + 0xff; |
||||
init(1 + max, max); |
||||
copy(1, max); |
||||
assertValidState(); |
||||
} |
||||
|
||||
public void testCopy_64k() throws IOException { |
||||
init(0x10000 + 2, 0x10000 + 1); |
||||
copy(1, 0x10000); |
||||
copy(0x10001, 1); |
||||
assertValidState(); |
||||
} |
||||
|
||||
public void testCopy_Gap() throws IOException { |
||||
init(256, 8); |
||||
copy(4, 4); |
||||
copy(128, 4); |
||||
assertValidState(); |
||||
} |
||||
|
||||
public void testCopy_OutOfOrder() throws IOException { |
||||
init((1 << 16) + 1, (1 << 16) + 1); |
||||
copy(1 << 8, 1 << 8); |
||||
copy(0, data.length - dataPtr); |
||||
assertValidState(); |
||||
} |
||||
|
||||
public void testInsert_SingleOp() throws IOException { |
||||
init((1 << 16) + 1, 2); |
||||
insert("hi"); |
||||
assertValidState(); |
||||
} |
||||
|
||||
public void testInsertAndCopy() throws IOException { |
||||
init(8, 512); |
||||
insert(new byte[127]); |
||||
insert(new byte[127]); |
||||
insert(new byte[127]); |
||||
insert(new byte[125]); |
||||
copy(2, 6); |
||||
assertValidState(); |
||||
} |
||||
|
||||
public void testSkip() throws IOException { |
||||
init(32, 15); |
||||
copy(2, 2); |
||||
insert("ab"); |
||||
insert("cd"); |
||||
copy(4, 4); |
||||
copy(0, 2); |
||||
insert("efg"); |
||||
assertValidState(); |
||||
|
||||
for (int p = 0; p < data.length; p++) { |
||||
byte[] act = new byte[data.length]; |
||||
System.arraycopy(data, 0, act, 0, p); |
||||
DeltaStream in = open(); |
||||
IO.skipFully(in, p); |
||||
assertEquals(data.length - p, in.read(act, p, data.length - p)); |
||||
assertEquals(-1, in.read()); |
||||
assertTrue("skipping " + p, Arrays.equals(data, act)); |
||||
} |
||||
|
||||
// Skip all the way to the end should still recognize EOF.
|
||||
DeltaStream in = open(); |
||||
IO.skipFully(in, data.length); |
||||
assertEquals(-1, in.read()); |
||||
assertEquals(0, in.skip(1)); |
||||
|
||||
// Skip should not open the base as we move past it, but it
|
||||
// will open when we need to start copying data from it.
|
||||
final boolean[] opened = new boolean[1]; |
||||
in = new DeltaStream(new ByteArrayInputStream(delta)) { |
||||
@Override |
||||
protected long getBaseSize() throws IOException { |
||||
return base.length; |
||||
} |
||||
|
||||
@Override |
||||
protected InputStream openBase() throws IOException { |
||||
opened[0] = true; |
||||
return new ByteArrayInputStream(base); |
||||
} |
||||
}; |
||||
IO.skipFully(in, 7); |
||||
assertFalse("not yet open", opened[0]); |
||||
assertEquals(data[7], in.read()); |
||||
assertTrue("now open", opened[0]); |
||||
} |
||||
|
||||
public void testIncorrectBaseSize() throws IOException { |
||||
init(4, 4); |
||||
copy(0, 4); |
||||
assertValidState(); |
||||
|
||||
DeltaStream in = new DeltaStream(new ByteArrayInputStream(delta)) { |
||||
@Override |
||||
protected long getBaseSize() throws IOException { |
||||
return 128; |
||||
} |
||||
|
||||
@Override |
||||
protected InputStream openBase() throws IOException { |
||||
return new ByteArrayInputStream(base); |
||||
} |
||||
}; |
||||
try { |
||||
in.read(new byte[4]); |
||||
fail("did not throw an exception"); |
||||
} catch (CorruptObjectException e) { |
||||
assertEquals(JGitText.get().baseLengthIncorrect, e.getMessage()); |
||||
} |
||||
|
||||
in = new DeltaStream(new ByteArrayInputStream(delta)) { |
||||
@Override |
||||
protected long getBaseSize() throws IOException { |
||||
return 4; |
||||
} |
||||
|
||||
@Override |
||||
protected InputStream openBase() throws IOException { |
||||
return new ByteArrayInputStream(new byte[0]); |
||||
} |
||||
}; |
||||
try { |
||||
in.read(new byte[4]); |
||||
fail("did not throw an exception"); |
||||
} catch (CorruptObjectException e) { |
||||
assertEquals(JGitText.get().baseLengthIncorrect, e.getMessage()); |
||||
} |
||||
} |
||||
|
||||
private void init(int baseSize, int dataSize) throws IOException { |
||||
base = rng.nextBytes(baseSize); |
||||
data = new byte[dataSize]; |
||||
deltaEnc = new DeltaEncoder(deltaBuf, baseSize, dataSize); |
||||
} |
||||
|
||||
private void copy(int offset, int len) throws IOException { |
||||
System.arraycopy(base, offset, data, dataPtr, len); |
||||
deltaEnc.copy(offset, len); |
||||
assertEquals(deltaBuf.size(), deltaEnc.getSize()); |
||||
dataPtr += len; |
||||
} |
||||
|
||||
private void insert(String text) throws IOException { |
||||
insert(Constants.encode(text)); |
||||
} |
||||
|
||||
private void insert(byte[] text) throws IOException { |
||||
System.arraycopy(text, 0, data, dataPtr, text.length); |
||||
deltaEnc.insert(text); |
||||
assertEquals(deltaBuf.size(), deltaEnc.getSize()); |
||||
dataPtr += text.length; |
||||
} |
||||
|
||||
private void assertValidState() throws IOException { |
||||
assertEquals("test filled example result", data.length, dataPtr); |
||||
|
||||
delta = deltaBuf.toByteArray(); |
||||
assertEquals(base.length, BinaryDelta.getBaseSize(delta)); |
||||
assertEquals(data.length, BinaryDelta.getResultSize(delta)); |
||||
assertTrue(Arrays.equals(data, BinaryDelta.apply(base, delta))); |
||||
|
||||
byte[] act = new byte[data.length]; |
||||
DeltaStream in = open(); |
||||
assertEquals(data.length, in.getSize()); |
||||
assertEquals(data.length, in.read(act)); |
||||
assertEquals(-1, in.read()); |
||||
assertTrue(Arrays.equals(data, act)); |
||||
} |
||||
|
||||
private DeltaStream open() throws IOException { |
||||
return new DeltaStream(new ByteArrayInputStream(delta)) { |
||||
@Override |
||||
protected long getBaseSize() throws IOException { |
||||
return base.length; |
||||
} |
||||
|
||||
@Override |
||||
protected InputStream openBase() throws IOException { |
||||
return new ByteArrayInputStream(base); |
||||
} |
||||
}; |
||||
} |
||||
} |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue