github
/
jgit
mirror of https://gitee.com/github-mirror-repo/jgit.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Remove streaming delta support from JGit 

Streaming packed deltas is so slow that it never feasibly completes
(it will take hours for it to stream a few hundred megabytes on
relatively fast systems with a large amount of storage).  This
was indicated as a "failed experiment" by Shawn in the following
mailing list post:
http://dev.eclipse.org/mhonarc/lists/jgit-dev/msg01674.html

Change-Id: Idc12f59e37b122f13856d7b533a5af9d8867a8a5
Signed-off-by: Doug Kelly <dougk.ff7@gmail.com>
stable-3.4
Doug Kelly 11 years ago committed by Shawn Pearce
parent
62bbde3393
commit
62697c8d33
6 changed files with 75 additions and 1000 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 124
      org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java
  2. 314
      org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/DeltaStreamTest.java
  3. 265
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedDeltaObject.java
  4. 25
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java
  5. 4
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/DeltaIndex.java
  6. 343
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/DeltaStream.java

124
org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java
Unescape View File

@ -47,20 +47,27 @@ import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.zip.Deflater; import java.util.zip.Deflater;
import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.pack.DeltaEncoder; import org.eclipse.jgit.internal.storage.pack.DeltaEncoder;
import org.eclipse.jgit.internal.storage.pack.PackExt;
import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.JGitTestUtil;
import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.junit.TestRepository;
@ -75,6 +82,7 @@ import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevBlob;
import org.eclipse.jgit.storage.file.WindowCacheConfig; import org.eclipse.jgit.storage.file.WindowCacheConfig;
import org.eclipse.jgit.transport.PackParser; import org.eclipse.jgit.transport.PackParser;
import org.eclipse.jgit.transport.PackedObjectInfo;
import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.IO;
import org.eclipse.jgit.util.NB; import org.eclipse.jgit.util.NB;
import org.eclipse.jgit.util.TemporaryBuffer; import org.eclipse.jgit.util.TemporaryBuffer;
@ -105,7 +113,7 @@ public class PackFileTest extends LocalDiskRepositoryTestCase {
WindowCacheConfig cfg = new WindowCacheConfig(); WindowCacheConfig cfg = new WindowCacheConfig();
cfg.setStreamFileThreshold(streamThreshold); cfg.setStreamFileThreshold(streamThreshold);
WindowCache.reconfigure(cfg); cfg.install();
repo = createBareRepository(); repo = createBareRepository();
tr = new TestRepository<Repository>(repo); tr = new TestRepository<Repository>(repo);
@ -116,7 +124,7 @@ public class PackFileTest extends LocalDiskRepositoryTestCase {
public void tearDown() throws Exception { public void tearDown() throws Exception {
if (wc != null) if (wc != null)
wc.release(); wc.release();
WindowCache.reconfigure(new WindowCacheConfig()); new WindowCacheConfig().install();
super.tearDown(); super.tearDown();
} }
@ -241,68 +249,65 @@ public class PackFileTest extends LocalDiskRepositoryTestCase {
} }
@Test @Test
public void testDelta_LargeObjectChain() throws Exception { public void testDelta_FailsOver2GiB() throws Exception {
ObjectInserter.Formatter fmt = new ObjectInserter.Formatter(); ObjectInserter.Formatter fmt = new ObjectInserter.Formatter();
byte[] data0 = new byte[streamThreshold + 5]; byte[] base = new byte[] { 'a' };
Arrays.fill(data0, (byte) 0xf3); ObjectId idA = fmt.idFor(Constants.OBJ_BLOB, base);
ObjectId id0 = fmt.idFor(Constants.OBJ_BLOB, data0); ObjectId idB = fmt.idFor(Constants.OBJ_BLOB, new byte[] { 'b' });
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); PackedObjectInfo a = new PackedObjectInfo(idA);
packHeader(pack, 4); PackedObjectInfo b = new PackedObjectInfo(idB);
objectHeader(pack, Constants.OBJ_BLOB, data0.length);
deflate(pack, data0);
byte[] data1 = clone(0x01, data0); TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024);
byte[] delta1 = delta(data0, data1); packHeader(pack, 2);
ObjectId id1 = fmt.idFor(Constants.OBJ_BLOB, data1); a.setOffset(pack.length());
objectHeader(pack, Constants.OBJ_REF_DELTA, delta1.length); objectHeader(pack, Constants.OBJ_BLOB, base.length);
id0.copyRawTo(pack); deflate(pack, base);
deflate(pack, delta1);
byte[] data2 = clone(0x02, data1);
byte[] delta2 = delta(data1, data2);
ObjectId id2 = fmt.idFor(Constants.OBJ_BLOB, data2);
objectHeader(pack, Constants.OBJ_REF_DELTA, delta2.length);
id1.copyRawTo(pack);
deflate(pack, delta2);
byte[] data3 = clone(0x03, data2);
byte[] delta3 = delta(data2, data3);
ObjectId id3 = fmt.idFor(Constants.OBJ_BLOB, data3);
objectHeader(pack, Constants.OBJ_REF_DELTA, delta3.length);
id2.copyRawTo(pack);
deflate(pack, delta3);
digest(pack);
PackParser ip = index(pack.toByteArray());
ip.setAllowThin(true);
ip.parse(NullProgressMonitor.INSTANCE);
assertTrue("has blob", wc.has(id3)); ByteArrayOutputStream tmp = new ByteArrayOutputStream();
DeltaEncoder de = new DeltaEncoder(tmp, base.length, 3L << 30);
de.copy(0, 1);
byte[] delta = tmp.toByteArray();
b.setOffset(pack.length());
objectHeader(pack, Constants.OBJ_REF_DELTA, delta.length);
idA.copyRawTo(pack);
deflate(pack, delta);
byte[] footer = digest(pack);
File packName = new File(new File(
((FileObjectDatabase) repo.getObjectDatabase()).getDirectory(),
"pack"), idA.name() + ".pack");
File idxName = new File(new File(
((FileObjectDatabase) repo.getObjectDatabase()).getDirectory(),
"pack"), idA.name() + ".idx");
FileOutputStream f = new FileOutputStream(packName);
try {
f.write(pack.toByteArray());
} finally {
f.close();
}
ObjectLoader ol = wc.open(id3); f = new FileOutputStream(idxName);
assertNotNull("created loader", ol);
assertEquals(Constants.OBJ_BLOB, ol.getType());
assertEquals(data3.length, ol.getSize());
assertTrue("is large", ol.isLarge());
try { try {
ol.getCachedBytes(); List<PackedObjectInfo> list = new ArrayList<PackedObjectInfo>();
fail("Should have thrown LargeObjectException"); list.add(a);
} catch (LargeObjectException tooBig) { list.add(b);
assertEquals(MessageFormat.format( Collections.sort(list);
JGitText.get().largeObjectException, id3.name()), tooBig new PackIndexWriterV1(f).write(list, footer);
.getMessage()); } finally {
f.close();
} }
ObjectStream in = ol.openStream(); PackFile packFile = new PackFile(packName, PackExt.INDEX.getBit());
assertNotNull("have stream", in); try {
assertEquals(Constants.OBJ_BLOB, in.getType()); packFile.get(wc, b);
assertEquals(data3.length, in.getSize()); fail("expected LargeObjectException.ExceedsByteArrayLimit");
byte[] act = new byte[data3.length]; } catch (LargeObjectException.ExceedsByteArrayLimit bad) {
IO.readFully(in, act, 0, data3.length); assertNull(bad.getObjectId());
assertTrue("same content", Arrays.equals(act, data3)); } finally {
assertEquals("stream at EOF", -1, in.read()); packFile.close();
in.close(); }
} }
private static byte[] clone(int first, byte[] base) { private static byte[] clone(int first, byte[] base) {
@ -358,10 +363,13 @@ public class PackFileTest extends LocalDiskRepositoryTestCase {
deflater.end(); deflater.end();
} }
private static void digest(TemporaryBuffer.Heap buf) throws IOException { private static byte[] digest(TemporaryBuffer.Heap buf)
throws IOException {
MessageDigest md = Constants.newMessageDigest(); MessageDigest md = Constants.newMessageDigest();
md.update(buf.toByteArray()); md.update(buf.toByteArray());
buf.write(md.digest()); byte[] footer = md.digest();
buf.write(footer);
return footer;
} }
private ObjectInserter inserter; private ObjectInserter inserter;

314
org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/DeltaStreamTest.java
Unescape View File

@ -1,314 +0,0 @@
/*
* Copyright (C) 2010, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.storage.pack;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
import org.eclipse.jgit.internal.storage.pack.DeltaEncoder;
import org.eclipse.jgit.internal.storage.pack.DeltaStream;
import org.eclipse.jgit.junit.JGitTestUtil;
import org.eclipse.jgit.junit.TestRng;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.util.IO;
import org.junit.Before;
import org.junit.Test;
public class DeltaStreamTest {
private TestRng rng;
private ByteArrayOutputStream deltaBuf;
private DeltaEncoder deltaEnc;
private byte[] base;
private byte[] data;
private int dataPtr;
private byte[] delta;
private TestRng getRng() {
if (rng == null)
rng = new TestRng(JGitTestUtil.getName());
return rng;
}
@Before
public void setUp() throws Exception {
deltaBuf = new ByteArrayOutputStream();
}
@Test
public void testCopy_SingleOp() throws IOException {
init((1 << 16) + 1, (1 << 8) + 1);
copy(0, data.length);
assertValidState();
}
@Test
public void testCopy_MaxSize() throws IOException {
int max = (0xff << 16) + (0xff << 8) + 0xff;
init(1 + max, max);
copy(1, max);
assertValidState();
}
@Test
public void testCopy_64k() throws IOException {
init(0x10000 + 2, 0x10000 + 1);
copy(1, 0x10000);
copy(0x10001, 1);
assertValidState();
}
@Test
public void testCopy_Gap() throws IOException {
init(256, 8);
copy(4, 4);
copy(128, 4);
assertValidState();
}
@Test
public void testCopy_OutOfOrder() throws IOException {
init((1 << 16) + 1, (1 << 16) + 1);
copy(1 << 8, 1 << 8);
copy(0, data.length - dataPtr);
assertValidState();
}
@Test
public void testInsert_SingleOp() throws IOException {
init((1 << 16) + 1, 2);
insert("hi");
assertValidState();
}
@Test
public void testInsertAndCopy() throws IOException {
init(8, 512);
insert(new byte[127]);
insert(new byte[127]);
insert(new byte[127]);
insert(new byte[125]);
copy(2, 6);
assertValidState();
}
@Test
public void testSkip() throws IOException {
init(32, 15);
copy(2, 2);
insert("ab");
insert("cd");
copy(4, 4);
copy(0, 2);
insert("efg");
assertValidState();
for (int p = 0; p < data.length; p++) {
byte[] act = new byte[data.length];
System.arraycopy(data, 0, act, 0, p);
DeltaStream in = open();
IO.skipFully(in, p);
assertEquals(data.length - p, in.read(act, p, data.length - p));
assertEquals(-1, in.read());
assertTrue("skipping " + p, Arrays.equals(data, act));
}
// Skip all the way to the end should still recognize EOF.
DeltaStream in = open();
IO.skipFully(in, data.length);
assertEquals(-1, in.read());
assertEquals(0, in.skip(1));
// Skip should not open the base as we move past it, but it
// will open when we need to start copying data from it.
final boolean[] opened = new boolean[1];
in = new DeltaStream(new ByteArrayInputStream(delta)) {
@Override
protected long getBaseSize() throws IOException {
return base.length;
}
@Override
protected InputStream openBase() throws IOException {
opened[0] = true;
return new ByteArrayInputStream(base);
}
};
IO.skipFully(in, 7);
assertFalse("not yet open", opened[0]);
assertEquals(data[7], in.read());
assertTrue("now open", opened[0]);
}
@Test
public void testIncorrectBaseSize() throws IOException {
init(4, 4);
copy(0, 4);
assertValidState();
DeltaStream in = new DeltaStream(new ByteArrayInputStream(delta)) {
@Override
protected long getBaseSize() throws IOException {
return 128;
}
@Override
protected InputStream openBase() throws IOException {
return new ByteArrayInputStream(base);
}
};
try {
in.read(new byte[4]);
fail("did not throw an exception");
} catch (CorruptObjectException e) {
assertEquals(JGitText.get().baseLengthIncorrect, e.getMessage());
}
in = new DeltaStream(new ByteArrayInputStream(delta)) {
@Override
protected long getBaseSize() throws IOException {
return 4;
}
@Override
protected InputStream openBase() throws IOException {
return new ByteArrayInputStream(new byte[0]);
}
};
try {
in.read(new byte[4]);
fail("did not throw an exception");
} catch (CorruptObjectException e) {
assertEquals(JGitText.get().baseLengthIncorrect, e.getMessage());
}
}
private void init(int baseSize, int dataSize) throws IOException {
base = getRng().nextBytes(baseSize);
data = new byte[dataSize];
deltaEnc = new DeltaEncoder(deltaBuf, baseSize, dataSize);
}
private void copy(int offset, int len) throws IOException {
System.arraycopy(base, offset, data, dataPtr, len);
deltaEnc.copy(offset, len);
assertEquals(deltaBuf.size(), deltaEnc.getSize());
dataPtr += len;
}
private void insert(String text) throws IOException {
insert(Constants.encode(text));
}
private void insert(byte[] text) throws IOException {
System.arraycopy(text, 0, data, dataPtr, text.length);
deltaEnc.insert(text);
assertEquals(deltaBuf.size(), deltaEnc.getSize());
dataPtr += text.length;
}
private void assertValidState() throws IOException {
assertEquals("test filled example result", data.length, dataPtr);
delta = deltaBuf.toByteArray();
assertEquals(base.length, BinaryDelta.getBaseSize(delta));
assertEquals(data.length, BinaryDelta.getResultSize(delta));
assertArrayEquals(data, BinaryDelta.apply(base, delta));
// Assert that a single bulk read produces the correct result.
//
byte[] act = new byte[data.length];
DeltaStream in = open();
assertEquals(data.length, in.getSize());
assertEquals(data.length, in.read(act));
assertEquals(-1, in.read());
assertTrue("bulk read has same content", Arrays.equals(data, act));
// Assert that smaller tiny reads have the same result too.
//
act = new byte[data.length];
in = open();
int read = 0;
while (read < data.length) {
int n = in.read(act, read, 128);
if (n <= 0)
break;
read += n;
}
assertEquals(data.length, read);
assertEquals(-1, in.read());
assertTrue("small reads have same content", Arrays.equals(data, act));
}
private DeltaStream open() throws IOException {
return new DeltaStream(new ByteArrayInputStream(delta)) {
@Override
protected long getBaseSize() throws IOException {
return base.length;
}
@Override
protected InputStream openBase() throws IOException {
return new ByteArrayInputStream(base);
}
};
}
}

265
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedDeltaObject.java
Unescape View File

@ -1,265 +0,0 @@
/*
* Copyright (C) 2010, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.storage.file;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.DataFormatException;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
import org.eclipse.jgit.internal.storage.pack.DeltaStream;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectStream;
import org.eclipse.jgit.util.io.TeeInputStream;
class LargePackedDeltaObject extends ObjectLoader {
private static final long SIZE_UNKNOWN = -1;
private int type;
private long size;
private final long objectOffset;
private final long baseOffset;
private final int headerLength;
private final PackFile pack;
private final FileObjectDatabase db;
LargePackedDeltaObject(long objectOffset,
long baseOffset, int headerLength, PackFile pack,
FileObjectDatabase db) {
this.type = Constants.OBJ_BAD;
this.size = SIZE_UNKNOWN;
this.objectOffset = objectOffset;
this.baseOffset = baseOffset;
this.headerLength = headerLength;
this.pack = pack;
this.db = db;
}
@Override
public int getType() {
if (type == Constants.OBJ_BAD) {
WindowCursor wc = new WindowCursor(db);
try {
type = pack.getObjectType(wc, objectOffset);
} catch (IOException packGone) {
// If the pack file cannot be pinned into the cursor, it
// probably was repacked recently. Go find the object
// again and get the type from that location instead.
//
try {
type = wc.open(getObjectId()).getType();
} catch (IOException packGone2) {
// "He's dead, Jim." We just can't discover the type
// and the interface isn't supposed to be lazy here.
// Report an invalid type code instead, callers will
// wind up bailing out with an error at some point.
}
} finally {
wc.release();
}
}
return type;
}
@Override
public long getSize() {
if (size == SIZE_UNKNOWN) {
WindowCursor wc = new WindowCursor(db);
try {
byte[] b = pack.getDeltaHeader(wc, objectOffset + headerLength);
size = BinaryDelta.getResultSize(b);
} catch (DataFormatException objectCorrupt) {
// The zlib stream for the delta is corrupt. We probably
// cannot access the object. Keep the size negative and
// report that bogus result to the caller.
} catch (IOException packGone) {
// If the pack file cannot be pinned into the cursor, it
// probably was repacked recently. Go find the object
// again and get the size from that location instead.
//
try {
size = wc.open(getObjectId()).getSize();
} catch (IOException packGone2) {
// "He's dead, Jim." We just can't discover the size
// and the interface isn't supposed to be lazy here.
// Report an invalid type code instead, callers will
// wind up bailing out with an error at some point.
}
} finally {
wc.release();
}
}
return size;
}
@Override
public boolean isLarge() {
return true;
}
@Override
public byte[] getCachedBytes() throws LargeObjectException {
try {
throw new LargeObjectException(getObjectId());
} catch (IOException cannotObtainId) {
LargeObjectException err = new LargeObjectException();
err.initCause(cannotObtainId);
throw err;
}
}
@Override
public ObjectStream openStream() throws MissingObjectException, IOException {
// If the object was recently unpacked, its available loose.
// The loose format is going to be faster to access than a
// delta applied on top of a base. Use that whenever we can.
//
final ObjectId myId = getObjectId();
final WindowCursor wc = new WindowCursor(db);
ObjectLoader ldr = db.openLooseObject(wc, myId);
if (ldr != null)
return ldr.openStream();
InputStream in = open(wc);
in = new BufferedInputStream(in, 8192);
// While we inflate the object, also deflate it back as a loose
// object. This will later be cleaned up by a gc pass, but until
// then we will reuse the loose form by the above code path.
//
int myType = getType();
long mySize = getSize();
final ObjectDirectoryInserter odi = db.newInserter();
final File tmp = odi.newTempFile();
DeflaterOutputStream dOut = odi.compress(new FileOutputStream(tmp));
odi.writeHeader(dOut, myType, mySize);
in = new TeeInputStream(in, dOut);
return new ObjectStream.Filter(myType, mySize, in) {
@Override
public void close() throws IOException {
super.close();
odi.release();
wc.release();
db.insertUnpackedObject(tmp, myId, true /* force creation */);
}
};
}
private InputStream open(final WindowCursor wc)
throws MissingObjectException, IOException,
IncorrectObjectTypeException {
InputStream delta;
try {
delta = new PackInputStream(pack, objectOffset + headerLength, wc);
} catch (IOException packGone) {
// If the pack file cannot be pinned into the cursor, it
// probably was repacked recently. Go find the object
// again and open the stream from that location instead.
//
return wc.open(getObjectId()).openStream();
}
delta = new InflaterInputStream(delta);
final ObjectLoader base = pack.load(wc, baseOffset);
DeltaStream ds = new DeltaStream(delta) {
private long baseSize = SIZE_UNKNOWN;
@Override
protected InputStream openBase() throws IOException {
InputStream in;
if (base instanceof LargePackedDeltaObject)
in = ((LargePackedDeltaObject) base).open(wc);
else
in = base.openStream();
if (baseSize == SIZE_UNKNOWN) {
if (in instanceof DeltaStream)
baseSize = ((DeltaStream) in).getSize();
else if (in instanceof ObjectStream)
baseSize = ((ObjectStream) in).getSize();
}
return in;
}
@Override
protected long getBaseSize() throws IOException {
if (baseSize == SIZE_UNKNOWN) {
// This code path should never be used as DeltaStream
// is supposed to open the stream first, which would
// initialize the size for us directly from the stream.
baseSize = base.getSize();
}
return baseSize;
}
};
if (type == Constants.OBJ_BAD) {
if (!(base instanceof LargePackedDeltaObject))
type = base.getType();
}
if (size == SIZE_UNKNOWN)
size = ds.getSize();
return ds;
}
private ObjectId getObjectId() throws IOException {
return pack.findObjectForOffset(objectOffset);
}
}

25
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java
Unescape View File

@ -65,6 +65,7 @@ import java.util.zip.DataFormatException;
import java.util.zip.Inflater; import java.util.zip.Inflater;
import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.errors.PackInvalidException; import org.eclipse.jgit.errors.PackInvalidException;
import org.eclipse.jgit.errors.PackMismatchException; import org.eclipse.jgit.errors.PackMismatchException;
@ -701,7 +702,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
} }
ObjectLoader load(final WindowCursor curs, long pos) ObjectLoader load(final WindowCursor curs, long pos)
throws IOException { throws IOException, LargeObjectException {
try { try {
final byte[] ib = curs.tempId; final byte[] ib = curs.tempId;
Delta delta = null; Delta delta = null;
@ -727,7 +728,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
case Constants.OBJ_TREE: case Constants.OBJ_TREE:
case Constants.OBJ_BLOB: case Constants.OBJ_BLOB:
case Constants.OBJ_TAG: { case Constants.OBJ_TAG: {
if (sz < curs.getStreamFileThreshold()) if (delta != null || sz < curs.getStreamFileThreshold())
data = decompress(pos + p, (int) sz, curs); data = decompress(pos + p, (int) sz, curs);
if (delta != null) { if (delta != null) {
@ -796,7 +797,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
// (Whole objects with no deltas to apply return early above.) // (Whole objects with no deltas to apply return early above.)
if (data == null) if (data == null)
return delta.large(this, curs); throw new IOException(JGitText.get().inMemoryBufferLimitExceeded);
do { do {
// Cache only the base immediately before desired object. // Cache only the base immediately before desired object.
@ -811,19 +812,19 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
delta.deltaSize, curs); delta.deltaSize, curs);
if (cmds == null) { if (cmds == null) {
data = null; // Discard base in case of OutOfMemoryError data = null; // Discard base in case of OutOfMemoryError
return delta.large(this, curs); throw new LargeObjectException.OutOfMemory(new OutOfMemoryError());
} }
final long sz = BinaryDelta.getResultSize(cmds); final long sz = BinaryDelta.getResultSize(cmds);
if (Integer.MAX_VALUE <= sz) if (Integer.MAX_VALUE <= sz)
return delta.large(this, curs); throw new LargeObjectException.ExceedsByteArrayLimit();
final byte[] result; final byte[] result;
try { try {
result = new byte[(int) sz]; result = new byte[(int) sz];
} catch (OutOfMemoryError tooBig) { } catch (OutOfMemoryError tooBig) {
data = null; // Discard base in case of OutOfMemoryError data = null; // Discard base in case of OutOfMemoryError
return delta.large(this, curs); throw new LargeObjectException.OutOfMemory(tooBig);
} }
BinaryDelta.apply(data, cmds, result); BinaryDelta.apply(data, cmds, result);
@ -875,18 +876,6 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
this.hdrLen = hdrLen; this.hdrLen = hdrLen;
this.basePos = baseOffset; this.basePos = baseOffset;
} }
ObjectLoader large(PackFile pack, WindowCursor wc) {
Delta d = this;
while (d.next != null)
d = d.next;
return d.newLargeLoader(pack, wc);
}
private ObjectLoader newLargeLoader(PackFile pack, WindowCursor wc) {
return new LargePackedDeltaObject(deltaPos, basePos, hdrLen,
pack, wc.db);
}
} }
byte[] getDeltaHeader(WindowCursor wc, long pos) byte[] getDeltaHeader(WindowCursor wc, long pos)

4
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/DeltaIndex.java
Unescape View File

@ -51,8 +51,8 @@ import java.io.OutputStream;
* <p> * <p>
* The index can be passed a result buffer, and output an instruction sequence * The index can be passed a result buffer, and output an instruction sequence
* that transforms the source buffer used by the index into the result buffer. * that transforms the source buffer used by the index into the result buffer.
* The instruction sequence can be executed by {@link BinaryDelta} or * The instruction sequence can be executed by {@link BinaryDelta} to recreate
* {@link DeltaStream} to recreate the result buffer. * the result buffer.
* <p> * <p>
* An index stores the entire contents of the source buffer, but also a table of * An index stores the entire contents of the source buffer, but also a table of
* block identities mapped to locations where the block appears in the source * block identities mapped to locations where the block appears in the source

343
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/DeltaStream.java
Unescape View File

@ -1,343 +0,0 @@
/*
* Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com>
* Copyright (C) 2006-2007, Shawn O. Pearce <spearce@spearce.org>
* Copyright (C) 2010, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.storage.pack;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.util.IO;
/**
* Inflates a delta in an incremental way.
* <p>
* Implementations must provide a means to access a stream for the base object.
* This stream may be accessed multiple times, in order to randomly position it
* to match the copy instructions. A {@code DeltaStream} performs an efficient
* skip by only moving through the delta stream, making restarts of stacked
* deltas reasonably efficient.
*/
public abstract class DeltaStream extends InputStream {
private static final int CMD_COPY = 0;
private static final int CMD_INSERT = 1;
private static final int CMD_EOF = 2;
private final InputStream deltaStream;
private long baseSize;
private long resultSize;
private final byte[] cmdbuf = new byte[512];
private int cmdptr;
private int cmdcnt;
/** Stream to read from the base object. */
private InputStream baseStream;
/** Current position within {@link #baseStream}. */
private long baseOffset;
private int curcmd;
/** If {@code curcmd == CMD_COPY}, position the base has to be at. */
private long copyOffset;
/** Total number of bytes in this current command. */
private int copySize;
/**
* Construct a delta application stream, reading instructions.
*
* @param deltaStream
* the stream to read delta instructions from.
* @throws IOException
* the delta instruction stream cannot be read, or is
* inconsistent with the the base object information.
*/
public DeltaStream(final InputStream deltaStream) throws IOException {
this.deltaStream = deltaStream;
if (!fill(cmdbuf.length))
throw new EOFException();
// Length of the base object.
//
int c, shift = 0;
do {
c = cmdbuf[cmdptr++] & 0xff;
baseSize |= ((long) (c & 0x7f)) << shift;
shift += 7;
} while ((c & 0x80) != 0);
// Length of the resulting object.
//
shift = 0;
do {
c = cmdbuf[cmdptr++] & 0xff;
resultSize |= ((long) (c & 0x7f)) << shift;
shift += 7;
} while ((c & 0x80) != 0);
curcmd = next();
}
/**
* Open the base stream.
* <p>
* The {@code DeltaStream} may close and reopen the base stream multiple
* times if copy instructions use offsets out of order. This can occur if a
* large block in the file was moved from near the top, to near the bottom.
* In such cases the reopened stream is skipped to the target offset, so
* {@code skip(long)} should be as efficient as possible.
*
* @return stream to read from the base object. This stream should not be
* buffered (or should be only minimally buffered), and does not
* need to support mark/reset.
* @throws IOException
* the base object cannot be opened for reading.
*/
protected abstract InputStream openBase() throws IOException;
/**
* @return length of the base object, in bytes.
* @throws IOException
* the length of the base cannot be determined.
*/
protected abstract long getBaseSize() throws IOException;
/** @return total size of this stream, in bytes. */
public long getSize() {
return resultSize;
}
@Override
public int read() throws IOException {
byte[] buf = new byte[1];
int n = read(buf, 0, 1);
return n == 1 ? buf[0] & 0xff : -1;
}
@Override
public void close() throws IOException {
deltaStream.close();
if (baseStream != null)
baseStream.close();
}
@Override
public long skip(long len) throws IOException {
long act = 0;
while (0 < len) {
long n = Math.min(len, copySize);
switch (curcmd) {
case CMD_COPY:
copyOffset += n;
break;
case CMD_INSERT:
cmdptr += n;
break;
case CMD_EOF:
return act;
default:
throw new CorruptObjectException(
JGitText.get().unsupportedCommand0);
}
act += n;
len -= n;
copySize -= n;
if (copySize == 0)
curcmd = next();
}
return act;
}
@Override
public int read(byte[] buf, int off, int len) throws IOException {
int act = 0;
while (0 < len) {
int n = Math.min(len, copySize);
switch (curcmd) {
case CMD_COPY:
seekBase();
n = baseStream.read(buf, off, n);
if (n < 0)
throw new CorruptObjectException(
JGitText.get().baseLengthIncorrect);
copyOffset += n;
baseOffset = copyOffset;
break;
case CMD_INSERT:
System.arraycopy(cmdbuf, cmdptr, buf, off, n);
cmdptr += n;
break;
case CMD_EOF:
return 0 < act ? act : -1;
default:
throw new CorruptObjectException(
JGitText.get().unsupportedCommand0);
}
act += n;
off += n;
len -= n;
copySize -= n;
if (copySize == 0)
curcmd = next();
}
return act;
}
private boolean fill(final int need) throws IOException {
int n = have();
if (need < n)
return true;
if (n == 0) {
cmdptr = 0;
cmdcnt = 0;
} else if (cmdbuf.length - cmdptr < need) {
// There isn't room for the entire worst-case copy command,
// so shift the array down to make sure we can use the entire
// command without having it span across the end of the array.
//
System.arraycopy(cmdbuf, cmdptr, cmdbuf, 0, n);
cmdptr = 0;
cmdcnt = n;
}
do {
n = deltaStream.read(cmdbuf, cmdcnt, cmdbuf.length - cmdcnt);
if (n < 0)
return 0 < have();
cmdcnt += n;
} while (cmdcnt < cmdbuf.length);
return true;
}
private int next() throws IOException {
if (!fill(8))
return CMD_EOF;
final int cmd = cmdbuf[cmdptr++] & 0xff;
if ((cmd & 0x80) != 0) {
// Determine the segment of the base which should
// be copied into the output. The segment is given
// as an offset and a length.
//
copyOffset = 0;
if ((cmd & 0x01) != 0)
copyOffset = cmdbuf[cmdptr++] & 0xff;
if ((cmd & 0x02) != 0)
copyOffset |= (cmdbuf[cmdptr++] & 0xff) << 8;
if ((cmd & 0x04) != 0)
copyOffset |= (cmdbuf[cmdptr++] & 0xff) << 16;
if ((cmd & 0x08) != 0)
copyOffset |= ((long) (cmdbuf[cmdptr++] & 0xff)) << 24;
copySize = 0;
if ((cmd & 0x10) != 0)
copySize = cmdbuf[cmdptr++] & 0xff;
if ((cmd & 0x20) != 0)
copySize |= (cmdbuf[cmdptr++] & 0xff) << 8;
if ((cmd & 0x40) != 0)
copySize |= (cmdbuf[cmdptr++] & 0xff) << 16;
if (copySize == 0)
copySize = 0x10000;
return CMD_COPY;
} else if (cmd != 0) {
// Anything else the data is literal within the delta
// itself. Page the entire thing into the cmdbuf, if
// its not already there.
//
fill(cmd);
copySize = cmd;
return CMD_INSERT;
} else {
// cmd == 0 has been reserved for future encoding but
// for now its not acceptable.
//
throw new CorruptObjectException(JGitText.get().unsupportedCommand0);
}
}
private int have() {
return cmdcnt - cmdptr;
}
private void seekBase() throws IOException {
if (baseStream == null) {
baseStream = openBase();
if (getBaseSize() != baseSize)
throw new CorruptObjectException(
JGitText.get().baseLengthIncorrect);
IO.skipFully(baseStream, copyOffset);
baseOffset = copyOffset;
} else if (baseOffset < copyOffset) {
IO.skipFully(baseStream, copyOffset - baseOffset);
baseOffset = copyOffset;
} else if (baseOffset > copyOffset) {
baseStream.close();
baseStream = openBase();
IO.skipFully(baseStream, copyOffset);
baseOffset = copyOffset;
}
}
}
Write Preview
Loading…
Cancel
Save