From b77ba049762e4ea3aadb756dad1d06c859bb3fe3 Mon Sep 17 00:00:00 2001 From: Colby Ranger Date: Mon, 13 Aug 2012 13:52:33 -0700 Subject: [PATCH 1/2] Do not delta compress objects that have already tried to compress. If an object is in a pack file already, delta compression will not attempt to re-compress it. This assumes that the previous packing already performed the optimal compression attempt, however, the subclasses of StoredObjectRepresentation may use other heuristics to determine if the stored format is optimal. Change-Id: I403de522f4b0dd2667d54f6faed621f392c07786 --- .../jgit/storage/pack/DeltaWindow.java | 2 +- .../jgit/storage/pack/ObjectToPack.java | 22 ++++++++++++++++++- .../eclipse/jgit/storage/pack/PackWriter.java | 1 + .../pack/StoredObjectRepresentation.java | 14 ++++++++++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java index 07dcd943c..fead2147b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java @@ -141,7 +141,7 @@ class DeltaWindow { } res.set(toSearch[off]); - if (res.object.isEdge()) { + if (res.object.isEdge() || res.object.doNotAttemptDelta()) { // We don't actually want to make a delta for // them, just need to push them into the window // so they can be read by other objects. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectToPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectToPack.java index 1f4e3e7e2..9c50f8d01 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectToPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectToPack.java @@ -66,6 +66,10 @@ public class ObjectToPack extends PackedObjectInfo { private static final int EDGE = 1 << 3; + private static final int DELTA_ATTEMPTED = 1 << 4; + + private static final int ATTEMPT_DELTA_MASK = REUSE_AS_IS | DELTA_ATTEMPTED; + private static final int TYPE_SHIFT = 5; private static final int EXT_SHIFT = 8; @@ -88,7 +92,7 @@ public class ObjectToPack extends PackedObjectInfo { *
  • 1 bit: canReuseAsIs
    • *
  • 1 bit: doNotDelta
    • *
  • 1 bit: edgeObject
    • - *
  • 1 bit: unused
    • + *
  • 1 bit: deltaAttempted
    • *
  • 3 bits: type
    • *
  • 4 bits: subclass flags (if any)
    • *
  • --
    • @@ -265,6 +269,22 @@ public class ObjectToPack extends PackedObjectInfo { flags |= EDGE; } + boolean doNotAttemptDelta() { + // Do not attempt if delta attempted and object reuse. + return (flags & ATTEMPT_DELTA_MASK) == ATTEMPT_DELTA_MASK; + } + + boolean isDeltaAttempted() { + return (flags & DELTA_ATTEMPTED) != 0; + } + + void setDeltaAttempted(boolean deltaAttempted) { + if (deltaAttempted) + flags |= DELTA_ATTEMPTED; + else + flags &= ~DELTA_ATTEMPTED; + } + /** @return the extended flags on this object, in the range [0x0, 0xf]. */ protected int getExtendedFlags() { return (flags >>> EXT_SHIFT) & EXT_MASK; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java index d93e2d680..99ec75cf9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java @@ -1870,6 +1870,7 @@ public class PackWriter { otp.clearReuseAsIs(); } + otp.setDeltaAttempted(next.wasDeltaAttempted()); otp.select(next); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/StoredObjectRepresentation.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/StoredObjectRepresentation.java index 334ea5ea1..543bc2f79 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/StoredObjectRepresentation.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/StoredObjectRepresentation.java @@ -72,8 +72,9 @@ public class StoredObjectRepresentation { } /** - * @return true if this is a delta against another object and this is stored - * in pack delta format. + * @return the storage format type, which must be one of + * {@link #PACK_DELTA}, {@link #PACK_WHOLE}, or + * {@link #FORMAT_OTHER}. */ public int getFormat() { return FORMAT_OTHER; @@ -87,4 +88,13 @@ public class StoredObjectRepresentation { public ObjectId getDeltaBase() { return null; } + + /** + * @return whether the current representation of the object has had delta + * compression attempted on it. + */ + public boolean wasDeltaAttempted() { + int fmt = getFormat(); + return fmt == PACK_DELTA || fmt == PACK_WHOLE; + } } From b777d7797d6fbb36f8bfeb6b018c2d0cbf0477e0 Mon Sep 17 00:00:00 2001 From: Colby Ranger Date: Mon, 13 Aug 2012 15:54:06 -0700 Subject: [PATCH 2/2] Implement wasDeltaAttempted() in DfsObjectRepresentation. In DFS, everything is stored in a pack but only objects in a pack with source GC or UNREACHABLE_GARBAGE have had delta compression attempted. Expose the PackSource setter and getter on DfsPackDescription in order to implement wasDeltaAttempted. Change-Id: Ie949f321147ad870f1c3f23b552343bbbda32152 --- .../storage/dfs/DfsObjectRepresentation.java | 13 +++++++++++++ .../jgit/storage/dfs/DfsPackDescription.java | 18 ++++++++++++++++++ .../jgit/storage/dfs/InMemoryRepository.java | 4 +++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsObjectRepresentation.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsObjectRepresentation.java index 1b8e3a3d4..2b45ffa20 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsObjectRepresentation.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsObjectRepresentation.java @@ -43,7 +43,11 @@ package org.eclipse.jgit.storage.dfs; +import static org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource.GC; +import static org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; + import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.storage.pack.ObjectToPack; import org.eclipse.jgit.storage.pack.StoredObjectRepresentation; @@ -87,4 +91,13 @@ class DfsObjectRepresentation extends StoredObjectRepresentation { public ObjectId getDeltaBase() { return baseId; } + + @Override + public boolean wasDeltaAttempted() { + if (pack != null) { + PackSource source = pack.getPackDescription().getPackSource(); + return source == GC || source == UNREACHABLE_GARBAGE; + } + return false; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackDescription.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackDescription.java index 1bd5a7827..6b90454b9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackDescription.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackDescription.java @@ -46,6 +46,7 @@ package org.eclipse.jgit.storage.dfs; import java.util.Set; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.storage.pack.PackWriter; /** @@ -61,6 +62,8 @@ public class DfsPackDescription implements Comparable { private final String packName; + private PackSource packSource; + private long lastModified; private long packSize; @@ -114,6 +117,21 @@ public class DfsPackDescription implements Comparable { return name.substring(0, dot) + ".idx"; } + /** @return the source of the pack. */ + public PackSource getPackSource() { + return packSource; + } + + /** + * @param source + * the source of the pack. + * @return {@code this} + */ + public DfsPackDescription setPackSource(PackSource source) { + packSource = source; + return this; + } + /** @return time the pack was created, in milliseconds. */ public long getLastModified() { return lastModified; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/InMemoryRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/InMemoryRepository.java index 270f23fae..d1ceae00c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/InMemoryRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/InMemoryRepository.java @@ -76,8 +76,10 @@ public class InMemoryRepository extends DfsRepository { @Override protected DfsPackDescription newPack(PackSource source) { int id = packId.incrementAndGet(); - return new MemPack("pack-" + id + "-" + source.name(), + DfsPackDescription desc = new MemPack( + "pack-" + id + "-" + source.name(), getRepository().getDescription()); + return desc.setPackSource(source); } @Override