diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java index c45a076b3..e548cc936 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java @@ -70,6 +70,19 @@ public class DeltaIndex { /** Number of bytes in a block. */ static final int BLKSZ = 16; // must be 16, see unrolled loop in hashBlock + /** + * Estimate the size of an index for a given source. + *

+ * This is roughly a worst-case estimate. The actual index may be smaller. + * + * @param sourceLength + * length of the source, in bytes. + * @return estimated size. Approximately {@code 1.75 * sourceLength}. + */ + public static long estimateIndexSize(int sourceLength) { + return sourceLength + (sourceLength * 3 / 4); + } + /** * Maximum number of positions to consider for a given content hash. *

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java index fa577b669..ee865fbe5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java @@ -68,9 +68,15 @@ class DeltaWindow { private final DeltaWindowEntry[] window; + /** Maximum number of bytes to admit to the window at once. */ + private final long maxMemory; + /** Maximum depth we should create for any delta chain. */ private final int maxDepth; + /** Amount of memory we have loaded right now. */ + private long loaded; + // The object we are currently considering needs a lot of state: /** Position of {@link #res} within {@link #window} array. */ @@ -115,6 +121,7 @@ class DeltaWindow { for (int i = 0; i < window.length; i++) window[i] = new DeltaWindowEntry(); + maxMemory = pw.getDeltaSearchMemoryLimit(); maxDepth = pw.getMaxDeltaDepth(); } @@ -125,6 +132,15 @@ class DeltaWindow { monitor.update(1); res = window[resSlot]; + if (0 < maxMemory) { + clear(res); + int tail = next(resSlot); + final long need = estimateSize(toSearch[off]); + while (maxMemory < loaded + need && tail != resSlot) { + clear(window[tail]); + tail = next(tail); + } + } res.set(toSearch[off]); if (res.object.isDoNotDelta()) { @@ -148,6 +164,18 @@ class DeltaWindow { } } + private static long estimateSize(ObjectToPack ent) { + return DeltaIndex.estimateIndexSize(ent.getWeight()); + } + + private void clear(DeltaWindowEntry ent) { + if (ent.index != null) + loaded -= ent.index.getIndexSize(); + else if (res.buffer != null) + loaded -= ent.buffer.length; + ent.set(null); + } + private void search() throws IOException { // TODO(spearce) If the object is used as a base for other // objects in this pack we should limit the depth we create @@ -336,8 +364,13 @@ class DeltaWindow { } private void keepInWindow() { - if (++resSlot == window.length) - resSlot = 0; + resSlot = next(resSlot); + } + + private int next(int slot) { + if (++slot == window.length) + return 0; + return slot; } private int prior(int slot) { @@ -397,6 +430,8 @@ class DeltaWindow { e.initCause(noMemory); throw e; } + if (0 < maxMemory) + loaded += idx.getIndexSize() - idx.getSourceSize(); ent.index = idx; } return idx; @@ -405,8 +440,12 @@ class DeltaWindow { private byte[] buffer(DeltaWindowEntry ent) throws MissingObjectException, IncorrectObjectTypeException, IOException, LargeObjectException { byte[] buf = ent.buffer; - if (buf == null) - ent.buffer = buf = writer.buffer(reader, ent.object); + if (buf == null) { + buf = writer.buffer(reader, ent.object); + if (0 < maxMemory) + loaded += buf.length; + ent.buffer = buf; + } return buf; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java index 3769d6b4b..38f00b573 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java @@ -229,6 +229,8 @@ public class PackWriter { private int deltaSearchWindowSize = DEFAULT_DELTA_SEARCH_WINDOW_SIZE; + private long deltaSearchMemoryLimit; + private long deltaCacheSize = DEFAULT_DELTA_CACHE_SIZE; private int deltaCacheLimit = DEFAULT_DELTA_CACHE_LIMIT; @@ -289,6 +291,7 @@ public class PackWriter { final PackConfig pc = configOf(repo).get(PackConfig.KEY); deltaSearchWindowSize = pc.deltaWindow; + deltaSearchMemoryLimit = pc.deltaWindowMemory; deltaCacheSize = pc.deltaCacheSize; deltaCacheLimit = pc.deltaCacheLimit; maxDeltaDepth = pc.deltaDepth; @@ -485,6 +488,36 @@ public class PackWriter { deltaSearchWindowSize = objectCount; } + /** + * Get maximum number of bytes to put into the delta search window. + *

+ * Default setting is 0, for an unlimited amount of memory usage. Actual + * memory used is the lower limit of either this setting, or the sum of + * space used by at most {@link #getDeltaSearchWindowSize()} objects. + *

+ * This limit is per thread, if 4 threads are used the actual memory + * limit will be 4 times this value. + * + * @return the memory limit. + */ + public long getDeltaSearchMemoryLimit() { + return deltaSearchMemoryLimit; + } + + /** + * Set the maximum number of bytes to put into the delta search window. + *

+ * Default setting is 0, for an unlimited amount of memory usage. If the + * memory limit is reached before {@link #getDeltaSearchWindowSize()} the + * window size is temporarily lowered. + * + * @param memoryLimit + * Maximum number of bytes to load at once, 0 for unlimited. + */ + public void setDeltaSearchMemoryLimit(long memoryLimit) { + deltaSearchMemoryLimit = memoryLimit; + } + /** * Get the size of the in-memory delta cache. *