From 97341949171dcdd7d6c30f3eef89ee2c63d8eb34 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 9 Jul 2010 18:32:33 -0700 Subject: [PATCH] Honor pack.windowlimit to cap memory usage during packing The pack.windowlimit configuration parameter places an upper bound on the number of bytes used by the DeltaWindow class as it scans through the object list. If memory usage would exceed the limit the window is temporarily decreased in size to keep memory used within that bound. Change-Id: I09521b8f335475d8aee6125826da8ba2e545060d Signed-off-by: Shawn O. Pearce --- .../eclipse/jgit/storage/pack/DeltaIndex.java | 13 +++++ .../jgit/storage/pack/DeltaWindow.java | 47 +++++++++++++++++-- .../eclipse/jgit/storage/pack/PackWriter.java | 33 +++++++++++++ 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java index c45a076b3..e548cc936 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java @@ -70,6 +70,19 @@ public class DeltaIndex { /** Number of bytes in a block. */ static final int BLKSZ = 16; // must be 16, see unrolled loop in hashBlock + /** + * Estimate the size of an index for a given source. + *

+ * This is roughly a worst-case estimate. The actual index may be smaller. + * + * @param sourceLength + * length of the source, in bytes. + * @return estimated size. Approximately {@code 1.75 * sourceLength}. + */ + public static long estimateIndexSize(int sourceLength) { + return sourceLength + (sourceLength * 3 / 4); + } + /** * Maximum number of positions to consider for a given content hash. *

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java index fa577b669..ee865fbe5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java @@ -68,9 +68,15 @@ class DeltaWindow { private final DeltaWindowEntry[] window; + /** Maximum number of bytes to admit to the window at once. */ + private final long maxMemory; + /** Maximum depth we should create for any delta chain. */ private final int maxDepth; + /** Amount of memory we have loaded right now. */ + private long loaded; + // The object we are currently considering needs a lot of state: /** Position of {@link #res} within {@link #window} array. */ @@ -115,6 +121,7 @@ class DeltaWindow { for (int i = 0; i < window.length; i++) window[i] = new DeltaWindowEntry(); + maxMemory = pw.getDeltaSearchMemoryLimit(); maxDepth = pw.getMaxDeltaDepth(); } @@ -125,6 +132,15 @@ class DeltaWindow { monitor.update(1); res = window[resSlot]; + if (0 < maxMemory) { + clear(res); + int tail = next(resSlot); + final long need = estimateSize(toSearch[off]); + while (maxMemory < loaded + need && tail != resSlot) { + clear(window[tail]); + tail = next(tail); + } + } res.set(toSearch[off]); if (res.object.isDoNotDelta()) { @@ -148,6 +164,18 @@ class DeltaWindow { } } + private static long estimateSize(ObjectToPack ent) { + return DeltaIndex.estimateIndexSize(ent.getWeight()); + } + + private void clear(DeltaWindowEntry ent) { + if (ent.index != null) + loaded -= ent.index.getIndexSize(); + else if (res.buffer != null) + loaded -= ent.buffer.length; + ent.set(null); + } + private void search() throws IOException { // TODO(spearce) If the object is used as a base for other // objects in this pack we should limit the depth we create @@ -336,8 +364,13 @@ class DeltaWindow { } private void keepInWindow() { - if (++resSlot == window.length) - resSlot = 0; + resSlot = next(resSlot); + } + + private int next(int slot) { + if (++slot == window.length) + return 0; + return slot; } private int prior(int slot) { @@ -397,6 +430,8 @@ class DeltaWindow { e.initCause(noMemory); throw e; } + if (0 < maxMemory) + loaded += idx.getIndexSize() - idx.getSourceSize(); ent.index = idx; } return idx; @@ -405,8 +440,12 @@ class DeltaWindow { private byte[] buffer(DeltaWindowEntry ent) throws MissingObjectException, IncorrectObjectTypeException, IOException, LargeObjectException { byte[] buf = ent.buffer; - if (buf == null) - ent.buffer = buf = writer.buffer(reader, ent.object); + if (buf == null) { + buf = writer.buffer(reader, ent.object); + if (0 < maxMemory) + loaded += buf.length; + ent.buffer = buf; + } return buf; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java index 3769d6b4b..38f00b573 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java @@ -229,6 +229,8 @@ public class PackWriter { private int deltaSearchWindowSize = DEFAULT_DELTA_SEARCH_WINDOW_SIZE; + private long deltaSearchMemoryLimit; + private long deltaCacheSize = DEFAULT_DELTA_CACHE_SIZE; private int deltaCacheLimit = DEFAULT_DELTA_CACHE_LIMIT; @@ -289,6 +291,7 @@ public class PackWriter { final PackConfig pc = configOf(repo).get(PackConfig.KEY); deltaSearchWindowSize = pc.deltaWindow; + deltaSearchMemoryLimit = pc.deltaWindowMemory; deltaCacheSize = pc.deltaCacheSize; deltaCacheLimit = pc.deltaCacheLimit; maxDeltaDepth = pc.deltaDepth; @@ -485,6 +488,36 @@ public class PackWriter { deltaSearchWindowSize = objectCount; } + /** + * Get maximum number of bytes to put into the delta search window. + *

+ * Default setting is 0, for an unlimited amount of memory usage. Actual + * memory used is the lower limit of either this setting, or the sum of + * space used by at most {@link #getDeltaSearchWindowSize()} objects. + *

+ * This limit is per thread, if 4 threads are used the actual memory + * limit will be 4 times this value. + * + * @return the memory limit. + */ + public long getDeltaSearchMemoryLimit() { + return deltaSearchMemoryLimit; + } + + /** + * Set the maximum number of bytes to put into the delta search window. + *

+ * Default setting is 0, for an unlimited amount of memory usage. If the + * memory limit is reached before {@link #getDeltaSearchWindowSize()} the + * window size is temporarily lowered. + * + * @param memoryLimit + * Maximum number of bytes to load at once, 0 for unlimited. + */ + public void setDeltaSearchMemoryLimit(long memoryLimit) { + deltaSearchMemoryLimit = memoryLimit; + } + /** * Get the size of the in-memory delta cache. *