Honor pack.windowlimit to cap memory usage during packing

The pack.windowlimit configuration parameter places an upper bound
on the number of bytes used by the DeltaWindow class as it scans
through the object list.  If memory usage would exceed the limit
the window is temporarily decreased in size to keep memory used
within that bound.

Change-Id: I09521b8f335475d8aee6125826da8ba2e545060d
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java

@@ -70,6 +70,19 @@ public class DeltaIndex {
 	/** Number of bytes in a block. */
 	static final int BLKSZ = 16; // must be 16, see unrolled loop in hashBlock
 
+	/**
+	 * Estimate the size of an index for a given source.
+	 * <p>
+	 * This is roughly a worst-case estimate. The actual index may be smaller.
+	 *
+	 * @param sourceLength
+	 *            length of the source, in bytes.
+	 * @return estimated size. Approximately {@code 1.75 * sourceLength}.
+	 */
+	public static long estimateIndexSize(int sourceLength) {
+		return sourceLength + (sourceLength * 3 / 4);
+	}
+
 	/**
 	 * Maximum number of positions to consider for a given content hash.
 	 * <p>

org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java

@@ -68,9 +68,15 @@ class DeltaWindow {
 
 	private final DeltaWindowEntry[] window;
 
+	/** Maximum number of bytes to admit to the window at once. */
+	private final long maxMemory;
+
 	/** Maximum depth we should create for any delta chain. */
 	private final int maxDepth;
 
+	/** Amount of memory we have loaded right now. */
+	private long loaded;
+
 	// The object we are currently considering needs a lot of state:
 
 	/** Position of {@link #res} within {@link #window} array. */
@@ -115,6 +121,7 @@ class DeltaWindow {
 		for (int i = 0; i < window.length; i++)
 			window[i] = new DeltaWindowEntry();
 
+		maxMemory = pw.getDeltaSearchMemoryLimit();
 		maxDepth = pw.getMaxDeltaDepth();
 	}
 
@@ -125,6 +132,15 @@ class DeltaWindow {
 				monitor.update(1);
 
 				res = window[resSlot];
+				if (0 < maxMemory) {
+					clear(res);
+					int tail = next(resSlot);
+					final long need = estimateSize(toSearch[off]);
+					while (maxMemory < loaded + need && tail != resSlot) {
+						clear(window[tail]);
+						tail = next(tail);
+					}
+				}
 				res.set(toSearch[off]);
 
 				if (res.object.isDoNotDelta()) {
@@ -148,6 +164,18 @@ class DeltaWindow {
 		}
 	}
 
+	private static long estimateSize(ObjectToPack ent) {
+		return DeltaIndex.estimateIndexSize(ent.getWeight());
+	}
+
+	private void clear(DeltaWindowEntry ent) {
+		if (ent.index != null)
+			loaded -= ent.index.getIndexSize();
+		else if (res.buffer != null)
+			loaded -= ent.buffer.length;
+		ent.set(null);
+	}
+
 	private void search() throws IOException {
 		// TODO(spearce) If the object is used as a base for other
 		// objects in this pack we should limit the depth we create
@@ -336,8 +364,13 @@ class DeltaWindow {
 	}
 
 	private void keepInWindow() {
-		if (++resSlot == window.length)
+		resSlot = next(resSlot);
-			resSlot = 0;
+	}
+
+	private int next(int slot) {
+		if (++slot == window.length)
+			return 0;
+		return slot;
 	}
 
 	private int prior(int slot) {
@@ -397,6 +430,8 @@ class DeltaWindow {
 				e.initCause(noMemory);
 				throw e;
 			}
+			if (0 < maxMemory)
+				loaded += idx.getIndexSize() - idx.getSourceSize();
 			ent.index = idx;
 		}
 		return idx;
@@ -405,8 +440,12 @@ class DeltaWindow {
 	private byte[] buffer(DeltaWindowEntry ent) throws MissingObjectException,
 			IncorrectObjectTypeException, IOException, LargeObjectException {
 		byte[] buf = ent.buffer;
-		if (buf == null)
+		if (buf == null) {
-			ent.buffer = buf = writer.buffer(reader, ent.object);
+			buf = writer.buffer(reader, ent.object);
+			if (0 < maxMemory)
+				loaded += buf.length;
+			ent.buffer = buf;
+		}
 		return buf;
 	}
 

org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java

@@ -229,6 +229,8 @@ public class PackWriter {
 
 	private int deltaSearchWindowSize = DEFAULT_DELTA_SEARCH_WINDOW_SIZE;
 
+	private long deltaSearchMemoryLimit;
+
 	private long deltaCacheSize = DEFAULT_DELTA_CACHE_SIZE;
 
 	private int deltaCacheLimit = DEFAULT_DELTA_CACHE_LIMIT;
@@ -289,6 +291,7 @@ public class PackWriter {
 
 		final PackConfig pc = configOf(repo).get(PackConfig.KEY);
 		deltaSearchWindowSize = pc.deltaWindow;
+		deltaSearchMemoryLimit = pc.deltaWindowMemory;
 		deltaCacheSize = pc.deltaCacheSize;
 		deltaCacheLimit = pc.deltaCacheLimit;
 		maxDeltaDepth = pc.deltaDepth;
@@ -485,6 +488,36 @@ public class PackWriter {
 			deltaSearchWindowSize = objectCount;
 	}
 
+	/**
+	 * Get maximum number of bytes to put into the delta search window.
+	 * <p>
+	 * Default setting is 0, for an unlimited amount of memory usage. Actual
+	 * memory used is the lower limit of either this setting, or the sum of
+	 * space used by at most {@link #getDeltaSearchWindowSize()} objects.
+	 * <p>
+	 * This limit is per thread, if 4 threads are used the actual memory
+	 * limit will be 4 times this value.
+	 *
+	 * @return the memory limit.
+	 */
+	public long getDeltaSearchMemoryLimit() {
+		return deltaSearchMemoryLimit;
+	}
+
+	/**
+	 * Set the maximum number of bytes to put into the delta search window.
+	 * <p>
+	 * Default setting is 0, for an unlimited amount of memory usage. If the
+	 * memory limit is reached before {@link #getDeltaSearchWindowSize()} the
+	 * window size is temporarily lowered.
+	 *
+	 * @param memoryLimit
+	 *            Maximum number of bytes to load at once, 0 for unlimited.
+	 */
+	public void setDeltaSearchMemoryLimit(long memoryLimit) {
+		deltaSearchMemoryLimit = memoryLimit;
+	}
+
 	/**
 	 * Get the size of the in-memory delta cache.
 	 * <p>