Implement async/batch lookup of object data

An ObjectReader implementation may be very slow for a single object,
but yet support bulk queries efficiently by batching multiple small
requests into a single larger request.  This easily happens when the
reader is built on top of a database that is stored on another host,
as the network round-trip time starts to dominate the operation cost.

RevWalk, ObjectWalk, UploadPack and PackWriter are the first major
users of this new bulk interface, with the goal being to support an
efficient way to pack a repository for a fetch/clone client when the
source repository is stored in a high-latency storage system.

Processing the want/have lists is now done in bulk, to remove
the high costs associated with common ancestor negotiation.

PackWriter already performs object reuse selection in bulk, but it
now can also do the object size lookup and object counting phases
with higher efficiency.  Actual object reuse, deltification, and
final output are still doing sequential lookups, making them a bit
more expensive to perform.

Change-Id: I4c966f84917482598012074c370b9831451404ee
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

org.eclipse.jgit/src/org/eclipse/jgit/lib/AsyncObjectLoaderQueue.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2010, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.lib;
+
+import java.io.IOException;
+
+import org.eclipse.jgit.errors.MissingObjectException;
+
+/**
+ * Queue to open objects asynchronously.
+ *
+ * A queue may perform background decompression of objects and supply them
+ * (possibly out-of-order) to the application.
+ *
+ * @param <T>
+ *            type of identifier supplied to the call that made the queue.
+ */
+public interface AsyncObjectLoaderQueue<T extends ObjectId> extends
+		AsyncOperation {
+
+	/**
+	 * Position this queue onto the next available result.
+	 *
+	 * Even if this method returns true, {@link #open()} may still throw
+	 * {@link MissingObjectException} if the underlying object database was
+	 * concurrently modified and the current object is no longer available.
+	 *
+	 * @return true if there is a result available; false if the queue has
+	 *         finished its input iteration.
+	 * @throws MissingObjectException
+	 *             the object does not exist. If the implementation is retaining
+	 *             the application's objects {@link #getCurrent()} will be the
+	 *             current object that is missing. There may be more results
+	 *             still available, so the caller should continue invoking next
+	 *             to examine another result.
+	 * @throws IOException
+	 *             the object store cannot be accessed.
+	 */
+	public boolean next() throws MissingObjectException, IOException;
+
+	/**
+	 * @return the current object, null if the implementation lost track.
+	 *         Implementations may for performance reasons discard the caller's
+	 *         ObjectId and provider their own through {@link #getObjectId()}.
+	 */
+	public T getCurrent();
+
+	/** @return the ObjectId of the current object. Never null. */
+	public ObjectId getObjectId();
+
+	/**
+	 * Obtain a loader to read the object.
+	 *
+	 * This method can only be invoked once per result
+	 *
+	 * Due to race conditions with a concurrent modification of the underlying
+	 * object database, an object may be unavailable when this method is
+	 * invoked, even though next returned successfully.
+	 *
+	 * @return the ObjectLoader to read this object. Never null.
+	 * @throws MissingObjectException
+	 *             the object does not exist. If the implementation is retaining
+	 *             the application's objects {@link #getCurrent()} will be the
+	 *             current object that is missing. There may be more results
+	 *             still available, so the caller should continue invoking next
+	 *             to examine another result.
+	 * @throws IOException
+	 *             the object store cannot be accessed.
+	 */
+	public ObjectLoader open() throws IOException;
+}

org.eclipse.jgit/src/org/eclipse/jgit/lib/AsyncObjectSizeQueue.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2010, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.lib;
+
+import java.io.IOException;
+
+import org.eclipse.jgit.errors.MissingObjectException;
+
+/**
+ * Queue to examine object sizes asynchronously.
+ *
+ * A queue may perform background lookup of object sizes and supply them
+ * (possibly out-of-order) to the application.
+ *
+ * @param <T>
+ *            type of identifier supplied to the call that made the queue.
+ */
+public interface AsyncObjectSizeQueue<T extends ObjectId> extends
+		AsyncOperation {
+
+	/**
+	 * Position this queue onto the next available result.
+	 *
+	 * @return true if there is a result available; false if the queue has
+	 *         finished its input iteration.
+	 * @throws MissingObjectException
+	 *             the object does not exist. If the implementation is retaining
+	 *             the application's objects {@link #getCurrent()} will be the
+	 *             current object that is missing. There may be more results
+	 *             still available, so the caller should continue invoking next
+	 *             to examine another result.
+	 * @throws IOException
+	 *             the object store cannot be accessed.
+	 */
+	public boolean next() throws MissingObjectException, IOException;
+
+	/**
+	 * @return the current object, null if the implementation lost track.
+	 *         Implementations may for performance reasons discard the caller's
+	 *         ObjectId and provider their own through {@link #getObjectId()}.
+	 */
+	public T getCurrent();
+
+	/** @return the ObjectId of the current object. Never null. */
+	public ObjectId getObjectId();
+
+	/** @return the size of the current object. */
+	public long getSize();
+}

org.eclipse.jgit/src/org/eclipse/jgit/lib/AsyncOperation.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2010, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.lib;
+
+/**
+ * Asynchronous operation handle.
+ *
+ * Callers that start an asynchronous operation are supplied with a handle that
+ * may be used to attempt cancellation of the operation if the caller does not
+ * wish to continue.
+ */
+public interface AsyncOperation {
+	/**
+	 * Cancels the running task.
+	 *
+	 * Attempts to cancel execution of this task. This attempt will fail if the
+	 * task has already completed, already been cancelled, or could not be
+	 * cancelled for some other reason. If successful, and this task has not
+	 * started when cancel is called, this task should never run. If the task
+	 * has already started, then the mayInterruptIfRunning parameter determines
+	 * whether the thread executing this task should be interrupted in an
+	 * attempt to stop the task.
+	 *
+	 * @param mayInterruptIfRunning
+	 *            true if the thread executing this task should be interrupted;
+	 *            otherwise, in-progress tasks are allowed to complete
+	 * @return false if the task could not be cancelled, typically because it
+	 *         has already completed normally; true otherwise
+	 */
+	public boolean cancel(boolean mayInterruptIfRunning);
+
+	/** Release resources used by the operation, including cancellation. */
+	public void release();
+}

org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java

@@ -45,6 +45,7 @@ package org.eclipse.jgit.lib;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Iterator;
 
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
@@ -149,6 +150,60 @@ public abstract class ObjectReader {
 			throws MissingObjectException, IncorrectObjectTypeException,
 			IOException;
 
+	/**
+	 * Asynchronous object opening.
+	 *
+	 * @param <T>
+	 *            type of identifier being supplied.
+	 * @param objectIds
+	 *            objects to open from the object store. The supplied collection
+	 *            must not be modified until the queue has finished.
+	 * @param reportMissing
+	 *            if true missing objects are reported by calling failure with a
+	 *            MissingObjectException. This may be more expensive for the
+	 *            implementation to guarantee. If false the implementation may
+	 *            choose to report MissingObjectException, or silently skip over
+	 *            the object with no warning.
+	 * @return queue to read the objects from.
+	 */
+	public <T extends ObjectId> AsyncObjectLoaderQueue<T> open(
+			Iterable<T> objectIds, final boolean reportMissing) {
+		final Iterator<T> idItr = objectIds.iterator();
+		return new AsyncObjectLoaderQueue<T>() {
+			private T cur;
+
+			public boolean next() throws MissingObjectException, IOException {
+				if (idItr.hasNext()) {
+					cur = idItr.next();
+					return true;
+				} else {
+					return false;
+				}
+			}
+
+			public T getCurrent() {
+				return cur;
+			}
+
+			public ObjectId getObjectId() {
+				return cur;
+			}
+
+			public ObjectLoader open() throws IOException {
+				return ObjectReader.this.open(cur, OBJ_ANY);
+			}
+
+			public boolean cancel(boolean mayInterruptIfRunning) {
+				return true;
+			}
+
+			public void release() {
+				// Since we are sequential by default, we don't
+				// have any state to clean up if we terminate early.
+			}
+		};
+	}
+
 	/**
 	 * Get only the size of an object.
 	 * <p>
@@ -177,6 +232,63 @@ public abstract class ObjectReader {
 		return open(objectId, typeHint).getSize();
 	}
 
+	/**
+	 * Asynchronous object size lookup.
+	 *
+	 * @param <T>
+	 *            type of identifier being supplied.
+	 * @param objectIds
+	 *            objects to get the size of from the object store. The supplied
+	 *            collection must not be modified until the queue has finished.
+	 * @param reportMissing
+	 *            if true missing objects are reported by calling failure with a
+	 *            MissingObjectException. This may be more expensive for the
+	 *            implementation to guarantee. If false the implementation may
+	 *            choose to report MissingObjectException, or silently skip over
+	 *            the object with no warning.
+	 * @return queue to read object sizes from.
+	 */
+	public <T extends ObjectId> AsyncObjectSizeQueue<T> getObjectSize(
+			Iterable<T> objectIds, final boolean reportMissing) {
+		final Iterator<T> idItr = objectIds.iterator();
+		return new AsyncObjectSizeQueue<T>() {
+			private T cur;
+
+			private long sz;
+
+			public boolean next() throws MissingObjectException, IOException {
+				if (idItr.hasNext()) {
+					cur = idItr.next();
+					sz = getObjectSize(cur, OBJ_ANY);
+					return true;
+				} else {
+					return false;
+				}
+			}
+
+			public T getCurrent() {
+				return cur;
+			}
+
+			public ObjectId getObjectId() {
+				return cur;
+			}
+
+			public long getSize() {
+				return sz;
+			}
+
+			public boolean cancel(boolean mayInterruptIfRunning) {
+				return true;
+			}
+
+			public void release() {
+				// Since we are sequential by default, we don't
+				// have any state to clean up if we terminate early.
+			}
+		};
+	}
+
 	/**
 	 * Advice from a {@link RevWalk} that a walk is starting from these roots.
 	 *

org.eclipse.jgit/src/org/eclipse/jgit/revwalk/AsyncRevObjectQueue.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2010, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.revwalk;
+
+import java.io.IOException;
+
+import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.lib.AsyncOperation;
+
+/**
+ * Queue to lookup and parse objects asynchronously.
+ *
+ * A queue may perform background lookup of objects and supply them (possibly
+ * out-of-order) to the application.
+ */
+public interface AsyncRevObjectQueue extends AsyncOperation {
+	/**
+	 * Obtain the next object.
+	 *
+	 * @return the object; null if there are no more objects remaining.
+	 * @throws MissingObjectException
+	 *             the object does not exist. There may be more objects
+	 *             remaining in the iteration, the application should call
+	 *             {@link #next()} again.
+	 * @throws IOException
+	 *             the object store cannot be accessed.
+	 */
+	public RevObject next() throws MissingObjectException, IOException;
+}

org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java

@@ -50,19 +50,23 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.Iterator;
+import java.util.List;
 
 import org.eclipse.jgit.JGitText;
+import org.eclipse.jgit.errors.CorruptObjectException;
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
+import org.eclipse.jgit.errors.LargeObjectException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.errors.RevWalkException;
 import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.AsyncObjectLoaderQueue;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.MutableObjectId;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectIdSubclassMap;
 import org.eclipse.jgit.lib.ObjectLoader;
-import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.ObjectReader;
+import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.filter.RevFilter;
 import org.eclipse.jgit.treewalk.filter.TreeFilter;
 
@@ -683,6 +687,18 @@ public class RevWalk implements Iterable<RevCommit> {
 		return r;
 	}
 
+	/**
+	 * Locate an object that was previously allocated in this walk.
+	 *
+	 * @param id
+	 *            name of the object.
+	 * @return reference to the object if it has been previously located;
+	 *         otherwise null.
+	 */
+	public RevObject lookupOrNull(AnyObjectId id) {
+		return objects.get(id);
+	}
+
 	/**
 	 * Locate a reference to a commit and immediately parse its content.
 	 * <p>
@@ -789,9 +805,17 @@ public class RevWalk implements Iterable<RevCommit> {
 	public RevObject parseAny(final AnyObjectId id)
 			throws MissingObjectException, IOException {
 		RevObject r = objects.get(id);
-		if (r == null) {
+		if (r == null)
-			final ObjectLoader ldr = reader.open(id);
+			r = parseNew(id, reader.open(id));
-			final int type = ldr.getType();
+		else
+			parseHeaders(r);
+		return r;
+	}
+
+	private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
+			throws CorruptObjectException, LargeObjectException {
+		RevObject r;
+		int type = ldr.getType();
 		switch (type) {
 		case Constants.OBJ_COMMIT: {
 			final RevCommit c = createCommit(id);
@@ -816,14 +840,93 @@ public class RevWalk implements Iterable<RevCommit> {
 			break;
 		}
 		default:
-				throw new IllegalArgumentException(MessageFormat.format(JGitText.get().badObjectType, type));
+			throw new IllegalArgumentException(MessageFormat.format(JGitText
+					.get().badObjectType, type));
 		}
 		objects.add(r);
+		return r;
+	}
+
+	/**
+	 * Asynchronous object parsing.
+	 *
+	 * @param <T>
+	 *            any ObjectId type.
+	 * @param objectIds
+	 *            objects to open from the object store. The supplied collection
+	 *            must not be modified until the queue has finished.
+	 * @param reportMissing
+	 *            if true missing objects are reported by calling failure with a
+	 *            MissingObjectException. This may be more expensive for the
+	 *            implementation to guarantee. If false the implementation may
+	 *            choose to report MissingObjectException, or silently skip over
+	 *            the object with no warning.
+	 * @return queue to read the objects from.
+	 */
+	public <T extends ObjectId> AsyncRevObjectQueue parseAny(
+			Iterable<T> objectIds, boolean reportMissing) {
+		List<T> need = new ArrayList<T>();
+		List<RevObject> have = new ArrayList<RevObject>();
+		for (T id : objectIds) {
+			RevObject r = objects.get(id);
+			if (r != null && (r.flags & PARSED) != 0)
+				have.add(r);
+			else
+				need.add(id);
+		}
+
+		final Iterator<RevObject> objItr = have.iterator();
+		if (need.isEmpty()) {
+			return new AsyncRevObjectQueue() {
+				public RevObject next() {
+					return objItr.hasNext() ? objItr.next() : null;
+				}
+
+				public boolean cancel(boolean mayInterruptIfRunning) {
+					return true;
+				}
+
+				public void release() {
+					// In-memory only, no action required.
+				}
+			};
+		}
+
+		final AsyncObjectLoaderQueue<T> lItr = reader.open(need, reportMissing);
+		return new AsyncRevObjectQueue() {
+			public RevObject next() throws MissingObjectException,
+					IncorrectObjectTypeException, IOException {
+				if (objItr.hasNext())
+					return objItr.next();
+				if (!lItr.next())
+					return null;
+
+				ObjectId id = lItr.getObjectId();
+				ObjectLoader ldr = lItr.open();
+				RevObject r = objects.get(id);
+				if (r == null)
+					r = parseNew(id, ldr);
+				else if (r instanceof RevCommit) {
+					byte[] raw = ldr.getCachedBytes();
+					((RevCommit) r).parseCanonical(RevWalk.this, raw);
+				} else if (r instanceof RevTag) {
+					byte[] raw = ldr.getCachedBytes();
+					((RevTag) r).parseCanonical(RevWalk.this, raw);
 				} else
-			parseHeaders(r);
+					r.flags |= PARSED;
 				return r;
 			}
 
+			public boolean cancel(boolean mayInterruptIfRunning) {
+				return lItr.cancel(mayInterruptIfRunning);
+			}
+
+			public void release() {
+				lItr.release();
+			}
+		};
+	}
+
 	/**
 	 * Ensure the object's critical headers have been parsed.
 	 * <p>

org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java

@@ -143,11 +143,8 @@ class DeltaWindow {
 				}
 				res.set(toSearch[off]);
 
-				if (res.object.isDoNotDelta()) {
+				if (res.object.isEdge()) {
-					// PackWriter marked edge objects with the
+					// We don't actually want to make a delta for
-					// do-not-delta flag. They are the only ones
-					// that appear in toSearch with it set, but
-					// we don't actually want to make a delta for
 					// them, just need to push them into the window
 					// so they can be read by other objects.
 					//
@@ -211,7 +208,7 @@ class DeltaWindow {
 		//
 		ObjectToPack srcObj = window[bestSlot].object;
 		ObjectToPack resObj = res.object;
-		if (srcObj.isDoNotDelta()) {
+		if (srcObj.isEdge()) {
 			// The source (the delta base) is an edge object outside of the
 			// pack. Its part of the common base set that the peer already
 			// has on hand, so we don't want to send it. We have to store
@@ -280,7 +277,7 @@ class DeltaWindow {
 			dropFromWindow(srcSlot);
 			return NEXT_SRC;
 		} catch (IOException notAvailable) {
-			if (src.object.isDoNotDelta()) {
+			if (src.object.isEdge()) {
 				// This is an edge that is suddenly not available.
 				dropFromWindow(srcSlot);
 				return NEXT_SRC;

org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectToPack.java

@@ -64,6 +64,8 @@ public class ObjectToPack extends PackedObjectInfo {
 
 	private static final int DO_NOT_DELTA = 1 << 2;
 
+	private static final int EDGE = 1 << 3;
+
 	private static final int TYPE_SHIFT = 5;
 
 	private static final int DELTA_SHIFT = 8;
@@ -79,7 +81,8 @@ public class ObjectToPack extends PackedObjectInfo {
 	 * <li>1 bit: wantWrite</li>
 	 * <li>1 bit: canReuseAsIs</li>
 	 * <li>1 bit: doNotDelta</li>
-	 * <li>2 bits: unused</li>
+	 * <li>1 bit: edgeObject</li>
+	 * <li>1 bit: unused</li>
 	 * <li>3 bits: type</li>
 	 * <li>--</li>
 	 * <li>24 bits: deltaDepth</li>
@@ -243,6 +246,14 @@ public class ObjectToPack extends PackedObjectInfo {
 			flags &= ~DO_NOT_DELTA;
 	}
 
+	boolean isEdge() {
+		return (flags & EDGE) != 0;
+	}
+
+	void setEdge() {
+		flags |= EDGE;
+	}
+
 	int getFormat() {
 		if (isReuseAsIs()) {
 			if (isDeltaRepresentation())
@@ -305,6 +316,8 @@ public class ObjectToPack extends PackedObjectInfo {
 			buf.append(" reuseAsIs");
 		if (isDoNotDelta())
 			buf.append(" doNotDelta");
+		if (isEdge())
+			buf.append(" edge");
 		if (getDeltaDepth() > 0)
 			buf.append(" depth=" + getDeltaDepth());
 		if (isDeltaRepresentation()) {

org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java

@@ -56,8 +56,10 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
@@ -75,6 +77,7 @@ import org.eclipse.jgit.errors.LargeObjectException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
 import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.AsyncObjectSizeQueue;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.NullProgressMonitor;
 import org.eclipse.jgit.lib.ObjectId;
@@ -84,6 +87,7 @@ import org.eclipse.jgit.lib.ObjectReader;
 import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.ThreadSafeProgressMonitor;
+import org.eclipse.jgit.revwalk.AsyncRevObjectQueue;
 import org.eclipse.jgit.revwalk.ObjectWalk;
 import org.eclipse.jgit.revwalk.RevFlag;
 import org.eclipse.jgit.revwalk.RevObject;
@@ -544,20 +548,66 @@ public class PackWriter {
 		// them in the search phase below.
 		//
 		for (ObjectToPack eo : edgeObjects) {
-			try {
+			eo.setWeight(0);
-				if (loadSize(eo))
 			list[cnt++] = eo;
-			} catch (IOException notAvailable) {
+		}
-				// Skip this object. Since we aren't going to write it out
+
-				// the only consequence of it being unavailable to us is we
+		// Compute the sizes of the objects so we can do a proper sort.
-				// may produce a larger data stream than we could have.
+		// We let the reader skip missing objects if it chooses. For
+		// some readers this can be a huge win. We detect missing objects
+		// by having set the weights above to 0 and allowing the delta
+		// search code to discover the missing object and skip over it, or
+		// abort with an exception if we actually had to have it.
 		//
-				if (!ignoreMissingUninteresting)
+		monitor.beginTask(JGitText.get().compressingObjects, cnt);
-					throw notAvailable;
+		AsyncObjectSizeQueue<ObjectToPack> sizeQueue = reader.getObjectSize(
+				Arrays.<ObjectToPack> asList(list).subList(0, cnt), false);
+		try {
+			final long limit = config.getBigFileThreshold();
+			for (;;) {
+				monitor.update(1);
+
+				try {
+					if (!sizeQueue.next())
+						break;
+				} catch (MissingObjectException notFound) {
+					if (ignoreMissingUninteresting) {
+						ObjectToPack otp = sizeQueue.getCurrent();
+						if (otp != null && otp.isEdge()) {
+							otp.setDoNotDelta(true);
+							continue;
+						}
+
+						otp = edgeObjects.get(notFound.getObjectId());
+						if (otp != null) {
+							otp.setDoNotDelta(true);
+							continue;
+						}
 					}
+					throw notFound;
 				}
 
-		monitor.beginTask(JGitText.get().compressingObjects, cnt);
+				ObjectToPack otp = sizeQueue.getCurrent();
+				if (otp == null) {
+					otp = objectsMap.get(sizeQueue.getObjectId());
+					if (otp == null)
+						otp = edgeObjects.get(sizeQueue.getObjectId());
+				}
+
+				long sz = sizeQueue.getSize();
+				if (limit <= sz || Integer.MAX_VALUE <= sz)
+					otp.setDoNotDelta(true); // too big, avoid costly files
+
+				else if (sz <= DeltaIndex.BLKSZ)
+					otp.setDoNotDelta(true); // too small, won't work
+
+				else
+					otp.setWeight((int) sz);
+			}
+		} finally {
+			sizeQueue.release();
+		}
+		monitor.endTask();
 
 		// Sort the objects by path hash so like files are near each other,
 		// and then by size descending so that bigger files are first. This
@@ -566,52 +616,51 @@ public class PackWriter {
 		//
 		Arrays.sort(list, 0, cnt, new Comparator<ObjectToPack>() {
 			public int compare(ObjectToPack a, ObjectToPack b) {
-				int cmp = a.getType() - b.getType();
+				int cmp = (a.isDoNotDelta() ? 1 : 0)
-				if (cmp == 0)
+						- (b.isDoNotDelta() ? 1 : 0);
+				if (cmp != 0)
+					return cmp;
+
+				cmp = a.getType() - b.getType();
+				if (cmp != 0)
+					return cmp;
+
 				cmp = (a.getPathHash() >>> 1) - (b.getPathHash() >>> 1);
-				if (cmp == 0)
+				if (cmp != 0)
+					return cmp;
+
 				cmp = (a.getPathHash() & 1) - (b.getPathHash() & 1);
-				if (cmp == 0)
+				if (cmp != 0)
-					cmp = b.getWeight() - a.getWeight();
 					return cmp;
+
+				return b.getWeight() - a.getWeight();
 			}
 		});
+
+		// Above we stored the objects we cannot delta onto the end.
+		// Remove them from the list so we don't waste time on them.
+		while (0 < cnt && list[cnt - 1].isDoNotDelta())
+			cnt--;
+		if (cnt == 0)
+			return;
+
+		monitor.beginTask(JGitText.get().compressingObjects, cnt);
 		searchForDeltas(monitor, list, cnt);
 		monitor.endTask();
 	}
 
-	private int findObjectsNeedingDelta(ObjectToPack[] list, int cnt, int type)
+	private int findObjectsNeedingDelta(ObjectToPack[] list, int cnt, int type) {
-			throws MissingObjectException, IncorrectObjectTypeException,
-			IOException {
 		for (ObjectToPack otp : objectsLists[type]) {
 			if (otp.isDoNotDelta()) // delta is disabled for this path
 				continue;
 			if (otp.isDeltaRepresentation()) // already reusing a delta
 				continue;
-			if (loadSize(otp))
+			otp.setWeight(0);
 			list[cnt++] = otp;
 		}
 		return cnt;
 	}
 
-	private boolean loadSize(ObjectToPack e) throws MissingObjectException,
-			IncorrectObjectTypeException, IOException {
-		long sz = reader.getObjectSize(e, e.getType());
-
-		// If its too big for us to handle, skip over it.
-		//
-		if (config.getBigFileThreshold() <= sz || Integer.MAX_VALUE <= sz)
-			return false;
-
-		// If its too tiny for the delta compression to work, skip it.
-		//
-		if (sz <= DeltaIndex.BLKSZ)
-			return false;
-
-		e.setWeight((int) sz);
-		return true;
-	}
-
 	private void searchForDeltas(final ProgressMonitor monitor,
 			final ObjectToPack[] list, final int cnt)
 			throws MissingObjectException, IncorrectObjectTypeException,
@@ -963,28 +1012,45 @@ public class PackWriter {
 			final Collection<? extends ObjectId> uninterestingObjects)
 			throws MissingObjectException, IOException,
 			IncorrectObjectTypeException {
+		List<ObjectId> all = new ArrayList<ObjectId>(interestingObjects.size());
+		for (ObjectId id : interestingObjects)
+			all.add(id.copy());
+
+		final Set<ObjectId> not;
+		if (uninterestingObjects != null && !uninterestingObjects.isEmpty()) {
+			not = new HashSet<ObjectId>();
+			for (ObjectId id : uninterestingObjects)
+				not.add(id.copy());
+			all.addAll(not);
+		} else
+			not = Collections.emptySet();
+
 		final ObjectWalk walker = new ObjectWalk(reader);
 		walker.setRetainBody(false);
 		walker.sort(RevSort.COMMIT_TIME_DESC);
-		if (thin)
+		if (thin && !not.isEmpty())
 			walker.sort(RevSort.BOUNDARY, true);
 
-		for (ObjectId id : interestingObjects) {
+		AsyncRevObjectQueue q = walker.parseAny(all, true);
-			RevObject o = walker.parseAny(id);
+		try {
-			walker.markStart(o);
+			for (;;) {
-		}
-		if (uninterestingObjects != null) {
-			for (ObjectId id : uninterestingObjects) {
-				final RevObject o;
 				try {
-					o = walker.parseAny(id);
+					RevObject o = q.next();
-				} catch (MissingObjectException x) {
+					if (o == null)
-					if (ignoreMissingUninteresting)
+						break;
+					if (not.contains(o.copy()))
+						walker.markUninteresting(o);
+					else
+						walker.markStart(o);
+				} catch (MissingObjectException e) {
+					if (ignoreMissingUninteresting
+							&& not.contains(e.getObjectId()))
 						continue;
-					throw x;
+					throw e;
 				}
-				walker.markUninteresting(o);
 			}
+		} finally {
+			q.release();
 		}
 		return walker;
 	}
@@ -1032,7 +1098,7 @@ public class PackWriter {
 			case Constants.OBJ_BLOB:
 				ObjectToPack otp = new ObjectToPack(object);
 				otp.setPathHash(pathHashCode);
-				otp.setDoNotDelta(true);
+				otp.setEdge();
 				edgeObjects.add(otp);
 				thin = true;
 				break;

org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

@@ -50,7 +50,6 @@ import java.io.OutputStream;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -63,6 +62,7 @@ import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.AsyncRevObjectQueue;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.revwalk.RevFlag;
 import org.eclipse.jgit.revwalk.RevFlagSet;
@@ -145,9 +145,6 @@ public class UploadPack {
 	/** Objects the client wants to obtain. */
 	private final List<RevObject> wantAll = new ArrayList<RevObject>();
 
-	/** Objects the client wants to obtain. */
-	private final List<RevCommit> wantCommits = new ArrayList<RevCommit>();
-
 	/** Objects on both sides, these don't have to be sent. */
 	private final List<RevObject> commonBase = new ArrayList<RevObject>();
 
@@ -166,6 +163,9 @@ public class UploadPack {
 	/** Marked on objects in {@link #commonBase}. */
 	private final RevFlag COMMON;
 
+	/** Objects where we found a path from the want list to a common base. */
+	private final RevFlag SATISFIED;
+
 	private final RevFlagSet SAVE;
 
 	private MultiAck multiAck = MultiAck.OFF;
@@ -185,6 +185,7 @@ public class UploadPack {
 		WANT = walk.newFlag("WANT");
 		PEER_HAS = walk.newFlag("PEER_HAS");
 		COMMON = walk.newFlag("COMMON");
+		SATISFIED = walk.newFlag("SATISFIED");
 		walk.carry(PEER_HAS);
 
 		SAVE = new RevFlagSet();
@@ -376,8 +377,9 @@ public class UploadPack {
 	}
 
 	private void recvWants() throws IOException {
+		HashSet<ObjectId> wantIds = new HashSet<ObjectId>();
 		boolean isFirst = true;
-		for (;; isFirst = false) {
+		for (;;) {
 			String line;
 			try {
 				line = pckIn.readString();
@@ -401,41 +403,54 @@ public class UploadPack {
 				line = line.substring(0, 45);
 			}
 
-			final ObjectId id = ObjectId.fromString(line.substring(5));
+			wantIds.add(ObjectId.fromString(line.substring(5)));
-			final RevObject o;
+			isFirst = false;
-			try {
-				o = walk.parseAny(id);
-			} catch (IOException e) {
-				throw new PackProtocolException(MessageFormat.format(JGitText.get().notValid, id.name()), e);
 		}
-			if (!o.has(ADVERTISED))
+
-				throw new PackProtocolException(MessageFormat.format(JGitText.get().notValid, id.name()));
+		if (wantIds.isEmpty())
+			return;
+
+		AsyncRevObjectQueue q = walk.parseAny(wantIds, true);
 		try {
-				want(o);
+			for (;;) {
-			} catch (IOException e) {
+				RevObject o;
-				throw new PackProtocolException(MessageFormat.format(JGitText.get().notValid, id.name()), e);
+				try {
-			}
+					o = q.next();
-		}
+				} catch (IOException error) {
+					throw new PackProtocolException(MessageFormat.format(
+							JGitText.get().notValid, error.getMessage()), error);
 				}
+				if (o == null)
+					break;
+				if (o.has(WANT)) {
+					// Already processed, the client repeated itself.
 
-	private void want(RevObject o) throws MissingObjectException, IOException {
+				} else if (o.has(ADVERTISED)) {
-		if (!o.has(WANT)) {
 					o.add(WANT);
 					wantAll.add(o);
 
-			if (o instanceof RevCommit)
+					if (o instanceof RevTag) {
-				wantCommits.add((RevCommit) o);
-
-			else if (o instanceof RevTag) {
 						o = walk.peel(o);
-				if (o instanceof RevCommit)
+						if (o instanceof RevCommit) {
-					want(o);
+							if (!o.has(WANT)) {
+								o.add(WANT);
+								wantAll.add(o);
+							}
 						}
 					}
+				} else {
+					throw new PackProtocolException(MessageFormat.format(
+							JGitText.get().notValid, o.name()));
+				}
+			}
+		} finally {
+			q.release();
+		}
 	}
 
 	private boolean negotiate() throws IOException {
 		ObjectId last = ObjectId.zeroId();
+		List<ObjectId> peerHas = new ArrayList<ObjectId>(64);
 		for (;;) {
 			String line;
 			try {
@@ -445,6 +460,7 @@ public class UploadPack {
 			}
 
 			if (line == PacketLineIn.END) {
+				last = processHaveLines(peerHas, last);
 				if (commonBase.isEmpty() || multiAck != MultiAck.OFF)
 					pckOut.writeString("NAK\n");
 				if (!biDirectionalPipe)
@@ -452,26 +468,78 @@ public class UploadPack {
 				pckOut.flush();
 
 			} else if (line.startsWith("have ") && line.length() == 45) {
-				final ObjectId id = ObjectId.fromString(line.substring(5));
+				peerHas.add(ObjectId.fromString(line.substring(5)));
-				if (matchHave(id)) {
+
-					// Both sides have the same object; let the client know.
+			} else if (line.equals("done")) {
+				last = processHaveLines(peerHas, last);
+
+				if (commonBase.isEmpty())
+					pckOut.writeString("NAK\n");
+
+				else if (multiAck != MultiAck.OFF)
+					pckOut.writeString("ACK " + last.name() + "\n");
+
+				return true;
+
+			} else {
+				throw new PackProtocolException(MessageFormat.format(JGitText.get().expectedGot, "have", line));
+			}
+		}
+	}
+
+	private ObjectId processHaveLines(List<ObjectId> peerHas, ObjectId last)
+			throws IOException {
+		if (peerHas.isEmpty())
+			return last;
+
+		// If both sides have the same object; let the client know.
 		//
-					last = id;
+		AsyncRevObjectQueue q = walk.parseAny(peerHas, false);
+		try {
+			for (;;) {
+				RevObject obj;
+				try {
+					obj = q.next();
+				} catch (MissingObjectException notFound) {
+					continue;
+				}
+				if (obj == null)
+					break;
+
+				last = obj;
+				if (obj.has(PEER_HAS))
+					continue;
+
+				obj.add(PEER_HAS);
+				if (obj instanceof RevCommit)
+					((RevCommit) obj).carry(PEER_HAS);
+				addCommonBase(obj);
+
 				switch (multiAck) {
 				case OFF:
 					if (commonBase.size() == 1)
-							pckOut.writeString("ACK " + id.name() + "\n");
+						pckOut.writeString("ACK " + obj.name() + "\n");
 					break;
 				case CONTINUE:
-						pckOut.writeString("ACK " + id.name() + " continue\n");
+					pckOut.writeString("ACK " + obj.name() + " continue\n");
 					break;
 				case DETAILED:
-						pckOut.writeString("ACK " + id.name() + " common\n");
+					pckOut.writeString("ACK " + obj.name() + " common\n");
 					break;
 				}
-				} else if (okToGiveUp()) {
+			}
-					// They have this object; we don't.
+		} finally {
+			q.release();
+		}
+
+		// If we don't have one of the objects but we're also willing to
+		// create a pack at this point, let the client know so it stops
+		// telling us about its history.
 		//
+		for (int i = peerHas.size() - 1; i >= 0; i--) {
+			ObjectId id = peerHas.get(i);
+			if (walk.lookupOrNull(id) == null) {
+				if (okToGiveUp()) {
 					switch (multiAck) {
 					case OFF:
 						break;
@@ -483,37 +551,12 @@ public class UploadPack {
 						break;
 					}
 				}
-
+				break;
-			} else if (line.equals("done")) {
-				if (commonBase.isEmpty())
-					pckOut.writeString("NAK\n");
-
-				else if (multiAck != MultiAck.OFF)
-					pckOut.writeString("ACK " + last.name() + "\n");
-
-				return true;
-
-			} else {
-				throw new PackProtocolException(MessageFormat.format(JGitText.get().expectedGot, "have", line));
-			}
-		}
 			}
-
-	private boolean matchHave(final ObjectId id) {
-		final RevObject o;
-		try {
-			o = walk.parseAny(id);
-		} catch (IOException err) {
-			return false;
 		}
 
-		if (!o.has(PEER_HAS)) {
+		peerHas.clear();
-			o.add(PEER_HAS);
+		return last;
-			if (o instanceof RevCommit)
-				((RevCommit) o).carry(PEER_HAS);
-			addCommonBase(o);
-		}
-		return true;
 	}
 
 	private void addCommonBase(final RevObject o) {
@@ -535,27 +578,34 @@ public class UploadPack {
 			return false;
 
 		try {
-			for (final Iterator<RevCommit> i = wantCommits.iterator(); i
+			for (RevObject obj : wantAll) {
-					.hasNext();) {
+				if (wantSatisfied(obj))
-				final RevCommit want = i.next();
+					return false;
-				if (wantSatisfied(want))
-					i.remove();
 			}
+			return true;
 		} catch (IOException e) {
 			throw new PackProtocolException(JGitText.get().internalRevisionError, e);
 		}
-		return wantCommits.isEmpty();
 	}
 
-	private boolean wantSatisfied(final RevCommit want) throws IOException {
+	private boolean wantSatisfied(final RevObject want) throws IOException {
+		if (want.has(SATISFIED))
+			return true;
+
+		if (!(want instanceof RevCommit)) {
+			want.add(SATISFIED);
+			return true;
+		}
+
 		walk.resetRetain(SAVE);
-		walk.markStart(want);
+		walk.markStart((RevCommit) want);
 		for (;;) {
 			final RevCommit c = walk.next();
 			if (c == null)
 				break;
 			if (c.has(PEER_HAS)) {
 				addCommonBase(c);
+				want.add(SATISFIED);
 				return true;
 			}
 		}