UploadPack: Add a PreUploadHook to monitor and control behavior

Embedding applications can use this hook to watch actions within
UploadPack and possibly reject them. This could be useful to prevent
clones of a large repository from this server, or to stop abusive
negotiation rounds that offer thousands of objects in a single batch.

Change-Id: Id96f1885ac4d61f22c80b6418fff54184b7348ba
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/SmartServiceInfoRefs.java

@@ -44,6 +44,7 @@
 package org.eclipse.jgit.http.server;
 
 import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE;
 import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED;
 import static org.eclipse.jgit.http.server.ServletUtils.ATTRIBUTE_HANDLER;
 import static org.eclipse.jgit.http.server.ServletUtils.getRepository;
@@ -63,6 +64,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.transport.PacketLineOut;
 import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser;
+import org.eclipse.jgit.transport.UploadPackMayNotContinueException;
 import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
 import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
 
@@ -119,10 +121,10 @@ abstract class SmartServiceInfoRefs implements Filter {
 			throws IOException {
 		final HttpServletRequest req = (HttpServletRequest) request;
 		final HttpServletResponse rsp = (HttpServletResponse) response;
+		final SmartOutputStream buf = new SmartOutputStream(req, rsp);
 		try {
 			rsp.setContentType("application/x-" + svc + "-advertisement");
 
-			final SmartOutputStream buf = new SmartOutputStream(req, rsp);
 			final PacketLineOut out = new PacketLineOut(buf);
 			out.writeString("# service=" + svc + "\n");
 			out.end();
@@ -133,6 +135,12 @@ abstract class SmartServiceInfoRefs implements Filter {
 
 		} catch (ServiceNotEnabledException e) {
 			rsp.sendError(SC_FORBIDDEN);
+
+		} catch (UploadPackMayNotContinueException e) {
+			if (e.isOutput())
+				buf.close();
+			else
+				rsp.sendError(SC_SERVICE_UNAVAILABLE);
 		}
 	}
 

org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/UploadPackServlet.java

@@ -45,6 +45,7 @@ package org.eclipse.jgit.http.server;
 
 import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE;
 import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED;
 import static javax.servlet.http.HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE;
 import static org.eclipse.jgit.http.server.ServletUtils.ATTRIBUTE_HANDLER;
@@ -67,6 +68,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser;
 import org.eclipse.jgit.transport.UploadPack;
+import org.eclipse.jgit.transport.UploadPackMayNotContinueException;
 import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
 import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
 import org.eclipse.jgit.transport.resolver.UploadPackFactory;
@@ -171,6 +173,11 @@ class UploadPackServlet extends HttpServlet {
 			up.upload(getInputStream(req), out, null);
 			out.close();
 
+		} catch (UploadPackMayNotContinueException e) {
+			if (!e.isOutput())
+				rsp.sendError(SC_SERVICE_UNAVAILABLE);
+			return;
+
 		} catch (IOException e) {
 			getServletContext().log(HttpServerText.get().internalErrorDuringUploadPack, e);
 			rsp.reset();

org.eclipse.jgit/src/org/eclipse/jgit/transport/PreUploadHook.java

@@ -0,0 +1,158 @@
+/*
+ * Copyright (C) 2011, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.transport;
+
+import java.util.Collection;
+
+import org.eclipse.jgit.lib.ObjectId;
+
+/**
+ * Hook invoked by {@link UploadPack} before during critical phases.
+ * <p>
+ * If any hook function throws {@link UploadPackMayNotContinueException} then
+ * processing stops immediately and the exception is thrown up the call stack.
+ * Most phases of UploadPack will try to report the exception's message text to
+ * the end-user over the client's protocol connection.
+ */
+public interface PreUploadHook {
+	/** A simple no-op hook. */
+	public static final PreUploadHook NULL = new PreUploadHook() {
+		public void onPreAdvertiseRefs(UploadPack up)
+				throws UploadPackMayNotContinueException {
+			// Do nothing.
+		}
+
+		public void onBeginNegotiateRound(UploadPack up,
+				Collection<? extends ObjectId> wants, int cntOffered)
+				throws UploadPackMayNotContinueException {
+			// Do nothing.
+		}
+
+		public void onEndNegotiateRound(UploadPack up,
+				Collection<? extends ObjectId> wants, int cntCommon,
+				int cntNotFound, boolean ready)
+				throws UploadPackMayNotContinueException {
+			// Do nothing.
+		}
+
+		public void onSendPack(UploadPack up,
+				Collection<? extends ObjectId> wants,
+				Collection<? extends ObjectId> haves)
+				throws UploadPackMayNotContinueException {
+			// Do nothing.
+		}
+	};
+
+	/**
+	 * Invoked just before {@link UploadPack#sendAdvertisedRefs(RefAdvertiser)}.
+	 *
+	 * @param up
+	 *            the upload pack instance handling the connection.
+	 * @throws UploadPackMayNotContinueException
+	 *             abort; the message will be sent to the user.
+	 */
+	public void onPreAdvertiseRefs(UploadPack up)
+			throws UploadPackMayNotContinueException;
+
+	/**
+	 * Invoked before negotiation round is started.
+	 *
+	 * @param up
+	 *            the upload pack instance handling the connection.
+	 * @param wants
+	 *            the list of wanted objects.
+	 * @param cntOffered
+	 *            number of objects the client has offered.
+	 * @throws UploadPackMayNotContinueException
+	 *             abort; the message will be sent to the user.
+	 */
+	public void onBeginNegotiateRound(UploadPack up,
+			Collection<? extends ObjectId> wants, int cntOffered)
+			throws UploadPackMayNotContinueException;
+
+	/**
+	 * Invoked after a negotiation round is completed.
+	 *
+	 * @param up
+	 *            the upload pack instance handling the connection.
+	 * @param wants
+	 *            the list of wanted objects.
+	 * @param cntCommon
+	 *            number of objects this round found to be common. In a smart
+	 *            HTTP transaction this includes the objects that were
+	 *            previously found to be common.
+	 * @param cntNotFound
+	 *            number of objects in this round the local repository does not
+	 *            have, but that were offered as potential common bases.
+	 * @param ready
+	 *            true if a pack is ready to be sent (the commit graph was
+	 *            successfully cut).
+	 * @throws UploadPackMayNotContinueException
+	 *             abort; the message will be sent to the user.
+	 */
+	public void onEndNegotiateRound(UploadPack up,
+			Collection<? extends ObjectId> wants, int cntCommon,
+			int cntNotFound, boolean ready)
+			throws UploadPackMayNotContinueException;
+
+	/**
+	 * Invoked just before a pack will be sent to the client.
+	 *
+	 * @param up
+	 *            the upload pack instance handling the connection.
+	 * @param wants
+	 *            the list of wanted objects. These may be RevObject or
+	 *            RevCommit if the processed parsed them. Implementors should
+	 *            not rely on the values being parsed.
+	 * @param haves
+	 *            the list of common objects. Empty on an initial clone request.
+	 *            These may be RevObject or RevCommit if the processed parsed
+	 *            them. Implementors should not rely on the values being parsed.
+	 * @throws UploadPackMayNotContinueException
+	 *             abort; the message will be sent to the user.
+	 */
+	public void onSendPack(UploadPack up, Collection<? extends ObjectId> wants,
+			Collection<? extends ObjectId> haves)
+			throws UploadPackMayNotContinueException;
+}

org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

@@ -142,6 +142,9 @@ public class UploadPack {
 	/** Filter used while advertising the refs to the client. */
 	private RefFilter refFilter;
 
+	/** Hook handling the various upload phases. */
+	private PreUploadHook preUploadHook = PreUploadHook.NULL;
+
 	/** Capabilities requested by the client. */
 	private final Set<String> options = new HashSet<String>();
 
@@ -220,6 +223,9 @@ public class UploadPack {
 
 	/** @return all refs which were advertised to the client. */
 	public final Map<String, Ref> getAdvertisedRefs() {
+		if (refs == null) {
+			refs = refFilter.filter(db.getAllRefs());
+		}
 		return refs;
 	}
 
@@ -281,6 +287,21 @@ public class UploadPack {
 		this.refFilter = refFilter != null ? refFilter : RefFilter.DEFAULT;
 	}
 
+	/** @return the configured upload hook. */
+	public PreUploadHook getPreUploadHook() {
+		return preUploadHook;
+	}
+
+	/**
+	 * Set the hook that controls how this instance will behave.
+	 *
+	 * @param hook
+	 *            the hook; if null no special actions are taken.
+	 */
+	public void setPreUploadHook(PreUploadHook hook) {
+		preUploadHook = hook != null ? hook : PreUploadHook.NULL;
+	}
+
 	/**
 	 * Set the configuration used by the pack generator.
 	 *
@@ -367,16 +388,18 @@ public class UploadPack {
 			sendAdvertisedRefs(new PacketLineOutRefAdvertiser(pckOut));
 		else {
 			advertised = new HashSet<ObjectId>();
-			refs = refFilter.filter(db.getAllRefs());
-			for (Ref ref : refs.values()) {
+			for (Ref ref : getAdvertisedRefs().values()) {
 				if (ref.getObjectId() != null)
 					advertised.add(ref.getObjectId());
 			}
 		}
 
 		recvWants();
-		if (wantIds.isEmpty())
+		if (wantIds.isEmpty()) {
+			preUploadHook.onBeginNegotiateRound(this, wantIds, 0);
+			preUploadHook.onEndNegotiateRound(this, wantIds, 0, 0, false);
 			return;
+		}
 
 		if (options.contains(OPTION_MULTI_ACK_DETAILED))
 			multiAck = MultiAck.DETAILED;
@@ -396,8 +419,21 @@ public class UploadPack {
 	 *            the advertisement formatter.
 	 * @throws IOException
 	 *             the formatter failed to write an advertisement.
+	 * @throws UploadPackMayNotContinueException
+	 *             the hook denied advertisement.
 	 */
-	public void sendAdvertisedRefs(final RefAdvertiser adv) throws IOException {
+	public void sendAdvertisedRefs(final RefAdvertiser adv) throws IOException,
+			UploadPackMayNotContinueException {
+		try {
+			preUploadHook.onPreAdvertiseRefs(this);
+		} catch (UploadPackMayNotContinueException fail) {
+			if (fail.getMessage() != null) {
+				adv.writeOne("ERR " + fail.getMessage());
+				fail.setOutput();
+			}
+			throw fail;
+		}
+
 		adv.init(db);
 		adv.advertiseCapability(OPTION_INCLUDE_TAG);
 		adv.advertiseCapability(OPTION_MULTI_ACK_DETAILED);
@@ -408,8 +444,7 @@ public class UploadPack {
 		adv.advertiseCapability(OPTION_THIN_PACK);
 		adv.advertiseCapability(OPTION_NO_PROGRESS);
 		adv.setDerefTags(true);
-		refs = refFilter.filter(db.getAllRefs());
-		advertised = adv.send(refs);
+		advertised = adv.send(getAdvertisedRefs());
 		adv.end();
 	}
 
@@ -487,6 +522,16 @@ public class UploadPack {
 
 	private ObjectId processHaveLines(List<ObjectId> peerHas, ObjectId last)
 			throws IOException {
+		try {
+			preUploadHook.onBeginNegotiateRound(this, wantIds, peerHas.size());
+		} catch (UploadPackMayNotContinueException fail) {
+			if (fail.getMessage() != null) {
+				pckOut.writeString("ERR " + fail.getMessage() + "\n");
+				fail.setOutput();
+			}
+			throw fail;
+		}
+
 		if (peerHas.isEmpty())
 			return last;
 
@@ -504,6 +549,7 @@ public class UploadPack {
 			needMissing = true;
 		}
 
+		int haveCnt = 0;
 		AsyncRevObjectQueue q = walk.parseAny(toParse, needMissing);
 		try {
 			for (;;) {
@@ -557,6 +603,7 @@ public class UploadPack {
 				}
 
 				last = obj;
+				haveCnt++;
 
 				if (obj instanceof RevCommit) {
 					RevCommit c = (RevCommit) obj;
@@ -590,35 +637,52 @@ public class UploadPack {
 		} finally {
 			q.release();
 		}
+		int missCnt = peerHas.size() - haveCnt;
 
 		// If we don't have one of the objects but we're also willing to
 		// create a pack at this point, let the client know so it stops
 		// telling us about its history.
 		//
 		boolean didOkToGiveUp = false;
-		for (int i = peerHas.size() - 1; i >= 0; i--) {
-			ObjectId id = peerHas.get(i);
-			if (walk.lookupOrNull(id) == null) {
-				didOkToGiveUp = true;
-				if (okToGiveUp()) {
-					switch (multiAck) {
-					case OFF:
-						break;
-					case CONTINUE:
-						pckOut.writeString("ACK " + id.name() + " continue\n");
-						break;
-					case DETAILED:
-						pckOut.writeString("ACK " + id.name() + " ready\n");
-						break;
+		boolean sentReady = false;
+		if (0 < missCnt) {
+			for (int i = peerHas.size() - 1; i >= 0; i--) {
+				ObjectId id = peerHas.get(i);
+				if (walk.lookupOrNull(id) == null) {
+					didOkToGiveUp = true;
+					if (okToGiveUp()) {
+						switch (multiAck) {
+						case OFF:
+							break;
+						case CONTINUE:
+							pckOut.writeString("ACK " + id.name() + " continue\n");
+							break;
+						case DETAILED:
+							pckOut.writeString("ACK " + id.name() + " ready\n");
+							sentReady = true;
+							break;
+						}
 					}
+					break;
 				}
-				break;
 			}
 		}
 
 		if (multiAck == MultiAck.DETAILED && !didOkToGiveUp && okToGiveUp()) {
 			ObjectId id = peerHas.get(peerHas.size() - 1);
 			pckOut.writeString("ACK " + id.name() + " ready\n");
+			sentReady = true;
+		}
+
+		try {
+			preUploadHook.onEndNegotiateRound(this, wantAll, //
+					haveCnt, missCnt, sentReady);
+		} catch (UploadPackMayNotContinueException fail) {
+			if (fail.getMessage() != null) {
+				pckOut.writeString("ERR " + fail.getMessage() + "\n");
+				fail.setOutput();
+			}
+			throw fail;
 		}
 
 		peerHas.clear();
@@ -697,6 +761,24 @@ public class UploadPack {
 			}
 		}
 
+		try {
+			if (wantAll.isEmpty()) {
+				preUploadHook.onSendPack(this, wantIds, commonBase);
+			} else {
+				preUploadHook.onSendPack(this, wantAll, commonBase);
+			}
+		} catch (UploadPackMayNotContinueException noPack) {
+			if (sideband && noPack.getMessage() != null) {
+				noPack.setOutput();
+				SideBandOutputStream err = new SideBandOutputStream(
+						SideBandOutputStream.CH_ERROR,
+						SideBandOutputStream.SMALL_BUF, rawOut);
+				err.write(Constants.encode(noPack.getMessage()));
+				err.flush();
+			}
+			throw noPack;
+		}
+
 		PackConfig cfg = packConfig;
 		if (cfg == null)
 			cfg = new PackConfig(db);

org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackMayNotContinueException.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2011, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.transport;
+
+import java.io.IOException;
+
+/** Indicates UploadPack may not continue execution. */
+public class UploadPackMayNotContinueException extends IOException {
+	private static final long serialVersionUID = 1L;
+
+	private boolean output;
+
+	/** Initialize with no message. */
+	public UploadPackMayNotContinueException() {
+		// Do not set a message.
+	}
+
+	/**
+	 * @param msg
+	 *            a message explaining why it cannot continue. This message may
+	 *            be shown to an end-user.
+	 */
+	public UploadPackMayNotContinueException(String msg) {
+		super(msg);
+	}
+
+	/** @return true if the message was already output to the client. */
+	public boolean isOutput() {
+		return output;
+	}
+
+	/** Mark this message has being sent to the client. */
+	public void setOutput() {
+		output = true;
+	}
+}