Implement protocol v2 with no capabilities in UploadPack

Add initial support for protocol v2 of the fetch-pack/upload-pack
protocol. This protocol is described in the Git project in
"Documentation/technical/protocol-v2.txt".

This patch adds support for protocol v2 (without any capabilities) to
UploadPack. Adaptations of callers to make use of this support will
come in subsequent patches.

[jn: split from a larger patch; tweaked the API to make UploadPack
 handle parsing the extra parameters and config instead of requiring
 each caller to do such parsing]

Change-Id: I79399fa0dce533fdc8c1dbb6756748818cee45b0
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Jonathan Nieder <jrn@google.com>

org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java

@@ -1,9 +1,13 @@
 package org.eclipse.jgit.transport;
 
+import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 import java.util.Collections;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import org.eclipse.jgit.errors.TransportException;
 import org.eclipse.jgit.internal.storage.dfs.DfsGarbageCollector;
 import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription;
@@ -11,6 +15,7 @@ import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
 import org.eclipse.jgit.junit.TestRepository;
 import org.eclipse.jgit.lib.NullProgressMonitor;
 import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.Sets;
 import org.eclipse.jgit.revwalk.RevBlob;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.revwalk.RevTree;
@@ -316,4 +321,63 @@ public class UploadPackTest {
 					Collections.singletonList(new RefSpec(commit.name())));
 		}
 	}
+
+	private static ByteArrayInputStream send(String... lines) throws Exception {
+		ByteArrayOutputStream os = new ByteArrayOutputStream();
+		PacketLineOut pckOut = new PacketLineOut(os);
+		for (String line : lines) {
+			if (line == PacketLineIn.END) {
+				pckOut.end();
+			} else if (line == PacketLineIn.DELIM) {
+				pckOut.writeDelim();
+			} else {
+				pckOut.writeString(line);
+			}
+		}
+		byte[] a = os.toByteArray();
+		return new ByteArrayInputStream(a);
+	}
+
+	/*
+	 * Invokes UploadPack with protocol v2 and sends it the given lines.
+	 * Returns UploadPack's output stream, not including the capability
+	 * advertisement by the server.
+	 */
+	private ByteArrayInputStream uploadPackV2(String... inputLines) throws Exception {
+		ByteArrayOutputStream send = new ByteArrayOutputStream();
+		PacketLineOut pckOut = new PacketLineOut(send);
+		for (String line : inputLines) {
+			if (line == PacketLineIn.END) {
+				pckOut.end();
+			} else if (line == PacketLineIn.DELIM) {
+				pckOut.writeDelim();
+			} else {
+				pckOut.writeString(line);
+			}
+		}
+
+		server.getConfig().setString("protocol", null, "version", "2");
+		UploadPack up = new UploadPack(server);
+		up.setExtraParameters(Sets.of("version=2"));
+
+		ByteArrayOutputStream recv = new ByteArrayOutputStream();
+		up.upload(new ByteArrayInputStream(send.toByteArray()), recv, null);
+
+		ByteArrayInputStream recvStream = new ByteArrayInputStream(recv.toByteArray());
+		PacketLineIn pckIn = new PacketLineIn(recvStream);
+
+		// capability advertisement (always sent)
+		assertThat(pckIn.readString(), is("version 2"));
+		assertTrue(pckIn.readString() == PacketLineIn.END);
+		return recvStream;
+	}
+
+	@Test
+	public void testV2EmptyRequest() throws Exception {
+		ByteArrayInputStream recvStream = uploadPackV2(PacketLineIn.END);
+		// Verify that there is nothing more after the capability
+		// advertisement.
+		assertThat(recvStream.available(), is(0));
+	}
+
 }

org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java

@@ -62,8 +62,8 @@ import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.util.SystemReader;
 
 /**
- * The standard "transfer", "fetch", "receive", and "uploadpack" configuration
+ * The standard "transfer", "fetch", "protocol", "receive", and "uploadpack"
- * parameters.
+ * configuration parameters.
  */
 public class TransferConfig {
 	private static final String FSCK = "fsck"; //$NON-NLS-1$
@@ -92,6 +92,33 @@ public class TransferConfig {
 		IGNORE;
 	}
 
+	/**
+	 * A git configuration variable for which versions of the Git protocol to prefer.
+	 * Used in protocol.version.
+	 */
+	enum ProtocolVersion {
+		V0("0"),
+		V2("2");
+
+		final String name;
+
+		ProtocolVersion(String name) {
+			this.name = name;
+		}
+
+		static @Nullable ProtocolVersion parse(@Nullable String name) {
+			if (name == null) {
+				return null;
+			}
+			for (ProtocolVersion v : ProtocolVersion.values()) {
+				if (v.name.equals(name)) {
+					return v;
+				}
+			}
+			return null;
+		}
+	}
+
 	private final boolean fetchFsck;
 	private final boolean receiveFsck;
 	private final String fsckSkipList;
@@ -102,6 +129,7 @@ public class TransferConfig {
 	private final boolean allowTipSha1InWant;
 	private final boolean allowReachableSha1InWant;
 	private final boolean allowFilter;
+	final @Nullable ProtocolVersion protocolVersion;
 	final String[] hideRefs;
 
 	TransferConfig(final Repository db) {
@@ -156,6 +184,7 @@ public class TransferConfig {
 				"uploadpack", "allowreachablesha1inwant", false); //$NON-NLS-1$ //$NON-NLS-2$
 		allowFilter = rc.getBoolean(
 				"uploadpack", "allowfilter", false); //$NON-NLS-1$ //$NON-NLS-2$
+		protocolVersion = ProtocolVersion.parse(rc.getString("protocol", null, "version"));
 		hideRefs = rc.getStringList("uploadpack", null, "hiderefs"); //$NON-NLS-1$ //$NON-NLS-2$
 	}
 

org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

@@ -103,6 +103,7 @@ import org.eclipse.jgit.storage.pack.PackConfig;
 import org.eclipse.jgit.storage.pack.PackStatistics;
 import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck;
 import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser;
+import org.eclipse.jgit.transport.TransferConfig.ProtocolVersion;
 import org.eclipse.jgit.util.io.InterruptTimer;
 import org.eclipse.jgit.util.io.NullOutputStream;
 import org.eclipse.jgit.util.io.TimeoutInputStream;
@@ -112,6 +113,12 @@ import org.eclipse.jgit.util.io.TimeoutOutputStream;
  * Implements the server side of a fetch connection, transmitting objects.
  */
 public class UploadPack {
+	// UploadPack sends these lines as the first response to a client that
+	// supports protocol version 2.
+	private static final String[] v2CapabilityAdvertisement = {
+		"version 2",
+	};
+
 	/** Policy the server uses to validate client requests */
 	public static enum RequestPolicy {
 		/** Client may only ask for objects the server advertised a reference for. */
@@ -238,6 +245,12 @@ public class UploadPack {
 	/** Timer to manage {@link #timeout}. */
 	private InterruptTimer timer;
 
+	/**
+	 * Whether the client requested to use protocol V2 through a side
+	 * channel (such as the Git-Protocol HTTP header).
+	 */
+	private boolean clientRequestedV2;
+
 	private InputStream rawIn;
 
 	private ResponseBufferedOutputStream rawOut;
@@ -656,9 +669,35 @@ public class UploadPack {
 				|| options.contains(OPTION_SIDE_BAND_64K));
 	}
 
+	/**
+	 * Set the Extra Parameters provided by the client.
+	 *
+	 * <p>These are parameters passed by the client through a side channel
+	 * such as the Git-Protocol HTTP header, to allow a client to request
+	 * a newer response format while remaining compatible with older servers
+	 * that do not understand different request formats.
+	 *
+	 * @param params
+	 *            parameters supplied by the client, split at colons or NUL
+	 *            bytes.
+	 * @since 5.0
+	 */
+	public void setExtraParameters(Collection<String> params) {
+		this.clientRequestedV2 = params.contains("version=2"); // $NON-NLS-1$
+	}
+
+	private boolean useProtocolV2() {
+		return ProtocolVersion.V2.equals(transferConfig.protocolVersion)
+				&& clientRequestedV2;
+	}
+
 	/**
 	 * Execute the upload task on the socket.
 	 *
+	 * <p>If the client passed extra parameters (e.g., "version=2") through a
+	 * side channel, the caller must call setExtraParameters first to supply
+	 * them.
+	 *
 	 * @param input
 	 *            raw input to read client commands from. Caller must ensure the
 	 *            input is buffered, otherwise read performance may suffer.
@@ -699,7 +738,11 @@ public class UploadPack {
 
 			pckIn = new PacketLineIn(rawIn);
 			pckOut = new PacketLineOut(rawOut);
-			service();
+			if (useProtocolV2()) {
+				serviceV2();
+			} else {
+				service();
+			}
 		} finally {
 			msgOut = NullOutputStream.INSTANCE;
 			walk.close();
@@ -821,6 +864,54 @@ public class UploadPack {
 			sendPack(accumulator);
 	}
 
+	/*
+	 * Returns true if this is the last command and we should tear down the
+	 * connection.
+	 */
+	private boolean serveOneCommandV2() throws IOException {
+		String command;
+		try {
+			command = pckIn.readString();
+		} catch (EOFException eof) {
+			/* EOF when awaiting command is fine */
+			return true;
+		}
+		if (command == PacketLineIn.END) {
+			// A blank request is valid according
+			// to the protocol; do nothing in this
+			// case.
+			return true;
+		}
+		throw new PackProtocolException("unknown command " + command);
+	}
+
+	private void serviceV2() throws IOException {
+		if (biDirectionalPipe) {
+			// Just like in service(), the capability advertisement
+			// is sent only if this is a bidirectional pipe. (If
+			// not, the client is expected to call
+			// sendAdvertisedRefs() on its own.)
+			for (String s : v2CapabilityAdvertisement) {
+				pckOut.writeString(s + "\n");
+			}
+			pckOut.end();
+
+			while (!serveOneCommandV2()) {
+				// Repeat until an empty command or EOF.
+			}
+			return;
+		}
+
+		try {
+			serveOneCommandV2();
+		} finally {
+			while (0 < rawIn.skip(2048) || 0 <= rawIn.read()) {
+				// Discard until EOF.
+			}
+			rawOut.stopBuffering();
+		}
+	}
+
 	private static Set<ObjectId> refIdSet(Collection<Ref> refs) {
 		Set<ObjectId> ids = new HashSet<>(refs.size());
 		for (Ref ref : refs) {
@@ -923,6 +1014,16 @@ public class UploadPack {
 			throw fail;
 		}
 
+		if (useProtocolV2()) {
+			// The equivalent in v2 is only the capabilities
+			// advertisement.
+			for (String s : v2CapabilityAdvertisement) {
+				adv.writeOne(s);
+			}
+			adv.end();
+			return;
+		}
+
 		adv.init(db);
 		adv.advertiseCapability(OPTION_INCLUDE_TAG);
 		adv.advertiseCapability(OPTION_MULTI_ACK_DETAILED);