Decide whether to "Accept-Encoding: gzip" on a request-by-request basis

When the reply is already compressed (e.g. a packfile fetched using dumb
HTTP), "Content-Encoding: gzip" wastes bandwidth relative to sending the
content raw. So don't "Accept-Encoding: gzip" for such requests.

Change-Id: Id25702c0b0ed2895df8e9790052c3417d713572c
Signed-off-by: Zhen Chen <czhen@google.com>

org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java

@@ -45,6 +45,7 @@
 
 package org.eclipse.jgit.transport;
 
+import static org.eclipse.jgit.lib.Constants.HEAD;
 import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
 import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP;
 import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
@@ -129,6 +130,25 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 
 	private static final String SVC_RECEIVE_PACK = "git-receive-pack"; //$NON-NLS-1$
 
+	/**
+	 * Accept-Encoding header in the HTTP request
+	 * (https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html).
+	 *
+	 * @since 4.6
+	 */
+	public enum AcceptEncoding {
+		/**
+		 * Do not specify an Accept-Encoding header. In most servers this
+		 * results in the content being transmitted as-is.
+		 */
+		UNSPECIFIED,
+
+		/**
+		 * Accept gzip content encoding.
+		 */
+		GZIP
+	}
+
 	static final TransportProtocol PROTO_HTTP = new TransportProtocol() {
 		private final String[] schemeNames = { "http", "https" }; //$NON-NLS-1$ //$NON-NLS-2$
 
@@ -325,12 +345,15 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 			br.close();
 		}
 
-		if (!refs.containsKey(Constants.HEAD)) {
+		if (!refs.containsKey(HEAD)) {
 			// If HEAD was not published in the info/refs file (it usually
 			// is not there) download HEAD by itself as a loose file and do
 			// the resolution by hand.
 			//
-			HttpConnection conn = httpOpen(new URL(baseUrl, Constants.HEAD));
+			HttpConnection conn = httpOpen(
+					METHOD_GET,
+					new URL(baseUrl, HEAD),
+					AcceptEncoding.GZIP);
 			int status = HttpSupport.response(conn);
 			switch (status) {
 			case HttpConnection.HTTP_OK: {
@@ -342,11 +365,11 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 						Ref r = refs.get(target);
 						if (r == null)
 							r = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null);
-						r = new SymbolicRef(Constants.HEAD, r);
+						r = new SymbolicRef(HEAD, r);
 						refs.put(r.getName(), r);
 					} else if (line != null && ObjectId.isId(line)) {
 						Ref r = new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK,
-								Constants.HEAD, ObjectId.fromString(line));
+								HEAD, ObjectId.fromString(line));
 						refs.put(r.getName(), r);
 					}
 				} finally {
@@ -456,7 +479,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 		Collection<Type> ignoreTypes = null;
 		for (;;) {
 			try {
-				final HttpConnection conn = httpOpen(u);
+				final HttpConnection conn = httpOpen(METHOD_GET, u, AcceptEncoding.GZIP);
 				if (useSmartHttp) {
 					String exp = "application/x-" + service + "-advertisement"; //$NON-NLS-1$ //$NON-NLS-2$
 					conn.setRequestProperty(HDR_ACCEPT, exp + ", */*"); //$NON-NLS-1$
@@ -530,21 +553,37 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 		}
 	}
 
-	final HttpConnection httpOpen(URL u) throws IOException {
+	/**
-		return httpOpen(METHOD_GET, u);
+	 * Open an HTTP connection, setting the accept-encoding request header to gzip.
+	 *
+	 * @param method HTTP request method
+	 * @param u url of the HTTP connection
+	 * @return the HTTP connection
+	 * @throws IOException
+	 * @since 3.3
+	 * @deprecated use {@link #httpOpen(String, URL, AcceptEncoding)} instead.
+	 */
+	@Deprecated
+	protected HttpConnection httpOpen(String method, URL u) throws IOException {
+		return httpOpen(method, u, AcceptEncoding.GZIP);
 	}
 
 	/**
 	 * Open an HTTP connection.
 	 *
-	 * @param method
+	 * @param method HTTP request method
-	 * @param u
+	 * @param u url of the HTTP connection
-	 * @return the connection
+	 * @param acceptEncoding accept-encoding header option
+	 * @return the HTTP connection
 	 * @throws IOException
-	 * @since 3.3
+	 * @since 4.6
 	 */
-	protected HttpConnection httpOpen(String method, URL u)
+	protected HttpConnection httpOpen(String method, URL u,
-			throws IOException {
+			AcceptEncoding acceptEncoding) throws IOException {
+		if (method == null || u == null || acceptEncoding == null) {
+			throw new NullPointerException();
+		}
+
 		final Proxy proxy = HttpSupport.proxyFor(proxySelector, u);
 		HttpConnection conn = connectionFactory.create(u, proxy);
 
@@ -554,7 +593,9 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 
 		conn.setRequestMethod(method);
 		conn.setUseCaches(false);
+		if (acceptEncoding == AcceptEncoding.GZIP) {
 			conn.setRequestProperty(HDR_ACCEPT_ENCODING, ENCODING_GZIP);
+		}
 		conn.setRequestProperty(HDR_PRAGMA, "no-cache"); //$NON-NLS-1$
 		if (UserAgent.get() != null) {
 			conn.setRequestProperty(HDR_USER_AGENT, UserAgent.get());
@@ -660,6 +701,14 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 			return new HttpObjectDB(new URL(httpObjectsUrl, location));
 		}
 
+		@Override
+		BufferedReader openReader(String path) throws IOException {
+			// Line oriented readable content is likely to compress well.
+			// Request gzip encoding.
+			InputStream is = open(path, AcceptEncoding.GZIP).in;
+			return new BufferedReader(new InputStreamReader(is, Constants.CHARSET));
+		}
+
 		@Override
 		Collection<String> getPackNames() throws IOException {
 			final Collection<String> packs = new ArrayList<String>();
@@ -685,9 +734,14 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 
 		@Override
 		FileStream open(final String path) throws IOException {
+			return open(path, AcceptEncoding.UNSPECIFIED);
+		}
+
+		FileStream open(String path, AcceptEncoding acceptEncoding)
+				throws IOException {
 			final URL base = httpObjectsUrl;
 			final URL u = new URL(base, path);
-			final HttpConnection c = httpOpen(u);
+			final HttpConnection c = httpOpen(METHOD_GET, u, acceptEncoding);
 			switch (HttpSupport.response(c)) {
 			case HttpConnection.HTTP_OK:
 				final InputStream in = openInputStream(c);
@@ -844,7 +898,10 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 		}
 
 		void openStream() throws IOException {
-			conn = httpOpen(METHOD_POST, new URL(baseUrl, serviceName));
+			conn = httpOpen(
+					METHOD_POST,
+					new URL(baseUrl, serviceName),
+					AcceptEncoding.GZIP);
 			conn.setInstanceFollowRedirects(false);
 			conn.setDoOutput(true);
 			conn.setRequestProperty(HDR_CONTENT_TYPE, requestType);

org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java

@@ -133,6 +133,9 @@ abstract class WalkRemoteObjectDatabase {
 	 * Callers such as {@link WalkFetchConnection} are prepared to handle this
 	 * by validating the content received, and assuming content that fails to
 	 * match its hash is an incorrectly phrased FileNotFoundException.
+	 * <p>
+	 * This method is recommended for already compressed files like loose objects
+	 * and pack files. For text files, see {@link #openReader(String)}.
 	 *
 	 * @param path
 	 *            location of the file to read, relative to this objects
@@ -346,8 +349,8 @@ abstract class WalkRemoteObjectDatabase {
 	/**
 	 * Open a buffered reader around a file.
 	 * <p>
-	 * This is shorthand for calling {@link #open(String)} and then wrapping it
+	 * This method is suitable for for reading line-oriented resources like
-	 * in a reader suitable for line oriented files like the alternates list.
+	 * <code>info/packs</code>, <code>info/refs</code>, and the alternates list.
 	 *
 	 * @return a stream to read from the file. Never null.
 	 * @param path