UploadPack: support custom packfile-to-URI mapping

Teach UploadPack to take a provider of URIs corresponding to cached
packs. When fetching, if the client supports the packfile-uri feature,
and if such a cached pack were to be streamed, instead send the
corresponding URI.

This packfile-uri feature is implemented in the jt/fetch-cdn-offload
branch of Git. There is interest in this feature [1], but it is not yet
merged.

[1] https://public-inbox.org/git/cover.1552073690.git.jonathantanmy@google.com/

Change-Id: I9a32dae131c9c56ad2ff4a8a9638ae3b5e44dc15
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>

org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java

@@ -35,6 +35,8 @@ import org.eclipse.jgit.internal.storage.dfs.DfsGarbageCollector;
 import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription;
 import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
 import org.eclipse.jgit.internal.storage.file.PackLock;
+import org.eclipse.jgit.internal.storage.pack.CachedPack;
+import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider;
 import org.eclipse.jgit.junit.TestRepository;
 import org.eclipse.jgit.lib.NullProgressMonitor;
 import org.eclipse.jgit.lib.ObjectId;
@@ -2149,6 +2151,58 @@ public class UploadPackTest {
 		parsePack(recvStream);
 	}
 
+	@Test
+	public void testV2FetchPackfileUris() throws Exception {
+		// Inside the pack
+		RevCommit commit = remote.commit().message("x").create();
+		remote.update("master", commit);
+		generateBitmaps(server);
+
+		// Outside the pack
+		RevCommit commit2 = remote.commit().message("x").parent(commit).create();
+		remote.update("master", commit2);
+
+		server.getConfig().setBoolean("uploadpack", null, "allowsidebandall", true);
+
+		ByteArrayInputStream recvStream = uploadPackV2(
+			(UploadPack up) -> {
+				up.setCachedPackUriProvider(new CachedPackUriProvider() {
+					@Override
+					public PackInfo getInfo(CachedPack pack,
+							Collection<String> protocolsSupported)
+							throws IOException {
+						assertThat(protocolsSupported, hasItems("https"));
+						if (!protocolsSupported.contains("https"))
+							return null;
+						return new PackInfo("myhash", "myuri");
+					}
+
+				});
+			},
+			"command=fetch\n",
+			PacketLineIn.DELIM,
+			"want " + commit2.getName() + "\n",
+			"sideband-all\n",
+			"packfile-uris https\n",
+			"done\n",
+			PacketLineIn.END);
+		PacketLineIn pckIn = new PacketLineIn(recvStream);
+
+		String s;
+		// skip all \002 strings
+		for (s = pckIn.readString(); s.startsWith("\002"); s = pckIn.readString()) {
+			// do nothing
+		}
+		assertThat(s, is("\001packfile-uris"));
+		assertThat(pckIn.readString(), is("\001myhash myuri"));
+		assertTrue(PacketLineIn.isDelimiter(pckIn.readString()));
+		assertThat(pckIn.readString(), is("\001packfile"));
+		parsePack(recvStream);
+
+		assertFalse(client.getObjectDatabase().has(commit.toObjectId()));
+		assertTrue(client.getObjectDatabase().has(commit2.toObjectId()));
+	}
+
 	@Test
 	public void testGetPeerAgentProtocolV0() throws Exception {
 		RevCommit one = remote.commit().message("1").create();

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java

@@ -312,6 +312,14 @@ public abstract class PackIndex
 	public abstract void resolve(Set<ObjectId> matches, AbbreviatedObjectId id,
 			int matchLimit) throws IOException;
 
+	/**
+	 * @return the checksum of the pack; caller must not modify it
+	 * @since 5.5
+	 */
+	public byte[] getChecksum() {
+		return packChecksum;
+	}
+
 	/**
 	 * Represent mutable entry of pack index consisting of object id and offset
 	 * in pack (both mutable).

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/CachedPackUriProvider.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2019, Google LLC.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.internal.storage.pack;
+
+import java.io.IOException;
+import java.util.Collection;
+import org.eclipse.jgit.annotations.Nullable;
+
+/**
+ * Provider of URIs corresponding to cached packs. For use with the
+ * "packfile-uris" feature.
+ * @since 5.5
+ */
+public interface CachedPackUriProvider {
+
+	/**
+	 * @param pack the cached pack for which to check if a corresponding URI
+	 *	exists
+	 * @param protocolsSupported the protocols that the client has declared
+	 *	support for; if a URI is returned, it must be of one of these
+	 *	protocols
+	 * @throws IOException implementations may throw this
+	 * @return if a URI corresponds to the cached pack, an object
+	 *	containing the URI and some other information; null otherwise
+	 * @since 5.5
+	 */
+	@Nullable
+	PackInfo getInfo(CachedPack pack, Collection<String> protocolsSupported)
+		throws IOException;
+
+	/**
+	 * Information about a packfile.
+	 * @since 5.5
+	 */
+	public static class PackInfo {
+		private final String hash;
+		private final String uri;
+
+		/**
+		 * Constructs an object containing information about a packfile.
+		 * @param hash the hash of the packfile as a hexadecimal string
+		 * @param uri the URI corresponding to the packfile
+		 */
+		public PackInfo(String hash, String uri) {
+			this.hash = hash;
+			this.uri = uri;
+		}
+
+		/**
+		 * @return the hash of the packfile as a hexadecimal string
+		 */
+		public String getHash() {
+			return hash;
+		}
+
+		/**
+		 * @return the URI corresponding to the packfile
+		 */
+		public String getUri() {
+			return uri;
+		}
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java

@@ -122,6 +122,7 @@ import org.eclipse.jgit.storage.pack.PackConfig;
 import org.eclipse.jgit.storage.pack.PackStatistics;
 import org.eclipse.jgit.transport.FilterSpec;
 import org.eclipse.jgit.transport.ObjectCountCallback;
+import org.eclipse.jgit.transport.PacketLineOut;
 import org.eclipse.jgit.transport.WriteAbortedException;
 import org.eclipse.jgit.util.BlockList;
 import org.eclipse.jgit.util.TemporaryBuffer;
@@ -307,6 +308,8 @@ public class PackWriter implements AutoCloseable {
 
 	private FilterSpec filterSpec = FilterSpec.NO_FILTER;
 
+	private PackfileUriConfig packfileUriConfig;
+
 	/**
 	 * Create writer for specified repository.
 	 * <p>
@@ -650,6 +653,14 @@ public class PackWriter implements AutoCloseable {
 		filterSpec = requireNonNull(filter);
 	}
 
+	/**
+	 * @param config configuration related to packfile URIs	
+	 * @since 5.5
+	 */
+	public void setPackfileUriConfig(PackfileUriConfig config) {
+		packfileUriConfig = config;
+	}
+
 	/**
 	 * Returns objects number in a pack file that was created by this writer.
 	 *
@@ -673,6 +684,26 @@ public class PackWriter implements AutoCloseable {
 		return stats.totalObjects;
 	}
 
+	private long getUnoffloadedObjectCount() throws IOException {
+		long objCnt = 0;
+
+		objCnt += objectsLists[OBJ_COMMIT].size();
+		objCnt += objectsLists[OBJ_TREE].size();
+		objCnt += objectsLists[OBJ_BLOB].size();
+		objCnt += objectsLists[OBJ_TAG].size();
+
+		for (CachedPack pack : cachedPacks) {
+			CachedPackUriProvider.PackInfo packInfo =
+				packfileUriConfig.cachedPackUriProvider.getInfo(
+					pack, packfileUriConfig.protocolsSupported);
+			if (packInfo == null) {
+				objCnt += pack.getObjectCount();
+			}
+		}
+
+		return objCnt;
+	}
+
 	/**
 	 * Returns the object ids in the pack file that was created by this writer.
 	 * <p>
@@ -1177,13 +1208,38 @@ public class PackWriter implements AutoCloseable {
 				: new CheckedOutputStream(packStream, crc32),
 			this);
 
-		long objCnt = getObjectCount();
+		long objCnt = packfileUriConfig == null ? getObjectCount() :
+			getUnoffloadedObjectCount();
 		stats.totalObjects = objCnt;
 		if (callback != null)
 			callback.setObjectCount(objCnt);
 		beginPhase(PackingPhase.WRITING, writeMonitor, objCnt);
 		long writeStart = System.currentTimeMillis();
 		try {
+			List<CachedPack> unwrittenCachedPacks;
+
+			if (packfileUriConfig != null) {
+				unwrittenCachedPacks = new ArrayList<>();
+				CachedPackUriProvider p = packfileUriConfig.cachedPackUriProvider;
+				PacketLineOut o = packfileUriConfig.pckOut;
+
+				o.writeString("packfile-uris\n");
+				for (CachedPack pack : cachedPacks) {
+					CachedPackUriProvider.PackInfo packInfo = p.getInfo(
+							pack, packfileUriConfig.protocolsSupported);
+					if (packInfo != null) {
+						o.writeString(packInfo.getHash() + ' ' +
+								packInfo.getUri() + '\n');
+					} else {
+						unwrittenCachedPacks.add(pack);
+					}
+				}
+				packfileUriConfig.pckOut.writeDelim();
+				packfileUriConfig.pckOut.writeString("packfile\n");
+			} else {
+				unwrittenCachedPacks = cachedPacks;
+			}
+
 			out.writeFileHeader(PACK_VERSION_GENERATED, objCnt);
 			out.flush();
 
@@ -1197,7 +1253,7 @@ public class PackWriter implements AutoCloseable {
 			}
 
 			stats.reusedPacks = Collections.unmodifiableList(cachedPacks);
-			for (CachedPack pack : cachedPacks) {
+			for (CachedPack pack : unwrittenCachedPacks) {
 				long deltaCnt = pack.getDeltaCount();
 				stats.reusedObjects += pack.getObjectCount();
 				stats.reusedDeltas += deltaCnt;
@@ -2426,4 +2482,37 @@ public class PackWriter implements AutoCloseable {
 			return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]";
 		}
 	}
+
+	/**
+	 * Configuration related to the packfile URI feature.
+	 *
+	 * @since 5.5
+	 */
+	public static class PackfileUriConfig {
+		@NonNull
+		private final PacketLineOut pckOut;
+
+		@NonNull
+		private final Collection<String> protocolsSupported;
+
+		@NonNull
+		private final CachedPackUriProvider cachedPackUriProvider;
+
+		/**
+		 * @param pckOut where to write "packfile-uri" lines to (should
+		 *     output to the same stream as the one passed to
+		 *     PackWriter#writePack)
+		 * @param protocolsSupported list of protocols supported (e.g. "https")
+		 * @param cachedPackUriProvider provider of URIs corresponding
+		 *     to cached packs
+		 * @since 5.5
+		 */
+		public PackfileUriConfig(@NonNull PacketLineOut pckOut,
+				@NonNull Collection<String> protocolsSupported,
+				@NonNull CachedPackUriProvider cachedPackUriProvider) {
+			this.pckOut = pckOut;
+			this.protocolsSupported = protocolsSupported;
+			this.cachedPackUriProvider = cachedPackUriProvider;
+		}
+	}
 }

org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java

@@ -74,6 +74,9 @@ public final class FetchV2Request extends FetchRequest {
 
 	private final boolean sidebandAll;
 
+	@NonNull
+	private final List<String> packfileUriProtocols;
+
 	FetchV2Request(@NonNull List<ObjectId> peerHas,
 			@NonNull List<String> wantedRefs,
 			@NonNull Set<ObjectId> wantIds,
@@ -82,7 +85,7 @@ public final class FetchV2Request extends FetchRequest {
 			@NonNull FilterSpec filterSpec,
 			boolean doneReceived, @NonNull Set<String> clientCapabilities,
 			@Nullable String agent, @NonNull List<String> serverOptions,
-			boolean sidebandAll) {
+			boolean sidebandAll, @NonNull List<String> packfileUriProtocols) {
 		super(wantIds, depth, clientShallowCommits, filterSpec,
 				clientCapabilities, deepenSince,
 				deepenNotRefs, agent);
@@ -91,6 +94,7 @@ public final class FetchV2Request extends FetchRequest {
 		this.doneReceived = doneReceived;
 		this.serverOptions = requireNonNull(serverOptions);
 		this.sidebandAll = sidebandAll;
+		this.packfileUriProtocols = packfileUriProtocols;
 	}
 
 	/**
@@ -138,6 +142,11 @@ public final class FetchV2Request extends FetchRequest {
 		return sidebandAll;
 	}
 
+	@NonNull
+	List<String> getPackfileUriProtocols() {
+		return packfileUriProtocols;
+	}
+
 	/** @return A builder of {@link FetchV2Request}. */
 	static Builder builder() {
 		return new Builder();
@@ -172,6 +181,8 @@ public final class FetchV2Request extends FetchRequest {
 
 		boolean sidebandAll;
 
+		final List<String> packfileUriProtocols = new ArrayList<>();
+
 		private Builder() {
 		}
 
@@ -339,6 +350,11 @@ public final class FetchV2Request extends FetchRequest {
 			return this;
 		}
 
+		Builder addPackfileUriProtocol(@NonNull String value) {
+			packfileUriProtocols.add(value);
+			return this;
+		}
+
 		/**
 		 * @return Initialized fetch request
 		 */
@@ -347,7 +363,8 @@ public final class FetchV2Request extends FetchRequest {
 					clientShallowCommits, deepenSince, deepenNotRefs,
 					depth, filterSpec, doneReceived, clientCapabilities,
 					agent, Collections.unmodifiableList(serverOptions),
-					sidebandAll);
+					sidebandAll,
+					Collections.unmodifiableList(packfileUriProtocols));
 		}
 	}
 }

org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java

@@ -214,6 +214,10 @@ final class ProtocolV2Parser {
 			} else if (transferConfig.isAllowSidebandAll()
 					&& line2.equals(OPTION_SIDEBAND_ALL)) {
 				reqBuilder.setSidebandAll(true);
+			} else if (line2.startsWith("packfile-uris ")) { //$NON-NLS-1$
+				for (String s : line2.substring(14).split(",")) {
+					reqBuilder.addPackfileUriProtocol(s);
+				}
 			} else {
 				throw new PackProtocolException(MessageFormat
 						.format(JGitText.get().unexpectedPacketLine, line2));

org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

@@ -93,6 +93,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.errors.PackProtocolException;
 import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider;
 import org.eclipse.jgit.internal.storage.pack.PackWriter;
 import org.eclipse.jgit.internal.transport.parser.FirstWant;
 import org.eclipse.jgit.lib.BitmapIndex;
@@ -359,6 +360,8 @@ public class UploadPack {
 	 */
 	private FetchRequest currentRequest;
 
+	private CachedPackUriProvider cachedPackUriProvider;
+
 	/**
 	 * Create a new pack upload for an open repository.
 	 *
@@ -736,6 +739,15 @@ public class UploadPack {
 		this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$
 	}
 
+	/**
+	 * @param p provider of URIs corresponding to cached packs (to support
+	 *     the packfile URIs feature)
+	 * @since 5.5
+	 */
+	public void setCachedPackUriProvider(@Nullable CachedPackUriProvider p) {
+		cachedPackUriProvider = p;
+	}
+
 	private boolean useProtocolV2() {
 		return ProtocolVersion.V2.equals(transferConfig.protocolVersion)
 				&& clientRequestedV2;
@@ -1285,6 +1297,7 @@ public class UploadPack {
 				(transferConfig.isAllowFilter() ? OPTION_FILTER + ' ' : "") + //$NON-NLS-1$
 				(advertiseRefInWant ? CAPABILITY_REF_IN_WANT + ' ' : "") + //$NON-NLS-1$
 				(transferConfig.isAllowSidebandAll() ? OPTION_SIDEBAND_ALL + ' ' : "") + //$NON-NLS-1$
+				(cachedPackUriProvider != null ? "packfile-uris " : "") + // $NON-NLS-1$
 				OPTION_SHALLOW);
 		caps.add(CAPABILITY_SERVER_OPTION);
 		return caps;
@@ -2298,7 +2311,21 @@ public class UploadPack {
 			}
 
 			if (pckOut.isUsingSideband()) {
-				pckOut.writeString("packfile\n"); //$NON-NLS-1$
+				if (req instanceof FetchV2Request &&
+						cachedPackUriProvider != null &&
+						!((FetchV2Request) req).getPackfileUriProtocols().isEmpty()) {
+					FetchV2Request reqV2 = (FetchV2Request) req;
+					pw.setPackfileUriConfig(new PackWriter.PackfileUriConfig(
+							pckOut,
+							reqV2.getPackfileUriProtocols(),
+							cachedPackUriProvider));
+				} else {
+					// PackWriter will write "packfile-uris\n" and "packfile\n"
+					// for us if provided a PackfileUriConfig. In this case, we
+					// are not providing a PackfileUriConfig, so we have to
+					// write this line ourselves.
+					pckOut.writeString("packfile\n"); //$NON-NLS-1$
+				}
 			}
 			pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut);