From a004820858b54d18c6f72fc94dc33bce8b606d66 Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Tue, 2 Apr 2019 10:08:24 -0700 Subject: [PATCH] UploadPack: support custom packfile-to-URI mapping Teach UploadPack to take a provider of URIs corresponding to cached packs. When fetching, if the client supports the packfile-uri feature, and if such a cached pack were to be streamed, instead send the corresponding URI. This packfile-uri feature is implemented in the jt/fetch-cdn-offload branch of Git. There is interest in this feature [1], but it is not yet merged. [1] https://public-inbox.org/git/cover.1552073690.git.jonathantanmy@google.com/ Change-Id: I9a32dae131c9c56ad2ff4a8a9638ae3b5e44dc15 Signed-off-by: Jonathan Tan --- .../jgit/transport/UploadPackTest.java | 54 +++++++++ .../jgit/internal/storage/file/PackIndex.java | 8 ++ .../storage/pack/CachedPackUriProvider.java | 103 ++++++++++++++++++ .../internal/storage/pack/PackWriter.java | 93 +++++++++++++++- .../jgit/transport/FetchV2Request.java | 21 +++- .../jgit/transport/ProtocolV2Parser.java | 4 + .../eclipse/jgit/transport/UploadPack.java | 29 ++++- 7 files changed, 307 insertions(+), 5 deletions(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/CachedPackUriProvider.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java index 22c67c101..528a63f9c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java @@ -35,6 +35,8 @@ import org.eclipse.jgit.internal.storage.dfs.DfsGarbageCollector; import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; import org.eclipse.jgit.internal.storage.file.PackLock; +import org.eclipse.jgit.internal.storage.pack.CachedPack; +import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; @@ -2149,6 +2151,58 @@ public class UploadPackTest { parsePack(recvStream); } + @Test + public void testV2FetchPackfileUris() throws Exception { + // Inside the pack + RevCommit commit = remote.commit().message("x").create(); + remote.update("master", commit); + generateBitmaps(server); + + // Outside the pack + RevCommit commit2 = remote.commit().message("x").parent(commit).create(); + remote.update("master", commit2); + + server.getConfig().setBoolean("uploadpack", null, "allowsidebandall", true); + + ByteArrayInputStream recvStream = uploadPackV2( + (UploadPack up) -> { + up.setCachedPackUriProvider(new CachedPackUriProvider() { + @Override + public PackInfo getInfo(CachedPack pack, + Collection protocolsSupported) + throws IOException { + assertThat(protocolsSupported, hasItems("https")); + if (!protocolsSupported.contains("https")) + return null; + return new PackInfo("myhash", "myuri"); + } + + }); + }, + "command=fetch\n", + PacketLineIn.DELIM, + "want " + commit2.getName() + "\n", + "sideband-all\n", + "packfile-uris https\n", + "done\n", + PacketLineIn.END); + PacketLineIn pckIn = new PacketLineIn(recvStream); + + String s; + // skip all \002 strings + for (s = pckIn.readString(); s.startsWith("\002"); s = pckIn.readString()) { + // do nothing + } + assertThat(s, is("\001packfile-uris")); + assertThat(pckIn.readString(), is("\001myhash myuri")); + assertTrue(PacketLineIn.isDelimiter(pckIn.readString())); + assertThat(pckIn.readString(), is("\001packfile")); + parsePack(recvStream); + + assertFalse(client.getObjectDatabase().has(commit.toObjectId())); + assertTrue(client.getObjectDatabase().has(commit2.toObjectId())); + } + @Test public void testGetPeerAgentProtocolV0() throws Exception { RevCommit one = remote.commit().message("1").create(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index 72699b043..43bd9d4f5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -312,6 +312,14 @@ public abstract class PackIndex public abstract void resolve(Set matches, AbbreviatedObjectId id, int matchLimit) throws IOException; + /** + * @return the checksum of the pack; caller must not modify it + * @since 5.5 + */ + public byte[] getChecksum() { + return packChecksum; + } + /** * Represent mutable entry of pack index consisting of object id and offset * in pack (both mutable). diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/CachedPackUriProvider.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/CachedPackUriProvider.java new file mode 100644 index 000000000..5cbc2baeb --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/CachedPackUriProvider.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2019, Google LLC. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.internal.storage.pack; + +import java.io.IOException; +import java.util.Collection; +import org.eclipse.jgit.annotations.Nullable; + +/** + * Provider of URIs corresponding to cached packs. For use with the + * "packfile-uris" feature. + * @since 5.5 + */ +public interface CachedPackUriProvider { + + /** + * @param pack the cached pack for which to check if a corresponding URI + * exists + * @param protocolsSupported the protocols that the client has declared + * support for; if a URI is returned, it must be of one of these + * protocols + * @throws IOException implementations may throw this + * @return if a URI corresponds to the cached pack, an object + * containing the URI and some other information; null otherwise + * @since 5.5 + */ + @Nullable + PackInfo getInfo(CachedPack pack, Collection protocolsSupported) + throws IOException; + + /** + * Information about a packfile. + * @since 5.5 + */ + public static class PackInfo { + private final String hash; + private final String uri; + + /** + * Constructs an object containing information about a packfile. + * @param hash the hash of the packfile as a hexadecimal string + * @param uri the URI corresponding to the packfile + */ + public PackInfo(String hash, String uri) { + this.hash = hash; + this.uri = uri; + } + + /** + * @return the hash of the packfile as a hexadecimal string + */ + public String getHash() { + return hash; + } + + /** + * @return the URI corresponding to the packfile + */ + public String getUri() { + return uri; + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 650678921..02cfe9049 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -122,6 +122,7 @@ import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.storage.pack.PackStatistics; import org.eclipse.jgit.transport.FilterSpec; import org.eclipse.jgit.transport.ObjectCountCallback; +import org.eclipse.jgit.transport.PacketLineOut; import org.eclipse.jgit.transport.WriteAbortedException; import org.eclipse.jgit.util.BlockList; import org.eclipse.jgit.util.TemporaryBuffer; @@ -307,6 +308,8 @@ public class PackWriter implements AutoCloseable { private FilterSpec filterSpec = FilterSpec.NO_FILTER; + private PackfileUriConfig packfileUriConfig; + /** * Create writer for specified repository. *

@@ -650,6 +653,14 @@ public class PackWriter implements AutoCloseable { filterSpec = requireNonNull(filter); } + /** + * @param config configuration related to packfile URIs + * @since 5.5 + */ + public void setPackfileUriConfig(PackfileUriConfig config) { + packfileUriConfig = config; + } + /** * Returns objects number in a pack file that was created by this writer. * @@ -673,6 +684,26 @@ public class PackWriter implements AutoCloseable { return stats.totalObjects; } + private long getUnoffloadedObjectCount() throws IOException { + long objCnt = 0; + + objCnt += objectsLists[OBJ_COMMIT].size(); + objCnt += objectsLists[OBJ_TREE].size(); + objCnt += objectsLists[OBJ_BLOB].size(); + objCnt += objectsLists[OBJ_TAG].size(); + + for (CachedPack pack : cachedPacks) { + CachedPackUriProvider.PackInfo packInfo = + packfileUriConfig.cachedPackUriProvider.getInfo( + pack, packfileUriConfig.protocolsSupported); + if (packInfo == null) { + objCnt += pack.getObjectCount(); + } + } + + return objCnt; + } + /** * Returns the object ids in the pack file that was created by this writer. *

@@ -1177,13 +1208,38 @@ public class PackWriter implements AutoCloseable { : new CheckedOutputStream(packStream, crc32), this); - long objCnt = getObjectCount(); + long objCnt = packfileUriConfig == null ? getObjectCount() : + getUnoffloadedObjectCount(); stats.totalObjects = objCnt; if (callback != null) callback.setObjectCount(objCnt); beginPhase(PackingPhase.WRITING, writeMonitor, objCnt); long writeStart = System.currentTimeMillis(); try { + List unwrittenCachedPacks; + + if (packfileUriConfig != null) { + unwrittenCachedPacks = new ArrayList<>(); + CachedPackUriProvider p = packfileUriConfig.cachedPackUriProvider; + PacketLineOut o = packfileUriConfig.pckOut; + + o.writeString("packfile-uris\n"); + for (CachedPack pack : cachedPacks) { + CachedPackUriProvider.PackInfo packInfo = p.getInfo( + pack, packfileUriConfig.protocolsSupported); + if (packInfo != null) { + o.writeString(packInfo.getHash() + ' ' + + packInfo.getUri() + '\n'); + } else { + unwrittenCachedPacks.add(pack); + } + } + packfileUriConfig.pckOut.writeDelim(); + packfileUriConfig.pckOut.writeString("packfile\n"); + } else { + unwrittenCachedPacks = cachedPacks; + } + out.writeFileHeader(PACK_VERSION_GENERATED, objCnt); out.flush(); @@ -1197,7 +1253,7 @@ public class PackWriter implements AutoCloseable { } stats.reusedPacks = Collections.unmodifiableList(cachedPacks); - for (CachedPack pack : cachedPacks) { + for (CachedPack pack : unwrittenCachedPacks) { long deltaCnt = pack.getDeltaCount(); stats.reusedObjects += pack.getObjectCount(); stats.reusedDeltas += deltaCnt; @@ -2426,4 +2482,37 @@ public class PackWriter implements AutoCloseable { return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]"; } } + + /** + * Configuration related to the packfile URI feature. + * + * @since 5.5 + */ + public static class PackfileUriConfig { + @NonNull + private final PacketLineOut pckOut; + + @NonNull + private final Collection protocolsSupported; + + @NonNull + private final CachedPackUriProvider cachedPackUriProvider; + + /** + * @param pckOut where to write "packfile-uri" lines to (should + * output to the same stream as the one passed to + * PackWriter#writePack) + * @param protocolsSupported list of protocols supported (e.g. "https") + * @param cachedPackUriProvider provider of URIs corresponding + * to cached packs + * @since 5.5 + */ + public PackfileUriConfig(@NonNull PacketLineOut pckOut, + @NonNull Collection protocolsSupported, + @NonNull CachedPackUriProvider cachedPackUriProvider) { + this.pckOut = pckOut; + this.protocolsSupported = protocolsSupported; + this.cachedPackUriProvider = cachedPackUriProvider; + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java index 86574c14e..fe1b69761 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java @@ -74,6 +74,9 @@ public final class FetchV2Request extends FetchRequest { private final boolean sidebandAll; + @NonNull + private final List packfileUriProtocols; + FetchV2Request(@NonNull List peerHas, @NonNull List wantedRefs, @NonNull Set wantIds, @@ -82,7 +85,7 @@ public final class FetchV2Request extends FetchRequest { @NonNull FilterSpec filterSpec, boolean doneReceived, @NonNull Set clientCapabilities, @Nullable String agent, @NonNull List serverOptions, - boolean sidebandAll) { + boolean sidebandAll, @NonNull List packfileUriProtocols) { super(wantIds, depth, clientShallowCommits, filterSpec, clientCapabilities, deepenSince, deepenNotRefs, agent); @@ -91,6 +94,7 @@ public final class FetchV2Request extends FetchRequest { this.doneReceived = doneReceived; this.serverOptions = requireNonNull(serverOptions); this.sidebandAll = sidebandAll; + this.packfileUriProtocols = packfileUriProtocols; } /** @@ -138,6 +142,11 @@ public final class FetchV2Request extends FetchRequest { return sidebandAll; } + @NonNull + List getPackfileUriProtocols() { + return packfileUriProtocols; + } + /** @return A builder of {@link FetchV2Request}. */ static Builder builder() { return new Builder(); @@ -172,6 +181,8 @@ public final class FetchV2Request extends FetchRequest { boolean sidebandAll; + final List packfileUriProtocols = new ArrayList<>(); + private Builder() { } @@ -339,6 +350,11 @@ public final class FetchV2Request extends FetchRequest { return this; } + Builder addPackfileUriProtocol(@NonNull String value) { + packfileUriProtocols.add(value); + return this; + } + /** * @return Initialized fetch request */ @@ -347,7 +363,8 @@ public final class FetchV2Request extends FetchRequest { clientShallowCommits, deepenSince, deepenNotRefs, depth, filterSpec, doneReceived, clientCapabilities, agent, Collections.unmodifiableList(serverOptions), - sidebandAll); + sidebandAll, + Collections.unmodifiableList(packfileUriProtocols)); } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java index 453be7f8c..14ccddfb6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java @@ -214,6 +214,10 @@ final class ProtocolV2Parser { } else if (transferConfig.isAllowSidebandAll() && line2.equals(OPTION_SIDEBAND_ALL)) { reqBuilder.setSidebandAll(true); + } else if (line2.startsWith("packfile-uris ")) { //$NON-NLS-1$ + for (String s : line2.substring(14).split(",")) { + reqBuilder.addPackfileUriProtocol(s); + } } else { throw new PackProtocolException(MessageFormat .format(JGitText.get().unexpectedPacketLine, line2)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 2194f2f30..1e49c7b01 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -93,6 +93,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.internal.transport.parser.FirstWant; import org.eclipse.jgit.lib.BitmapIndex; @@ -359,6 +360,8 @@ public class UploadPack { */ private FetchRequest currentRequest; + private CachedPackUriProvider cachedPackUriProvider; + /** * Create a new pack upload for an open repository. * @@ -736,6 +739,15 @@ public class UploadPack { this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$ } + /** + * @param p provider of URIs corresponding to cached packs (to support + * the packfile URIs feature) + * @since 5.5 + */ + public void setCachedPackUriProvider(@Nullable CachedPackUriProvider p) { + cachedPackUriProvider = p; + } + private boolean useProtocolV2() { return ProtocolVersion.V2.equals(transferConfig.protocolVersion) && clientRequestedV2; @@ -1285,6 +1297,7 @@ public class UploadPack { (transferConfig.isAllowFilter() ? OPTION_FILTER + ' ' : "") + //$NON-NLS-1$ (advertiseRefInWant ? CAPABILITY_REF_IN_WANT + ' ' : "") + //$NON-NLS-1$ (transferConfig.isAllowSidebandAll() ? OPTION_SIDEBAND_ALL + ' ' : "") + //$NON-NLS-1$ + (cachedPackUriProvider != null ? "packfile-uris " : "") + // $NON-NLS-1$ OPTION_SHALLOW); caps.add(CAPABILITY_SERVER_OPTION); return caps; @@ -2298,7 +2311,21 @@ public class UploadPack { } if (pckOut.isUsingSideband()) { - pckOut.writeString("packfile\n"); //$NON-NLS-1$ + if (req instanceof FetchV2Request && + cachedPackUriProvider != null && + !((FetchV2Request) req).getPackfileUriProtocols().isEmpty()) { + FetchV2Request reqV2 = (FetchV2Request) req; + pw.setPackfileUriConfig(new PackWriter.PackfileUriConfig( + pckOut, + reqV2.getPackfileUriProtocols(), + cachedPackUriProvider)); + } else { + // PackWriter will write "packfile-uris\n" and "packfile\n" + // for us if provided a PackfileUriConfig. In this case, we + // are not providing a PackfileUriConfig, so we have to + // write this line ourselves. + pckOut.writeString("packfile\n"); //$NON-NLS-1$ + } } pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut);