Browse Source

UploadPack: support custom packfile-to-URI mapping

Teach UploadPack to take a provider of URIs corresponding to cached
packs. When fetching, if the client supports the packfile-uri feature,
and if such a cached pack were to be streamed, instead send the
corresponding URI.

This packfile-uri feature is implemented in the jt/fetch-cdn-offload
branch of Git. There is interest in this feature [1], but it is not yet
merged.

[1] https://public-inbox.org/git/cover.1552073690.git.jonathantanmy@google.com/

Change-Id: I9a32dae131c9c56ad2ff4a8a9638ae3b5e44dc15
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
stable-5.5
Jonathan Tan 6 years ago
parent
commit
a004820858
  1. 54
      org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java
  2. 8
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java
  3. 103
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/CachedPackUriProvider.java
  4. 93
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
  5. 21
      org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java
  6. 4
      org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java
  7. 29
      org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

54
org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java

@ -35,6 +35,8 @@ import org.eclipse.jgit.internal.storage.dfs.DfsGarbageCollector;
import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription;
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
import org.eclipse.jgit.internal.storage.file.PackLock; import org.eclipse.jgit.internal.storage.file.PackLock;
import org.eclipse.jgit.internal.storage.pack.CachedPack;
import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider;
import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectId;
@ -2149,6 +2151,58 @@ public class UploadPackTest {
parsePack(recvStream); parsePack(recvStream);
} }
@Test
public void testV2FetchPackfileUris() throws Exception {
// Inside the pack
RevCommit commit = remote.commit().message("x").create();
remote.update("master", commit);
generateBitmaps(server);
// Outside the pack
RevCommit commit2 = remote.commit().message("x").parent(commit).create();
remote.update("master", commit2);
server.getConfig().setBoolean("uploadpack", null, "allowsidebandall", true);
ByteArrayInputStream recvStream = uploadPackV2(
(UploadPack up) -> {
up.setCachedPackUriProvider(new CachedPackUriProvider() {
@Override
public PackInfo getInfo(CachedPack pack,
Collection<String> protocolsSupported)
throws IOException {
assertThat(protocolsSupported, hasItems("https"));
if (!protocolsSupported.contains("https"))
return null;
return new PackInfo("myhash", "myuri");
}
});
},
"command=fetch\n",
PacketLineIn.DELIM,
"want " + commit2.getName() + "\n",
"sideband-all\n",
"packfile-uris https\n",
"done\n",
PacketLineIn.END);
PacketLineIn pckIn = new PacketLineIn(recvStream);
String s;
// skip all \002 strings
for (s = pckIn.readString(); s.startsWith("\002"); s = pckIn.readString()) {
// do nothing
}
assertThat(s, is("\001packfile-uris"));
assertThat(pckIn.readString(), is("\001myhash myuri"));
assertTrue(PacketLineIn.isDelimiter(pckIn.readString()));
assertThat(pckIn.readString(), is("\001packfile"));
parsePack(recvStream);
assertFalse(client.getObjectDatabase().has(commit.toObjectId()));
assertTrue(client.getObjectDatabase().has(commit2.toObjectId()));
}
@Test @Test
public void testGetPeerAgentProtocolV0() throws Exception { public void testGetPeerAgentProtocolV0() throws Exception {
RevCommit one = remote.commit().message("1").create(); RevCommit one = remote.commit().message("1").create();

8
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java

@ -312,6 +312,14 @@ public abstract class PackIndex
public abstract void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, public abstract void resolve(Set<ObjectId> matches, AbbreviatedObjectId id,
int matchLimit) throws IOException; int matchLimit) throws IOException;
/**
* @return the checksum of the pack; caller must not modify it
* @since 5.5
*/
public byte[] getChecksum() {
return packChecksum;
}
/** /**
* Represent mutable entry of pack index consisting of object id and offset * Represent mutable entry of pack index consisting of object id and offset
* in pack (both mutable). * in pack (both mutable).

103
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/CachedPackUriProvider.java

@ -0,0 +1,103 @@
/*
* Copyright (C) 2019, Google LLC.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.storage.pack;
import java.io.IOException;
import java.util.Collection;
import org.eclipse.jgit.annotations.Nullable;
/**
* Provider of URIs corresponding to cached packs. For use with the
* "packfile-uris" feature.
* @since 5.5
*/
public interface CachedPackUriProvider {
/**
* @param pack the cached pack for which to check if a corresponding URI
* exists
* @param protocolsSupported the protocols that the client has declared
* support for; if a URI is returned, it must be of one of these
* protocols
* @throws IOException implementations may throw this
* @return if a URI corresponds to the cached pack, an object
* containing the URI and some other information; null otherwise
* @since 5.5
*/
@Nullable
PackInfo getInfo(CachedPack pack, Collection<String> protocolsSupported)
throws IOException;
/**
* Information about a packfile.
* @since 5.5
*/
public static class PackInfo {
private final String hash;
private final String uri;
/**
* Constructs an object containing information about a packfile.
* @param hash the hash of the packfile as a hexadecimal string
* @param uri the URI corresponding to the packfile
*/
public PackInfo(String hash, String uri) {
this.hash = hash;
this.uri = uri;
}
/**
* @return the hash of the packfile as a hexadecimal string
*/
public String getHash() {
return hash;
}
/**
* @return the URI corresponding to the packfile
*/
public String getUri() {
return uri;
}
}
}

93
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java

@ -122,6 +122,7 @@ import org.eclipse.jgit.storage.pack.PackConfig;
import org.eclipse.jgit.storage.pack.PackStatistics; import org.eclipse.jgit.storage.pack.PackStatistics;
import org.eclipse.jgit.transport.FilterSpec; import org.eclipse.jgit.transport.FilterSpec;
import org.eclipse.jgit.transport.ObjectCountCallback; import org.eclipse.jgit.transport.ObjectCountCallback;
import org.eclipse.jgit.transport.PacketLineOut;
import org.eclipse.jgit.transport.WriteAbortedException; import org.eclipse.jgit.transport.WriteAbortedException;
import org.eclipse.jgit.util.BlockList; import org.eclipse.jgit.util.BlockList;
import org.eclipse.jgit.util.TemporaryBuffer; import org.eclipse.jgit.util.TemporaryBuffer;
@ -307,6 +308,8 @@ public class PackWriter implements AutoCloseable {
private FilterSpec filterSpec = FilterSpec.NO_FILTER; private FilterSpec filterSpec = FilterSpec.NO_FILTER;
private PackfileUriConfig packfileUriConfig;
/** /**
* Create writer for specified repository. * Create writer for specified repository.
* <p> * <p>
@ -650,6 +653,14 @@ public class PackWriter implements AutoCloseable {
filterSpec = requireNonNull(filter); filterSpec = requireNonNull(filter);
} }
/**
* @param config configuration related to packfile URIs
* @since 5.5
*/
public void setPackfileUriConfig(PackfileUriConfig config) {
packfileUriConfig = config;
}
/** /**
* Returns objects number in a pack file that was created by this writer. * Returns objects number in a pack file that was created by this writer.
* *
@ -673,6 +684,26 @@ public class PackWriter implements AutoCloseable {
return stats.totalObjects; return stats.totalObjects;
} }
private long getUnoffloadedObjectCount() throws IOException {
long objCnt = 0;
objCnt += objectsLists[OBJ_COMMIT].size();
objCnt += objectsLists[OBJ_TREE].size();
objCnt += objectsLists[OBJ_BLOB].size();
objCnt += objectsLists[OBJ_TAG].size();
for (CachedPack pack : cachedPacks) {
CachedPackUriProvider.PackInfo packInfo =
packfileUriConfig.cachedPackUriProvider.getInfo(
pack, packfileUriConfig.protocolsSupported);
if (packInfo == null) {
objCnt += pack.getObjectCount();
}
}
return objCnt;
}
/** /**
* Returns the object ids in the pack file that was created by this writer. * Returns the object ids in the pack file that was created by this writer.
* <p> * <p>
@ -1177,13 +1208,38 @@ public class PackWriter implements AutoCloseable {
: new CheckedOutputStream(packStream, crc32), : new CheckedOutputStream(packStream, crc32),
this); this);
long objCnt = getObjectCount(); long objCnt = packfileUriConfig == null ? getObjectCount() :
getUnoffloadedObjectCount();
stats.totalObjects = objCnt; stats.totalObjects = objCnt;
if (callback != null) if (callback != null)
callback.setObjectCount(objCnt); callback.setObjectCount(objCnt);
beginPhase(PackingPhase.WRITING, writeMonitor, objCnt); beginPhase(PackingPhase.WRITING, writeMonitor, objCnt);
long writeStart = System.currentTimeMillis(); long writeStart = System.currentTimeMillis();
try { try {
List<CachedPack> unwrittenCachedPacks;
if (packfileUriConfig != null) {
unwrittenCachedPacks = new ArrayList<>();
CachedPackUriProvider p = packfileUriConfig.cachedPackUriProvider;
PacketLineOut o = packfileUriConfig.pckOut;
o.writeString("packfile-uris\n");
for (CachedPack pack : cachedPacks) {
CachedPackUriProvider.PackInfo packInfo = p.getInfo(
pack, packfileUriConfig.protocolsSupported);
if (packInfo != null) {
o.writeString(packInfo.getHash() + ' ' +
packInfo.getUri() + '\n');
} else {
unwrittenCachedPacks.add(pack);
}
}
packfileUriConfig.pckOut.writeDelim();
packfileUriConfig.pckOut.writeString("packfile\n");
} else {
unwrittenCachedPacks = cachedPacks;
}
out.writeFileHeader(PACK_VERSION_GENERATED, objCnt); out.writeFileHeader(PACK_VERSION_GENERATED, objCnt);
out.flush(); out.flush();
@ -1197,7 +1253,7 @@ public class PackWriter implements AutoCloseable {
} }
stats.reusedPacks = Collections.unmodifiableList(cachedPacks); stats.reusedPacks = Collections.unmodifiableList(cachedPacks);
for (CachedPack pack : cachedPacks) { for (CachedPack pack : unwrittenCachedPacks) {
long deltaCnt = pack.getDeltaCount(); long deltaCnt = pack.getDeltaCount();
stats.reusedObjects += pack.getObjectCount(); stats.reusedObjects += pack.getObjectCount();
stats.reusedDeltas += deltaCnt; stats.reusedDeltas += deltaCnt;
@ -2426,4 +2482,37 @@ public class PackWriter implements AutoCloseable {
return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]"; return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]";
} }
} }
/**
* Configuration related to the packfile URI feature.
*
* @since 5.5
*/
public static class PackfileUriConfig {
@NonNull
private final PacketLineOut pckOut;
@NonNull
private final Collection<String> protocolsSupported;
@NonNull
private final CachedPackUriProvider cachedPackUriProvider;
/**
* @param pckOut where to write "packfile-uri" lines to (should
* output to the same stream as the one passed to
* PackWriter#writePack)
* @param protocolsSupported list of protocols supported (e.g. "https")
* @param cachedPackUriProvider provider of URIs corresponding
* to cached packs
* @since 5.5
*/
public PackfileUriConfig(@NonNull PacketLineOut pckOut,
@NonNull Collection<String> protocolsSupported,
@NonNull CachedPackUriProvider cachedPackUriProvider) {
this.pckOut = pckOut;
this.protocolsSupported = protocolsSupported;
this.cachedPackUriProvider = cachedPackUriProvider;
}
}
} }

21
org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java

@ -74,6 +74,9 @@ public final class FetchV2Request extends FetchRequest {
private final boolean sidebandAll; private final boolean sidebandAll;
@NonNull
private final List<String> packfileUriProtocols;
FetchV2Request(@NonNull List<ObjectId> peerHas, FetchV2Request(@NonNull List<ObjectId> peerHas,
@NonNull List<String> wantedRefs, @NonNull List<String> wantedRefs,
@NonNull Set<ObjectId> wantIds, @NonNull Set<ObjectId> wantIds,
@ -82,7 +85,7 @@ public final class FetchV2Request extends FetchRequest {
@NonNull FilterSpec filterSpec, @NonNull FilterSpec filterSpec,
boolean doneReceived, @NonNull Set<String> clientCapabilities, boolean doneReceived, @NonNull Set<String> clientCapabilities,
@Nullable String agent, @NonNull List<String> serverOptions, @Nullable String agent, @NonNull List<String> serverOptions,
boolean sidebandAll) { boolean sidebandAll, @NonNull List<String> packfileUriProtocols) {
super(wantIds, depth, clientShallowCommits, filterSpec, super(wantIds, depth, clientShallowCommits, filterSpec,
clientCapabilities, deepenSince, clientCapabilities, deepenSince,
deepenNotRefs, agent); deepenNotRefs, agent);
@ -91,6 +94,7 @@ public final class FetchV2Request extends FetchRequest {
this.doneReceived = doneReceived; this.doneReceived = doneReceived;
this.serverOptions = requireNonNull(serverOptions); this.serverOptions = requireNonNull(serverOptions);
this.sidebandAll = sidebandAll; this.sidebandAll = sidebandAll;
this.packfileUriProtocols = packfileUriProtocols;
} }
/** /**
@ -138,6 +142,11 @@ public final class FetchV2Request extends FetchRequest {
return sidebandAll; return sidebandAll;
} }
@NonNull
List<String> getPackfileUriProtocols() {
return packfileUriProtocols;
}
/** @return A builder of {@link FetchV2Request}. */ /** @return A builder of {@link FetchV2Request}. */
static Builder builder() { static Builder builder() {
return new Builder(); return new Builder();
@ -172,6 +181,8 @@ public final class FetchV2Request extends FetchRequest {
boolean sidebandAll; boolean sidebandAll;
final List<String> packfileUriProtocols = new ArrayList<>();
private Builder() { private Builder() {
} }
@ -339,6 +350,11 @@ public final class FetchV2Request extends FetchRequest {
return this; return this;
} }
Builder addPackfileUriProtocol(@NonNull String value) {
packfileUriProtocols.add(value);
return this;
}
/** /**
* @return Initialized fetch request * @return Initialized fetch request
*/ */
@ -347,7 +363,8 @@ public final class FetchV2Request extends FetchRequest {
clientShallowCommits, deepenSince, deepenNotRefs, clientShallowCommits, deepenSince, deepenNotRefs,
depth, filterSpec, doneReceived, clientCapabilities, depth, filterSpec, doneReceived, clientCapabilities,
agent, Collections.unmodifiableList(serverOptions), agent, Collections.unmodifiableList(serverOptions),
sidebandAll); sidebandAll,
Collections.unmodifiableList(packfileUriProtocols));
} }
} }
} }

4
org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java

@ -214,6 +214,10 @@ final class ProtocolV2Parser {
} else if (transferConfig.isAllowSidebandAll() } else if (transferConfig.isAllowSidebandAll()
&& line2.equals(OPTION_SIDEBAND_ALL)) { && line2.equals(OPTION_SIDEBAND_ALL)) {
reqBuilder.setSidebandAll(true); reqBuilder.setSidebandAll(true);
} else if (line2.startsWith("packfile-uris ")) { //$NON-NLS-1$
for (String s : line2.substring(14).split(",")) {
reqBuilder.addPackfileUriProtocol(s);
}
} else { } else {
throw new PackProtocolException(MessageFormat throw new PackProtocolException(MessageFormat
.format(JGitText.get().unexpectedPacketLine, line2)); .format(JGitText.get().unexpectedPacketLine, line2));

29
org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

@ -93,6 +93,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.errors.PackProtocolException;
import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider;
import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.internal.storage.pack.PackWriter;
import org.eclipse.jgit.internal.transport.parser.FirstWant; import org.eclipse.jgit.internal.transport.parser.FirstWant;
import org.eclipse.jgit.lib.BitmapIndex; import org.eclipse.jgit.lib.BitmapIndex;
@ -359,6 +360,8 @@ public class UploadPack {
*/ */
private FetchRequest currentRequest; private FetchRequest currentRequest;
private CachedPackUriProvider cachedPackUriProvider;
/** /**
* Create a new pack upload for an open repository. * Create a new pack upload for an open repository.
* *
@ -736,6 +739,15 @@ public class UploadPack {
this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$ this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$
} }
/**
* @param p provider of URIs corresponding to cached packs (to support
* the packfile URIs feature)
* @since 5.5
*/
public void setCachedPackUriProvider(@Nullable CachedPackUriProvider p) {
cachedPackUriProvider = p;
}
private boolean useProtocolV2() { private boolean useProtocolV2() {
return ProtocolVersion.V2.equals(transferConfig.protocolVersion) return ProtocolVersion.V2.equals(transferConfig.protocolVersion)
&& clientRequestedV2; && clientRequestedV2;
@ -1285,6 +1297,7 @@ public class UploadPack {
(transferConfig.isAllowFilter() ? OPTION_FILTER + ' ' : "") + //$NON-NLS-1$ (transferConfig.isAllowFilter() ? OPTION_FILTER + ' ' : "") + //$NON-NLS-1$
(advertiseRefInWant ? CAPABILITY_REF_IN_WANT + ' ' : "") + //$NON-NLS-1$ (advertiseRefInWant ? CAPABILITY_REF_IN_WANT + ' ' : "") + //$NON-NLS-1$
(transferConfig.isAllowSidebandAll() ? OPTION_SIDEBAND_ALL + ' ' : "") + //$NON-NLS-1$ (transferConfig.isAllowSidebandAll() ? OPTION_SIDEBAND_ALL + ' ' : "") + //$NON-NLS-1$
(cachedPackUriProvider != null ? "packfile-uris " : "") + // $NON-NLS-1$
OPTION_SHALLOW); OPTION_SHALLOW);
caps.add(CAPABILITY_SERVER_OPTION); caps.add(CAPABILITY_SERVER_OPTION);
return caps; return caps;
@ -2298,7 +2311,21 @@ public class UploadPack {
} }
if (pckOut.isUsingSideband()) { if (pckOut.isUsingSideband()) {
pckOut.writeString("packfile\n"); //$NON-NLS-1$ if (req instanceof FetchV2Request &&
cachedPackUriProvider != null &&
!((FetchV2Request) req).getPackfileUriProtocols().isEmpty()) {
FetchV2Request reqV2 = (FetchV2Request) req;
pw.setPackfileUriConfig(new PackWriter.PackfileUriConfig(
pckOut,
reqV2.getPackfileUriProtocols(),
cachedPackUriProvider));
} else {
// PackWriter will write "packfile-uris\n" and "packfile\n"
// for us if provided a PackfileUriConfig. In this case, we
// are not providing a PackfileUriConfig, so we have to
// write this line ourselves.
pckOut.writeString("packfile\n"); //$NON-NLS-1$
}
} }
pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut); pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut);

Loading…
Cancel
Save