Browse Source

LFS: Dramatically improve checkout speed with SSH authentication

SSH Authentication is quite expensive (~120ms on localhost against
Gerrit with LFS plugin). The SSH authentication typically also sends a
validity time of the returned token, which allows to re-use it for a
certain time, avoiding the expensive authentication on every download
request. This improves checkout times by large factors depending on the
LFS object amount/sizes.

Also make sure that all instances of Gson used by LFS are configured in
the same way.

Change-Id: I422c94c37021b4322789b3829fa0185e25d683f2
Signed-off-by: Markus Duft <markus.duft@ssi-schaefer.com>
stable-4.11
Markus Duft 7 years ago
parent
commit
ea2f7e93c7
  1. 5
      org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPrePushHook.java
  2. 35
      org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java
  3. 2
      org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java
  4. 81
      org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java

5
org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPrePushHook.java

@ -43,8 +43,8 @@
package org.eclipse.jgit.lfs;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.eclipse.jgit.lfs.internal.LfsConnectionFactory.toRequest;
import static org.eclipse.jgit.lfs.Protocol.OPERATION_UPLOAD;
import static org.eclipse.jgit.lfs.internal.LfsConnectionFactory.toRequest;
import static org.eclipse.jgit.transport.http.HttpConnection.HTTP_OK;
import static org.eclipse.jgit.util.HttpSupport.METHOD_POST;
import static org.eclipse.jgit.util.HttpSupport.METHOD_PUT;
@ -123,6 +123,7 @@ public class LfsPrePushHook extends PrePushHook {
Map<String, LfsPointer> oid2ptr = requestBatchUpload(api, toPush);
uploadContents(api, oid2ptr);
return EMPTY;
}
private Set<LfsPointer> findObjectsToPush() throws IOException,
@ -201,7 +202,7 @@ public class LfsPrePushHook extends PrePushHook {
for (LfsPointer p : res) {
oidStr2ptr.put(p.getOid().name(), p);
}
Gson gson = new Gson();
Gson gson = Protocol.gson();
api.getOutputStream().write(
gson.toJson(toRequest(OPERATION_UPLOAD, res)).getBytes(UTF_8));
int responseCode = api.getResponseCode();

35
org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java

@ -46,6 +46,10 @@ package org.eclipse.jgit.lfs;
import java.util.List;
import java.util.Map;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
/**
* This interface describes the network protocol used between lfs client and lfs
* server
@ -97,6 +101,24 @@ public interface Protocol {
public Map<String, String> header;
}
/**
* An action with an additional expiration timestamp
*
* @since 4.11
*/
class ExpiringAction extends Action {
/**
* Absolute date/time in format "yyyy-MM-dd'T'HH:mm:ss.SSSX"
*/
public String expiresAt;
/**
* Validity time in milliseconds (preferred over expiresAt as specified:
* https://github.com/git-lfs/git-lfs/blob/master/docs/api/authentication.md)
*/
public String expiresIn;
}
/** Describes an error to be returned by the LFS batch API */
class Error {
public int code;
@ -138,4 +160,17 @@ public interface Protocol {
* Path to the LFS objects servlet.
*/
String OBJECTS_LFS_ENDPOINT = "/objects/batch"; //$NON-NLS-1$
/**
* @return a {@link Gson} instance suitable for handling this
* {@link Protocol}
*
* @since 4.11
*/
public static Gson gson() {
return new GsonBuilder()
.setFieldNamingPolicy(
FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
.disableHtmlEscaping().create();
}
}

2
org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java

@ -158,7 +158,7 @@ public class SmudgeFilter extends FilterCommand {
}
HttpConnection lfsServerConn = LfsConnectionFactory.getLfsConnection(db,
HttpSupport.METHOD_POST, Protocol.OPERATION_DOWNLOAD);
Gson gson = new Gson();
Gson gson = Protocol.gson();
lfsServerConn.getOutputStream()
.write(gson
.toJson(LfsConnectionFactory

81
org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java

@ -52,6 +52,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.net.ProxySelector;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.LinkedList;
import java.util.Map;
import java.util.TreeMap;
@ -75,8 +76,6 @@ import org.eclipse.jgit.util.HttpSupport;
import org.eclipse.jgit.util.io.MessageWriter;
import org.eclipse.jgit.util.io.StreamCopyThread;
import com.google.gson.Gson;
/**
* Provides means to get a valid LFS connection for a given repository.
*/
@ -84,6 +83,7 @@ public class LfsConnectionFactory {
private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$
private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$
private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>();
/**
* Determine URL of LFS server by looking into config parameters lfs.url,
@ -166,17 +166,9 @@ public class LfsConnectionFactory {
Map<String, String> additionalHeaders, String remoteUrl) {
try {
URIish u = new URIish(remoteUrl);
if (SCHEME_SSH.equals(u.getScheme())) {
// discover and authenticate; git-lfs does "ssh -p
// <port> -- <host> git-lfs-authenticate <project>
// <upload/download>"
String json = runSshCommand(u.setPath(""), db.getFS(), //$NON-NLS-1$
"git-lfs-authenticate " + extractProjectName(u) //$NON-NLS-1$
+ " " + purpose); //$NON-NLS-1$
Protocol.Action action = new Gson().fromJson(json,
Protocol.Action.class);
Protocol.ExpiringAction action = getSshAuthentication(
db, purpose, remoteUrl, u);
additionalHeaders.putAll(action.header);
return action.href;
} else {
@ -187,6 +179,34 @@ public class LfsConnectionFactory {
}
}
private static Protocol.ExpiringAction getSshAuthentication(
Repository db, String purpose, String remoteUrl, URIish u)
throws IOException {
AuthCache cached = sshAuthCache.get(remoteUrl);
Protocol.ExpiringAction action = null;
if (cached != null && cached.validUntil > System.currentTimeMillis()) {
action = cached.cachedAction;
}
if (action == null) {
// discover and authenticate; git-lfs does "ssh
// -p <port> -- <host> git-lfs-authenticate
// <project> <upload/download>"
String json = runSshCommand(u.setPath(""), //$NON-NLS-1$
db.getFS(),
"git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$
+ purpose);
action = Protocol.gson().fromJson(json,
Protocol.ExpiringAction.class);
// cache the result as long as possible.
AuthCache c = new AuthCache(action);
sshAuthCache.put(remoteUrl, c);
}
return action;
}
/**
* Create a connection for the specified
* {@link org.eclipse.jgit.lfs.Protocol.Action}.
@ -291,4 +311,41 @@ public class LfsConnectionFactory {
return req;
}
private static final class AuthCache {
private static final long AUTH_CACHE_EAGER_TIMEOUT = 100;
private static final SimpleDateFormat ISO_FORMAT = new SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$
/**
* Creates a cache entry for an authentication response.
* <p>
* The timeout of the cache token is extracted from the given action. If
* no timeout can be determined, the token will be used only once.
*
* @param action
*/
public AuthCache(Protocol.ExpiringAction action) {
this.cachedAction = action;
try {
if (action.expiresIn != null && !action.expiresIn.isEmpty()) {
this.validUntil = System.currentTimeMillis()
+ Long.parseLong(action.expiresIn);
} else if (action.expiresAt != null
&& !action.expiresAt.isEmpty()) {
this.validUntil = ISO_FORMAT.parse(action.expiresAt)
.getTime() - AUTH_CACHE_EAGER_TIMEOUT;
} else {
this.validUntil = System.currentTimeMillis();
}
} catch (Exception e) {
this.validUntil = System.currentTimeMillis();
}
}
long validUntil;
Protocol.ExpiringAction cachedAction;
}
}

Loading…
Cancel
Save