Browse Source
If JGit built in LFS support is enabled for the current repository (or user/system), any existing pre-push hook will cause an exception for the time beeing, as only a single pre-push hook is supported. Thus either native pre-push hooks OR JGit built-in LFS support may be enabled currently, but not both. Change-Id: Ie7d2b90e26e948d9cca3d05a7a19489488c75895 Signed-off-by: Markus Duft <markus.duft@ssi-schaefer.com> Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>stable-4.11
Markus Duft
7 years ago
committed by
Matthias Sohn
15 changed files with 837 additions and 184 deletions
@ -0,0 +1,165 @@
|
||||
/* |
||||
* Copyright (C) 2018, Markus Duft <markus.duft@ssi-schaefer.com> |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
package org.eclipse.jgit.lfs.server.fs; |
||||
|
||||
import static org.junit.Assert.assertEquals; |
||||
|
||||
import java.io.InputStream; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Path; |
||||
|
||||
import org.eclipse.jgit.api.Git; |
||||
import org.eclipse.jgit.api.RemoteAddCommand; |
||||
import org.eclipse.jgit.junit.JGitTestUtil; |
||||
import org.eclipse.jgit.junit.TestRepository; |
||||
import org.eclipse.jgit.lfs.CleanFilter; |
||||
import org.eclipse.jgit.lfs.SmudgeFilter; |
||||
import org.eclipse.jgit.lib.Constants; |
||||
import org.eclipse.jgit.lib.ObjectId; |
||||
import org.eclipse.jgit.lib.ObjectLoader; |
||||
import org.eclipse.jgit.lib.Repository; |
||||
import org.eclipse.jgit.lib.StoredConfig; |
||||
import org.eclipse.jgit.revwalk.RevCommit; |
||||
import org.eclipse.jgit.revwalk.RevWalk; |
||||
import org.eclipse.jgit.storage.file.FileRepositoryBuilder; |
||||
import org.eclipse.jgit.transport.URIish; |
||||
import org.eclipse.jgit.treewalk.TreeWalk; |
||||
import org.eclipse.jgit.treewalk.filter.PathFilter; |
||||
import org.eclipse.jgit.util.FileUtils; |
||||
import org.eclipse.jgit.util.IO; |
||||
import org.junit.After; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
|
||||
public class PushTest extends LfsServerTest { |
||||
|
||||
Git git; |
||||
|
||||
private TestRepository localDb; |
||||
|
||||
private Repository remoteDb; |
||||
|
||||
@Override |
||||
@Before |
||||
public void setup() throws Exception { |
||||
super.setup(); |
||||
|
||||
SmudgeFilter.register(); |
||||
CleanFilter.register(); |
||||
|
||||
Path rtmp = Files.createTempDirectory("jgit_test_"); |
||||
remoteDb = FileRepositoryBuilder.create(rtmp.toFile()); |
||||
remoteDb.create(true); |
||||
|
||||
Path tmp = Files.createTempDirectory("jgit_test_"); |
||||
Repository db = FileRepositoryBuilder |
||||
.create(tmp.resolve(".git").toFile()); |
||||
db.create(false); |
||||
StoredConfig cfg = db.getConfig(); |
||||
cfg.setString("filter", "lfs", "usejgitbuiltin", "true"); |
||||
cfg.setString("lfs", null, "url", server.getURI().toString() + "/lfs"); |
||||
cfg.save(); |
||||
|
||||
localDb = new TestRepository<>(db); |
||||
localDb.branch("master").commit().add(".gitattributes", |
||||
"*.bin filter=lfs diff=lfs merge=lfs -text ").create(); |
||||
git = Git.wrap(db); |
||||
|
||||
URIish uri = new URIish( |
||||
"file://" + remoteDb.getDirectory()); |
||||
RemoteAddCommand radd = git.remoteAdd(); |
||||
radd.setUri(uri); |
||||
radd.setName(Constants.DEFAULT_REMOTE_NAME); |
||||
radd.call(); |
||||
|
||||
git.checkout().setName("master").call(); |
||||
git.push().call(); |
||||
} |
||||
|
||||
@After |
||||
public void cleanup() throws Exception { |
||||
remoteDb.close(); |
||||
localDb.getRepository().close(); |
||||
FileUtils.delete(localDb.getRepository().getWorkTree(), |
||||
FileUtils.RECURSIVE); |
||||
FileUtils.delete(remoteDb.getDirectory(), FileUtils.RECURSIVE); |
||||
} |
||||
|
||||
@Test |
||||
public void testPushSimple() throws Exception { |
||||
JGitTestUtil.writeTrashFile(localDb.getRepository(), "a.bin", |
||||
"1234567"); |
||||
git.add().addFilepattern("a.bin").call(); |
||||
RevCommit commit = git.commit().setMessage("add lfs blob").call(); |
||||
git.push().call(); |
||||
|
||||
// check object in remote db, should be LFS pointer
|
||||
ObjectId id = commit.getId(); |
||||
try (RevWalk walk = new RevWalk(remoteDb)) { |
||||
RevCommit rc = walk.parseCommit(id); |
||||
try (TreeWalk tw = new TreeWalk(walk.getObjectReader())) { |
||||
tw.addTree(rc.getTree()); |
||||
tw.setFilter(PathFilter.create("a.bin")); |
||||
tw.next(); |
||||
|
||||
assertEquals(tw.getPathString(), "a.bin"); |
||||
ObjectLoader ldr = walk.getObjectReader() |
||||
.open(tw.getObjectId(0), Constants.OBJ_BLOB); |
||||
try(InputStream is = ldr.openStream()) { |
||||
assertEquals( |
||||
"version https://git-lfs.github.com/spec/v1\noid sha256:8bb0cf6eb9b17d0f7d22b456f121257dc1254e1f01665370476383ea776df414\nsize 7\n", |
||||
new String(IO |
||||
.readWholeStream(is, |
||||
(int) ldr.getSize()) |
||||
.array())); |
||||
} |
||||
} |
||||
|
||||
} |
||||
|
||||
assertEquals( |
||||
"[POST /lfs/objects/batch 200, PUT /lfs/objects/8bb0cf6eb9b17d0f7d22b456f121257dc1254e1f01665370476383ea776df414 200]", |
||||
server.getRequests().toString()); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,11 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?> |
||||
<component id="org.eclipse.jgit.lfs" version="2"> |
||||
<resource path="src/org/eclipse/jgit/lfs/LfsPointer.java" type="org.eclipse.jgit.lfs.LfsPointer"> |
||||
<filter id="336658481"> |
||||
<message_arguments> |
||||
<message_argument value="org.eclipse.jgit.lfs.LfsPointer"/> |
||||
<message_argument value="SIZE_THRESHOLD"/> |
||||
</message_arguments> |
||||
</filter> |
||||
</resource> |
||||
</component> |
||||
@ -0,0 +1,272 @@
|
||||
/* |
||||
* Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
package org.eclipse.jgit.lfs; |
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8; |
||||
import static org.eclipse.jgit.lfs.internal.LfsConnectionFactory.toRequest; |
||||
import static org.eclipse.jgit.lfs.Protocol.OPERATION_UPLOAD; |
||||
import static org.eclipse.jgit.transport.http.HttpConnection.HTTP_OK; |
||||
import static org.eclipse.jgit.util.HttpSupport.METHOD_POST; |
||||
import static org.eclipse.jgit.util.HttpSupport.METHOD_PUT; |
||||
|
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.io.InputStreamReader; |
||||
import java.io.OutputStream; |
||||
import java.io.PrintStream; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Path; |
||||
import java.text.MessageFormat; |
||||
import java.util.Collection; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.Set; |
||||
import java.util.TreeSet; |
||||
|
||||
import org.eclipse.jgit.api.errors.AbortedByHookException; |
||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException; |
||||
import org.eclipse.jgit.errors.MissingObjectException; |
||||
import org.eclipse.jgit.hooks.PrePushHook; |
||||
import org.eclipse.jgit.lfs.Protocol.ObjectInfo; |
||||
import org.eclipse.jgit.lfs.errors.CorruptMediaFile; |
||||
import org.eclipse.jgit.lfs.internal.LfsConnectionFactory; |
||||
import org.eclipse.jgit.lfs.internal.LfsText; |
||||
import org.eclipse.jgit.lib.AnyObjectId; |
||||
import org.eclipse.jgit.lib.Constants; |
||||
import org.eclipse.jgit.lib.ObjectId; |
||||
import org.eclipse.jgit.lib.ObjectReader; |
||||
import org.eclipse.jgit.lib.Ref; |
||||
import org.eclipse.jgit.lib.RefDatabase; |
||||
import org.eclipse.jgit.lib.Repository; |
||||
import org.eclipse.jgit.revwalk.ObjectWalk; |
||||
import org.eclipse.jgit.revwalk.RevObject; |
||||
import org.eclipse.jgit.transport.RemoteRefUpdate; |
||||
import org.eclipse.jgit.transport.http.HttpConnection; |
||||
|
||||
import com.google.gson.Gson; |
||||
import com.google.gson.stream.JsonReader; |
||||
|
||||
/** |
||||
* Pre-push hook that handles uploading LFS artefacts. |
||||
* |
||||
* @since 4.11 |
||||
*/ |
||||
public class LfsPrePushHook extends PrePushHook { |
||||
|
||||
private static final String EMPTY = ""; //$NON-NLS-1$
|
||||
private Collection<RemoteRefUpdate> refs; |
||||
|
||||
/** |
||||
* @param repo |
||||
* the repository |
||||
* @param outputStream |
||||
* not used by this implementation |
||||
*/ |
||||
public LfsPrePushHook(Repository repo, PrintStream outputStream) { |
||||
super(repo, outputStream); |
||||
} |
||||
|
||||
@Override |
||||
public void setRefs(Collection<RemoteRefUpdate> toRefs) { |
||||
this.refs = toRefs; |
||||
} |
||||
|
||||
@Override |
||||
public String call() throws IOException, AbortedByHookException { |
||||
Set<LfsPointer> toPush = findObjectsToPush(); |
||||
if (toPush.isEmpty()) { |
||||
return EMPTY; |
||||
} |
||||
HttpConnection api = LfsConnectionFactory.getLfsConnection( |
||||
getRepository(), METHOD_POST, OPERATION_UPLOAD); |
||||
Map<String, LfsPointer> oid2ptr = requestBatchUpload(api, toPush); |
||||
uploadContents(api, oid2ptr); |
||||
return EMPTY; |
||||
} |
||||
|
||||
private Set<LfsPointer> findObjectsToPush() throws IOException, |
||||
MissingObjectException, IncorrectObjectTypeException { |
||||
Set<LfsPointer> toPush = new TreeSet<>(); |
||||
|
||||
try (ObjectWalk walk = new ObjectWalk(getRepository())) { |
||||
for (RemoteRefUpdate up : refs) { |
||||
walk.setRewriteParents(false); |
||||
excludeRemoteRefs(walk); |
||||
walk.markStart(walk.parseCommit(up.getNewObjectId())); |
||||
while (walk.next() != null) { |
||||
// walk all commits to populate objects
|
||||
} |
||||
findLfsPointers(toPush, walk); |
||||
} |
||||
} |
||||
return toPush; |
||||
} |
||||
|
||||
private static void findLfsPointers(Set<LfsPointer> toPush, ObjectWalk walk) |
||||
throws MissingObjectException, IncorrectObjectTypeException, |
||||
IOException { |
||||
RevObject obj; |
||||
ObjectReader r = walk.getObjectReader(); |
||||
while ((obj = walk.nextObject()) != null) { |
||||
if (obj.getType() == Constants.OBJ_BLOB |
||||
&& getObjectSize(r, obj) < LfsPointer.SIZE_THRESHOLD) { |
||||
LfsPointer ptr = loadLfsPointer(r, obj); |
||||
if (ptr != null) { |
||||
toPush.add(ptr); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
private static long getObjectSize(ObjectReader r, RevObject obj) |
||||
throws IOException { |
||||
return r.getObjectSize(obj.getId(), Constants.OBJ_BLOB); |
||||
} |
||||
|
||||
private static LfsPointer loadLfsPointer(ObjectReader r, AnyObjectId obj) |
||||
throws IOException { |
||||
try (InputStream is = r.open(obj, Constants.OBJ_BLOB).openStream()) { |
||||
return LfsPointer.parseLfsPointer(is); |
||||
} |
||||
} |
||||
|
||||
private void excludeRemoteRefs(ObjectWalk walk) throws IOException { |
||||
RefDatabase refDatabase = getRepository().getRefDatabase(); |
||||
Map<String, Ref> remoteRefs = refDatabase.getRefs(remote()); |
||||
for (Ref r : remoteRefs.values()) { |
||||
ObjectId oid = r.getPeeledObjectId(); |
||||
if (oid == null) { |
||||
oid = r.getObjectId(); |
||||
} |
||||
RevObject o = walk.parseAny(oid); |
||||
if (o.getType() == Constants.OBJ_COMMIT |
||||
|| o.getType() == Constants.OBJ_TAG) { |
||||
walk.markUninteresting(o); |
||||
} |
||||
} |
||||
} |
||||
|
||||
private String remote() { |
||||
String remoteName = getRemoteName() == null |
||||
? Constants.DEFAULT_REMOTE_NAME |
||||
: getRemoteName(); |
||||
return Constants.R_REMOTES + remoteName; |
||||
} |
||||
|
||||
private Map<String, LfsPointer> requestBatchUpload(HttpConnection api, |
||||
Set<LfsPointer> toPush) throws IOException { |
||||
LfsPointer[] res = toPush.toArray(new LfsPointer[toPush.size()]); |
||||
Map<String, LfsPointer> oidStr2ptr = new HashMap<>(); |
||||
for (LfsPointer p : res) { |
||||
oidStr2ptr.put(p.getOid().name(), p); |
||||
} |
||||
Gson gson = new Gson(); |
||||
api.getOutputStream().write( |
||||
gson.toJson(toRequest(OPERATION_UPLOAD, res)).getBytes(UTF_8)); |
||||
int responseCode = api.getResponseCode(); |
||||
if (responseCode != HTTP_OK) { |
||||
throw new IOException( |
||||
MessageFormat.format(LfsText.get().serverFailure, |
||||
api.getURL(), Integer.valueOf(responseCode))); |
||||
} |
||||
return oidStr2ptr; |
||||
} |
||||
|
||||
private void uploadContents(HttpConnection api, |
||||
Map<String, LfsPointer> oid2ptr) throws IOException { |
||||
try (JsonReader reader = new JsonReader( |
||||
new InputStreamReader(api.getInputStream()))) { |
||||
for (Protocol.ObjectInfo o : parseObjects(reader)) { |
||||
if (o.actions == null) { |
||||
continue; |
||||
} |
||||
LfsPointer ptr = oid2ptr.get(o.oid); |
||||
if (ptr == null) { |
||||
// received an object we didn't request
|
||||
continue; |
||||
} |
||||
Protocol.Action uploadAction = o.actions.get(OPERATION_UPLOAD); |
||||
if (uploadAction == null || uploadAction.href == null) { |
||||
continue; |
||||
} |
||||
|
||||
Lfs lfs = new Lfs(getRepository()); |
||||
Path path = lfs.getMediaFile(ptr.getOid()); |
||||
if (!Files.exists(path)) { |
||||
throw new IOException(MessageFormat |
||||
.format(LfsText.get().missingLocalObject, path)); |
||||
} |
||||
uploadFile(o, uploadAction, path); |
||||
} |
||||
} |
||||
} |
||||
|
||||
private List<ObjectInfo> parseObjects(JsonReader reader) { |
||||
Gson gson = new Gson(); |
||||
Protocol.Response resp = gson.fromJson(reader, Protocol.Response.class); |
||||
return resp.objects; |
||||
} |
||||
|
||||
private void uploadFile(Protocol.ObjectInfo o, |
||||
Protocol.Action uploadAction, Path path) |
||||
throws IOException, CorruptMediaFile { |
||||
HttpConnection contentServer = LfsConnectionFactory |
||||
.getLfsContentConnection(getRepository(), uploadAction, |
||||
METHOD_PUT); |
||||
contentServer.setDoOutput(true); |
||||
try (OutputStream out = contentServer |
||||
.getOutputStream()) { |
||||
long size = Files.copy(path, out); |
||||
if (size != o.size) { |
||||
throw new CorruptMediaFile(path, o.size, size); |
||||
} |
||||
} |
||||
int responseCode = contentServer.getResponseCode(); |
||||
if (responseCode != HTTP_OK) { |
||||
throw new IOException(MessageFormat.format( |
||||
LfsText.get().serverFailure, contentServer.getURL(), |
||||
Integer.valueOf(responseCode))); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,294 @@
|
||||
/* |
||||
* Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
package org.eclipse.jgit.lfs.internal; |
||||
|
||||
import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP; |
||||
import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT; |
||||
import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING; |
||||
import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE; |
||||
|
||||
import java.io.BufferedReader; |
||||
import java.io.IOException; |
||||
import java.io.InputStreamReader; |
||||
import java.net.ProxySelector; |
||||
import java.net.URL; |
||||
import java.util.LinkedList; |
||||
import java.util.Map; |
||||
import java.util.TreeMap; |
||||
|
||||
import org.eclipse.jgit.annotations.NonNull; |
||||
import org.eclipse.jgit.lfs.LfsPointer; |
||||
import org.eclipse.jgit.lfs.Protocol; |
||||
import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException; |
||||
import org.eclipse.jgit.lfs.lib.Constants; |
||||
import org.eclipse.jgit.lib.ConfigConstants; |
||||
import org.eclipse.jgit.lib.Repository; |
||||
import org.eclipse.jgit.lib.StoredConfig; |
||||
import org.eclipse.jgit.transport.HttpConfig; |
||||
import org.eclipse.jgit.transport.HttpTransport; |
||||
import org.eclipse.jgit.transport.RemoteSession; |
||||
import org.eclipse.jgit.transport.SshSessionFactory; |
||||
import org.eclipse.jgit.transport.URIish; |
||||
import org.eclipse.jgit.transport.http.HttpConnection; |
||||
import org.eclipse.jgit.util.FS; |
||||
import org.eclipse.jgit.util.HttpSupport; |
||||
import org.eclipse.jgit.util.io.MessageWriter; |
||||
import org.eclipse.jgit.util.io.StreamCopyThread; |
||||
|
||||
import com.google.gson.Gson; |
||||
|
||||
/** |
||||
* Provides means to get a valid LFS connection for a given repository. |
||||
*/ |
||||
public class LfsConnectionFactory { |
||||
|
||||
private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$
|
||||
private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$
|
||||
|
||||
/** |
||||
* Determine URL of LFS server by looking into config parameters lfs.url, |
||||
* lfs.[remote].url or remote.[remote].url. The LFS server URL is computed |
||||
* from remote.[remote].url by appending "/info/lfs". In case there is no |
||||
* URL configured, a SSH remote URI can be used to auto-detect the LFS URI |
||||
* by using the remote "git-lfs-authenticate" command. |
||||
* |
||||
* @param db |
||||
* the repository to work with |
||||
* @param method |
||||
* the method (GET,PUT,...) of the request this connection will |
||||
* be used for |
||||
* @param purpose |
||||
* the action, e.g. Protocol.OPERATION_DOWNLOAD |
||||
* @return the url for the lfs server. e.g. |
||||
* "https://github.com/github/git-lfs.git/info/lfs" |
||||
* @throws IOException |
||||
*/ |
||||
public static HttpConnection getLfsConnection(Repository db, String method, |
||||
String purpose) throws IOException { |
||||
StoredConfig config = db.getConfig(); |
||||
Map<String, String> additionalHeaders = new TreeMap<>(); |
||||
String lfsUrl = getLfsUrl(db, purpose, additionalHeaders); |
||||
URL url = new URL(lfsUrl + Protocol.OBJECTS_LFS_ENDPOINT); |
||||
HttpConnection connection = HttpTransport.getConnectionFactory().create( |
||||
url, HttpSupport.proxyFor(ProxySelector.getDefault(), url)); |
||||
connection.setDoOutput(true); |
||||
if (url.getProtocol().equals(SCHEME_HTTPS) |
||||
&& !config.getBoolean(HttpConfig.HTTP, |
||||
HttpConfig.SSL_VERIFY_KEY, true)) { |
||||
HttpSupport.disableSslVerify(connection); |
||||
} |
||||
connection.setRequestMethod(method); |
||||
connection.setRequestProperty(HDR_ACCEPT, |
||||
Protocol.CONTENTTYPE_VND_GIT_LFS_JSON); |
||||
connection.setRequestProperty(HDR_CONTENT_TYPE, |
||||
Protocol.CONTENTTYPE_VND_GIT_LFS_JSON); |
||||
additionalHeaders |
||||
.forEach((k, v) -> connection.setRequestProperty(k, v)); |
||||
return connection; |
||||
} |
||||
|
||||
private static String getLfsUrl(Repository db, String purpose, |
||||
Map<String, String> additionalHeaders) |
||||
throws LfsConfigInvalidException { |
||||
StoredConfig config = db.getConfig(); |
||||
String lfsUrl = config.getString(Constants.LFS, null, |
||||
ConfigConstants.CONFIG_KEY_URL); |
||||
if (lfsUrl == null) { |
||||
String remoteUrl = null; |
||||
for (String remote : db.getRemoteNames()) { |
||||
lfsUrl = config.getString(Constants.LFS, remote, |
||||
ConfigConstants.CONFIG_KEY_URL); |
||||
// This could be done better (more precise logic), but according
|
||||
// to https://github.com/git-lfs/git-lfs/issues/1759 git-lfs
|
||||
// generally only supports 'origin' in an integrated workflow.
|
||||
if (lfsUrl == null && (remote.equals( |
||||
org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME))) { |
||||
remoteUrl = config.getString( |
||||
ConfigConstants.CONFIG_KEY_REMOTE, remote, |
||||
ConfigConstants.CONFIG_KEY_URL); |
||||
} |
||||
break; |
||||
} |
||||
if (lfsUrl == null && remoteUrl != null) { |
||||
lfsUrl = discoverLfsUrl(db, purpose, additionalHeaders, |
||||
remoteUrl); |
||||
} else { |
||||
lfsUrl = lfsUrl + Protocol.INFO_LFS_ENDPOINT; |
||||
} |
||||
} |
||||
if (lfsUrl == null) { |
||||
throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl); |
||||
} |
||||
return lfsUrl; |
||||
} |
||||
|
||||
private static String discoverLfsUrl(Repository db, String purpose, |
||||
Map<String, String> additionalHeaders, String remoteUrl) { |
||||
try { |
||||
URIish u = new URIish(remoteUrl); |
||||
|
||||
if (SCHEME_SSH.equals(u.getScheme())) { |
||||
// discover and authenticate; git-lfs does "ssh -p
|
||||
// <port> -- <host> git-lfs-authenticate <project>
|
||||
// <upload/download>"
|
||||
String json = runSshCommand(u.setPath(""), db.getFS(), //$NON-NLS-1$
|
||||
"git-lfs-authenticate " + extractProjectName(u) //$NON-NLS-1$
|
||||
+ " " + purpose); //$NON-NLS-1$
|
||||
|
||||
Protocol.Action action = new Gson().fromJson(json, |
||||
Protocol.Action.class); |
||||
additionalHeaders.putAll(action.header); |
||||
return action.href; |
||||
} else { |
||||
return remoteUrl + Protocol.INFO_LFS_ENDPOINT; |
||||
} |
||||
} catch (Exception e) { |
||||
return null; // could not discover
|
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Create a connection for the specified |
||||
* {@link org.eclipse.jgit.lfs.Protocol.Action}. |
||||
* |
||||
* @param repo |
||||
* the repo to fetch required configuration from |
||||
* @param action |
||||
* the action for which to create a connection |
||||
* @param method |
||||
* the target method (GET or PUT) |
||||
* @return a connection. output mode is not set. |
||||
* @throws IOException |
||||
* in case of any error. |
||||
*/ |
||||
public static @NonNull HttpConnection getLfsContentConnection( |
||||
Repository repo, Protocol.Action action, String method) |
||||
throws IOException { |
||||
URL contentUrl = new URL(action.href); |
||||
HttpConnection contentServerConn = HttpTransport.getConnectionFactory() |
||||
.create(contentUrl, HttpSupport |
||||
.proxyFor(ProxySelector.getDefault(), contentUrl)); |
||||
contentServerConn.setRequestMethod(method); |
||||
action.header |
||||
.forEach((k, v) -> contentServerConn.setRequestProperty(k, v)); |
||||
if (contentUrl.getProtocol().equals(SCHEME_HTTPS) |
||||
&& !repo.getConfig().getBoolean(HttpConfig.HTTP, |
||||
HttpConfig.SSL_VERIFY_KEY, true)) { |
||||
HttpSupport.disableSslVerify(contentServerConn); |
||||
} |
||||
|
||||
contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING, |
||||
ENCODING_GZIP); |
||||
|
||||
return contentServerConn; |
||||
} |
||||
|
||||
private static String extractProjectName(URIish u) { |
||||
String path = u.getPath().substring(1); |
||||
if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) { |
||||
return path.substring(0, path.length() - 4); |
||||
} else { |
||||
return path; |
||||
} |
||||
} |
||||
|
||||
private static String runSshCommand(URIish sshUri, FS fs, String command) |
||||
throws IOException { |
||||
RemoteSession session = null; |
||||
Process process = null; |
||||
StreamCopyThread errorThread = null; |
||||
try (MessageWriter stderr = new MessageWriter()) { |
||||
session = SshSessionFactory.getInstance().getSession(sshUri, null, |
||||
fs, 5_000); |
||||
process = session.exec(command, 0); |
||||
errorThread = new StreamCopyThread(process.getErrorStream(), |
||||
stderr.getRawStream()); |
||||
errorThread.start(); |
||||
try (BufferedReader reader = new BufferedReader( |
||||
new InputStreamReader(process.getInputStream(), |
||||
org.eclipse.jgit.lib.Constants.CHARSET))) { |
||||
return reader.readLine(); |
||||
} |
||||
} finally { |
||||
if (process != null) { |
||||
process.destroy(); |
||||
} |
||||
if (errorThread != null) { |
||||
try { |
||||
errorThread.halt(); |
||||
} catch (InterruptedException e) { |
||||
// Stop waiting and return anyway.
|
||||
} finally { |
||||
errorThread = null; |
||||
} |
||||
} |
||||
if (session != null) { |
||||
SshSessionFactory.getInstance().releaseSession(session); |
||||
} |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* @param operation |
||||
* the operation to perform, e.g. Protocol.OPERATION_DOWNLOAD |
||||
* @param resources |
||||
* the LFS resources affected |
||||
* @return a request that can be serialized to JSON |
||||
*/ |
||||
public static Protocol.Request toRequest(String operation, |
||||
LfsPointer... resources) { |
||||
Protocol.Request req = new Protocol.Request(); |
||||
req.operation = operation; |
||||
if (resources != null) { |
||||
req.objects = new LinkedList<>(); |
||||
for (LfsPointer res : resources) { |
||||
Protocol.ObjectSpec o = new Protocol.ObjectSpec(); |
||||
o.oid = res.getOid().getName(); |
||||
o.size = res.getSize(); |
||||
req.objects.add(o); |
||||
} |
||||
} |
||||
return req; |
||||
} |
||||
|
||||
} |
||||
Loading…
Reference in new issue