From d3ed64bcd467e3e8976b018095e71ed3e3033eae Mon Sep 17 00:00:00 2001 From: Markus Duft Date: Fri, 2 Mar 2018 10:11:42 +0100 Subject: [PATCH] LFS: support merge/rebase/cherry-pick/diff/compare with LFS files Respect merge=lfs and diff=lfs attributes where required to replace (in memory) the content of LFS pointers with the actual blob content from the LFS storage (and vice versa when staging/merging). Does not implement general support for merge/diff attributes for any other use case apart from LFS. Change-Id: Ibad8875de1e0bee8fe3a1dffb1add93111534cae Signed-off-by: Markus Duft Signed-off-by: Matthias Sohn --- .../jgit/lfs/server/fs/CheckoutTest.java | 6 +- .../eclipse/jgit/lfs/server/fs/PushTest.java | 6 +- org.eclipse.jgit.lfs/.settings/.api_filters | 16 + org.eclipse.jgit.lfs/META-INF/MANIFEST.MF | 2 + .../src/org/eclipse/jgit/lfs/BuiltinLFS.java | 140 +++++++++ .../src/org/eclipse/jgit/lfs/CleanFilter.java | 2 +- .../org/eclipse/jgit/lfs/LfsBlobFilter.java | 130 ++++++++ .../org/eclipse/jgit/lfs/LfsBlobLoader.java | 119 ++++++++ .../org/eclipse/jgit/lfs/SmudgeFilter.java | 17 +- .../org/eclipse/jgit/lfs/lib/Constants.java | 2 +- .../src/org/eclipse/jgit/pgm/Main.java | 6 +- .../org/eclipse/jgit/api/AddCommandTest.java | 6 +- .../eclipse/jgit/api/CheckoutCommandTest.java | 6 +- org.eclipse.jgit/.settings/.api_filters | 8 + .../eclipse/jgit/internal/JGitText.properties | 1 + .../eclipse/jgit/blame/BlameGenerator.java | 20 +- .../src/org/eclipse/jgit/blame/Candidate.java | 33 +- .../src/org/eclipse/jgit/diff/DiffEntry.java | 25 ++ .../org/eclipse/jgit/diff/DiffFormatter.java | 7 +- .../src/org/eclipse/jgit/diff/RawText.java | 8 + .../src/org/eclipse/jgit/hooks/Hooks.java | 34 +-- .../org/eclipse/jgit/internal/JGitText.java | 1 + .../src/org/eclipse/jgit/lib/Constants.java | 7 + .../org/eclipse/jgit/merge/ResolveMerger.java | 55 ++-- .../src/org/eclipse/jgit/util/LfsFactory.java | 284 ++++++++++++++++++ 25 files changed, 848 insertions(+), 93 deletions(-) create mode 100644 org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/BuiltinLFS.java create mode 100644 org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java create mode 100644 org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobLoader.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/LfsFactory.java diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/CheckoutTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/CheckoutTest.java index ab99e94ee..df43ccf41 100644 --- a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/CheckoutTest.java +++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/CheckoutTest.java @@ -51,8 +51,7 @@ import java.nio.file.Path; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lfs.CleanFilter; -import org.eclipse.jgit.lfs.SmudgeFilter; +import org.eclipse.jgit.lfs.BuiltinLFS; import org.eclipse.jgit.lfs.lib.LongObjectId; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; @@ -72,8 +71,7 @@ public class CheckoutTest extends LfsServerTest { public void setup() throws Exception { super.setup(); - SmudgeFilter.register(); - CleanFilter.register(); + BuiltinLFS.register(); Path tmp = Files.createTempDirectory("jgit_test_"); Repository db = FileRepositoryBuilder diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/PushTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/PushTest.java index 82566f351..b081a8ef7 100644 --- a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/PushTest.java +++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/PushTest.java @@ -52,8 +52,7 @@ import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.RemoteAddCommand; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lfs.CleanFilter; -import org.eclipse.jgit.lfs.SmudgeFilter; +import org.eclipse.jgit.lfs.BuiltinLFS; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; @@ -84,8 +83,7 @@ public class PushTest extends LfsServerTest { public void setup() throws Exception { super.setup(); - SmudgeFilter.register(); - CleanFilter.register(); + BuiltinLFS.register(); Path rtmp = Files.createTempDirectory("jgit_test_"); remoteDb = FileRepositoryBuilder.create(rtmp.toFile()); diff --git a/org.eclipse.jgit.lfs/.settings/.api_filters b/org.eclipse.jgit.lfs/.settings/.api_filters index 097fd2065..f4887e272 100644 --- a/org.eclipse.jgit.lfs/.settings/.api_filters +++ b/org.eclipse.jgit.lfs/.settings/.api_filters @@ -1,5 +1,13 @@ + + + + + + + + @@ -8,4 +16,12 @@ + + + + + + + + diff --git a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF index 798fccdfc..6d9487714 100644 --- a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF @@ -18,6 +18,7 @@ Import-Package: com.google.gson;version="[2.8.2,3.0.0)", org.eclipse.jgit.annotations;version="[4.11.0,4.12.0)";resolution:=optional, org.eclipse.jgit.api.errors;version="[4.11.0,4.12.0)", org.eclipse.jgit.attributes;version="[4.11.0,4.12.0)", + org.eclipse.jgit.diff;version="[4.11.0,4.12.0)", org.eclipse.jgit.errors;version="[4.11.0,4.12.0)", org.eclipse.jgit.hooks;version="[4.11.0,4.12.0)", org.eclipse.jgit.internal.storage.file;version="[4.11.0,4.12.0)", @@ -25,6 +26,7 @@ Import-Package: com.google.gson;version="[2.8.2,3.0.0)", org.eclipse.jgit.nls;version="[4.11.0,4.12.0)", org.eclipse.jgit.revwalk;version="[4.11.0,4.12.0)", org.eclipse.jgit.storage.file;version="[4.11.0,4.12.0)", + org.eclipse.jgit.storage.pack;version="[4.11.0,4.12.0)", org.eclipse.jgit.transport;version="[4.11.0,4.12.0)", org.eclipse.jgit.transport.http;version="[4.11.0,4.12.0)", org.eclipse.jgit.treewalk;version="[4.11.0,4.12.0)", diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/BuiltinLFS.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/BuiltinLFS.java new file mode 100644 index 000000000..e1b9e34ed --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/BuiltinLFS.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2017, Markus Duft + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.attributes.Attribute; +import org.eclipse.jgit.hooks.PrePushHook; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.LfsFactory; + +/** + * Implementation of {@link LfsFactory}, using built-in (optional) LFS support. + * + * @since 4.11 + */ +public class BuiltinLFS extends LfsFactory { + + private BuiltinLFS() { + SmudgeFilter.register(); + CleanFilter.register(); + } + + /** + * Activates the built-in LFS support. + */ + public static void register() { + setInstance(new BuiltinLFS()); + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + public ObjectLoader applySmudgeFilter(Repository db, ObjectLoader loader, + Attribute attribute) throws IOException { + if (isEnabled(db) && (attribute == null || isEnabled(db, attribute))) { + return LfsBlobFilter.smudgeLfsBlob(db, loader); + } else { + return loader; + } + } + + @Override + public LfsInputStream applyCleanFilter(Repository db, InputStream input, + long length, Attribute attribute) throws IOException { + if (isEnabled(db, attribute)) { + return new LfsInputStream(LfsBlobFilter.cleanLfsBlob(db, input)); + } else { + return new LfsInputStream(input, length); + } + } + + @Override + public @Nullable PrePushHook getPrePushHook(Repository repo, + PrintStream outputStream) { + if (isEnabled(repo)) { + return new LfsPrePushHook(repo, outputStream); + } + return null; + } + + /** + * @param db + * the repository + * @return whether LFS is requested for the given repo. + */ + private boolean isEnabled(Repository db) { + if (db == null) { + return false; + } + return db.getConfig().getBoolean(ConfigConstants.CONFIG_FILTER_SECTION, + Constants.LFS, ConfigConstants.CONFIG_KEY_USEJGITBUILTIN, + false); + } + + /** + * @param db + * the repository + * @param attribute + * the attribute to check + * @return whether LFS filter is enabled for the given .gitattribute + * attribute. + */ + private boolean isEnabled(Repository db, Attribute attribute) { + if (attribute == null) { + return false; + } + return isEnabled(db) && Constants.LFS.equals(attribute.getValue()); + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java index 3e6f9961a..fccbae795 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java @@ -91,7 +91,7 @@ public class CleanFilter extends FilterCommand { * Registers this filter by calling * {@link FilterCommandRegistry#register(String, FilterCommandFactory)} */ - public final static void register() { + static void register() { FilterCommandRegistry .register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + Constants.ATTR_FILTER_DRIVER_PREFIX diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java new file mode 100644 index 000000000..a7d218f80 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2017, Markus Duft + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.TemporaryBuffer; +import org.eclipse.jgit.util.TemporaryBuffer.LocalFile; + +/** + * Provides transparently either a stream to the blob or a LFS media file if + * managed by LFS. + * + * @since 4.11 + */ +public class LfsBlobFilter { + + /** + * In case the given {@link ObjectLoader} points to a LFS pointer file + * replace the loader with one pointing to the LFS media file contents. + * Missing LFS files are downloaded on the fly - same logic as the smudge + * filter. + * + * @param db + * the repo + * @param loader + * the loader for the blob + * @return either the original loader, or a loader for the LFS media file if + * managed by LFS. Files are downloaded on demand if required. + * @throws IOException + * in case of an error + */ + public static ObjectLoader smudgeLfsBlob(Repository db, ObjectLoader loader) + throws IOException { + if (loader.getSize() > LfsPointer.SIZE_THRESHOLD) { + return loader; + } + + try (InputStream is = loader.openStream()) { + LfsPointer ptr = LfsPointer.parseLfsPointer(is); + if (ptr != null) { + Lfs lfs = new Lfs(db); + AnyLongObjectId oid = ptr.getOid(); + Path mediaFile = lfs.getMediaFile(oid); + if (!Files.exists(mediaFile)) { + SmudgeFilter.downloadLfsResource(lfs, db, ptr); + } + + return new LfsBlobLoader(mediaFile); + } + } + + return loader; + } + + /** + * Run the LFS clean filter on the given stream and return a stream to the + * LFS pointer file buffer. Used when inserting objects. + * + * @param db + * the {@link Repository} + * @param originalContent + * the {@link InputStream} to the original content + * @return a {@link TemporaryBuffer} representing the LFS pointer. The + * caller is responsible to destroy the buffer. + * @throws IOException + * in case of any error. + */ + public static TemporaryBuffer cleanLfsBlob(Repository db, + InputStream originalContent) throws IOException { + LocalFile buffer = new TemporaryBuffer.LocalFile(null); + CleanFilter f = new CleanFilter(db, originalContent, buffer); + try { + while (f.run() != -1) { + // loop as long as f.run() tells there is work to do + } + } catch (IOException e) { + buffer.destroy(); + throw e; + } + return buffer; + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobLoader.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobLoader.java new file mode 100644 index 000000000..697ae47a6 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobLoader.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2017, Markus Duft + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; + +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.eclipse.jgit.util.IO; + +/** + * An {@link ObjectLoader} implementation that reads a media file from the LFS + * storage. + * + * @since 4.11 + */ +public class LfsBlobLoader extends ObjectLoader { + + private Path mediaFile; + + private BasicFileAttributes attributes; + + private byte[] cached; + + /** + * Create a loader for the LFS media file at the given path. + * + * @param mediaFile + * path to the file + * @throws IOException + * in case of an error reading attributes + */ + public LfsBlobLoader(Path mediaFile) throws IOException { + this.mediaFile = mediaFile; + this.attributes = Files.readAttributes(mediaFile, + BasicFileAttributes.class); + } + + @Override + public int getType() { + return Constants.OBJ_BLOB; + } + + @Override + public long getSize() { + return attributes.size(); + } + + @Override + public byte[] getCachedBytes() throws LargeObjectException { + if (getSize() > PackConfig.DEFAULT_BIG_FILE_THRESHOLD) { + throw new LargeObjectException(); + } + + if (cached == null) { + try { + cached = IO.readFully(mediaFile.toFile()); + } catch (IOException ioe) { + throw new LargeObjectException(ioe); + } + } + return cached; + } + + @Override + public ObjectStream openStream() + throws MissingObjectException, IOException { + return new ObjectStream.Filter(getType(), getSize(), + Files.newInputStream(mediaFile)); + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java index ae7fab83a..142e74df6 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java @@ -100,9 +100,9 @@ public class SmudgeFilter extends FilterCommand { }; /** - * Registers this filter in JGit by calling + * Register this filter in JGit */ - public final static void register() { + static void register() { FilterCommandRegistry .register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + Constants.ATTR_FILTER_DRIVER_PREFIX @@ -110,8 +110,6 @@ public class SmudgeFilter extends FilterCommand { FACTORY); } - private Lfs lfs; - /** * Constructor for SmudgeFilter. * @@ -126,13 +124,13 @@ public class SmudgeFilter extends FilterCommand { public SmudgeFilter(Repository db, InputStream in, OutputStream out) throws IOException { super(in, out); - lfs = new Lfs(db); + Lfs lfs = new Lfs(db); LfsPointer res = LfsPointer.parseLfsPointer(in); if (res != null) { AnyLongObjectId oid = res.getOid(); Path mediaFile = lfs.getMediaFile(oid); if (!Files.exists(mediaFile)) { - downloadLfsResource(db, res); + downloadLfsResource(lfs, db, res); } this.in = Files.newInputStream(mediaFile); } @@ -141,6 +139,8 @@ public class SmudgeFilter extends FilterCommand { /** * Download content which is hosted on a LFS server * + * @param lfs + * local {@link Lfs} storage. * @param db * the repository to work with * @param res @@ -148,9 +148,8 @@ public class SmudgeFilter extends FilterCommand { * @return the paths of all mediafiles which have been downloaded * @throws IOException */ - private Collection downloadLfsResource(Repository db, - LfsPointer... res) - throws IOException { + public static Collection downloadLfsResource(Lfs lfs, Repository db, + LfsPointer... res) throws IOException { Collection downloadedPaths = new ArrayList<>(); Map oidStr2ptr = new HashMap<>(); for (LfsPointer p : res) { diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java index fbfbf377b..835d7be02 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java @@ -56,7 +56,7 @@ import org.eclipse.jgit.lfs.internal.LfsText; @SuppressWarnings("nls") public final class Constants { /** - * lfs folder + * lfs folder/section/filter name * * @since 4.6 */ diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java index b29b09747..189a5acbf 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java @@ -65,8 +65,7 @@ import java.util.concurrent.TimeUnit; import org.eclipse.jgit.awtui.AwtAuthenticator; import org.eclipse.jgit.awtui.AwtCredentialsProvider; import org.eclipse.jgit.errors.TransportException; -import org.eclipse.jgit.lfs.CleanFilter; -import org.eclipse.jgit.lfs.SmudgeFilter; +import org.eclipse.jgit.lfs.BuiltinLFS; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryBuilder; import org.eclipse.jgit.pgm.internal.CLIText; @@ -111,8 +110,7 @@ public class Main { */ public Main() { HttpTransport.setConnectionFactory(new HttpClientConnectionFactory()); - CleanFilter.register(); - SmudgeFilter.register(); + BuiltinLFS.register(); gcExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() { private final ThreadFactory baseFactory = Executors .defaultThreadFactory(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java index df4be4523..7d34e0d3f 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java @@ -63,8 +63,7 @@ import org.eclipse.jgit.dircache.DirCacheBuilder; import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; -import org.eclipse.jgit.lfs.CleanFilter; -import org.eclipse.jgit.lfs.SmudgeFilter; +import org.eclipse.jgit.lfs.BuiltinLFS; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; @@ -92,8 +91,7 @@ public class AddCommandTest extends RepositoryTestCase { @Override public void setUp() throws Exception { - CleanFilter.register(); - SmudgeFilter.register(); + BuiltinLFS.register(); super.setUp(); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java index f5a9130c1..4f0f2a7fb 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java @@ -74,8 +74,7 @@ import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; -import org.eclipse.jgit.lfs.CleanFilter; -import org.eclipse.jgit.lfs.SmudgeFilter; +import org.eclipse.jgit.lfs.BuiltinLFS; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; @@ -102,8 +101,7 @@ public class CheckoutCommandTest extends RepositoryTestCase { @Override @Before public void setUp() throws Exception { - CleanFilter.register(); - SmudgeFilter.register(); + BuiltinLFS.register(); super.setUp(); git = new Git(db); // commit something diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 3adb2f976..86c13b470 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -1,5 +1,13 @@ + + + + + + + + diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index fdaee3ee0..10139f9b8 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -462,6 +462,7 @@ noHMACsupport=No {0} support: {1} noMergeBase=No merge base could be determined. Reason={0}. {1} noMergeHeadSpecified=No merge head specified nonBareLinkFilesNotSupported=Link files are not supported with nonbare repos +noPathAttributesFound=No Attributes found for {0}. noSuchRef=no such ref noSuchSubmodule=no such submodule {0} notABoolean=Not a boolean: {0} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java index e2411d604..1ad7a3055 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java @@ -303,7 +303,8 @@ public class BlameGenerator implements AutoCloseable { throws IOException { if (description == null) description = JGitText.get().blameNotCommittedYet; - BlobCandidate c = new BlobCandidate(description, resultPath); + BlobCandidate c = new BlobCandidate(getRepository(), description, + resultPath); c.sourceText = contents; c.regionList = new Region(0, 0, contents.size()); remaining = contents.size(); @@ -333,7 +334,8 @@ public class BlameGenerator implements AutoCloseable { if (ldr.getType() == OBJ_BLOB) { if (description == null) description = JGitText.get().blameNotCommittedYet; - BlobCandidate c = new BlobCandidate(description, resultPath); + BlobCandidate c = new BlobCandidate(getRepository(), description, + resultPath); c.sourceBlob = id.toObjectId(); c.sourceText = new RawText(ldr.getCachedBytes(Integer.MAX_VALUE)); c.regionList = new Region(0, 0, c.sourceText.size()); @@ -346,7 +348,7 @@ public class BlameGenerator implements AutoCloseable { if (!find(commit, resultPath)) return this; - Candidate c = new Candidate(commit, resultPath); + Candidate c = new Candidate(getRepository(), commit, resultPath); c.sourceBlob = idBuf.toObjectId(); c.loadText(reader); c.regionList = new Region(0, 0, c.sourceText.size()); @@ -430,7 +432,8 @@ public class BlameGenerator implements AutoCloseable { // just pump the queue } - ReverseCandidate c = new ReverseCandidate(result, resultPath); + ReverseCandidate c = new ReverseCandidate(getRepository(), result, + resultPath); c.sourceBlob = idBuf.toObjectId(); c.loadText(reader); c.regionList = new Region(0, 0, c.sourceText.size()); @@ -637,7 +640,8 @@ public class BlameGenerator implements AutoCloseable { return false; } - Candidate next = n.create(parent, PathFilter.create(r.getOldPath())); + Candidate next = n.create(getRepository(), parent, + PathFilter.create(r.getOldPath())); next.sourceBlob = r.getOldId().toObjectId(); next.renameScore = r.getScore(); next.loadText(reader); @@ -653,7 +657,7 @@ public class BlameGenerator implements AutoCloseable { private boolean splitBlameWithParent(Candidate n, RevCommit parent) throws IOException { - Candidate next = n.create(parent, n.sourcePath); + Candidate next = n.create(getRepository(), parent, n.sourcePath); next.sourceBlob = idBuf.toObjectId(); next.loadText(reader); return split(next, n); @@ -740,12 +744,12 @@ public class BlameGenerator implements AutoCloseable { Candidate p; if (renames != null && renames[pIdx] != null) { - p = n.create(parent, + p = n.create(getRepository(), parent, PathFilter.create(renames[pIdx].getOldPath())); p.renameScore = renames[pIdx].getScore(); p.sourceBlob = renames[pIdx].getOldId().toObjectId(); } else if (ids != null && ids[pIdx] != null) { - p = n.create(parent, n.sourcePath); + p = n.create(getRepository(), parent, n.sourcePath); p.sourceBlob = ids[pIdx]; } else { continue; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/Candidate.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/Candidate.java index 855ef7825..457d1d2ce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/blame/Candidate.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/Candidate.java @@ -55,10 +55,12 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevFlag; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.eclipse.jgit.util.LfsFactory; /** * A source that may have supplied some (or all) of the result file. @@ -109,7 +111,11 @@ class Candidate { */ int renameScore; - Candidate(RevCommit commit, PathFilter path) { + /** repository used for LFS blob handling */ + private Repository sourceRepository; + + Candidate(Repository repo, RevCommit commit, PathFilter path) { + sourceRepository = repo; sourceCommit = commit; sourcePath = path; } @@ -150,12 +156,12 @@ class Candidate { return sourceCommit.getAuthorIdent(); } - Candidate create(RevCommit commit, PathFilter path) { - return new Candidate(commit, path); + Candidate create(Repository repo, RevCommit commit, PathFilter path) { + return new Candidate(repo, commit, path); } Candidate copy(RevCommit commit) { - Candidate r = create(commit, sourcePath); + Candidate r = create(sourceRepository, commit, sourcePath); r.sourceBlob = sourceBlob; r.sourceText = sourceText; r.regionList = regionList; @@ -164,7 +170,11 @@ class Candidate { } void loadText(ObjectReader reader) throws IOException { - ObjectLoader ldr = reader.open(sourceBlob, Constants.OBJ_BLOB); + ObjectLoader ldr = LfsFactory.getInstance().applySmudgeFilter(sourceRepository, + reader.open(sourceBlob, Constants.OBJ_BLOB), + LfsFactory.getAttributesForPath(sourceRepository, + sourcePath.getPath(), sourceCommit) + .get(Constants.ATTR_DIFF)); sourceText = new RawText(ldr.getCachedBytes(Integer.MAX_VALUE)); } @@ -349,8 +359,9 @@ class Candidate { * children pointers, allowing reverse navigation of history. */ static final class ReverseCandidate extends Candidate { - ReverseCandidate(ReverseCommit commit, PathFilter path) { - super(commit, path); + ReverseCandidate(Repository repo, ReverseCommit commit, + PathFilter path) { + super(repo, commit, path); } @Override @@ -370,8 +381,8 @@ class Candidate { } @Override - Candidate create(RevCommit commit, PathFilter path) { - return new ReverseCandidate((ReverseCommit) commit, path); + Candidate create(Repository repo, RevCommit commit, PathFilter path) { + return new ReverseCandidate(repo, (ReverseCommit) commit, path); } @Override @@ -400,8 +411,8 @@ class Candidate { /** Author name to refer to this blob with. */ String description; - BlobCandidate(String name, PathFilter path) { - super(null, path); + BlobCandidate(Repository repo, String name, PathFilter path) { + super(repo, null, path); description = name; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffEntry.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffEntry.java index 0f5ea7651..5c8343f92 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffEntry.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffEntry.java @@ -48,9 +48,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.eclipse.jgit.attributes.Attribute; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.ObjectId; @@ -196,6 +198,11 @@ public class DiffEntry { entry.newMode = walk.getFileMode(1); entry.newPath = entry.oldPath = walk.getPathString(); + if (walk.getAttributesNodeProvider() != null) { + entry.diffAttribute = walk.getAttributes() + .get(Constants.ATTR_DIFF); + } + if (treeFilterMarker != null) entry.treeFilterMarks = treeFilterMarker.getMarks(walk); @@ -282,6 +289,7 @@ public class DiffEntry { del.newMode = FileMode.MISSING; del.newPath = DiffEntry.DEV_NULL; del.changeType = ChangeType.DELETE; + del.diffAttribute = entry.diffAttribute; DiffEntry add = new DiffEntry(); add.oldId = A_ZERO; @@ -292,6 +300,7 @@ public class DiffEntry { add.newMode = entry.getNewMode(); add.newPath = entry.getNewPath(); add.changeType = ChangeType.ADD; + add.diffAttribute = entry.diffAttribute; return Arrays.asList(del, add); } @@ -306,6 +315,7 @@ public class DiffEntry { r.newId = dst.newId; r.newMode = dst.newMode; r.newPath = dst.newPath; + r.diffAttribute = dst.diffAttribute; r.changeType = changeType; r.score = score; @@ -321,6 +331,13 @@ public class DiffEntry { /** File name of the new (post-image). */ protected String newPath; + /** + * diff filter attribute + * + * @since 4.11 + */ + protected Attribute diffAttribute; + /** Old mode of the file, if described by the patch, else null. */ protected FileMode oldMode; @@ -394,6 +411,14 @@ public class DiffEntry { return side == Side.OLD ? getOldPath() : getNewPath(); } + /** + * @return the {@link Attribute} determining filters to be applied. + * @since 4.11 + */ + public Attribute getDiffAttribute() { + return diffAttribute; + } + /** * Get the old file mode * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java index bf9a27bdc..2e29b8177 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java @@ -98,6 +98,7 @@ import org.eclipse.jgit.treewalk.filter.IndexDiffFilter; import org.eclipse.jgit.treewalk.filter.NotIgnoredFilter; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.eclipse.jgit.treewalk.filter.TreeFilter; +import org.eclipse.jgit.util.LfsFactory; import org.eclipse.jgit.util.QuotedString; /** @@ -141,6 +142,8 @@ public class DiffFormatter implements AutoCloseable { private ContentSource.Pair source; + private Repository repository; + /** * Create a new formatter with a default level of context. * @@ -172,6 +175,7 @@ public class DiffFormatter implements AutoCloseable { * source repository holding referenced objects. */ public void setRepository(Repository repository) { + this.repository = repository; setReader(repository.newObjectReader(), repository.getConfig(), true); } @@ -1057,7 +1061,8 @@ public class DiffFormatter implements AutoCloseable { throw new AmbiguousObjectException(id, ids); } - ObjectLoader ldr = source.open(side, entry); + ObjectLoader ldr = LfsFactory.getInstance().applySmudgeFilter(repository, + source.open(side, entry), entry.getDiffAttribute()); return RawText.load(ldr, binaryFileThreshold); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java index 7be165910..1c9807493 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java @@ -110,6 +110,14 @@ public class RawText extends Sequence { this(IO.readFully(file)); } + /** + * @return the raw, unprocessed content read. + * @since 4.11 + */ + public byte[] getRawContent() { + return content; + } + /** @return total number of items in the sequence. */ /** {@inheritDoc} */ @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/Hooks.java b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/Hooks.java index a5eeb6428..79395ed23 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/Hooks.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/Hooks.java @@ -43,13 +43,11 @@ package org.eclipse.jgit.hooks; import java.io.PrintStream; -import java.lang.reflect.Constructor; import java.text.MessageFormat; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.util.LfsFactory; /** * Factory class for instantiating supported hooks. @@ -112,28 +110,16 @@ public class Hooks { * @since 4.2 */ public static PrePushHook prePush(Repository repo, PrintStream outputStream) { - PrePushHook lfsHook = null; - try { - StoredConfig cfg = repo.getConfig(); - if (cfg.getBoolean(ConfigConstants.CONFIG_FILTER_SECTION, "lfs", //$NON-NLS-1$ - ConfigConstants.CONFIG_KEY_USEJGITBUILTIN, false)) { - @SuppressWarnings("unchecked") - Class cls = (Class) Class - .forName("org.eclipse.jgit.lfs.LfsPrePushHook"); //$NON-NLS-1$ - Constructor constructor = cls - .getConstructor(Repository.class, PrintStream.class); - - lfsHook = constructor.newInstance(repo, outputStream); - } - } catch (Exception e) { - // no problem :) no LFS support present - } - if (lfsHook != null) { - if (lfsHook.isNativeHookPresent()) { - throw new IllegalStateException(MessageFormat - .format(JGitText.get().lfsHookConflict, repo)); + if (LfsFactory.getInstance().isAvailable()) { + PrePushHook hook = LfsFactory.getInstance().getPrePushHook(repo, + outputStream); + if (hook != null) { + if (hook.isNativeHookPresent()) { + throw new IllegalStateException(MessageFormat + .format(JGitText.get().lfsHookConflict, repo)); + } + return hook; } - return lfsHook; } return new PrePushHook(repo, outputStream); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 41f0c5c43..753a7f9a8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -523,6 +523,7 @@ public class JGitText extends TranslationBundle { /***/ public String noMergeBase; /***/ public String noMergeHeadSpecified; /***/ public String nonBareLinkFilesNotSupported; + /***/ public String noPathAttributesFound; /***/ public String noSuchRef; /***/ public String noSuchSubmodule; /***/ public String notABoolean; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index d2160011b..bb85229f8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -437,6 +437,13 @@ public final class Constants { */ public static final String ATTR_MERGE = "merge"; //$NON-NLS-1$ + /** + * Diff attribute. + * + * @since 4.11 + */ + public static final String ATTR_DIFF = "diff"; //$NON-NLS-1$ + /** * Binary value for custom merger. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index a9c139aad..6462608f5 100755 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -87,12 +87,12 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.NoWorkTreeException; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectLoader; -import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.storage.pack.PackConfig; @@ -106,6 +106,8 @@ import org.eclipse.jgit.treewalk.WorkingTreeIterator; import org.eclipse.jgit.treewalk.WorkingTreeOptions; import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.LfsFactory; +import org.eclipse.jgit.util.LfsFactory.LfsInputStream; import org.eclipse.jgit.util.TemporaryBuffer; import org.eclipse.jgit.util.io.EolStreamTypeUtil; @@ -769,11 +771,12 @@ public class ResolveMerger extends ThreeWayMerger { return false; } - MergeResult result = contentMerge(base, ours, theirs); + MergeResult result = contentMerge(base, ours, theirs, + attributes); if (ignoreConflicts) { result.setContainsConflicts(false); } - updateIndex(base, ours, theirs, result); + updateIndex(base, ours, theirs, result, attributes); if (result.containsConflicts() && !ignoreConflicts) unmergedPaths.add(tw.getPathString()); modifiedFiles.add(tw.getPathString()); @@ -781,7 +784,8 @@ public class ResolveMerger extends ThreeWayMerger { // OURS or THEIRS has been deleted if (((modeO != 0 && !tw.idEqual(T_BASE, T_OURS)) || (modeT != 0 && !tw .idEqual(T_BASE, T_THEIRS)))) { - MergeResult result = contentMerge(base, ours, theirs); + MergeResult result = contentMerge(base, ours, theirs, + attributes); add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, 0, 0); add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, 0, 0); @@ -816,12 +820,14 @@ public class ResolveMerger extends ThreeWayMerger { * @param base * @param ours * @param theirs + * @param attributes * * @return the result of the content merge * @throws IOException */ private MergeResult contentMerge(CanonicalTreeParser base, - CanonicalTreeParser ours, CanonicalTreeParser theirs) + CanonicalTreeParser ours, CanonicalTreeParser theirs, + Attributes attributes) throws IOException { RawText baseText; RawText ourText; @@ -829,11 +835,11 @@ public class ResolveMerger extends ThreeWayMerger { try { baseText = base == null ? RawText.EMPTY_TEXT : getRawText( - base.getEntryObjectId(), reader); + base.getEntryObjectId(), attributes); ourText = ours == null ? RawText.EMPTY_TEXT : getRawText( - ours.getEntryObjectId(), reader); + ours.getEntryObjectId(), attributes); theirsText = theirs == null ? RawText.EMPTY_TEXT : getRawText( - theirs.getEntryObjectId(), reader); + theirs.getEntryObjectId(), attributes); } catch (BinaryBlobException e) { MergeResult r = new MergeResult<>(Collections.emptyList()); r.setContainsConflicts(true); @@ -897,17 +903,20 @@ public class ResolveMerger extends ThreeWayMerger { * @param ours * @param theirs * @param result + * @param attributes * @throws FileNotFoundException * @throws IOException */ private void updateIndex(CanonicalTreeParser base, CanonicalTreeParser ours, CanonicalTreeParser theirs, - MergeResult result) throws FileNotFoundException, + MergeResult result, Attributes attributes) + throws FileNotFoundException, IOException { TemporaryBuffer rawMerged = null; try { rawMerged = doMerge(result); - File mergedFile = inCore ? null : writeMergedFile(rawMerged); + File mergedFile = inCore ? null + : writeMergedFile(rawMerged, attributes); if (result.containsConflicts()) { // A conflict occurred, the file will contain conflict markers // the index will be populated with the three stages and the @@ -934,7 +943,7 @@ public class ResolveMerger extends ThreeWayMerger { nonNullRepo().getFS().lastModified(mergedFile)); dce.setLength((int) mergedFile.length()); } - dce.setObjectId(insertMergeResult(rawMerged)); + dce.setObjectId(insertMergeResult(rawMerged, attributes)); builder.add(dce); } finally { if (rawMerged != null) { @@ -948,11 +957,14 @@ public class ResolveMerger extends ThreeWayMerger { * * @param rawMerged * the raw merged content + * @param attributes + * the files .gitattributes entries * @return the working tree file to which the merged content was written. * @throws FileNotFoundException * @throws IOException */ - private File writeMergedFile(TemporaryBuffer rawMerged) + private File writeMergedFile(TemporaryBuffer rawMerged, + Attributes attributes) throws FileNotFoundException, IOException { File workTree = nonNullRepo().getWorkTree(); FS fs = nonNullRepo().getFS(); @@ -963,7 +975,7 @@ public class ResolveMerger extends ThreeWayMerger { } EolStreamType streamType = EolStreamTypeUtil.detectStreamType( OperationType.CHECKOUT_OP, workingTreeOptions, - tw.getAttributes()); + attributes); try (OutputStream os = EolStreamTypeUtil.wrapOutputStream( new BufferedOutputStream(new FileOutputStream(of)), streamType)) { @@ -987,9 +999,13 @@ public class ResolveMerger extends ThreeWayMerger { return buf; } - private ObjectId insertMergeResult(TemporaryBuffer buf) throws IOException { - try (InputStream in = buf.openInputStream()) { - return getObjectInserter().insert(OBJ_BLOB, buf.length(), in); + private ObjectId insertMergeResult(TemporaryBuffer buf, + Attributes attributes) throws IOException { + InputStream in = buf.openInputStream(); + try (LfsInputStream is = LfsFactory.getInstance().applyCleanFilter( + getRepository(), in, + buf.length(), attributes.get(Constants.ATTR_MERGE))) { + return getObjectInserter().insert(OBJ_BLOB, is.getLength(), is); } } @@ -1021,12 +1037,15 @@ public class ResolveMerger extends ThreeWayMerger { return FileMode.MISSING.getBits(); } - private static RawText getRawText(ObjectId id, ObjectReader reader) + private RawText getRawText(ObjectId id, + Attributes attributes) throws IOException, BinaryBlobException { if (id.equals(ObjectId.zeroId())) return new RawText(new byte[] {}); - ObjectLoader loader = reader.open(id, OBJ_BLOB); + ObjectLoader loader = LfsFactory.getInstance().applySmudgeFilter( + getRepository(), reader.open(id, OBJ_BLOB), + attributes.get(Constants.ATTR_MERGE)); int threshold = PackConfig.DEFAULT_BIG_FILE_THRESHOLD; return RawText.load(loader, threshold); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/LfsFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/LfsFactory.java new file mode 100644 index 000000000..10fe5642a --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/LfsFactory.java @@ -0,0 +1,284 @@ +/* + * Copyright (C) 2018, Markus Duft + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.text.MessageFormat; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.attributes.Attribute; +import org.eclipse.jgit.attributes.Attributes; +import org.eclipse.jgit.hooks.PrePushHook; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.treewalk.FileTreeIterator; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; + +/** + * Represents an optionally present LFS support implementation + * + * @since 4.11 + */ +public class LfsFactory { + + private static LfsFactory instance = new LfsFactory(); + + /** + * Constructor + */ + protected LfsFactory() { + } + + /** + * @return the current LFS implementation + */ + public static LfsFactory getInstance() { + return instance; + } + + /** + * @param instance + * register a {@link LfsFactory} instance as the + * {@link LfsFactory} implementation to use. + */ + public static void setInstance(LfsFactory instance) { + LfsFactory.instance = instance; + } + + /** + * @return whether LFS support is available + */ + public boolean isAvailable() { + return false; + } + + /** + * Apply clean filtering to the given stream, writing the file content to + * the LFS storage if required and returning a stream to the LFS pointer + * instead. + * + * @param db + * the repository + * @param input + * the original input + * @param length + * the expected input stream length + * @param attribute + * the attribute used to check for LFS enablement (i.e. "merge", + * "diff", "filter" from .gitattributes). + * @return a stream to the content that should be written to the object + * store along with the expected length of the stream. the original + * stream is not applicable. + * @throws IOException + * in case of an error + */ + public LfsInputStream applyCleanFilter(Repository db, + InputStream input, long length, Attribute attribute) + throws IOException { + return new LfsInputStream(input, length); + } + + /** + * Apply smudge filtering to a given loader, potentially redirecting it to a + * LFS blob which is downloaded on demand. + * + * @param db + * the repository + * @param loader + * the loader for the blob + * @param attribute + * the attribute used to check for LFS enablement (i.e. "merge", + * "diff", "filter" from .gitattributes). + * @return a loader for the actual data of a blob, or the original loader in + * case LFS is not applicable. + * @throws IOException + */ + public ObjectLoader applySmudgeFilter(Repository db, + ObjectLoader loader, Attribute attribute) throws IOException { + return loader; + } + + /** + * Retrieve a pre-push hook to be applied. + * + * @param repo + * the {@link Repository} the hook is applied to. + * @param outputStream + * @return a {@link PrePushHook} implementation or null + */ + public @Nullable PrePushHook getPrePushHook(Repository repo, + PrintStream outputStream) { + return null; + } + + /** + * @param db + * the repository + * @param path + * the path to find attributes for + * @return the {@link Attributes} for the given path. + * @throws IOException + * in case of an error + */ + public static Attributes getAttributesForPath(Repository db, String path) + throws IOException { + try (TreeWalk walk = new TreeWalk(db)) { + walk.addTree(new FileTreeIterator(db)); + PathFilter f = PathFilter.create(path); + walk.setFilter(f); + walk.setRecursive(false); + Attributes attr = null; + while (walk.next()) { + if (f.isDone(walk)) { + attr = walk.getAttributes(); + break; + } else if (walk.isSubtree()) { + walk.enterSubtree(); + } + } + if (attr == null) { + throw new IOException(MessageFormat + .format(JGitText.get().noPathAttributesFound, path)); + } + + return attr; + } + } + + /** + * Get attributes for given path and commit + * + * @param db + * the repository + * @param path + * the path to find attributes for + * @param commit + * the commit to inspect. + * @return the {@link Attributes} for the given path. + * @throws IOException + * in case of an error + */ + public static Attributes getAttributesForPath(Repository db, String path, + RevCommit commit) throws IOException { + if (commit == null) { + return getAttributesForPath(db, path); + } + + try (TreeWalk walk = TreeWalk.forPath(db, path, commit.getTree())) { + Attributes attr = walk == null ? null : walk.getAttributes(); + if (attr == null) { + throw new IOException(MessageFormat + .format(JGitText.get().noPathAttributesFound, path)); + } + + return attr; + } + } + + /** + * Encapsulate a potentially exchanged {@link InputStream} along with the + * expected stream content length. + */ + public static final class LfsInputStream extends InputStream { + /** + * The actual stream. + */ + private InputStream stream; + + /** + * The expected stream content length. + */ + private long length; + + /** + * Create a new wrapper around a certain stream + * + * @param stream + * the stream to wrap. the stream will be closed on + * {@link #close()}. + * @param length + * the expected length of the stream + */ + public LfsInputStream(InputStream stream, long length) { + this.stream = stream; + this.length = length; + } + + /** + * Create a new wrapper around a temporary buffer. + * + * @param buffer + * the buffer to initialize stream and length from. The + * buffer will be destroyed on {@link #close()} + * @throws IOException + * in case of an error opening the stream to the buffer. + */ + public LfsInputStream(TemporaryBuffer buffer) throws IOException { + this.stream = buffer.openInputStream(); + this.length = buffer.length(); + } + + @Override + public void close() throws IOException { + stream.close(); + } + + @Override + public int read() throws IOException { + return stream.read(); + } + + /** + * @return the length of the stream + */ + public long getLength() { + return length; + } + } + +}