From bd31555ee507cd29a11060756297031354b9b0c3 Mon Sep 17 00:00:00 2001 From: Christian Halstrick Date: Thu, 29 Oct 2015 14:15:08 +0100 Subject: [PATCH] Add support for smudge filters If defined in .gitattributes call smudge filter during checkout. To support checkout where current HEAD,index do not contain attributes we need to also consider attributes from the tree we checkout. Therefore CanonicalTreeParser has to learn how to provide attributes. Change-Id: I168fdb81a8e1a9f991587b3e95a36550ea845f0a Signed-off-by: Matthias Sohn --- .../eclipse/jgit/api/CheckoutCommandTest.java | 124 ++++++++++++++++++ .../jgit/lib/DirCacheCheckoutTest.java | 7 +- .../jgit/dircache/DirCacheCheckout.java | 115 ++++++++++++++-- .../src/org/eclipse/jgit/lib/Constants.java | 7 + 4 files changed, 234 insertions(+), 19 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java index 82d509f6f..4bfb128cb 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java @@ -72,12 +72,14 @@ import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.api.errors.TransportException; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.Sets; import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.FileBasedConfig; @@ -86,6 +88,7 @@ import org.eclipse.jgit.transport.RemoteConfig; import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.util.FileUtils; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class CheckoutCommandTest extends RepositoryTestCase { @@ -556,4 +559,125 @@ public class CheckoutCommandTest extends RepositoryTestCase { } org.junit.Assume.assumeTrue(foundUnsmudged); } + + @Test + public void testSmudgeFilter_modifyExisting() throws IOException, GitAPIException { + File script = writeTempFile("sed s/o/e/g"); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "smudge", + "sh " + slashify(script.getPath())); + config.save(); + + writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("add filter").call(); + + writeTrashFile("src/a.tmp", "x"); + // Caution: we need a trailing '\n' since sed on mac always appends + // linefeeds if missing + writeTrashFile("src/a.txt", "x\n"); + git.add().addFilepattern("src/a.tmp").addFilepattern("src/a.txt") + .call(); + RevCommit content1 = git.commit().setMessage("add content").call(); + + writeTrashFile("src/a.tmp", "foo"); + writeTrashFile("src/a.txt", "foo\n"); + git.add().addFilepattern("src/a.tmp").addFilepattern("src/a.txt") + .call(); + RevCommit content2 = git.commit().setMessage("changed content").call(); + + git.checkout().setName(content1.getName()).call(); + git.checkout().setName(content2.getName()).call(); + + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", + indexState(CONTENT)); + assertEquals(Sets.of("src/a.txt"), git.status().call().getModified()); + assertEquals("foo", read("src/a.tmp")); + assertEquals("fee\n", read("src/a.txt")); + } + + @Test + public void testSmudgeFilter_createNew() + throws IOException, GitAPIException { + File script = writeTempFile("sed s/o/e/g"); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "smudge", + "sh " + slashify(script.getPath())); + config.save(); + + writeTrashFile("foo", "foo"); + git.add().addFilepattern("foo").call(); + RevCommit initial = git.commit().setMessage("initial").call(); + + writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("add filter").call(); + + writeTrashFile("src/a.tmp", "foo"); + // Caution: we need a trailing '\n' since sed on mac always appends + // linefeeds if missing + writeTrashFile("src/a.txt", "foo\n"); + git.add().addFilepattern("src/a.tmp").addFilepattern("src/a.txt") + .call(); + RevCommit content = git.commit().setMessage("added content").call(); + + git.checkout().setName(initial.getName()).call(); + git.checkout().setName(content.getName()).call(); + + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", + indexState(CONTENT)); + assertEquals("foo", read("src/a.tmp")); + assertEquals("fee\n", read("src/a.txt")); + } + + @Test + @Ignore + public void testSmudgeAndClean() throws IOException, GitAPIException { + // @TODO: fix this test + File clean_filter = writeTempFile("sed s/V1/@version/g -"); + File smudge_filter = writeTempFile("sed s/@version/V1/g -"); + + Git git = new Git(db); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "smudge", + "sh " + slashify(smudge_filter.getPath())); + config.setString("filter", "tstFilter", "clean", + "sh " + slashify(clean_filter.getPath())); + config.save(); + writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("add attributes").call(); + + writeTrashFile("filterTest.txt", "hello world, V1"); + git.add().addFilepattern("filterTest.txt").call(); + git.commit().setMessage("add filterText.txt").call(); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some other change][filterTest.txt, mode:100644, content:hello world, @version]", + indexState(CONTENT)); + + git.checkout().setCreateBranch(true).setName("test2").call(); + writeTrashFile("filterTest.txt", "bon giorno world, V1"); + git.add().addFilepattern("filterTest.txt").call(); + git.commit().setMessage("modified filterText.txt").call(); + + assertTrue(git.status().call().isClean()); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some other change][filterTest.txt, mode:100644, content:bon giorno world, @version]", + indexState(CONTENT)); + + git.checkout().setName("refs/heads/test").call(); + assertTrue(git.status().call().isClean()); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some other change][filterTest.txt, mode:100644, content:hello world, @version]", + indexState(CONTENT)); + assertEquals("hello world, V1", read("filterTest.txt")); + } + + private File writeTempFile(String body) throws IOException { + File f = File.createTempFile("AddCommandTest_", ""); + JGitTestUtil.write(f, body); + return f; + } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java index 51a6b5a8f..d768e0fa0 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java @@ -113,7 +113,7 @@ public class DirCacheCheckoutTest extends RepositoryTestCase { return dco.getRemoved(); } - private Map getUpdated() { + private Map getUpdated() { return dco.getUpdated(); } @@ -268,8 +268,6 @@ public class DirCacheCheckoutTest extends RepositoryTestCase { @Test public void testRules1thru3_NoIndexEntry() throws IOException { ObjectId head = buildTree(mk("foo")); - TreeWalk tw = TreeWalk.forPath(db, "foo", head); - ObjectId objectId = tw.getObjectId(0); ObjectId merge = db.newObjectInserter().insert(Constants.OBJ_TREE, new byte[0]); @@ -279,10 +277,9 @@ public class DirCacheCheckoutTest extends RepositoryTestCase { prescanTwoTrees(merge, head); - assertEquals(objectId, getUpdated().get("foo")); + assertTrue(getUpdated().containsKey("foo")); merge = buildTree(mkmap("foo", "a")); - tw = TreeWalk.forPath(db, "foo", merge); prescanTwoTrees(head, merge); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index 0036ab508..393a85494 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -52,15 +52,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.eclipse.jgit.api.errors.FilterFailedException; import org.eclipse.jgit.errors.CheckoutConflictException; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.IndexWriteException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; import org.eclipse.jgit.lib.CoreConfig.SymLinks; import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectChecker; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; @@ -76,6 +79,7 @@ import org.eclipse.jgit.treewalk.WorkingTreeIterator; import org.eclipse.jgit.treewalk.WorkingTreeOptions; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.ExecutionResult; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.RawParseUtils; import org.eclipse.jgit.util.SystemReader; @@ -85,9 +89,10 @@ import org.eclipse.jgit.util.io.AutoCRLFOutputStream; * This class handles checking out one or two trees merging with the index. */ public class DirCacheCheckout { + private static final int MAX_EXCEPTION_TEXT_SIZE = 10 * 1024; private Repository repo; - private HashMap updated = new HashMap(); + private HashMap updated = new HashMap(); private ArrayList conflicts = new ArrayList(); @@ -112,9 +117,9 @@ public class DirCacheCheckout { private boolean emptyDirCache; /** - * @return a list of updated paths and objectIds + * @return a list of updated paths and smudgeFilterCommands */ - public Map getUpdated() { + public Map getUpdated() { return updated; } @@ -447,7 +452,8 @@ public class DirCacheCheckout { for (String path : updated.keySet()) { DirCacheEntry entry = dc.getEntry(path); if (!FileMode.GITLINK.equals(entry.getRawMode())) - checkoutEntry(repo, entry, objectReader, false); + checkoutEntry(repo, entry, objectReader, false, + updated.get(path)); } // commit the index builder - a new index is persisted @@ -996,9 +1002,12 @@ public class DirCacheCheckout { removed.add(path); } - private void update(String path, ObjectId mId, FileMode mode) { + private void update(String path, ObjectId mId, FileMode mode) + throws IOException { if (!FileMode.TREE.equals(mode)) { - updated.put(path, mId); + updated.put(path, + walk.getFilterCommand(Constants.ATTR_FILTER_TYPE_SMUDGE)); + DirCacheEntry entry = new DirCacheEntry(path, DirCacheEntry.STAGE_0); entry.setObjectId(mId); entry.setFileMode(mode); @@ -1150,7 +1159,7 @@ public class DirCacheCheckout { */ public static void checkoutEntry(Repository repo, DirCacheEntry entry, ObjectReader or) throws IOException { - checkoutEntry(repo, entry, or, false); + checkoutEntry(repo, entry, or, false, null); } /** @@ -1186,6 +1195,46 @@ public class DirCacheCheckout { */ public static void checkoutEntry(Repository repo, DirCacheEntry entry, ObjectReader or, boolean deleteRecursive) throws IOException { + checkoutEntry(repo, entry, or, deleteRecursive, null); + } + + /** + * Updates the file in the working tree with content and mode from an entry + * in the index. The new content is first written to a new temporary file in + * the same directory as the real file. Then that new file is renamed to the + * final filename. + * + *

+ * Note: if the entry path on local file system exists as a file, it + * will be deleted and if it exists as a directory, it will be deleted + * recursively, independently if has any content. + *

+ * + *

+ * TODO: this method works directly on File IO, we may need another + * abstraction (like WorkingTreeIterator). This way we could tell e.g. + * Eclipse that Files in the workspace got changed + *

+ * + * @param repo + * repository managing the destination work tree. + * @param entry + * the entry containing new mode and content + * @param or + * object reader to use for checkout + * @param deleteRecursive + * true to recursively delete final path if it exists on the file + * system + * @param smudgeFilterCommand + * the filter command to be run for smudging the entry to be + * checked out + * + * @throws IOException + * @since 4.2 + */ + public static void checkoutEntry(Repository repo, DirCacheEntry entry, + ObjectReader or, boolean deleteRecursive, + String smudgeFilterCommand) throws IOException { ObjectLoader ol = or.open(entry.getObjectId()); File f = new File(repo.getWorkTree(), entry.getPathString()); File parentDir = f.getParentFile(); @@ -1210,14 +1259,52 @@ public class DirCacheCheckout { OutputStream channel = new FileOutputStream(tmpFile); if (opt.getAutoCRLF() == AutoCRLF.TRUE) channel = new AutoCRLFOutputStream(channel); - try { - ol.copyTo(channel); - } finally { - channel.close(); + if (smudgeFilterCommand != null) { + ProcessBuilder filterProcessBuilder = fs + .runInShell(smudgeFilterCommand, new String[0]); + filterProcessBuilder.directory(repo.getWorkTree()); + filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, + repo.getDirectory().getAbsolutePath()); + ExecutionResult result; + int rc; + try { + // TODO: wire correctly with AUTOCRLF + result = fs.execute(filterProcessBuilder, ol.openStream()); + rc = result.getRc(); + if (rc == 0) { + result.getStdout().writeTo(channel, + NullProgressMonitor.INSTANCE); + } + } catch (IOException | InterruptedException e) { + throw new IOException(new FilterFailedException(e, + smudgeFilterCommand, entry.getPathString())); + + } finally { + channel.close(); + } + if (rc != 0) { + throw new IOException(new FilterFailedException(rc, + smudgeFilterCommand, entry.getPathString(), + result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE), + RawParseUtils.decode(result.getStderr() + .toByteArray(MAX_EXCEPTION_TEXT_SIZE)))); + } + } else { + try { + ol.copyTo(channel); + } finally { + channel.close(); + } + } + // The entry needs to correspond to the on-disk filesize. If the content + // was filtered (either by autocrlf handling or smudge filters) ask the + // filesystem again for the length. Otherwise the objectloader knows the + // size + if (opt.getAutoCRLF() == AutoCRLF.TRUE || smudgeFilterCommand != null) { + entry.setLength(tmpFile.length()); + } else { + entry.setLength(ol.getSize()); } - entry.setLength(opt.getAutoCRLF() == AutoCRLF.TRUE ? // - tmpFile.length() // AutoCRLF wants on-disk-size - : (int) ol.getSize()); if (opt.isFileMode() && fs.supportsExecute()) { if (FileMode.EXECUTABLE_FILE.equals(entry.getRawMode())) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 1a3111ab4..d30edaf41 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -384,6 +384,13 @@ public final class Constants { */ public static final String ATTR_FILTER_TYPE_CLEAN = "clean"; + /** + * smudge command name, used to call filter driver + * + * @since 4.2 + */ + public static final String ATTR_FILTER_TYPE_SMUDGE = "smudge"; + /** Name of the ignore file */ public static final String DOT_GIT_IGNORE = ".gitignore";