Add support for built-in smudge filters

JGit supports smudge filters defined in repository configuration. The
filters are implemented as external programs filtering content by
accepting the original content (as seen in git's object database) on
stdin and which emit the filtered content on stdout. This content is
then written to the file in the working tree. To run such a filter JGit
has to start an external process and pump data into/from this process.

This commit adds support for built-in smudge filters which are
implemented in Java and which are executed by jgit's main thread. When a
filter is defined in the configuration as
"jgit://builtin/<filterDriverName>/smudge" then JGit will lookup in a
static map whether a builtin filter is registered under this name. If
found such a filter is called to do the filtering.

The functionality in this commit requires that a program using JGit
explicitly calls the JGit API to register built-in implementations for
specific smudge filters. In follow-up commits configuration parameters
will be added which trigger such registrations.

Change-Id: Ia743aa0dbed795e71e5792f35ae55660e0eb3c24

org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FilterCommandsTest.java

@@ -154,4 +154,98 @@ public class FilterCommandsTest extends RepositoryTestCase {
 		config.setString("filter", "test", "clean", null);
 		config.save();
 	}
+
+	@Test
+	public void testBuiltinSmudgeFilter() throws IOException, GitAPIException {
+		String builtinCommandName = "jgit://builtin/test/smudge";
+		FilterCommandRegistry.register(builtinCommandName,
+				new TestCommandFactory('s'));
+		StoredConfig config = git.getRepository().getConfig();
+		config.setString("filter", "test", "smudge", builtinCommandName);
+		config.save();
+
+		writeTrashFile(".gitattributes", "*.txt filter=test");
+		git.add().addFilepattern(".gitattributes").call();
+		git.commit().setMessage("add filter").call();
+
+		writeTrashFile("Test.txt", "Hello again");
+		git.add().addFilepattern("Test.txt").call();
+		assertEquals(
+				"[.gitattributes, mode:100644, content:*.txt filter=test][Test.txt, mode:100644, content:Hello again]",
+				indexState(CONTENT));
+		assertEquals("Hello again", read("Test.txt"));
+		deleteTrashFile("Test.txt");
+		git.checkout().addPath("Test.txt").call();
+		assertEquals("sHseslslsos sasgsasisn", read("Test.txt"));
+
+		writeTrashFile("Test.bin", "Hello again");
+		git.add().addFilepattern("Test.bin").call();
+		assertEquals(
+				"[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:Hello again]",
+				indexState(CONTENT));
+		deleteTrashFile("Test.bin");
+		git.checkout().addPath("Test.bin").call();
+		assertEquals("Hello again", read("Test.bin"));
+
+		config.setString("filter", "test", "clean", null);
+		config.save();
+
+		git.add().addFilepattern("Test.txt").call();
+		assertEquals(
+				"[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:sHseslslsos sasgsasisn]",
+				indexState(CONTENT));
+
+		config.setString("filter", "test", "clean", null);
+		config.save();
+	}
+
+	@Test
+	public void testBuiltinCleanAndSmudgeFilter() throws IOException, GitAPIException {
+		String builtinCommandPrefix = "jgit://builtin/test/";
+		FilterCommandRegistry.register(builtinCommandPrefix + "smudge",
+				new TestCommandFactory('s'));
+		FilterCommandRegistry.register(builtinCommandPrefix + "clean",
+				new TestCommandFactory('c'));
+		StoredConfig config = git.getRepository().getConfig();
+		config.setString("filter", "test", "smudge", builtinCommandPrefix+"smudge");
+		config.setString("filter", "test", "clean",
+				builtinCommandPrefix + "clean");
+		config.save();
+
+		writeTrashFile(".gitattributes", "*.txt filter=test");
+		git.add().addFilepattern(".gitattributes").call();
+		git.commit().setMessage("add filter").call();
+
+		writeTrashFile("Test.txt", "Hello again");
+		git.add().addFilepattern("Test.txt").call();
+		assertEquals(
+				"[.gitattributes, mode:100644, content:*.txt filter=test][Test.txt, mode:100644, content:cHceclclcoc cacgcacicn]",
+				indexState(CONTENT));
+		assertEquals("Hello again", read("Test.txt"));
+		deleteTrashFile("Test.txt");
+		git.checkout().addPath("Test.txt").call();
+		assertEquals("scsHscsescslscslscsoscs scsascsgscsascsiscsn",
+				read("Test.txt"));
+
+		writeTrashFile("Test.bin", "Hello again");
+		git.add().addFilepattern("Test.bin").call();
+		assertEquals(
+				"[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:cHceclclcoc cacgcacicn]",
+				indexState(CONTENT));
+		deleteTrashFile("Test.bin");
+		git.checkout().addPath("Test.bin").call();
+		assertEquals("Hello again", read("Test.bin"));
+
+		config.setString("filter", "test", "clean", null);
+		config.save();
+
+		git.add().addFilepattern("Test.txt").call();
+		assertEquals(
+				"[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:scsHscsescslscslscsoscs scsascsgscsascsiscsn]",
+				indexState(CONTENT));
+
+		config.setString("filter", "test", "clean", null);
+		config.save();
+	}
+
 }

org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties

@@ -279,6 +279,7 @@ expectedLessThanGot=expected less than ''{0}'', got ''{1}''
 expectedPktLineWithService=expected pkt-line with ''# service=-'', got ''{0}''
 expectedReceivedContentType=expected Content-Type {0}; received Content-Type {1}
 expectedReportForRefNotReceived={0}: expected report for ref {1} not received
+failedToDetermineFilterDefinition=An exception occured while determining filter definitions
 failedUpdatingRefs=failed updating refs
 failureDueToOneOfTheFollowing=Failure due to one of the following:
 failureUpdatingFETCH_HEAD=Failure updating FETCH_HEAD: {0}

org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java

@@ -54,6 +54,8 @@ import java.util.List;
 import java.util.Map;
 
 import org.eclipse.jgit.api.errors.FilterFailedException;
+import org.eclipse.jgit.attributes.FilterCommand;
+import org.eclipse.jgit.attributes.FilterCommandRegistry;
 import org.eclipse.jgit.errors.CheckoutConflictException;
 import org.eclipse.jgit.errors.CorruptObjectException;
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
@@ -86,11 +88,15 @@ import org.eclipse.jgit.util.FileUtils;
 import org.eclipse.jgit.util.RawParseUtils;
 import org.eclipse.jgit.util.SystemReader;
 import org.eclipse.jgit.util.io.EolStreamTypeUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class handles checking out one or two trees merging with the index.
  */
 public class DirCacheCheckout {
+	private static Logger LOG = LoggerFactory.getLogger(DirCacheCheckout.class);
+
 	private static final int MAX_EXCEPTION_TEXT_SIZE = 10 * 1024;
 
 	/**
@@ -1303,45 +1309,19 @@ public class DirCacheCheckout {
 		} else {
 			nonNullEolStreamType = EolStreamType.DIRECT;
 		}
-		OutputStream channel = EolStreamTypeUtil.wrapOutputStream(
+		try (OutputStream channel = EolStreamTypeUtil.wrapOutputStream(
-				new FileOutputStream(tmpFile), nonNullEolStreamType);
+				new FileOutputStream(tmpFile), nonNullEolStreamType)) {
-		if (checkoutMetadata.smudgeFilterCommand != null) {
+			if (checkoutMetadata.smudgeFilterCommand != null) {
-			ProcessBuilder filterProcessBuilder = fs.runInShell(
+				if (FilterCommandRegistry
-					checkoutMetadata.smudgeFilterCommand, new String[0]);
+						.isRegistered(checkoutMetadata.smudgeFilterCommand)) {
-			filterProcessBuilder.directory(repo.getWorkTree());
+					runBuiltinFilterCommand(repo, checkoutMetadata, ol,
-			filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY,
+							channel);
-					repo.getDirectory().getAbsolutePath());
+				} else {
-			ExecutionResult result;
+					runExternalFilterCommand(repo, entry, checkoutMetadata, ol,
-			int rc;
+							fs, channel);
-			try {
-				// TODO: wire correctly with AUTOCRLF
-				result = fs.execute(filterProcessBuilder, ol.openStream());
-				rc = result.getRc();
-				if (rc == 0) {
-					result.getStdout().writeTo(channel,
-							NullProgressMonitor.INSTANCE);
 				}
-			} catch (IOException | InterruptedException e) {
+			} else {
-				throw new IOException(new FilterFailedException(e,
-						checkoutMetadata.smudgeFilterCommand,
-						entry.getPathString()));
-
-			} finally {
-				channel.close();
-			}
-			if (rc != 0) {
-				throw new IOException(new FilterFailedException(rc,
-						checkoutMetadata.smudgeFilterCommand,
-						entry.getPathString(),
-						result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE),
-						RawParseUtils.decode(result.getStderr()
-								.toByteArray(MAX_EXCEPTION_TEXT_SIZE))));
-			}
-		} else {
-			try {
 				ol.copyTo(channel);
-			} finally {
-				channel.close();
 			}
 		}
 		// The entry needs to correspond to the on-disk filesize. If the content
@@ -1382,6 +1362,63 @@ public class DirCacheCheckout {
 		entry.setLastModified(fs.lastModified(f));
 	}
 
+	// Run an external filter command
+	private static void runExternalFilterCommand(Repository repo,
+			DirCacheEntry entry,
+			CheckoutMetadata checkoutMetadata, ObjectLoader ol, FS fs,
+			OutputStream channel) throws IOException {
+		ProcessBuilder filterProcessBuilder = fs.runInShell(
+				checkoutMetadata.smudgeFilterCommand, new String[0]);
+		filterProcessBuilder.directory(repo.getWorkTree());
+		filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY,
+				repo.getDirectory().getAbsolutePath());
+		ExecutionResult result;
+		int rc;
+		try {
+			// TODO: wire correctly with AUTOCRLF
+			result = fs.execute(filterProcessBuilder, ol.openStream());
+			rc = result.getRc();
+			if (rc == 0) {
+				result.getStdout().writeTo(channel,
+						NullProgressMonitor.INSTANCE);
+			}
+		} catch (IOException | InterruptedException e) {
+			throw new IOException(new FilterFailedException(e,
+					checkoutMetadata.smudgeFilterCommand,
+					entry.getPathString()));
+		}
+		if (rc != 0) {
+			throw new IOException(new FilterFailedException(rc,
+					checkoutMetadata.smudgeFilterCommand,
+					entry.getPathString(),
+					result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE),
+					RawParseUtils.decode(result.getStderr()
+							.toByteArray(MAX_EXCEPTION_TEXT_SIZE))));
+		}
+	}
+
+	// Run a builtin filter command
+	private static void runBuiltinFilterCommand(Repository repo,
+			CheckoutMetadata checkoutMetadata, ObjectLoader ol,
+			OutputStream channel) throws MissingObjectException, IOException {
+		FilterCommand command = null;
+		try {
+			command = FilterCommandRegistry.createFilterCommand(
+					checkoutMetadata.smudgeFilterCommand, repo, ol.openStream(),
+					channel);
+		} catch (IOException e) {
+			LOG.error(JGitText.get().failedToDetermineFilterDefinition, e);
+			// In case an IOException occurred during creating of the command
+			// then proceed as if there would not have been a builtin filter.
+			ol.copyTo(channel);
+		}
+		if (command != null) {
+			while (command.run() != -1) {
+				// loop as long as command.run() tells there is work to do
+			}
+		}
+	}
+
 	@SuppressWarnings("deprecation")
 	private static void checkValidPath(CanonicalTreeParser t)
 			throws InvalidPathException {

org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java

@@ -338,6 +338,7 @@ public class JGitText extends TranslationBundle {
 	/***/ public String expectedPktLineWithService;
 	/***/ public String expectedReceivedContentType;
 	/***/ public String expectedReportForRefNotReceived;
+	/***/ public String failedToDetermineFilterDefinition;
 	/***/ public String failedUpdatingRefs;
 	/***/ public String failureDueToOneOfTheFollowing;
 	/***/ public String failureUpdatingFETCH_HEAD;