Add support for clean filters

When filters are defined for certain paths in gitattributes make
sure that clean filters are processed when adding new content to the
object database.

Change-Id: Iffd72914cec5b434ba4d0de232e285b7492db868
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java

@@ -282,6 +282,19 @@ public abstract class RepositoryTestCase extends LocalDiskRepositoryTestCase {
 		return name;
 	}
 
+	/**
+	 * Replaces '\' by '/'
+	 *
+	 * @param str
+	 *            the string in which backslashes should be replaced
+	 * @return the resulting string with slashes
+         * @since 4.2
+	 */
+	public static String slashify(String str) {
+		str = str.replace('\\', '/');
+		return str;
+	}
+
 	/**
 	 * Waits until it is guaranteed that a subsequent file modification has a
 	 * younger modification timestamp than the modification timestamp of the

org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java

@@ -45,6 +45,7 @@ package org.eclipse.jgit.api;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.File;
@@ -52,11 +53,13 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
 
+import org.eclipse.jgit.api.errors.FilterFailedException;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.NoFilepatternException;
 import org.eclipse.jgit.dircache.DirCache;
 import org.eclipse.jgit.dircache.DirCacheBuilder;
 import org.eclipse.jgit.dircache.DirCacheEntry;
+import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.RepositoryTestCase;
 import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
@@ -111,6 +114,191 @@ public class AddCommandTest extends RepositoryTestCase {
 				indexState(CONTENT));
 	}
 
+	@Test
+	public void testCleanFilter() throws IOException,
+			GitAPIException {
+		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile("src/a.tmp", "foo");
+		// Caution: we need a trailing '\n' since sed on mac always appends
+		// linefeeds if missing
+		writeTrashFile("src/a.txt", "foo\n");
+		File script = writeTempFile("sed s/o/e/g");
+
+		Git git = new Git(db);
+		StoredConfig config = git.getRepository().getConfig();
+		config.setString("filter", "tstFilter", "clean",
+				"sh " + slashify(script.getPath()));
+		config.save();
+
+		git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp")
+				.call();
+
+		assertEquals(
+				"[src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:fee\n]",
+				indexState(CONTENT));
+	}
+
+	@Test
+	public void testCleanFilterEnvironment()
+			throws IOException, GitAPIException {
+		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile("src/a.txt", "foo");
+		File script = writeTempFile("echo $GIT_DIR; echo 1 >xyz");
+
+		Git git = new Git(db);
+		StoredConfig config = git.getRepository().getConfig();
+		config.setString("filter", "tstFilter", "clean",
+				"sh " + slashify(script.getPath()));
+		config.save();
+		git.add().addFilepattern("src/a.txt").call();
+
+		String gitDir = db.getDirectory().getAbsolutePath();
+		assertEquals("[src/a.txt, mode:100644, content:" + gitDir
+				+ "\n]", indexState(CONTENT));
+		assertTrue(new File(db.getWorkTree(), "xyz").exists());
+	}
+
+	@Test
+	public void testMultipleCleanFilter() throws IOException, GitAPIException {
+		writeTrashFile(".gitattributes",
+				"*.txt filter=tstFilter\n*.tmp filter=tstFilter2");
+		// Caution: we need a trailing '\n' since sed on mac always appends
+		// linefeeds if missing
+		writeTrashFile("src/a.tmp", "foo\n");
+		writeTrashFile("src/a.txt", "foo\n");
+		File script = writeTempFile("sed s/o/e/g");
+		File script2 = writeTempFile("sed s/f/x/g");
+
+		Git git = new Git(db);
+		StoredConfig config = git.getRepository().getConfig();
+		config.setString("filter", "tstFilter", "clean",
+				"sh " + slashify(script.getPath()));
+		config.setString("filter", "tstFilter2", "clean",
+				"sh " + slashify(script2.getPath()));
+		config.save();
+
+		git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp")
+				.call();
+
+		assertEquals(
+				"[src/a.tmp, mode:100644, content:xoo\n][src/a.txt, mode:100644, content:fee\n]",
+				indexState(CONTENT));
+
+		// TODO: multiple clean filters for one file???
+	}
+
+	/**
+	 * The path of an added file name contains ';' and afterwards malicious
+	 * commands. Make sure when calling filter commands to properly escape the
+	 * filenames
+	 *
+	 * @throws IOException
+	 * @throws GitAPIException
+	 */
+	@Test
+	public void testCommandInjection() throws IOException, GitAPIException {
+		// Caution: we need a trailing '\n' since sed on mac always appends
+		// linefeeds if missing
+		writeTrashFile("; echo virus", "foo\n");
+		File script = writeTempFile("sed s/o/e/g");
+
+		Git git = new Git(db);
+		StoredConfig config = git.getRepository().getConfig();
+		config.setString("filter", "tstFilter", "clean",
+				"sh " + slashify(script.getPath()) + " %f");
+		writeTrashFile(".gitattributes", "* filter=tstFilter");
+
+		git.add().addFilepattern("; echo virus").call();
+		// Without proper escaping the content would be "feovirus". The sed
+		// command and the "echo virus" would contribute to the content
+		assertEquals("[; echo virus, mode:100644, content:fee\n]",
+				indexState(CONTENT));
+	}
+
+	@Test
+	public void testBadCleanFilter() throws IOException, GitAPIException {
+		writeTrashFile("a.txt", "foo");
+		File script = writeTempFile("sedfoo s/o/e/g");
+
+		Git git = new Git(db);
+		StoredConfig config = git.getRepository().getConfig();
+		config.setString("filter", "tstFilter", "clean",
+				"sh " + script.getPath());
+		config.save();
+		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+
+		try {
+			git.add().addFilepattern("a.txt").call();
+			fail("Didn't received the expected exception");
+		} catch (FilterFailedException e) {
+			assertEquals(127, e.getReturnCode());
+		}
+	}
+
+	@Test
+	public void testBadCleanFilter2() throws IOException, GitAPIException {
+		writeTrashFile("a.txt", "foo");
+		File script = writeTempFile("sed s/o/e/g");
+
+		Git git = new Git(db);
+		StoredConfig config = git.getRepository().getConfig();
+		config.setString("filter", "tstFilter", "clean",
+				"shfoo " + script.getPath());
+		config.save();
+		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+
+		try {
+			git.add().addFilepattern("a.txt").call();
+			fail("Didn't received the expected exception");
+		} catch (FilterFailedException e) {
+			assertEquals(127, e.getReturnCode());
+		}
+	}
+
+	@Test
+	public void testCleanFilterReturning12() throws IOException,
+			GitAPIException {
+		writeTrashFile("a.txt", "foo");
+		File script = writeTempFile("exit 12");
+
+		Git git = new Git(db);
+		StoredConfig config = git.getRepository().getConfig();
+		config.setString("filter", "tstFilter", "clean",
+				"sh " + slashify(script.getPath()));
+		config.save();
+		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+
+		try {
+			git.add().addFilepattern("a.txt").call();
+			fail("Didn't received the expected exception");
+		} catch (FilterFailedException e) {
+			assertEquals(12, e.getReturnCode());
+		}
+	}
+
+	@Test
+	public void testNotApplicableFilter() throws IOException, GitAPIException {
+		writeTrashFile("a.txt", "foo");
+		File script = writeTempFile("sed s/o/e/g");
+
+		Git git = new Git(db);
+		StoredConfig config = git.getRepository().getConfig();
+		config.setString("filter", "tstFilter", "something",
+				"sh " + script.getPath());
+		config.save();
+		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+
+		git.add().addFilepattern("a.txt").call();
+
+		assertEquals("[a.txt, mode:100644, content:foo]", indexState(CONTENT));
+	}
+
+	private File writeTempFile(String body) throws IOException {
+		File f = File.createTempFile("AddCommandTest_", "");
+		JGitTestUtil.write(f, body);
+		return f;
+	}
+
 	@Test
 	public void testAddExistingSingleSmallFileWithNewLine() throws IOException,
 			GitAPIException {

org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties

@@ -281,6 +281,8 @@ fileCannotBeDeleted=File cannot be deleted: {0}
 fileIsTooBigForThisConvenienceMethod=File is too big for this convenience method ({0} bytes).
 fileIsTooLarge=File is too large: {0}
 fileModeNotSetForPath=FileMode not set for path {0}
+filterExecutionFailed=Execution of filter command ''{0}'' on file ''{1}'' failed
+filterExecutionFailedRc=Execution of filter command ''{0}'' on file ''{1}'' failed with return code ''{2}'', message on stderr: ''{3}''
 findingGarbage=Finding garbage
 flagIsDisposed={0} is disposed.
 flagNotFromThis={0} not from this.

org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java

@@ -48,6 +48,7 @@ import java.io.InputStream;
 import java.util.Collection;
 import java.util.LinkedList;
 
+import org.eclipse.jgit.api.errors.FilterFailedException;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.JGitInternalException;
 import org.eclipse.jgit.api.errors.NoFilepatternException;
@@ -211,6 +212,9 @@ public class AddCommand extends GitCommand<DirCache> {
 			builder.commit();
 			setCallable(false);
 		} catch (IOException e) {
+			Throwable cause = e.getCause();
+			if (cause != null && cause instanceof FilterFailedException)
+				throw (FilterFailedException) cause;
 			throw new JGitInternalException(
 					JGitText.get().exceptionCaughtDuringExecutionOfAddCommand, e);
 		} finally {

org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java

@@ -332,7 +332,9 @@ public class CommitCommand extends GitCommand<RevCommit> {
 			treeWalk.setOperationType(OperationType.CHECKIN_OP);
 			int dcIdx = treeWalk
 					.addTree(new DirCacheBuildIterator(existingBuilder));
-			int fIdx = treeWalk.addTree(new FileTreeIterator(repo));
+			FileTreeIterator fti = new FileTreeIterator(repo);
+			fti.setDirCacheIterator(treeWalk, 0);
+			int fIdx = treeWalk.addTree(fti);
 			int hIdx = -1;
 			if (headId != null)
 				hIdx = treeWalk.addTree(rw.parseTree(headId));

org.eclipse.jgit/src/org/eclipse/jgit/api/errors/FilterFailedException.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2015, Christian Halstrick <christian.halstrick@sap.com> and
+ * other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v1.0 which accompanies this
+ * distribution, is reproduced below, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.api.errors;
+
+import java.text.MessageFormat;
+
+import org.eclipse.jgit.internal.JGitText;
+
+/**
+ * Exception thrown when the execution of a filter command failed
+ *
+ * @since 4.2
+ */
+public class FilterFailedException extends GitAPIException {
+	private static final long serialVersionUID = 1L;
+
+	private String filterCommand;
+
+	private String path;
+
+	private byte[] stdout;
+
+	private String stderr;
+
+	private int rc;
+
+	/**
+	 * Thrown if during execution of filter command an exception occurred
+	 *
+	 * @param cause
+	 *            the exception
+	 * @param filterCommand
+	 *            the command which failed
+	 * @param path
+	 *            the path processed by the filter
+	 */
+	public FilterFailedException(Exception cause, String filterCommand,
+			String path) {
+		super(MessageFormat.format(JGitText.get().filterExecutionFailed,
+				filterCommand, path), cause);
+		this.filterCommand = filterCommand;
+		this.path = path;
+	}
+
+	/**
+	 * Thrown if a filter command returns a non-zero return code
+	 *
+	 * @param rc
+	 *            the return code
+	 * @param filterCommand
+	 *            the command which failed
+	 * @param path
+	 *            the path processed by the filter
+	 * @param stdout
+	 *            the output the filter generated so far. This should be limited
+	 *            to reasonable size.
+	 * @param stderr
+	 *            the stderr output of the filter
+	 */
+	@SuppressWarnings("boxing")
+	public FilterFailedException(int rc, String filterCommand, String path,
+			byte[] stdout, String stderr) {
+		super(MessageFormat.format(JGitText.get().filterExecutionFailedRc,
+				filterCommand, path, rc, stderr));
+		this.rc = rc;
+		this.filterCommand = filterCommand;
+		this.path = path;
+		this.stdout = stdout;
+		this.stderr = stderr;
+	}
+
+	/**
+	 * @return the filterCommand
+	 */
+	public String getFilterCommand() {
+		return filterCommand;
+	}
+
+	/**
+	 * @return the path of the file processed by the filter command
+	 */
+	public String getPath() {
+		return path;
+	}
+
+	/**
+	 * @return the output generated by the filter command. Might be truncated to
+	 *         limit memory consumption.
+	 */
+	public byte[] getOutput() {
+		return stdout;
+	}
+
+	/**
+	 * @return the error output returned by the filter command
+	 */
+	public String getError() {
+		return stderr;
+	}
+
+	/**
+	 * @return the return code returned by the filter command
+	 */
+	public int getReturnCode() {
+		return rc;
+	}
+
+}

org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java

@@ -983,6 +983,7 @@ public class DirCache {
 			FileTreeIterator fIter = new FileTreeIterator(repository);
 			walk.addTree(iIter);
 			walk.addTree(fIter);
+			fIter.setDirCacheIterator(walk, 0);
 			walk.setRecursive(true);
 			while (walk.next()) {
 				iIter = walk.getTree(0, DirCacheIterator.class);

org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java

@@ -340,6 +340,8 @@ public class JGitText extends TranslationBundle {
 	/***/ public String fileIsTooBigForThisConvenienceMethod;
 	/***/ public String fileIsTooLarge;
 	/***/ public String fileModeNotSetForPath;
+	/***/ public String filterExecutionFailed;
+	/***/ public String filterExecutionFailedRc;
 	/***/ public String findingGarbage;
 	/***/ public String flagIsDisposed;
 	/***/ public String flagNotFromThis;

org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java

@@ -370,6 +370,20 @@ public final class Constants {
 	 */
 	public static final String DOT_GIT_ATTRIBUTES = ".gitattributes";
 
+	/**
+	 * Key for filters in .gitattributes
+	 *
+	 * @since 4.2
+	 */
+	public static final String ATTR_FILTER = "filter";
+
+	/**
+	 * clean command name, used to call filter driver
+	 *
+	 * @since 4.2
+	 */
+	public static final String ATTR_FILTER_TYPE_CLEAN = "clean";
+
 	/** Name of the ignore file */
 	public static final String DOT_GIT_IGNORE = ".gitignore";
 

org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java

@@ -413,6 +413,7 @@ public class IndexDiff {
 				treeWalk.addTree(new EmptyTreeIterator());
 			treeWalk.addTree(new DirCacheIterator(dirCache));
 			treeWalk.addTree(initialWorkingTreeIterator);
+			initialWorkingTreeIterator.setDirCacheIterator(treeWalk, 1);
 			Collection<TreeFilter> filters = new ArrayList<TreeFilter>(4);
 
 			if (monitor != null) {

org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java

@@ -45,22 +45,25 @@
 package org.eclipse.jgit.treewalk;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Set;
 
 import org.eclipse.jgit.annotations.Nullable;
 import org.eclipse.jgit.api.errors.JGitInternalException;
 import org.eclipse.jgit.attributes.Attribute;
+import org.eclipse.jgit.attributes.Attribute.State;
 import org.eclipse.jgit.attributes.Attributes;
 import org.eclipse.jgit.attributes.AttributesNode;
 import org.eclipse.jgit.attributes.AttributesNodeProvider;
 import org.eclipse.jgit.attributes.AttributesProvider;
-import org.eclipse.jgit.attributes.Attribute.State;
 import org.eclipse.jgit.dircache.DirCacheIterator;
 import org.eclipse.jgit.errors.CorruptObjectException;
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.errors.StopWalkException;
 import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.MutableObjectId;
@@ -70,6 +73,7 @@ import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.RevTree;
 import org.eclipse.jgit.treewalk.filter.PathFilter;
 import org.eclipse.jgit.treewalk.filter.TreeFilter;
+import org.eclipse.jgit.util.QuotedString;
 import org.eclipse.jgit.util.RawParseUtils;
 
 /**
@@ -116,6 +120,12 @@ public class TreeWalk implements AutoCloseable, AttributesProvider {
 	 */
 	private OperationType operationType = OperationType.CHECKOUT_OP;
 
+	/**
+	 * The filter command as defined in gitattributes. The keys are
+	 * filterName+"."+filterCommandType. E.g. "lfs.clean"
+	 */
+	private Map<String, String> filterCommandsByNameDotType = new HashMap<String, String>();
+
 	/**
 	 * @param operationType
 	 * @since 4.2
@@ -259,6 +269,8 @@ public class TreeWalk implements AutoCloseable, AttributesProvider {
 	/** Cached attribute for the current entry */
 	private Attributes attrs = null;
 
+	private Config config;
+
 	/**
 	 * Create a new tree walker for a given repository.
 	 *
@@ -269,6 +281,7 @@ public class TreeWalk implements AutoCloseable, AttributesProvider {
 	 */
 	public TreeWalk(final Repository repo) {
 		this(repo.newObjectReader(), true);
+		config = repo.getConfig();
 		attributesNodeProvider = repo.createAttributesNodeProvider();
 	}
 
@@ -1308,4 +1321,66 @@ public class TreeWalk implements AutoCloseable, AttributesProvider {
 			AttributesNode defaultValue) {
 		return (value == null) ? defaultValue : value;
 	}
+
+	/**
+	 * Inspect config and attributes to return a filtercommand applicable for
+	 * the current path
+	 *
+	 * @param filterCommandType
+	 *            which type of filterCommand should be executed. E.g. "clean",
+	 *            "smudge"
+	 * @return a filter command
+	 * @throws IOException
+	 * @since 4.2
+	 */
+	public String getFilterCommand(String filterCommandType)
+			throws IOException {
+		Attributes attributes = getAttributes();
+
+		Attribute f = attributes.get(Constants.ATTR_FILTER);
+		if (f == null) {
+			return null;
+		}
+		String filterValue = f.getValue();
+		if (filterValue == null) {
+			return null;
+		}
+
+		String filterCommand = getFilterCommandDefinition(filterValue,
+				filterCommandType);
+		if (filterCommand == null) {
+			return null;
+		}
+		return filterCommand.replaceAll("%f", //$NON-NLS-1$
+				QuotedString.BOURNE.quote((getPathString())));
+	}
+
+	/**
+	 * Get the filter command how it is defined in gitconfig. The returned
+	 * string may contain "%f" which needs to be replaced by the current path
+	 * before executing the filter command. These filter definitions are cached
+	 * for better performance.
+	 *
+	 * @param filterDriverName
+	 *            The name of the filter driver as it is referenced in the
+	 *            gitattributes file. E.g. "lfs". For each filter driver there
+	 *            may be many commands defined in the .gitconfig
+	 * @param filterCommandType
+	 *            The type of the filter command for a specific filter driver.
+	 *            May be "clean" or "smudge".
+	 * @return the definition of the command to be executed for this filter
+	 *         driver and filter command
+	 */
+	private String getFilterCommandDefinition(String filterDriverName,
+			String filterCommandType) {
+		String key = filterDriverName + "." + filterCommandType; //$NON-NLS-1$
+		String filterCommand = filterCommandsByNameDotType.get(key);
+		if (filterCommand != null)
+			return filterCommand;
+		filterCommand = config.getString(Constants.ATTR_FILTER,
+				filterDriverName, filterCommandType);
+		if (filterCommand != null)
+			filterCommandsByNameDotType.put(key, filterCommand);
+		return filterCommand;
+	}
 }

org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java

@@ -62,6 +62,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 
+import org.eclipse.jgit.api.errors.FilterFailedException;
 import org.eclipse.jgit.attributes.AttributesNode;
 import org.eclipse.jgit.attributes.AttributesRule;
 import org.eclipse.jgit.diff.RawText;
@@ -76,6 +77,7 @@ import org.eclipse.jgit.ignore.IgnoreNode;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.CoreConfig;
+import org.eclipse.jgit.lib.CoreConfig.AutoCRLF;
 import org.eclipse.jgit.lib.CoreConfig.CheckStat;
 import org.eclipse.jgit.lib.CoreConfig.SymLinks;
 import org.eclipse.jgit.lib.FileMode;
@@ -85,6 +87,7 @@ import org.eclipse.jgit.lib.ObjectReader;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.submodule.SubmoduleWalk;
 import org.eclipse.jgit.util.FS;
+import org.eclipse.jgit.util.FS.ExecutionResult;
 import org.eclipse.jgit.util.IO;
 import org.eclipse.jgit.util.RawParseUtils;
 import org.eclipse.jgit.util.io.EolCanonicalizingInputStream;
@@ -101,6 +104,8 @@ import org.eclipse.jgit.util.io.EolCanonicalizingInputStream;
  * @see FileTreeIterator
  */
 public abstract class WorkingTreeIterator extends AbstractTreeIterator {
+	private static final int MAX_EXCEPTION_TEXT_SIZE = 10 * 1024;
+
 	/** An empty entry array, suitable for {@link #init(Entry[])}. */
 	protected static final Entry[] EOF = {};
 
@@ -134,6 +139,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	/** If there is a .gitignore file present, the parsed rules from it. */
 	private IgnoreNode ignoreNode;
 
+	private String cleanFilterCommand;
+
 	/** Repository that is the root level being iterated over */
 	protected Repository repository;
 
@@ -186,6 +193,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	protected WorkingTreeIterator(final WorkingTreeIterator p) {
 		super(p);
 		state = p.state;
+		repository = p.repository;
 	}
 
 	/**
@@ -348,7 +356,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 
 	private InputStream possiblyFilteredInputStream(final Entry e,
 			final InputStream is, final long len) throws IOException {
-		if (!mightNeedCleaning()) {
+		boolean mightNeedCleaning = mightNeedCleaning();
+		if (!mightNeedCleaning) {
 			canonLen = len;
 			return is;
 		}
@@ -366,7 +375,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 			return new ByteArrayInputStream(raw, 0, n);
 		}
 
-		if (isBinary(e)) {
+		// TODO: fix autocrlf causing mightneedcleaning
+		if (!mightNeedCleaning && isBinary(e)) {
 			canonLen = len;
 			return is;
 		}
@@ -390,10 +400,12 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 		}
 	}
 
-	private boolean mightNeedCleaning() {
+	private boolean mightNeedCleaning() throws IOException {
 		switch (getOptions().getAutoCRLF()) {
 		case FALSE:
 		default:
+			if (getCleanFilterCommand() != null)
+				return true;
 			return false;
 
 		case TRUE:
@@ -415,8 +427,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 		}
 	}
 
-	private static ByteBuffer filterClean(byte[] src, int n)
+	private ByteBuffer filterClean(byte[] src, int n) throws IOException {
-			throws IOException {
 		InputStream in = new ByteArrayInputStream(src);
 		try {
 			return IO.readWholeStream(filterClean(in), n);
@@ -425,8 +436,42 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 		}
 	}
 
-	private static InputStream filterClean(InputStream in) {
+	private InputStream filterClean(InputStream in) throws IOException {
-		return new EolCanonicalizingInputStream(in, true);
+		in = handleAutoCRLF(in);
+		String filterCommand = getCleanFilterCommand();
+		if (filterCommand != null) {
+			FS fs = repository.getFS();
+			ProcessBuilder filterProcessBuilder = fs.runInShell(filterCommand,
+					new String[0]);
+			filterProcessBuilder.directory(repository.getWorkTree());
+			filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY,
+					repository.getDirectory().getAbsolutePath());
+			ExecutionResult result;
+			try {
+				result = fs.execute(filterProcessBuilder, in);
+			} catch (IOException | InterruptedException e) {
+				throw new IOException(new FilterFailedException(e,
+						filterCommand, getEntryPathString()));
+			}
+			int rc = result.getRc();
+			if (rc != 0) {
+				throw new IOException(new FilterFailedException(rc,
+						filterCommand, getEntryPathString(),
+						result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE),
+						RawParseUtils.decode(result.getStderr()
+								.toByteArray(MAX_EXCEPTION_TEXT_SIZE))));
+			}
+			return result.getStdout().openInputStream();
+		}
+		return in;
+	}
+
+	private InputStream handleAutoCRLF(InputStream in) {
+		AutoCRLF autoCRLF = getOptions().getAutoCRLF();
+		if (autoCRLF == AutoCRLF.TRUE || autoCRLF == AutoCRLF.INPUT) {
+			in = new EolCanonicalizingInputStream(in, true);
+		}
+		return in;
 	}
 
 	/**
@@ -485,6 +530,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 		System.arraycopy(e.encodedName, 0, path, pathOffset, nameLen);
 		pathLen = pathOffset + nameLen;
 		canonLen = -1;
+		cleanFilterCommand = null;
 	}
 
 	/**
@@ -1271,4 +1317,18 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 			}
 		}
 	}
+
+	/**
+	 * @return the clean filter command for the current entry or
+	 *         <code>null</code> if no such command is defined
+	 * @throws IOException
+	 * @since 4.2
+	 */
+	public String getCleanFilterCommand() throws IOException {
+		if (cleanFilterCommand == null && state.walk != null) {
+			cleanFilterCommand = state.walk
+					.getFilterCommand(Constants.ATTR_FILTER_TYPE_CLEAN);
+		}
+		return cleanFilterCommand;
+	}
 }

org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java

@@ -246,6 +246,37 @@ public abstract class TemporaryBuffer extends OutputStream {
 		return out;
 	}
 
+	/**
+	 * Convert this buffer's contents into a contiguous byte array. If this size
+	 * of the buffer exceeds the limit only return the first {@code limit} bytes
+	 * <p>
+	 * The buffer is only complete after {@link #close()} has been invoked.
+	 *
+	 * @param limit
+	 *            the maximum number of bytes to be returned
+	 *
+	 * @return the byte array limited to {@code limit} bytes.
+	 * @throws IOException
+	 *             an error occurred reading from a local temporary file
+	 * @throws OutOfMemoryError
+	 *             the buffer cannot fit in memory
+	 *
+	 * @since 4.2
+	 */
+	public byte[] toByteArray(int limit) throws IOException {
+		final long len = Math.min(length(), limit);
+		if (Integer.MAX_VALUE < len)
+			throw new OutOfMemoryError(
+					JGitText.get().lengthExceedsMaximumArraySize);
+		final byte[] out = new byte[(int) len];
+		int outPtr = 0;
+		for (final Block b : blocks) {
+			System.arraycopy(b.buffer, 0, out, outPtr, b.count);
+			outPtr += b.count;
+		}
+		return out;
+	}
+
 	/**
 	 * Send this buffer to an output stream.
 	 * <p>