Handle content length in WorkingTreeIterator

Content length is computed and cached (short term) in the working
tree iterator when core.autocrlf is set.

Hopefully this is a cleaner fix than my previous attempt to make
autocrlf work.

Change-Id: I1b6bbb643101a00db94e5514b5e2b069f338907a

org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java

@@ -44,9 +44,7 @@
 package org.eclipse.jgit.api;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.File;
@@ -63,7 +61,6 @@ import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectInserter;
-import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.RepositoryTestCase;
 import org.eclipse.jgit.lib.StoredConfig;
 import org.eclipse.jgit.revwalk.RevCommit;
@@ -114,7 +111,7 @@ public class AddCommandTest extends RepositoryTestCase {
 	}
 
 	@Test
-	public void testAddExistingSingleFileWithNewLine() throws IOException,
+	public void testAddExistingSingleSmallFileWithNewLine() throws IOException,
 			NoFilepatternException {
 		File file = new File(db.getWorkTree(), "a.txt");
 		FileUtils.createNewFile(file);
@@ -137,6 +134,35 @@ public class AddCommandTest extends RepositoryTestCase {
 				indexState(CONTENT));
 	}
 
+	@Test
+	public void testAddExistingSingleMediumSizeFileWithNewLine()
+			throws IOException, NoFilepatternException {
+		File file = new File(db.getWorkTree(), "a.txt");
+		FileUtils.createNewFile(file);
+		StringBuilder data = new StringBuilder();
+		for (int i = 0; i < 1000; ++i) {
+			data.append("row1\r\nrow2");
+		}
+		String crData = data.toString();
+		PrintWriter writer = new PrintWriter(file);
+		writer.print(crData);
+		writer.close();
+		String lfData = data.toString().replaceAll("\r", "");
+		Git git = new Git(db);
+		db.getConfig().setString("core", null, "autocrlf", "false");
+		git.add().addFilepattern("a.txt").call();
+		assertEquals("[a.txt, mode:100644, content:" + data + "]",
+				indexState(CONTENT));
+		db.getConfig().setString("core", null, "autocrlf", "true");
+		git.add().addFilepattern("a.txt").call();
+		assertEquals("[a.txt, mode:100644, content:" + lfData + "]",
+				indexState(CONTENT));
+		db.getConfig().setString("core", null, "autocrlf", "input");
+		git.add().addFilepattern("a.txt").call();
+		assertEquals("[a.txt, mode:100644, content:" + lfData + "]",
+				indexState(CONTENT));
+	}
+
 	@Test
 	public void testAddExistingSingleBinaryFile() throws IOException,
 			NoFilepatternException {
@@ -658,40 +684,6 @@ public class AddCommandTest extends RepositoryTestCase {
 		assertEquals(FileMode.EXECUTABLE_FILE, walk.getFileMode(0));
 	}
 
-	@Test
-	public void testSubmoduleDeleteNotStagedWithUpdate() throws Exception {
-		Git git = new Git(db);
-		writeTrashFile("file.txt", "content");
-		git.add().addFilepattern("file.txt").call();
-		assertNotNull(git.commit().setMessage("create file").call());
-
-		SubmoduleAddCommand command = new SubmoduleAddCommand(db);
-		String path = "sub";
-		command.setPath(path);
-		String uri = db.getDirectory().toURI().toString();
-		command.setURI(uri);
-		Repository repo = command.call();
-		assertNotNull(repo);
-		assertNotNull(git.commit().setMessage("add submodule").call());
-
-		assertTrue(git.status().call().isClean());
-
-		FileUtils.delete(repo.getWorkTree(), FileUtils.RECURSIVE);
-		FileUtils.mkdir(new File(db.getWorkTree(), path), false);
-
-		assertNotNull(git.add().addFilepattern(".").setUpdate(true).call());
-
-		Status status = git.status().call();
-		assertFalse(status.isClean());
-		assertTrue(status.getAdded().isEmpty());
-		assertTrue(status.getChanged().isEmpty());
-		assertTrue(status.getRemoved().isEmpty());
-		assertTrue(status.getUntracked().isEmpty());
-		assertTrue(status.getModified().isEmpty());
-		assertEquals(1, status.getMissing().size());
-		assertEquals(path, status.getMissing().iterator().next());
-	}
-
 	private DirCacheEntry addEntryToBuilder(String path, File file,
 			ObjectInserter newObjectInserter, DirCacheBuilder builder, int stage)
 			throws IOException {

org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java

@@ -176,10 +176,12 @@ public class AddCommand extends GitCommand<DirCache> {
 									entry.setLength(sz);
 									entry.setLastModified(f
 											.getEntryLastModified());
+									long contentSize = f
+											.getEntryContentLength();
 									InputStream in = f.openEntryStream();
 									try {
 										entry.setObjectId(inserter.insert(
-												Constants.OBJ_BLOB, sz, in));
+												Constants.OBJ_BLOB, contentSize, in));
 									} finally {
 										in.close();
 									}

org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java

@@ -357,11 +357,11 @@ public class CommitCommand extends GitCommand<RevCommit> {
 							// insert object
 							if (inserter == null)
 								inserter = repo.newObjectInserter();
-
+							long contentLength = fTree.getEntryContentLength();
 							InputStream inputStream = fTree.openEntryStream();
 							try {
 								dcEntry.setObjectId(inserter.insert(
-										Constants.OBJ_BLOB, entryLength,
+										Constants.OBJ_BLOB, contentLength,
 										inputStream));
 							} finally {
 								inputStream.close();

org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java

@@ -254,11 +254,11 @@ public class StashCreateCommand extends GitCommand<RevCommit> {
 						entry.setLength(wtIter.getEntryLength());
 						entry.setLastModified(wtIter.getEntryLastModified());
 						entry.setFileMode(wtIter.getEntryFileMode());
+						long contentLength = wtIter.getEntryContentLength();
 						InputStream in = wtIter.openEntryStream();
 						try {
 							entry.setObjectId(inserter.insert(
-									Constants.OBJ_BLOB,
+									Constants.OBJ_BLOB, contentLength, in));
-									wtIter.getEntryLength(), in));
 						} finally {
 							in.close();
 						}

org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java

@@ -184,9 +184,10 @@ public abstract class ContentSource {
 				@Override
 				public ObjectStream openStream() throws MissingObjectException,
 						IOException {
+					long contentLength = ptr.getEntryContentLength();
 					InputStream in = ptr.openEntryStream();
 					in = new BufferedInputStream(in);
-					return new ObjectStream.Filter(getType(), getSize(), in);
+					return new ObjectStream.Filter(getType(), contentLength, in);
 				}
 
 				@Override

org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java

@@ -72,7 +72,6 @@ import org.eclipse.jgit.ignore.IgnoreRule;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.CoreConfig;
-import org.eclipse.jgit.lib.CoreConfig.AutoCRLF;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.Repository;
@@ -129,6 +128,9 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	/** Repository that is the root level being iterated over */
 	protected Repository repository;
 
+	/** Cached canonical length, initialized from {@link #idBuffer()} */
+	private long canonLen = -1;
+
 	/**
 	 * Create a new iterator with no parent.
 	 *
@@ -320,33 +322,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 				state.initializeDigestAndReadBuffer();
 
 				final long len = e.getLength();
-				if (!mightNeedCleaning())
+				InputStream filteredIs = possiblyFilteredInputStream(e, is, len);
-					return computeHash(is, len);
+				return computeHash(filteredIs, canonLen);
-
-				if (len <= MAXIMUM_FILE_SIZE_TO_READ_FULLY) {
-					ByteBuffer rawbuf = IO.readWholeStream(is, (int) len);
-					byte[] raw = rawbuf.array();
-					int n = rawbuf.limit();
-					if (!isBinary(raw, n)) {
-						rawbuf = filterClean(raw, n);
-						raw = rawbuf.array();
-						n = rawbuf.limit();
-					}
-					return computeHash(new ByteArrayInputStream(raw, 0, n), n);
-				}
-
-				if (isBinary(e))
-					return computeHash(is, len);
-
-				final long canonLen;
-				final InputStream lenIs = filterClean(e.openInputStream());
-				try {
-					canonLen = computeLength(lenIs);
-				} finally {
-					safeClose(lenIs);
-				}
-
-				return computeHash(filterClean(is), canonLen);
 			} finally {
 				safeClose(is);
 			}
@@ -356,6 +333,43 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 		}
 	}
 
+	private InputStream possiblyFilteredInputStream(final Entry e,
+			final InputStream is, final long len) throws IOException {
+		InputStream filteredIs;
+		if (!mightNeedCleaning()) {
+			filteredIs = is;
+			canonLen = len;
+		} else {
+			if (len <= MAXIMUM_FILE_SIZE_TO_READ_FULLY) {
+				ByteBuffer rawbuf = IO.readWholeStream(is, (int) len);
+				byte[] raw = rawbuf.array();
+				int n = rawbuf.limit();
+				if (!isBinary(raw, n)) {
+					rawbuf = filterClean(raw, n);
+					raw = rawbuf.array();
+					n = rawbuf.limit();
+				}
+				filteredIs = new ByteArrayInputStream(raw, 0, n);
+				canonLen = n;
+			} else {
+				if (isBinary(e)) {
+					filteredIs = is;
+					canonLen = len;
+				} else {
+					final InputStream lenIs = filterClean(e
+							.openInputStream());
+					try {
+						canonLen = computeLength(lenIs);
+					} finally {
+						safeClose(lenIs);
+					}
+					filteredIs = filterClean(is);
+				}
+			}
+		}
+		return filteredIs;
+	}
+
 	private static void safeClose(final InputStream in) {
 		try {
 			in.close();
@@ -441,8 +455,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	@Override
 	public void next(final int delta) throws CorruptObjectException {
 		ptr += delta;
-		if (!eof())
+		if (!eof()) {
+			canonLen = -1;
 			parseEntry();
+		}
 	}
 
 	@Override
@@ -462,7 +478,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	}
 
 	/**
-	 * Get the byte length of this entry.
+	 * Get the raw byte length of this entry.
 	 *
 	 * @return size of this file, in bytes.
 	 */
@@ -470,6 +486,29 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 		return current().getLength();
 	}
 
+	/**
+	 * Get the filtered input length of this entry
+	 *
+	 * @return size of the content, in bytes
+	 * @throws IOException
+	 */
+	public long getEntryContentLength() throws IOException {
+		if (canonLen == -1) {
+			long rawLen = getEntryLength();
+			if (rawLen == 0)
+				canonLen = 0;
+			InputStream is = current().openInputStream();
+			try {
+				// canonLen gets updated here
+				possiblyFilteredInputStream(current(), is, current()
+						.getLength());
+			} finally {
+				safeClose(is);
+			}
+		}
+		return canonLen;
+	}
+
 	/**
 	 * Get the last modified time of this entry.
 	 *
@@ -498,12 +537,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	 */
 	public InputStream openEntryStream() throws IOException {
 		InputStream rawis = current().openInputStream();
-		InputStream is;
+		if (mightNeedCleaning())
-		if (getOptions().getAutoCRLF() != AutoCRLF.FALSE)
+			return filterClean(rawis);
-			is = new EolCanonicalizingInputStream(rawis, true);
 		else
-			is = rawis;
+			return rawis;
-		return is;
 	}
 
 	/**