From 3f4725c179c176560937d756682fcd6cfbf685fe Mon Sep 17 00:00:00 2001 From: Robin Rosenberg Date: Sun, 25 Mar 2012 14:42:54 +0200 Subject: [PATCH] Handle content length in WorkingTreeIterator Content length is computed and cached (short term) in the working tree iterator when core.autocrlf is set. Hopefully this is a cleaner fix than my previous attempt to make autocrlf work. Change-Id: I1b6bbb643101a00db94e5514b5e2b069f338907a --- .../org/eclipse/jgit/api/AddCommandTest.java | 68 +++++------ .../src/org/eclipse/jgit/api/AddCommand.java | 4 +- .../org/eclipse/jgit/api/CommitCommand.java | 4 +- .../eclipse/jgit/api/StashCreateCommand.java | 4 +- .../org/eclipse/jgit/diff/ContentSource.java | 3 +- .../jgit/treewalk/WorkingTreeIterator.java | 107 ++++++++++++------ 6 files changed, 111 insertions(+), 79 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java index d989b63b2..2fb228e01 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java @@ -44,9 +44,7 @@ package org.eclipse.jgit.api; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; @@ -63,7 +61,6 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryTestCase; import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevCommit; @@ -114,7 +111,7 @@ public class AddCommandTest extends RepositoryTestCase { } @Test - public void testAddExistingSingleFileWithNewLine() throws IOException, + public void testAddExistingSingleSmallFileWithNewLine() throws IOException, NoFilepatternException { File file = new File(db.getWorkTree(), "a.txt"); FileUtils.createNewFile(file); @@ -137,6 +134,35 @@ public class AddCommandTest extends RepositoryTestCase { indexState(CONTENT)); } + @Test + public void testAddExistingSingleMediumSizeFileWithNewLine() + throws IOException, NoFilepatternException { + File file = new File(db.getWorkTree(), "a.txt"); + FileUtils.createNewFile(file); + StringBuilder data = new StringBuilder(); + for (int i = 0; i < 1000; ++i) { + data.append("row1\r\nrow2"); + } + String crData = data.toString(); + PrintWriter writer = new PrintWriter(file); + writer.print(crData); + writer.close(); + String lfData = data.toString().replaceAll("\r", ""); + Git git = new Git(db); + db.getConfig().setString("core", null, "autocrlf", "false"); + git.add().addFilepattern("a.txt").call(); + assertEquals("[a.txt, mode:100644, content:" + data + "]", + indexState(CONTENT)); + db.getConfig().setString("core", null, "autocrlf", "true"); + git.add().addFilepattern("a.txt").call(); + assertEquals("[a.txt, mode:100644, content:" + lfData + "]", + indexState(CONTENT)); + db.getConfig().setString("core", null, "autocrlf", "input"); + git.add().addFilepattern("a.txt").call(); + assertEquals("[a.txt, mode:100644, content:" + lfData + "]", + indexState(CONTENT)); + } + @Test public void testAddExistingSingleBinaryFile() throws IOException, NoFilepatternException { @@ -658,40 +684,6 @@ public class AddCommandTest extends RepositoryTestCase { assertEquals(FileMode.EXECUTABLE_FILE, walk.getFileMode(0)); } - @Test - public void testSubmoduleDeleteNotStagedWithUpdate() throws Exception { - Git git = new Git(db); - writeTrashFile("file.txt", "content"); - git.add().addFilepattern("file.txt").call(); - assertNotNull(git.commit().setMessage("create file").call()); - - SubmoduleAddCommand command = new SubmoduleAddCommand(db); - String path = "sub"; - command.setPath(path); - String uri = db.getDirectory().toURI().toString(); - command.setURI(uri); - Repository repo = command.call(); - assertNotNull(repo); - assertNotNull(git.commit().setMessage("add submodule").call()); - - assertTrue(git.status().call().isClean()); - - FileUtils.delete(repo.getWorkTree(), FileUtils.RECURSIVE); - FileUtils.mkdir(new File(db.getWorkTree(), path), false); - - assertNotNull(git.add().addFilepattern(".").setUpdate(true).call()); - - Status status = git.status().call(); - assertFalse(status.isClean()); - assertTrue(status.getAdded().isEmpty()); - assertTrue(status.getChanged().isEmpty()); - assertTrue(status.getRemoved().isEmpty()); - assertTrue(status.getUntracked().isEmpty()); - assertTrue(status.getModified().isEmpty()); - assertEquals(1, status.getMissing().size()); - assertEquals(path, status.getMissing().iterator().next()); - } - private DirCacheEntry addEntryToBuilder(String path, File file, ObjectInserter newObjectInserter, DirCacheBuilder builder, int stage) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java index 206d0876a..3da9640b4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java @@ -176,10 +176,12 @@ public class AddCommand extends GitCommand { entry.setLength(sz); entry.setLastModified(f .getEntryLastModified()); + long contentSize = f + .getEntryContentLength(); InputStream in = f.openEntryStream(); try { entry.setObjectId(inserter.insert( - Constants.OBJ_BLOB, sz, in)); + Constants.OBJ_BLOB, contentSize, in)); } finally { in.close(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java index 87edaddc6..242d11efa 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java @@ -357,11 +357,11 @@ public class CommitCommand extends GitCommand { // insert object if (inserter == null) inserter = repo.newObjectInserter(); - + long contentLength = fTree.getEntryContentLength(); InputStream inputStream = fTree.openEntryStream(); try { dcEntry.setObjectId(inserter.insert( - Constants.OBJ_BLOB, entryLength, + Constants.OBJ_BLOB, contentLength, inputStream)); } finally { inputStream.close(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java index 07d74b266..104c22f2a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java @@ -254,11 +254,11 @@ public class StashCreateCommand extends GitCommand { entry.setLength(wtIter.getEntryLength()); entry.setLastModified(wtIter.getEntryLastModified()); entry.setFileMode(wtIter.getEntryFileMode()); + long contentLength = wtIter.getEntryContentLength(); InputStream in = wtIter.openEntryStream(); try { entry.setObjectId(inserter.insert( - Constants.OBJ_BLOB, - wtIter.getEntryLength(), in)); + Constants.OBJ_BLOB, contentLength, in)); } finally { in.close(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java index 6fdab6bf8..6e28f854a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java @@ -184,9 +184,10 @@ public abstract class ContentSource { @Override public ObjectStream openStream() throws MissingObjectException, IOException { + long contentLength = ptr.getEntryContentLength(); InputStream in = ptr.openEntryStream(); in = new BufferedInputStream(in); - return new ObjectStream.Filter(getType(), getSize(), in); + return new ObjectStream.Filter(getType(), contentLength, in); } @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index ac92eb5c2..955d7d857 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -72,7 +72,6 @@ import org.eclipse.jgit.ignore.IgnoreRule; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; -import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Repository; @@ -129,6 +128,9 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { /** Repository that is the root level being iterated over */ protected Repository repository; + /** Cached canonical length, initialized from {@link #idBuffer()} */ + private long canonLen = -1; + /** * Create a new iterator with no parent. * @@ -320,33 +322,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { state.initializeDigestAndReadBuffer(); final long len = e.getLength(); - if (!mightNeedCleaning()) - return computeHash(is, len); - - if (len <= MAXIMUM_FILE_SIZE_TO_READ_FULLY) { - ByteBuffer rawbuf = IO.readWholeStream(is, (int) len); - byte[] raw = rawbuf.array(); - int n = rawbuf.limit(); - if (!isBinary(raw, n)) { - rawbuf = filterClean(raw, n); - raw = rawbuf.array(); - n = rawbuf.limit(); - } - return computeHash(new ByteArrayInputStream(raw, 0, n), n); - } - - if (isBinary(e)) - return computeHash(is, len); - - final long canonLen; - final InputStream lenIs = filterClean(e.openInputStream()); - try { - canonLen = computeLength(lenIs); - } finally { - safeClose(lenIs); - } - - return computeHash(filterClean(is), canonLen); + InputStream filteredIs = possiblyFilteredInputStream(e, is, len); + return computeHash(filteredIs, canonLen); } finally { safeClose(is); } @@ -356,6 +333,43 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { } } + private InputStream possiblyFilteredInputStream(final Entry e, + final InputStream is, final long len) throws IOException { + InputStream filteredIs; + if (!mightNeedCleaning()) { + filteredIs = is; + canonLen = len; + } else { + if (len <= MAXIMUM_FILE_SIZE_TO_READ_FULLY) { + ByteBuffer rawbuf = IO.readWholeStream(is, (int) len); + byte[] raw = rawbuf.array(); + int n = rawbuf.limit(); + if (!isBinary(raw, n)) { + rawbuf = filterClean(raw, n); + raw = rawbuf.array(); + n = rawbuf.limit(); + } + filteredIs = new ByteArrayInputStream(raw, 0, n); + canonLen = n; + } else { + if (isBinary(e)) { + filteredIs = is; + canonLen = len; + } else { + final InputStream lenIs = filterClean(e + .openInputStream()); + try { + canonLen = computeLength(lenIs); + } finally { + safeClose(lenIs); + } + filteredIs = filterClean(is); + } + } + } + return filteredIs; + } + private static void safeClose(final InputStream in) { try { in.close(); @@ -441,8 +455,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { @Override public void next(final int delta) throws CorruptObjectException { ptr += delta; - if (!eof()) + if (!eof()) { + canonLen = -1; parseEntry(); + } } @Override @@ -462,7 +478,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { } /** - * Get the byte length of this entry. + * Get the raw byte length of this entry. * * @return size of this file, in bytes. */ @@ -470,6 +486,29 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { return current().getLength(); } + /** + * Get the filtered input length of this entry + * + * @return size of the content, in bytes + * @throws IOException + */ + public long getEntryContentLength() throws IOException { + if (canonLen == -1) { + long rawLen = getEntryLength(); + if (rawLen == 0) + canonLen = 0; + InputStream is = current().openInputStream(); + try { + // canonLen gets updated here + possiblyFilteredInputStream(current(), is, current() + .getLength()); + } finally { + safeClose(is); + } + } + return canonLen; + } + /** * Get the last modified time of this entry. * @@ -498,12 +537,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { */ public InputStream openEntryStream() throws IOException { InputStream rawis = current().openInputStream(); - InputStream is; - if (getOptions().getAutoCRLF() != AutoCRLF.FALSE) - is = new EolCanonicalizingInputStream(rawis, true); + if (mightNeedCleaning()) + return filterClean(rawis); else - is = rawis; - return is; + return rawis; } /**