From e3f535cb15a75d6b2448bfadbc9839da8085f1ec Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Thu, 12 Sep 2019 21:05:19 +0200 Subject: [PATCH] WorkingTreeIterator: handle different timestamp resolutions Older JGit stored only milliseconds timestamps in the index. Newer JGit may get finer timestamps from the file system. This leads to slow index diffs when a new JGit runs against an index produced by older JGit because many timestamps will differ and JGit will then do many content checks. See [1]. Handle this migration case by only comparing milliseconds if the index entry has only millisecond precision. The inverse may also occur; also compare only milliseconds if the file timestamp has only millisecond precision. Do the same also for microsecond resolution. On Windows, NTFS may provide 100ns resolution and may be used by external programs writing the index, but Java's WindowsFileAttributes may provide only microseconds. File timestamp precision in Java depends not only on the Java APIs used by different JGit versions but may also change when running the same Java code on different VMs. And of course the resolution may vary among operating and file systems. Moreover, timestamp precision in the index depends on the program that wrote the index. Canonical git may use a different resolution, maybe even different between git versions. [1] https://www.eclipse.org/forums/index.php/t/1100344/ Change-Id: Idfd08606c883cb98787b2138f9baf0cc89a57b56 Signed-off-by: Thomas Wolf Signed-off-by: Matthias Sohn --- .../jgit/treewalk/InstantComparatorTest.java | 182 ++++++++++++++++++ .../jgit/treewalk/InstantComparator.java | 112 +++++++++++ .../jgit/treewalk/WorkingTreeIterator.java | 31 ++- 3 files changed, 307 insertions(+), 18 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/InstantComparatorTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/treewalk/InstantComparator.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/InstantComparatorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/InstantComparatorTest.java new file mode 100644 index 000000000..271b4ea3c --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/InstantComparatorTest.java @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2019, Thomas Wolf + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.treewalk; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.time.Instant; + +import org.junit.Test; + +public class InstantComparatorTest { + + private final InstantComparator cmp = new InstantComparator(); + + @Test + public void compareNow() { + Instant now = Instant.now(); + assertEquals(0, cmp.compare(now, now)); + assertEquals(0, cmp.compare(now, now, true)); + } + + @Test + public void compareSeconds() { + Instant now = Instant.now(); + Instant t = Instant.ofEpochSecond(now.getEpochSecond()); + Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456789); + assertEquals(0, cmp.compare(t, s)); + assertEquals(0, cmp.compare(t, t)); + assertEquals(0, cmp.compare(s, t)); + } + + @Test + public void compareSecondsOnly() { + Instant now = Instant.now(); + Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 987654321); + Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456789); + assertEquals(0, cmp.compare(t, s, true)); + assertEquals(0, cmp.compare(t, t, true)); + assertEquals(0, cmp.compare(s, t, true)); + } + + @Test + public void compareSecondsUnequal() { + Instant now = Instant.now(); + Instant t = Instant.ofEpochSecond(now.getEpochSecond()); + Instant s = Instant.ofEpochSecond(now.getEpochSecond() - 1L); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + } + + @Test + public void compareMillisEqual() { + Instant now = Instant.now(); + Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123000000); + Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456789); + assertEquals(0, cmp.compare(s, t)); + assertEquals(0, cmp.compare(t, t)); + assertEquals(0, cmp.compare(t, s)); + s = Instant.ofEpochSecond(now.getEpochSecond(), 123456000); + assertEquals(0, cmp.compare(s, t)); + assertEquals(0, cmp.compare(t, s)); + s = Instant.ofEpochSecond(now.getEpochSecond(), 123400000); + assertEquals(0, cmp.compare(s, t)); + assertEquals(0, cmp.compare(t, s)); + } + + @Test + public void compareMillisUnequal() { + Instant now = Instant.now(); + Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123000000); + Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 122000000); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + t = Instant.ofEpochSecond(now.getEpochSecond(), 130000000); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + t = Instant.ofEpochSecond(now.getEpochSecond(), 200000000); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + s = Instant.ofEpochSecond(now.getEpochSecond() - 1L, 123000000); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + } + + @Test + public void compareMicrosEqual() { + Instant now = Instant.now(); + Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123456000); + Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456789); + assertEquals(0, cmp.compare(s, t)); + assertEquals(0, cmp.compare(t, s)); + s = Instant.ofEpochSecond(now.getEpochSecond(), 123456700); + assertEquals(0, cmp.compare(s, t)); + assertEquals(0, cmp.compare(t, s)); + } + + @Test + public void compareMicrosUnequal() { + Instant now = Instant.now(); + Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123456000); + Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123455000); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + t = Instant.ofEpochSecond(now.getEpochSecond(), 123460000); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + t = Instant.ofEpochSecond(now.getEpochSecond(), 123500000); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + s = Instant.ofEpochSecond(now.getEpochSecond() - 1L, 123456000); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + } + + @Test + public void compareNanosEqual() { + Instant now = Instant.now(); + Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123456789); + Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456789); + assertEquals(0, cmp.compare(s, t)); + assertEquals(0, cmp.compare(t, s)); + } + + @Test + public void compareNanosUnequal() { + Instant now = Instant.now(); + Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123456789); + Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456700); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + t = Instant.ofEpochSecond(now.getEpochSecond(), 123456800); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + s = Instant.ofEpochSecond(now.getEpochSecond() - 1L, 123456789); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + s = Instant.ofEpochSecond(now.getEpochSecond(), 123456788); + assertTrue(cmp.compare(s, t) < 0); + assertTrue(cmp.compare(t, s) > 0); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/InstantComparator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/InstantComparator.java new file mode 100644 index 000000000..ffcb4b46e --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/InstantComparator.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2019, Thomas Wolf + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.treewalk; + +import java.time.Instant; +import java.util.Comparator; + +/** + * Specialized comparator for {@link Instant}s. If either timestamp has a zero + * fraction, compares only seconds. If either timestamp has no time fraction + * smaller than a millisecond, compares only milliseconds. If either timestamp + * has no fraction smaller than a microsecond, compares only microseconds. + */ +class InstantComparator implements Comparator { + + @Override + public int compare(Instant a, Instant b) { + return compare(a, b, false); + } + + /** + * Compares two {@link Instant}s to the lower resolution of the two + * instants. See {@link InstantComparator}. + * + * @param a + * first {@link Instant} to compare + * @param b + * second {@link Instant} to compare + * @param forceSecondsOnly + * whether to omit all fraction comparison + * @return a value < 0 if a < b, a value > 0 if a > b, and 0 if + * a == b + */ + public int compare(Instant a, Instant b, boolean forceSecondsOnly) { + long aSeconds = a.getEpochSecond(); + long bSeconds = b.getEpochSecond(); + int result = Long.compare(aSeconds, bSeconds); + if (result != 0) { + return result; + } + int aSubSecond = a.getNano(); + int bSubSecond = b.getNano(); + if (forceSecondsOnly || (aSubSecond == 0) + || (bSubSecond == 0)) { + // Don't check the subseconds part. + return 0; + } else if (aSubSecond != bSubSecond) { + // If either has nothing smaller than a millisecond, compare only + // milliseconds. + int aSubMillis = aSubSecond % 1_000_000; + int bSubMillis = bSubSecond % 1_000_000; + if (aSubMillis == 0) { + bSubSecond -= bSubMillis; + } else if (bSubMillis == 0) { + aSubSecond -= aSubMillis; + } else { + // Same again, but for microsecond resolution. NTFS has 100ns + // resolution, but WindowsFileAttributes may provide only + // microseconds (1000ns). Similar for some Unix file systems. + int aSubMicros = aSubSecond % 1000; + int bSubMicros = bSubSecond % 1000; + if (aSubMicros == 0) { + bSubSecond -= bSubMicros; + } else if (bSubMicros == 0) { + aSubSecond -= aSubMicros; + } + } + } + return Integer.compare(aSubSecond, bSubSecond); + } + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index a83165daa..83be8f6e0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -174,6 +174,9 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { /** The offset of the content id in {@link #idBuffer()} */ private int contentIdOffset; + /** A comparator for {@link Instant}s. */ + private final InstantComparator timestampComparator = new InstantComparator(); + /** * Create a new iterator with no parent. * @@ -935,27 +938,19 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { if (!entry.isSmudged() && entry.getLength() != (int) getEntryLength()) return MetadataDiff.DIFFER_BY_METADATA; - // Git under windows only stores seconds so we round the timestamp - // Java gives us if it looks like the timestamp in index is seconds - // only. Otherwise we compare the timestamp at nanosecond precision, - // unless core.checkstat is set to "minimal", in which case we only - // compare the whole second part. + // Cache and file timestamps may differ in resolution. Therefore don't + // compare instants directly but use a comparator that compares only + // up to the lower apparent resolution of either timestamp. + // + // If core.checkstat is set to "minimal", compare only the seconds part. Instant cacheLastModified = entry.getLastModifiedInstant(); Instant fileLastModified = getEntryLastModifiedInstant(); - if ((getOptions().getCheckStat() == CheckStat.MINIMAL) - || (cacheLastModified.getNano() == 0) - // Some Java version on Linux return whole seconds only even - // when the file systems supports more precision. - || (fileLastModified.getNano() == 0)) { - if (fileLastModified.getEpochSecond() != cacheLastModified - .getEpochSecond()) { - return MetadataDiff.DIFFER_BY_TIMESTAMP; - } else if (entry.isSmudged()) { - return MetadataDiff.SMUDGED; - } - } else if (!fileLastModified.equals(cacheLastModified)) { + if (timestampComparator.compare(cacheLastModified, fileLastModified, + getOptions().getCheckStat() == CheckStat.MINIMAL) != 0) { return MetadataDiff.DIFFER_BY_TIMESTAMP; - } else if (entry.isSmudged()) { + } + + if (entry.isSmudged()) { return MetadataDiff.SMUDGED; } // The file is clean when when comparing timestamps