diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 1dea47084..9eb55db09 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -42,9 +42,13 @@ */ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.junit.JGitTestUtil.read; +import static org.eclipse.jgit.junit.JGitTestUtil.write; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; @@ -54,17 +58,23 @@ import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileTime; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; +import org.eclipse.jgit.util.Stats; import org.eclipse.jgit.util.SystemReader; import org.junit.After; import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FileSnapshotTest { + private static final Logger LOG = LoggerFactory + .getLogger(FileSnapshotTest.class); private Path trash; @@ -205,6 +215,48 @@ public class FileSnapshotTest { assertTrue(fs2.equals(fs1)); } + @SuppressWarnings("boxing") + @Test + public void detectFileModified() throws IOException { + int failures = 0; + long racyNanos = 0; + final int COUNT = 10000; + ArrayList deltas = new ArrayList<>(); + File f = createFile("test").toFile(); + for (int i = 0; i < COUNT; i++) { + write(f, "a"); + FileSnapshot snapshot = FileSnapshot.save(f); + assertEquals("file should contain 'a'", "a", read(f)); + write(f, "b"); + if (!snapshot.isModified(f)) { + deltas.add(snapshot.lastDelta()); + racyNanos = snapshot.lastRacyNanos(); + failures++; + } + assertEquals("file should contain 'b'", "b", read(f)); + } + if (failures > 0) { + Stats stats = new Stats(); + LOG.debug( + "delta [ns] since modification FileSnapshot failed to detect"); + for (Long d : deltas) { + stats.add(d); + LOG.debug(String.format("%,d", d)); + } + LOG.error( + "count, failures, racy limit [ns], delta min [ns]," + + " delta max [ns], delta avg [ns]," + + " delta stddev [ns]"); + LOG.error(String.format( + "%,d, %,d, %,d, %,.0f, %,.0f, %,.0f, %,.0f", COUNT, + failures, racyNanos, stats.min(), stats.max(), + stats.avg(), stats.stddev())); + } + assertTrue( + "FileSnapshot: number of failures to detect file modifications should be 0", + failures == 0); + } + private Path createFile(String string) throws IOException { Files.createDirectories(trash); return Files.createTempFile(trash, string, "tdat"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StatsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StatsTest.java new file mode 100644 index 000000000..8b253828c --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StatsTest.java @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2019, Matthias Sohn + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.eclipse.jgit.util.Stats; +import org.junit.Test; + +public class StatsTest { + @Test + public void testStatsTrivial() { + Stats s = new Stats(); + s.add(1); + s.add(1); + s.add(1); + assertEquals(3, s.count()); + assertEquals(1.0, s.min(), 1E-6); + assertEquals(1.0, s.max(), 1E-6); + assertEquals(1.0, s.avg(), 1E-6); + assertEquals(0.0, s.var(), 1E-6); + assertEquals(0.0, s.stddev(), 1E-6); + } + + @Test + public void testStats() { + Stats s = new Stats(); + s.add(1); + s.add(2); + s.add(3); + s.add(4); + assertEquals(4, s.count()); + assertEquals(1.0, s.min(), 1E-6); + assertEquals(4.0, s.max(), 1E-6); + assertEquals(2.5, s.avg(), 1E-6); + assertEquals(1.666667, s.var(), 1E-6); + assertEquals(1.290994, s.stddev(), 1E-6); + } + + @Test + /** + * see + * https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Example + */ + public void testStatsCancellationExample1() { + Stats s = new Stats(); + s.add(1E8 + 4); + s.add(1E8 + 7); + s.add(1E8 + 13); + s.add(1E8 + 16); + assertEquals(4, s.count()); + assertEquals(1E8 + 4, s.min(), 1E-6); + assertEquals(1E8 + 16, s.max(), 1E-6); + assertEquals(1E8 + 10, s.avg(), 1E-6); + assertEquals(30, s.var(), 1E-6); + assertEquals(5.477226, s.stddev(), 1E-6); + } + + @Test + /** + * see + * https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Example + */ + public void testStatsCancellationExample2() { + Stats s = new Stats(); + s.add(1E9 + 4); + s.add(1E9 + 7); + s.add(1E9 + 13); + s.add(1E9 + 16); + assertEquals(4, s.count()); + assertEquals(1E9 + 4, s.min(), 1E-6); + assertEquals(1E9 + 16, s.max(), 1E-6); + assertEquals(1E9 + 10, s.avg(), 1E-6); + assertEquals(30, s.var(), 1E-6); + assertEquals(5.477226, s.stddev(), 1E-6); + } + + @Test + public void testNoValues() { + Stats s = new Stats(); + assertTrue(Double.isNaN(s.var())); + assertTrue(Double.isNaN(s.stddev())); + assertTrue(Double.isNaN(s.avg())); + assertTrue(Double.isNaN(s.min())); + assertTrue(Double.isNaN(s.max())); + s.add(42.3); + assertTrue(Double.isNaN(s.var())); + assertTrue(Double.isNaN(s.stddev())); + assertEquals(42.3, s.avg(), 1E-6); + assertEquals(42.3, s.max(), 1E-6); + assertEquals(42.3, s.min(), 1E-6); + s.add(42.3); + assertEquals(0, s.var(), 1E-6); + assertEquals(0, s.stddev(), 1E-6); + } +} diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 01215a8c7..59bafc52e 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -226,4 +226,12 @@ + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index ec71783c6..2a490a4a1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -283,6 +283,10 @@ public class FileSnapshot { private boolean wasRacyClean; + private long delta; + + private long racyNanos; + private FileSnapshot(Instant read, Instant modified, long size, @NonNull Duration fsTimestampResolution, @NonNull Object fileKey) { this.file = null; @@ -466,6 +470,21 @@ public class FileSnapshot { return wasRacyClean; } + /** + * @return the delta in nanoseconds between lastModified and lastRead during + * last racy check + */ + long lastDelta() { + return delta; + } + + /** + * @return the racyNanos threshold in nanoseconds during last racy check + */ + long lastRacyNanos() { + return racyNanos; + } + /** {@inheritDoc} */ @SuppressWarnings("nls") @Override @@ -483,8 +502,8 @@ public class FileSnapshot { private boolean isRacyClean(Instant read) { // add a 10% safety margin - long racyNanos = (fsTimestampResolution.toNanos() + 1) * 11 / 10; - long delta = Duration.between(lastModified, read).toNanos(); + racyNanos = (fsTimestampResolution.toNanos() + 1) * 11 / 10; + delta = Duration.between(lastModified, read).toNanos(); wasRacyClean = delta <= racyNanos; if (LOG.isDebugEnabled()) { LOG.debug( diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java new file mode 100644 index 000000000..e9307d342 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2019, Matthias Sohn + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.util; + +/** + * Simple double statistics, computed incrementally, variance and standard + * deviation using Welford's online algorithm, see + * https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + * + * @since 5.1.9 + */ +public class Stats { + private int n = 0; + + private double avg = 0.0; + + private double min = 0.0; + + private double max = 0.0; + + private double sum = 0.0; + + /** + * Add a value + * + * @param x + * value + */ + public void add(double x) { + n++; + min = n == 1 ? x : Math.min(min, x); + max = n == 1 ? x : Math.max(max, x); + double d = x - avg; + avg += d / n; + sum += d * d * (n - 1) / n; + } + + /** + * @return number of the added values + */ + public int count() { + return n; + } + + /** + * @return minimum of the added values + */ + public double min() { + if (n < 1) { + return Double.NaN; + } + return min; + } + + /** + * @return maximum of the added values + */ + public double max() { + if (n < 1) { + return Double.NaN; + } + return max; + } + + /** + * @return average of the added values + */ + + public double avg() { + if (n < 1) { + return Double.NaN; + } + return avg; + } + + /** + * @return variance of the added values + */ + public double var() { + if (n < 2) { + return Double.NaN; + } + return sum / (n - 1); + } + + /** + * @return standard deviation of the added values + */ + public double stddev() { + return Math.sqrt(this.var()); + } +}